Commit fffcbae8 authored by can eldem's avatar can eldem Committed by mo khan

Replace undefined confidence with unknown severity for vulnerabilities

Add index for large set of data
Add test for testing data
parent 340e78b8
# frozen_string_literal: true
class UpdateUndefinedConfidenceFromVulnerabilities < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
INDEX_NAME = 'index_vulnerability_on_id_and_confidence_eq_zero'
DOWNTIME = false
disable_ddl_transaction!
BATCH_SIZE = 1_000
INTERVAL = 2.minutes
# 87_602 records to be updated on GitLab.com
def up
# create temporary index for undefined vulnerabilities
add_concurrent_index(:vulnerabilities, :id, where: 'confidence = 0', name: INDEX_NAME)
return unless Gitlab.ee?
migration = Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel
migration_name = migration.to_s.demodulize
relation = migration::Vulnerability.undefined_confidence
queue_background_migration_jobs_by_range_at_intervals(relation,
migration_name,
INTERVAL,
batch_size: BATCH_SIZE)
end
def down
# no-op
# temporary index is to be dropped in a different migration in an upcoming release
remove_concurrent_index(:vulnerabilities, :id, where: 'confidence = 0', name: INDEX_NAME)
# This migration can not be reversed because we can not know which records had undefined confidence
end
end
...@@ -10843,6 +10843,8 @@ CREATE UNIQUE INDEX index_vulnerability_occurrences_on_uuid ON public.vulnerabil ...@@ -10843,6 +10843,8 @@ CREATE UNIQUE INDEX index_vulnerability_occurrences_on_uuid ON public.vulnerabil
CREATE INDEX index_vulnerability_occurrences_on_vulnerability_id ON public.vulnerability_occurrences USING btree (vulnerability_id); CREATE INDEX index_vulnerability_occurrences_on_vulnerability_id ON public.vulnerability_occurrences USING btree (vulnerability_id);
CREATE INDEX index_vulnerability_on_id_and_confidence_eq_zero ON public.vulnerabilities USING btree (id) WHERE (confidence = 0);
CREATE UNIQUE INDEX index_vulnerability_scanners_on_project_id_and_external_id ON public.vulnerability_scanners USING btree (project_id, external_id); CREATE UNIQUE INDEX index_vulnerability_scanners_on_project_id_and_external_id ON public.vulnerability_scanners USING btree (project_id, external_id);
CREATE UNIQUE INDEX index_vulnerability_user_mentions_on_note_id ON public.vulnerability_user_mentions USING btree (note_id) WHERE (note_id IS NOT NULL); CREATE UNIQUE INDEX index_vulnerability_user_mentions_on_note_id ON public.vulnerability_user_mentions USING btree (note_id) WHERE (note_id IS NOT NULL);
...@@ -13764,6 +13766,7 @@ COPY "schema_migrations" (version) FROM STDIN; ...@@ -13764,6 +13766,7 @@ COPY "schema_migrations" (version) FROM STDIN;
20200506154421 20200506154421
20200507221434 20200507221434
20200508091106 20200508091106
20200511092714
20200511145545 20200511145545
\. \.
---
title: Replace undefined confidence with unknown severity for vulnerabilities
merge_request: 31593
author:
type: other
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module RemoveUndefinedVulnerabilityConfidenceLevel
extend ::Gitlab::Utils::Override
class Vulnerability < ActiveRecord::Base
include ::EachBatch
self.table_name = 'vulnerabilities'
CONFIDENCE_LEVELS = {
undefined: 0,
unknown: 2
}.with_indifferent_access.freeze
enum confidence: CONFIDENCE_LEVELS
def self.undefined_confidence
where(confidence: Vulnerability.confidences[:undefined])
end
end
override :perform
def perform(start_id, stop_id)
Vulnerability.undefined_confidence
.where(id: start_id..stop_id)
.update_all(confidence: Vulnerability.confidences[:unknown])
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel, :migration, schema: 20200511092714 do
let(:vulnerabilities) { table(:vulnerabilities) }
let(:identifiers) { table(:vulnerability_identifiers) }
let(:projects) { table(:projects) }
let(:users) { table(:users) }
it 'updates undefined confidence level to unknown' do
projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab')
users.create!(id: 13, email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active')
vul1 = vulnerabilities.create!(vuln_params)
vulnerabilities.create!(vuln_params)
vul3 = vulnerabilities.create!(vuln_params.merge(confidence: 2))
expect(vulnerabilities.where(confidence: 2).count). to eq(1)
expect(vulnerabilities.where(severity: 5).count). to eq(3)
described_class.new.perform(vul1.id, vul3.id)
expect(vulnerabilities.where(confidence: 2).count).to eq(3)
end
def vuln_params
{
title: 'title',
state: 1,
severity: 5,
confidence: 0,
report_type: 2,
project_id: 123,
author_id: 13
}
end
end
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20200511092714_update_undefined_confidence_from_vulnerabilities.rb')
describe UpdateUndefinedConfidenceFromVulnerabilities, :migration do
let(:vulnerabilities) { table(:vulnerabilities) }
let(:identifiers) { table(:vulnerability_identifiers) }
let(:projects) { table(:projects) }
let(:users) { table(:users) }
before do
projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab')
users.create!(id: 13, email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active')
stub_const("#{described_class}::BATCH_SIZE", 2)
end
it 'updates undefined confidence levels to unkown', :sidekiq_might_not_need_inline do
allow_any_instance_of(Gitlab).to receive(:ee?).and_return(true)
vulnerabilities.create!(vuln_params)
vulnerabilities.create!(vuln_params.merge(confidence: 2))
expect(vulnerabilities.where(confidence: 0).count). to eq(1)
migrate!
expect(vulnerabilities.exists?(confidence: 0)).to be_falsy
expect(vulnerabilities.where(confidence: 2).count).to eq(2)
end
it 'skips migration for ce' do
allow_any_instance_of(Gitlab).to receive(:ee?).and_return(false)
vulnerabilities.create!(vuln_params)
expect(vulnerabilities.where(confidence: 0).count). to eq(1)
migrate!
expect(vulnerabilities.exists?(confidence: 0)).to be_truthy
end
def vuln_params
{
title: 'title',
state: 1,
confidence: 0,
severity: 5,
report_type: 2,
project_id: 123,
author_id: 13
}
end
end
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class RemoveUndefinedVulnerabilityConfidenceLevel
def perform(start_id, stop_id)
end
end
end
end
Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel')
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment