Commit 88326bac authored by Michał Zając's avatar Michał Zając Committed by Patrick Bair

Schedule recalculating UUID for all `Vulnerabilities::Finding` records

There are two major reasons for this:

1. We made a mistake in the algorithm when recalculating this for the
first time.
See: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67578

2. The algorithm changed yet again to improve the tracking method
See: https://gitlab.com/gitlab-org/gitlab/-/issues/322044

Changes:

* Update background migration to use signatures

* Make previous reschedule a no-op so customers doing major version
  leaps will not schedule the same job multiple times

* Remove ALL jobs from `background_migration_jobs` table. Succeeded jobs
  should be removed anyway and pending jobs were not finished because we
  had records for which the recalculation yielded identical UUIDv5. The
  reason we had such records was due to mistake outlined in point 1.
  This situation was fixed by
  https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74008

* Schedule `RecalculateVulnerabilitiesOccurrencesUuid` over entire
  `vulnerability_occurrences` table so we can leverage our new algorithm
  and finish the migration path from UUIDv4 to UUIDv5.

Changelog: added
parent 1f693f1c
---
name: migrate_vulnerability_finding_uuids
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75546
rollout_issue_url:
milestone: '14.7'
type: development
group: group::threat insights
default_enabled: true
# frozen_string_literal: true # frozen_string_literal: true
class ScheduleRecalculateUuidOnVulnerabilitiesOccurrences2 < ActiveRecord::Migration[6.0] class ScheduleRecalculateUuidOnVulnerabilitiesOccurrences2 < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
MIGRATION = 'RecalculateVulnerabilitiesOccurrencesUuid'
DELAY_INTERVAL = 2.minutes.to_i
BATCH_SIZE = 2_500
disable_ddl_transaction!
class VulnerabilitiesFinding < ActiveRecord::Base
include ::EachBatch
self.inheritance_column = :_type_disabled
self.table_name = "vulnerability_occurrences"
end
def up def up
# Make sure that RemoveDuplicateVulnerabilitiesFindings has finished running # no-op
# so that we don't run into duplicate UUID issues # superseded by db/post_migrate/20211207125231_schedule_recalculate_uuid_on_vulnerabilities_occurrences4.rb
Gitlab::BackgroundMigration.steal('RemoveDuplicateVulnerabilitiesFindings')
say "Scheduling #{MIGRATION} jobs"
queue_background_migration_jobs_by_range_at_intervals(
VulnerabilitiesFinding,
MIGRATION,
DELAY_INTERVAL,
batch_size: BATCH_SIZE,
track_jobs: true
)
end end
def down def down
......
# frozen_string_literal: true # frozen_string_literal: true
class ScheduleRecalculateUuidOnVulnerabilitiesOccurrences3 < ActiveRecord::Migration[6.0] class ScheduleRecalculateUuidOnVulnerabilitiesOccurrences3 < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
MIGRATION = 'RecalculateVulnerabilitiesOccurrencesUuid'
DELAY_INTERVAL = 2.minutes.to_i
BATCH_SIZE = 2_500
disable_ddl_transaction!
def up def up
# Make sure that RemoveDuplicateVulnerabilitiesFindings has finished running # no-op
# so that we don't run into duplicate UUID issues # superseded by db/post_migrate/20211207125231_schedule_recalculate_uuid_on_vulnerabilities_occurrences4.rb
Gitlab::BackgroundMigration.steal('RemoveDuplicateVulnerabilitiesFindings')
say "Scheduling #{MIGRATION} jobs"
queue_background_migration_jobs_by_range_at_intervals(
define_batchable_model('vulnerability_occurrences'),
MIGRATION,
DELAY_INTERVAL,
batch_size: BATCH_SIZE,
track_jobs: true
)
end end
def down def down
......
# frozen_string_literal: true # frozen_string_literal: true
class ReschedulePendingJobsForRecalculateVulnerabilitiesOccurrencesUuid < Gitlab::Database::Migration[1.0] class ReschedulePendingJobsForRecalculateVulnerabilitiesOccurrencesUuid < Gitlab::Database::Migration[1.0]
MIGRATION = "RecalculateVulnerabilitiesOccurrencesUuid"
DELAY_INTERVAL = 2.minutes
disable_ddl_transaction!
def up def up
delete_queued_jobs(MIGRATION) # no-op
# no replacement because we will reschedule this for the whole table
requeue_background_migration_jobs_by_range_at_intervals(MIGRATION, DELAY_INTERVAL)
end end
def down def down
......
# frozen_string_literal: true
class RemoveJobsForRecalculateVulnerabilitiesOccurrencesUuid < Gitlab::Database::Migration[1.0]
MIGRATION_NAME = 'RecalculateVulnerabilitiesOccurrencesUuid'
def up
delete_job_tracking(
MIGRATION_NAME,
status: %w[pending succeeded]
)
end
def down
# no-op
end
end
# frozen_string_literal: true
class ScheduleRecalculateUuidOnVulnerabilitiesOccurrences4 < Gitlab::Database::Migration[1.0]
MIGRATION = 'RecalculateVulnerabilitiesOccurrencesUuid'
DELAY_INTERVAL = 2.minutes.to_i
BATCH_SIZE = 2_500
disable_ddl_transaction!
def up
# Make sure the migration removing Findings with attributes for which UUID would be identical
# has finished
# https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74008
Gitlab::BackgroundMigration.steal('RemoveOccurrencePipelinesAndDuplicateVulnerabilitiesFindings')
queue_background_migration_jobs_by_range_at_intervals(
define_batchable_model('vulnerability_occurrences'),
MIGRATION,
DELAY_INTERVAL,
batch_size: BATCH_SIZE,
track_jobs: true
)
end
def down
# no-op
end
end
5ead867b7609248f702771078849c48c0558f5fe9a3021fbb32e4f9174af653a
\ No newline at end of file
3d9dcab49ee409da8c1ab398101041092e566b06a7bb2764db49a9201a0e5f0c
\ No newline at end of file
# frozen_string_literal: true # frozen_string_literal: true
# rubocop: disable Style/Documentation # rubocop: disable Style/Documentation
class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid # rubocop:disable Metrics/ClassLength
# rubocop: disable Gitlab/NamespacedClass # rubocop: disable Gitlab/NamespacedClass
class VulnerabilitiesIdentifier < ActiveRecord::Base class VulnerabilitiesIdentifier < ActiveRecord::Base
self.table_name = "vulnerability_identifiers" self.table_name = "vulnerability_identifiers"
...@@ -9,10 +9,14 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid ...@@ -9,10 +9,14 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid
end end
class VulnerabilitiesFinding < ActiveRecord::Base class VulnerabilitiesFinding < ActiveRecord::Base
include EachBatch
include ShaAttribute include ShaAttribute
self.table_name = "vulnerability_occurrences" self.table_name = "vulnerability_occurrences"
has_many :signatures, foreign_key: 'finding_id', class_name: 'VulnerabilityFindingSignature', inverse_of: :finding
belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id' belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id'
REPORT_TYPES = { REPORT_TYPES = {
sast: 0, sast: 0,
dependency_scanning: 1, dependency_scanning: 1,
...@@ -20,7 +24,9 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid ...@@ -20,7 +24,9 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid
dast: 3, dast: 3,
secret_detection: 4, secret_detection: 4,
coverage_fuzzing: 5, coverage_fuzzing: 5,
api_fuzzing: 6 api_fuzzing: 6,
cluster_image_scanning: 7,
generic: 99
}.with_indifferent_access.freeze }.with_indifferent_access.freeze
enum report_type: REPORT_TYPES enum report_type: REPORT_TYPES
...@@ -28,6 +34,25 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid ...@@ -28,6 +34,25 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid
sha_attribute :location_fingerprint sha_attribute :location_fingerprint
end end
class VulnerabilityFindingSignature < ActiveRecord::Base
include ShaAttribute
self.table_name = 'vulnerability_finding_signatures'
belongs_to :finding, foreign_key: 'finding_id', inverse_of: :signatures, class_name: 'VulnerabilitiesFinding'
sha_attribute :signature_sha
end
class VulnerabilitiesFindingPipeline < ActiveRecord::Base
include EachBatch
self.table_name = "vulnerability_occurrence_pipelines"
end
class Vulnerability < ActiveRecord::Base
include EachBatch
self.table_name = "vulnerabilities"
end
class CalculateFindingUUID class CalculateFindingUUID
FINDING_NAMESPACES_IDS = { FINDING_NAMESPACES_IDS = {
development: "a143e9e2-41b3-47bc-9a19-081d089229f4", development: "a143e9e2-41b3-47bc-9a19-081d089229f4",
...@@ -52,35 +77,122 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid ...@@ -52,35 +77,122 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid
end end
# rubocop: enable Gitlab/NamespacedClass # rubocop: enable Gitlab/NamespacedClass
# rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength
def perform(start_id, end_id) def perform(start_id, end_id)
findings = VulnerabilitiesFinding unless Feature.enabled?(:migrate_vulnerability_finding_uuids, default_enabled: true)
return log_info('Migration is disabled by the feature flag', start_id: start_id, end_id: end_id)
end
log_info('Migration started', start_id: start_id, end_id: end_id)
VulnerabilitiesFinding
.joins(:primary_identifier) .joins(:primary_identifier)
.select(:id, :report_type, :fingerprint, :location_fingerprint, :project_id) .includes(:signatures)
.select(:id, :report_type, :primary_identifier_id, :fingerprint, :location_fingerprint, :project_id, :created_at, :vulnerability_id, :uuid)
.where(id: start_id..end_id) .where(id: start_id..end_id)
.each_batch(of: 50) do |relation|
duplicates = find_duplicates(relation)
remove_findings(ids: duplicates) if duplicates.present?
to_update = relation.reject { |finding| duplicates.include?(finding.id) }
begin
known_uuids = Set.new
to_be_deleted = []
mappings = to_update.each_with_object({}) do |finding, hash|
uuid = calculate_uuid_v5_for_finding(finding)
if known_uuids.add?(uuid)
hash[finding] = { uuid: uuid }
else
to_be_deleted << finding.id
end
end
# It is technically still possible to have duplicate uuids
# if the data integrity is broken somehow and the primary identifiers of
# the findings are pointing to different projects with the same fingerprint values.
if to_be_deleted.present?
log_info('Conflicting UUIDs found within the batch', finding_ids: to_be_deleted)
mappings = findings.each_with_object({}) do |finding, hash| remove_findings(ids: to_be_deleted)
hash[finding] = { uuid: calculate_uuid_v5_for_finding(finding) }
end end
::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) if mappings.present?
logger.info(message: 'RecalculateVulnerabilitiesOccurrencesUuid Migration: recalculation is done for:', log_info('Recalculation is done', finding_ids: mappings.keys.pluck(:id))
finding_ids: mappings.keys.pluck(:id)) rescue ActiveRecord::RecordNotUnique => error
log_info('RecordNotUnique error received')
match_data = /\(uuid\)=\((?<uuid>\S{36})\)/.match(error.message)
# This exception returns the **correct** UUIDv5 which probably comes from a later record
# and it's the one we can drop in the easiest way before retrying the UPDATE query
if match_data
uuid = match_data[:uuid]
log_info('Conflicting UUID found', uuid: uuid)
id = VulnerabilitiesFinding.find_by(uuid: uuid)&.id
remove_findings(ids: id) if id
retry
else
log_error('Couldnt find conflicting uuid')
Gitlab::ErrorTracking.track_and_raise_exception(error)
end
end
end
mark_job_as_succeeded(start_id, end_id) mark_job_as_succeeded(start_id, end_id)
rescue StandardError => error rescue StandardError => error
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) log_error('An exception happened')
Gitlab::ErrorTracking.track_and_raise_exception(error)
end end
# rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength
private private
def find_duplicates(relation)
to_exclude = []
relation.flat_map do |record|
# Assuming we're scanning id 31 and the duplicate is id 40
# first we'd process 31 and add 40 to the list of ids to remove
# then we would process record 40 and add 31 to the list of removals
# so we would drop both records
to_exclude << record.id
VulnerabilitiesFinding.where(
report_type: record.report_type,
location_fingerprint: record.location_fingerprint,
primary_identifier_id: record.primary_identifier_id,
project_id: record.project_id
).where.not(id: to_exclude).pluck(:id)
end
end
def remove_findings(ids:)
ids = Array(ids)
log_info('Removing Findings and associated records', ids: ids)
vulnerability_ids = VulnerabilitiesFinding.where(id: ids).pluck(:vulnerability_id).uniq.compact
VulnerabilitiesFindingPipeline.where(occurrence_id: ids).each_batch { |batch| batch.delete_all }
Vulnerability.where(id: vulnerability_ids).each_batch { |batch| batch.delete_all }
VulnerabilitiesFinding.where(id: ids).delete_all
end
def calculate_uuid_v5_for_finding(vulnerability_finding) def calculate_uuid_v5_for_finding(vulnerability_finding)
return unless vulnerability_finding return unless vulnerability_finding
signatures = vulnerability_finding.signatures.sort_by { |signature| signature.algorithm_type_before_type_cast }
location_fingerprint = signatures.last&.signature_sha || vulnerability_finding.location_fingerprint
uuid_v5_name_components = { uuid_v5_name_components = {
report_type: vulnerability_finding.report_type, report_type: vulnerability_finding.report_type,
primary_identifier_fingerprint: vulnerability_finding.fingerprint, primary_identifier_fingerprint: vulnerability_finding.fingerprint,
location_fingerprint: vulnerability_finding.location_fingerprint, location_fingerprint: location_fingerprint,
project_id: vulnerability_finding.project_id project_id: vulnerability_finding.project_id
} }
...@@ -89,6 +201,14 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid ...@@ -89,6 +201,14 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid
CalculateFindingUUID.call(name) CalculateFindingUUID.call(name)
end end
def log_info(message, **extra)
logger.info(migrator: 'RecalculateVulnerabilitiesOccurrencesUuid', message: message, **extra)
end
def log_error(message, **extra)
logger.error(migrator: 'RecalculateVulnerabilitiesOccurrencesUuid', message: message, **extra)
end
def logger def logger
@logger ||= Gitlab::BackgroundMigration::Logger.build @logger ||= Gitlab::BackgroundMigration::Logger.build
end end
......
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20210918202855_reschedule_pending_jobs_for_recalculate_vulnerabilities_occurrences_uuid.rb')
RSpec.describe ReschedulePendingJobsForRecalculateVulnerabilitiesOccurrencesUuid, :migration do
let_it_be(:background_migration_jobs) { table(:background_migration_jobs) }
context 'when RecalculateVulnerabilitiesOccurrencesUuid jobs are pending' do
before do
background_migration_jobs.create!(
class_name: 'RecalculateVulnerabilitiesOccurrencesUuid',
arguments: [1, 2, 3],
status: Gitlab::Database::BackgroundMigrationJob.statuses['pending']
)
background_migration_jobs.create!(
class_name: 'RecalculateVulnerabilitiesOccurrencesUuid',
arguments: [4, 5, 6],
status: Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']
)
end
it 'queues pending jobs' do
migrate!
expect(BackgroundMigrationWorker.jobs.length).to eq(1)
expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['RecalculateVulnerabilitiesOccurrencesUuid', [1, 2, 3]])
expect(BackgroundMigrationWorker.jobs[0]['at']).to be_nil
end
end
end
# frozen_string_literal: true
require 'spec_helper'
require_migration!
def create_background_migration_jobs(ids, status, created_at)
proper_status = case status
when :pending
Gitlab::Database::BackgroundMigrationJob.statuses['pending']
when :succeeded
Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']
else
raise ArgumentError
end
background_migration_jobs.create!(
class_name: 'RecalculateVulnerabilitiesOccurrencesUuid',
arguments: Array(ids),
status: proper_status,
created_at: created_at
)
end
RSpec.describe RemoveJobsForRecalculateVulnerabilitiesOccurrencesUuid, :migration do
let_it_be(:background_migration_jobs) { table(:background_migration_jobs) }
context 'when RecalculateVulnerabilitiesOccurrencesUuid jobs are present' do
before do
create_background_migration_jobs([1, 2, 3], :succeeded, DateTime.new(2021, 5, 5, 0, 2))
create_background_migration_jobs([4, 5, 6], :pending, DateTime.new(2021, 5, 5, 0, 4))
create_background_migration_jobs([1, 2, 3], :succeeded, DateTime.new(2021, 8, 18, 0, 0))
create_background_migration_jobs([4, 5, 6], :pending, DateTime.new(2021, 8, 18, 0, 2))
create_background_migration_jobs([7, 8, 9], :pending, DateTime.new(2021, 8, 18, 0, 4))
end
it 'removes all jobs' do
expect(background_migration_jobs.count).to eq(5)
migrate!
expect(background_migration_jobs.count).to eq(0)
end
end
end
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
require 'spec_helper' require 'spec_helper'
require_migration! require_migration!
RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences2 do RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences4 do
let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
let(:users) { table(:users) } let(:users) { table(:users) }
let(:user) { create_user! } let(:user) { create_user! }
...@@ -13,6 +13,7 @@ RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences2 do ...@@ -13,6 +13,7 @@ RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences2 do
let(:different_scanner) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } let(:different_scanner) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') }
let(:vulnerabilities) { table(:vulnerabilities) } let(:vulnerabilities) { table(:vulnerabilities) }
let(:vulnerabilities_findings) { table(:vulnerability_occurrences) } let(:vulnerabilities_findings) { table(:vulnerability_occurrences) }
let(:vulnerability_finding_signatures) { table(:vulnerability_finding_signatures) }
let(:vulnerability_identifiers) { table(:vulnerability_identifiers) } let(:vulnerability_identifiers) { table(:vulnerability_identifiers) }
let(:vulnerability_identifier) do let(:vulnerability_identifier) do
vulnerability_identifiers.create!( vulnerability_identifiers.create!(
...@@ -32,6 +33,17 @@ RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences2 do ...@@ -32,6 +33,17 @@ RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences2 do
name: 'Identifier for UUIDv4') name: 'Identifier for UUIDv4')
end end
let!(:uuidv4_finding) do
create_finding!(
vulnerability_id: vulnerability_for_uuidv4.id,
project_id: project.id,
scanner_id: different_scanner.id,
primary_identifier_id: different_vulnerability_identifier.id,
location_fingerprint: Gitlab::Database::ShaAttribute.serialize('fa18f432f1d56675f4098d318739c3cd5b14eb3e'),
uuid: 'b3cc2518-5446-4dea-871c-89d5e999c1ac'
)
end
let(:vulnerability_for_uuidv4) do let(:vulnerability_for_uuidv4) do
create_vulnerability!( create_vulnerability!(
project_id: project.id, project_id: project.id,
...@@ -39,6 +51,17 @@ RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences2 do ...@@ -39,6 +51,17 @@ RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences2 do
) )
end end
let!(:uuidv5_finding) do
create_finding!(
vulnerability_id: vulnerability_for_uuidv5.id,
project_id: project.id,
scanner_id: scanner.id,
primary_identifier_id: vulnerability_identifier.id,
location_fingerprint: Gitlab::Database::ShaAttribute.serialize('838574be0210968bf6b9f569df9c2576242cbf0a'),
uuid: '77211ed6-7dff-5f6b-8c9a-da89ad0a9b60'
)
end
let(:vulnerability_for_uuidv5) do let(:vulnerability_for_uuidv5) do
create_vulnerability!( create_vulnerability!(
project_id: project.id, project_id: project.id,
...@@ -46,25 +69,22 @@ RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences2 do ...@@ -46,25 +69,22 @@ RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences2 do
) )
end end
let!(:finding1) do let(:vulnerability_for_finding_with_signature) do
create_finding!( create_vulnerability!(
vulnerability_id: vulnerability_for_uuidv4.id,
project_id: project.id, project_id: project.id,
scanner_id: different_scanner.id, author_id: user.id
primary_identifier_id: different_vulnerability_identifier.id,
location_fingerprint: 'fa18f432f1d56675f4098d318739c3cd5b14eb3e',
uuid: 'b3cc2518-5446-4dea-871c-89d5e999c1ac'
) )
end end
let!(:finding2) do let!(:finding_with_signature) do
create_finding!( create_finding!(
vulnerability_id: vulnerability_for_uuidv5.id, vulnerability_id: vulnerability_for_finding_with_signature.id,
project_id: project.id, project_id: project.id,
scanner_id: scanner.id, scanner_id: scanner.id,
primary_identifier_id: vulnerability_identifier.id, primary_identifier_id: vulnerability_identifier.id,
location_fingerprint: '838574be0210968bf6b9f569df9c2576242cbf0a', report_type: 0, # "sast"
uuid: '77211ed6-7dff-5f6b-8c9a-da89ad0a9b60' location_fingerprint: Gitlab::Database::ShaAttribute.serialize('123609eafffffa2207a9ca2425ba4337h34fga1b'),
uuid: '252aa474-d689-5d2b-ab42-7bbb5a100c02'
) )
end end
...@@ -79,9 +99,10 @@ RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences2 do ...@@ -79,9 +99,10 @@ RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences2 do
it 'schedules background migrations', :aggregate_failures do it 'schedules background migrations', :aggregate_failures do
migrate! migrate!
expect(BackgroundMigrationWorker.jobs.size).to eq(2) expect(BackgroundMigrationWorker.jobs.size).to eq(3)
expect(described_class::MIGRATION).to be_scheduled_delayed_migration(2.minutes, finding1.id, finding1.id) expect(described_class::MIGRATION).to be_scheduled_delayed_migration(2.minutes, uuidv4_finding.id, uuidv4_finding.id)
expect(described_class::MIGRATION).to be_scheduled_delayed_migration(4.minutes, finding2.id, finding2.id) expect(described_class::MIGRATION).to be_scheduled_delayed_migration(4.minutes, uuidv5_finding.id, uuidv5_finding.id)
expect(described_class::MIGRATION).to be_scheduled_delayed_migration(6.minutes, finding_with_signature.id, finding_with_signature.id)
end end
private private
...@@ -98,14 +119,14 @@ RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences2 do ...@@ -98,14 +119,14 @@ RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences2 do
end end
def create_finding!( def create_finding!(
vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, location_fingerprint:, uuid:) vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, location_fingerprint:, uuid:, report_type: 0)
vulnerabilities_findings.create!( vulnerabilities_findings.create!(
vulnerability_id: vulnerability_id, vulnerability_id: vulnerability_id,
project_id: project_id, project_id: project_id,
name: 'test', name: 'test',
severity: 7, severity: 7,
confidence: 7, confidence: 7,
report_type: 0, report_type: report_type,
project_fingerprint: '123qweasdzxc', project_fingerprint: '123qweasdzxc',
scanner_id: scanner_id, scanner_id: scanner_id,
primary_identifier_id: primary_identifier_id, primary_identifier_id: primary_identifier_id,
......
# frozen_string_literal: true
require 'spec_helper'
require_migration!
RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences3 do
let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
let(:users) { table(:users) }
let(:user) { create_user! }
let(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) }
let(:scanners) { table(:vulnerability_scanners) }
let(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
let(:different_scanner) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') }
let(:vulnerabilities) { table(:vulnerabilities) }
let(:vulnerabilities_findings) { table(:vulnerability_occurrences) }
let(:vulnerability_identifiers) { table(:vulnerability_identifiers) }
let(:vulnerability_identifier) do
vulnerability_identifiers.create!(
project_id: project.id,
external_type: 'uuid-v5',
external_id: 'uuid-v5',
fingerprint: '7e394d1b1eb461a7406d7b1e08f057a1cf11287a',
name: 'Identifier for UUIDv5')
end
let(:different_vulnerability_identifier) do
vulnerability_identifiers.create!(
project_id: project.id,
external_type: 'uuid-v4',
external_id: 'uuid-v4',
fingerprint: '772da93d34a1ba010bcb5efa9fb6f8e01bafcc89',
name: 'Identifier for UUIDv4')
end
let(:vulnerability_for_uuidv4) do
create_vulnerability!(
project_id: project.id,
author_id: user.id
)
end
let(:vulnerability_for_uuidv5) do
create_vulnerability!(
project_id: project.id,
author_id: user.id
)
end
let!(:finding1) do
create_finding!(
vulnerability_id: vulnerability_for_uuidv4.id,
project_id: project.id,
scanner_id: different_scanner.id,
primary_identifier_id: different_vulnerability_identifier.id,
location_fingerprint: 'fa18f432f1d56675f4098d318739c3cd5b14eb3e',
uuid: 'b3cc2518-5446-4dea-871c-89d5e999c1ac'
)
end
let!(:finding2) do
create_finding!(
vulnerability_id: vulnerability_for_uuidv5.id,
project_id: project.id,
scanner_id: scanner.id,
primary_identifier_id: vulnerability_identifier.id,
location_fingerprint: '838574be0210968bf6b9f569df9c2576242cbf0a',
uuid: '77211ed6-7dff-5f6b-8c9a-da89ad0a9b60'
)
end
before do
stub_const("#{described_class}::BATCH_SIZE", 1)
end
around do |example|
freeze_time { Sidekiq::Testing.fake! { example.run } }
end
it 'schedules background migrations', :aggregate_failures do
migrate!
expect(BackgroundMigrationWorker.jobs.size).to eq(2)
expect(described_class::MIGRATION).to be_scheduled_delayed_migration(2.minutes, finding1.id, finding1.id)
expect(described_class::MIGRATION).to be_scheduled_delayed_migration(4.minutes, finding2.id, finding2.id)
end
private
def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0)
vulnerabilities.create!(
project_id: project_id,
author_id: author_id,
title: title,
severity: severity,
confidence: confidence,
report_type: report_type
)
end
def create_finding!(
vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, location_fingerprint:, uuid:)
vulnerabilities_findings.create!(
vulnerability_id: vulnerability_id,
project_id: project_id,
name: 'test',
severity: 7,
confidence: 7,
report_type: 0,
project_fingerprint: '123qweasdzxc',
scanner_id: scanner_id,
primary_identifier_id: primary_identifier_id,
location_fingerprint: location_fingerprint,
metadata_version: 'test',
raw_metadata: 'test',
uuid: uuid
)
end
def create_user!(name: "Example User", email: "user@example.com", user_type: nil)
users.create!(
name: name,
email: email,
username: name,
projects_limit: 0
)
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment