Skip to content

G
gitlab-ce
Commit 6523bd4a authored by Thong Kuah's avatar Thong Kuah
Browse files
Options

Merge branch 'pl-revert-cs-finding-location' into 'master' 

Revert: Update location fingerprint for existing CS vulnerabilities

See merge request gitlab-org/gitlab!41737
parents f45c07d7 021d2665
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 324 deletions
db/post_migrate/20200824140259_update_location_fingerprint_for_cs_findings.rb deleted 100644 → 0
View file @ f45c07d7
# frozen_string_literal: true
class UpdateLocationFingerprintForCsFindings < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
BATCH_SIZE = 1_000
INTERVAL = 2.minutes
# 815_565 records
def up
# no-op
# There was a bug introduced with this migration for gitlab.com
# We created new migration to mitigate that VERISON=20200907123723
# and change this one to no-op to prevent running migration twice
end
def down
# no-op
# intentionally blank
end
end
db/post_migrate/20200907123723_fix_update_location_fingerprint_for_cs_findings.rb deleted 100644 → 0
View file @ f45c07d7
# frozen_string_literal: true
class FixUpdateLocationFingerprintForCsFindings < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
BATCH_SIZE = 1_000
INTERVAL = 2.minutes
# 815_565 records
def up
return unless Gitlab.ee?
migration = Gitlab::BackgroundMigration::UpdateLocationFingerprintForCsFindings
migration_name = migration.to_s.demodulize
Gitlab::BackgroundMigration.steal(migration_name)
relation = migration::Finding.container_scanning
queue_background_migration_jobs_by_range_at_intervals(relation,
migration_name,
INTERVAL,
batch_size: BATCH_SIZE)
end
def down
# no-op
# intentionally blank
end
end
db/schema_migrations/20200824140259 deleted 100644 → 0
View file @ f45c07d7
e24f8495b7458ce57e148017ef6cae901e7f3997fca247afeff524a43e739431
\ No newline at end of file
db/schema_migrations/20200907123723 deleted 100644 → 0
View file @ f45c07d7
74f6adf29b9293537d653b993ab0e2f31af2ab5a8581ca631e8a87fc589ca284
\ No newline at end of file
ee/changelogs/unreleased/add-sha-for-background-migration.yml deleted 100644 → 0
View file @ f45c07d7
---
title: Fix Update location fingerprint for existing CS vulnerabilities
merge_request: 41671
author:
type: fixed
ee/changelogs/unreleased/update-location-fingerprint-for-cs.yml deleted 100644 → 0
View file @ f45c07d7
---
title: Update location fingerprint for existing CS vulnerabilities
merge_request: 40497
author:
type: other
ee/lib/ee/gitlab/background_migration/update_location_fingerprint_for_cs_findings.rb deleted 100644 → 0
View file @ f45c07d7
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module UpdateLocationFingerprintForCsFindings
extend ::Gitlab::Utils::Override
class Finding < ActiveRecord::Base
include ::ShaAttribute
include ::EachBatch
self.table_name = 'vulnerability_occurrences'
REPORT_TYPES = {
container_scanning: 2
}.with_indifferent_access.freeze
enum report_type: REPORT_TYPES
sha_attribute :location_fingerprint
# Copied from Reports::Security::Locations
def calculate_new_fingerprint(image, package_name)
return if image.nil? || package_name.nil?
Digest::SHA1.hexdigest("#{docker_image_name_without_tag(image)}:#{package_name}")
end
private
def docker_image_name_without_tag(image)
base_name, version = image.split(':')
return image if version_semver_like?(version)
base_name
end
def version_semver_like?(version)
hash_like = /\A[0-9a-f]{32,128}\z/i
if Gem::Version.correct?(version)
!hash_like.match?(version)
else
false
end
end
end
override :perform
def perform(start_id, stop_id)
Finding.select(:id, "raw_metadata::json->'location' AS loc")
.where(id: start_id..stop_id)
.each do |finding|
next if finding.loc.nil?
package = finding.loc.dig('dependency', 'package', 'name')
image = finding.loc.dig('image')
new_fingerprint = finding.calculate_new_fingerprint(image, package)
next if new_fingerprint.nil?
begin
finding.update_column(:location_fingerprint, new_fingerprint)
rescue ActiveRecord::RecordNotUnique
Gitlab::BackgroundMigration::Logger.warn("Duplicate finding found with finding id #{finding.id}")
end
end
end
end
end
end
end
ee/spec/lib/ee/gitlab/background_migration/update_location_fingerprint_for_cs_findings_spec.rb deleted 100644 → 0
View file @ f45c07d7
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::UpdateLocationFingerprintForCsFindings, :migration, schema: 20200824140259 do
let(:namespaces) { table(:namespaces) }
let(:group) { namespaces.create!(name: 'foo', path: 'foo') }
let(:projects) { table(:projects) }
let(:findings) { table(:vulnerability_occurrences) }
let(:scanners) { table(:vulnerability_scanners) }
let(:identifiers) { table(:vulnerability_identifiers) }
let!(:project) { projects.create!(id: 123, namespace_id: group.id, name: 'gitlab', path: 'gitlab') }
let!(:scanner) do
scanners.create!(id: 6, project_id: project.id, external_id: 'clair', name: 'Security Scanner')
end
it 'updates location fingerprint' do
raw_metadata = [
"{ \"location\":{\"dependency\":{\"package\":{\"name\":\"apparmor\"},\"version\":\"2.10.95-0ubuntu2.11\"},\"operating_system\":\"ubuntu:16.04\",\"image\":\"registry.staging.gitlab.com/gitlab/alpine-ruby2/master:49dda736b6386592f7dd0367bcdd260cb84edfa8\"} }",
"{ \"location\":{\"dependency\":{\"package\":{\"name\":\"glibc\"},\"version\":\"2.10.95-0ubuntu2.11\"},\"operating_system\":\"ubuntu:16.04\",\"image\":\"registry.staging.gitlab.com/gitlab/docker/master:2.1\"} }",
"{ \"location\":{\"dependency\":{\"package\":{\"name\":\"apt\"},\"version\":\"2.10.95-0ubuntu2.11\"},\"operating_system\":\"ubuntu:16.04\",\"image\":\"registry.staging.gitlab.com/gitlab/gitlab/master:49dda73\"} }"
]
new_fingerprints = %w(6c871440eb9f7618b9aef25e5246acddff6ed7a1 9d1a47927875f1aee1e2b9f16c25a8ff7586f1a6 d7da2cc109c18d890ab239e833524d451cc45246)
create_identifier(3)
vul1 = findings.create!(finding_params(1).merge({ raw_metadata: raw_metadata[0] }))
findings.create!(finding_params(2).merge({ raw_metadata: raw_metadata[1] }))
vul3 = findings.create!(finding_params(3).merge({ raw_metadata: raw_metadata[2] }))
expect(findings.where(report_type: 2).count). to eq(3)
described_class.new.perform(vul1.id, vul3.id)
location_fingerprints = findings.pluck(:location_fingerprint).flat_map { |x| Gitlab::Database::ShaAttribute.new.deserialize(x) }
expect(location_fingerprints).to match_array(new_fingerprints)
end
def create_identifier(number_of)
(1..number_of).each do |identifier_id|
identifiers.create!(id: identifier_id,
project_id: 123,
fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c' + identifier_id.to_s,
external_type: 'SECURITY_ID',
external_id: 'SECURITY_0',
name: 'SECURITY_IDENTIFIER 0')
end
end
def finding_params(primary_identifier_id)
attrs = attributes_for(:vulnerabilities_finding)
{
severity: 0,
confidence: 5,
report_type: 2,
project_id: 123,
scanner_id: 6,
primary_identifier_id: primary_identifier_id,
project_fingerprint: attrs[:project_fingerprint],
location_fingerprint: Digest::SHA1.hexdigest(SecureRandom.hex(10)),
uuid: attrs[:uuid],
name: attrs[:name],
metadata_version: '1.3',
raw_metadata: attrs[:raw_metadata]
}
end
end
ee/spec/migrations/20200907123723_fix_update_location_fingerprint_for_cs_findings_spec.rb deleted 100644 → 0
View file @ f45c07d7
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20200907123723_fix_update_location_fingerprint_for_cs_findings.rb')
RSpec.describe FixUpdateLocationFingerprintForCsFindings, :migration do
let(:namespaces) { table(:namespaces) }
let(:users) { table(:users) }
let(:group) { namespaces.create!(name: 'foo', path: 'foo') }
let(:projects) { table(:projects) }
let(:findings) { table(:vulnerability_occurrences) }
let(:scanners) { table(:vulnerability_scanners) }
let(:identifiers) { table(:vulnerability_identifiers) }
let!(:project) { projects.create!(id: 123, namespace_id: group.id, name: 'gitlab', path: 'gitlab') }
let!(:scanner) do
scanners.create!(id: 6, project_id: project.id, external_id: 'clair', name: 'Security Scanner')
end
let!(:user) do
users.create!(id: 13, email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active')
end
before do
stub_const("#{described_class}::BATCH_SIZE", 2)
end
it 'updates location fingerprint for containter scanning findings', :sidekiq_might_not_need_inline do
raw_metadata = [
"{ \"location\":{\"dependency\":{\"package\":{\"name\":\"apparmor\"},\"version\":\"2.10.95-0ubuntu2.11\"},\"operating_system\":\"ubuntu:16.04\",\"image\":\"registry.staging.gitlab.com/gitlab/alpine-ruby2/master:49dda736b6386592f7dd0367bcdd260cb84edfa8\"} }",
"{ \"location\":{\"dependency\":{\"package\":{\"name\":\"glibc\"},\"version\":\"2.10.95-0ubuntu2.11\"},\"operating_system\":\"ubuntu:16.04\",\"image\":\"registry.staging.gitlab.com/gitlab/docker/master:2.1\"} }"
]
new_fingerprints = %w(6c871440eb9f7618b9aef25e5246acddff6ed7a1 9d1a47927875f1aee1e2b9f16c25a8ff7586f1a6)
allow_any_instance_of(Gitlab).to receive(:ee?).and_return(true)
create_identifier(2)
findings.create!(finding_params(1).merge({ raw_metadata: raw_metadata[0] }))
findings.create!(finding_params(2).merge({ raw_metadata: raw_metadata[1] }))
migrate!
location_fingerprints = findings.pluck(:location_fingerprint).flat_map { |x| Gitlab::Database::ShaAttribute.new.deserialize(x) }
expect(location_fingerprints).to match_array(new_fingerprints)
end
it 'skips migration for ce' do
allow_any_instance_of(Gitlab).to receive(:ee?).and_return(false)
create_identifier(2)
findings.create!(finding_params(1))
findings.create!(finding_params(2))
before_location_fingerprints = findings.pluck(:location_fingerprint).flat_map { |x| Gitlab::Database::ShaAttribute.new.deserialize(x) }
migrate!
after_location_fingerprints = findings.pluck(:location_fingerprint).flat_map { |x| Gitlab::Database::ShaAttribute.new.deserialize(x) }
expect(after_location_fingerprints).to match_array(before_location_fingerprints)
end
def create_identifier(number_of)
(1..number_of).each do |identifier_id|
identifiers.create!(id: identifier_id,
project_id: 123,
fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c' + identifier_id.to_s,
external_type: 'SECURITY_ID',
external_id: 'SECURITY_0',
name: 'SECURITY_IDENTIFIER 0')
end
end
def finding_params(primary_identifier_id)
attrs = attributes_for(:vulnerabilities_finding)
{
severity: 0,
confidence: 5,
report_type: 2,
project_id: 123,
scanner_id: 6,
primary_identifier_id: primary_identifier_id,
project_fingerprint: attrs[:project_fingerprint],
location_fingerprint: Digest::SHA1.hexdigest(SecureRandom.hex(10)),
uuid: attrs[:uuid],
name: attrs[:name],
metadata_version: '1.3',
raw_metadata: attrs[:raw_metadata]
}
end
end
lib/gitlab/background_migration/update_location_fingerprint_for_cs_findings.rb deleted 100644 → 0
View file @ f45c07d7
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class UpdateLocationFingerprintForCsFindings
def perform(start_id, stop_id)
end
end
end
end
Gitlab::BackgroundMigration::UpdateLocationFingerprintForCsFindings.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateLocationFingerprintForCsFindings')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7