Commit b8d0d450 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch '6717_add_security_reports_logic' into 'master'

Add Security Reports logic

See merge request gitlab-org/gitlab-ee!7578
parents d4ae2e43 3270e15c
......@@ -177,6 +177,8 @@
- geo:geo_repository_verification_primary_single
- geo:geo_repository_verification_secondary_single
- pipeline_default:store_security_reports
- admin_emails
- create_github_webhook
- elastic_batch_project_indexer
......
......@@ -11,9 +11,13 @@ class Gitlab::Seeder::Vulnerabilities
return unless pipeline
10.times do |rank|
occurrence = create_occurrence(rank)
create_occurrence_identifier(occurrence, rank, primary: true)
create_occurrence_identifier(occurrence, rank)
primary_identifier = create_identifier(rank)
occurrence = create_occurrence(rank, primary_identifier)
# Create occurrence_pipeline join model
occurrence.pipelines << pipeline
# Create occurrence_identifier join models
occurrence.identifiers << primary_identifier
occurrence.identifiers << create_identifier(rank) if rank % 3 == 0
if author
case rank % 3
......@@ -30,37 +34,28 @@ class Gitlab::Seeder::Vulnerabilities
private
def create_occurrence(rank)
def create_occurrence(rank, primary_identifier)
project.vulnerabilities.create!(
uuid: random_uuid,
name: 'Cipher with no integrity',
pipeline: pipeline,
ref: project.default_branch,
report_type: :sast,
severity: random_level,
confidence: random_level,
project_fingerprint: random_fingerprint,
primary_identifier_fingerprint: random_fingerprint,
location_fingerprint: random_fingerprint,
primary_identifier: primary_identifier,
raw_metadata: metadata(rank).to_json,
metadata_version: 'sast:1.0',
scanner: scanner)
end
def create_occurrence_identifier(occurrence, key, primary: false)
type = primary ? 'primary' : 'secondary'
fingerprint = if primary
occurrence.primary_identifier_fingerprint
else
Digest::SHA1.hexdigest("sid_fingerprint-#{project.id}-#{key}")
end
def create_identifier(rank)
project.vulnerability_identifiers.create!(
external_type: "#{type.upcase}_SECURITY_ID",
external_id: "#{type.upcase}_SECURITY_#{key}",
fingerprint: fingerprint,
name: "#{type.capitalize} #{key}",
url: "https://security.example.com/#{type.downcase}/#{key}"
external_type: "SECURITY_ID",
external_id: "SECURITY_#{rank}",
fingerprint: random_fingerprint,
name: "SECURITY_IDENTIFIER #{rank}",
url: "https://security.example.com/#{rank}"
)
end
......
......@@ -3052,27 +3052,35 @@ ActiveRecord::Schema.define(version: 20181017131623) do
add_index "vulnerability_occurrence_identifiers", ["identifier_id"], name: "index_vulnerability_occurrence_identifiers_on_identifier_id", using: :btree
add_index "vulnerability_occurrence_identifiers", ["occurrence_id", "identifier_id"], name: "index_vulnerability_occurrence_identifiers_on_unique_keys", unique: true, using: :btree
create_table "vulnerability_occurrence_pipelines", id: :bigserial, force: :cascade do |t|
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.integer "occurrence_id", limit: 8, null: false
t.integer "pipeline_id", null: false
end
add_index "vulnerability_occurrence_pipelines", ["occurrence_id", "pipeline_id"], name: "vulnerability_occurrence_pipelines_on_unique_keys", unique: true, using: :btree
add_index "vulnerability_occurrence_pipelines", ["pipeline_id"], name: "index_vulnerability_occurrence_pipelines_on_pipeline_id", using: :btree
create_table "vulnerability_occurrences", id: :bigserial, force: :cascade do |t|
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.integer "severity", limit: 2, null: false
t.integer "confidence", limit: 2, null: false
t.integer "report_type", limit: 2, null: false
t.integer "pipeline_id", null: false
t.integer "project_id", null: false
t.integer "scanner_id", limit: 8, null: false
t.integer "primary_identifier_id", limit: 8, null: false
t.binary "project_fingerprint", null: false
t.binary "location_fingerprint", null: false
t.binary "primary_identifier_fingerprint", null: false
t.string "uuid", limit: 36, null: false
t.string "ref", null: false
t.string "name", null: false
t.string "metadata_version", null: false
t.text "raw_metadata", null: false
end
add_index "vulnerability_occurrences", ["pipeline_id"], name: "index_vulnerability_occurrences_on_pipeline_id", using: :btree
add_index "vulnerability_occurrences", ["project_id", "ref", "primary_identifier_fingerprint", "location_fingerprint", "pipeline_id", "scanner_id"], name: "index_vulnerability_occurrences_on_unique_keys", unique: true, using: :btree
add_index "vulnerability_occurrences", ["primary_identifier_id"], name: "index_vulnerability_occurrences_on_primary_identifier_id", using: :btree
add_index "vulnerability_occurrences", ["project_id", "primary_identifier_id", "location_fingerprint", "scanner_id"], name: "index_vulnerability_occurrences_on_unique_keys", unique: true, using: :btree
add_index "vulnerability_occurrences", ["scanner_id"], name: "index_vulnerability_occurrences_on_scanner_id", using: :btree
add_index "vulnerability_occurrences", ["uuid"], name: "index_vulnerability_occurrences_on_uuid", unique: true, using: :btree
......@@ -3398,8 +3406,10 @@ ActiveRecord::Schema.define(version: 20181017131623) do
add_foreign_key "vulnerability_identifiers", "projects", on_delete: :cascade
add_foreign_key "vulnerability_occurrence_identifiers", "vulnerability_identifiers", column: "identifier_id", on_delete: :cascade
add_foreign_key "vulnerability_occurrence_identifiers", "vulnerability_occurrences", column: "occurrence_id", on_delete: :cascade
add_foreign_key "vulnerability_occurrences", "ci_pipelines", column: "pipeline_id", on_delete: :cascade
add_foreign_key "vulnerability_occurrence_pipelines", "ci_pipelines", column: "pipeline_id", on_delete: :cascade
add_foreign_key "vulnerability_occurrence_pipelines", "vulnerability_occurrences", column: "occurrence_id", on_delete: :cascade
add_foreign_key "vulnerability_occurrences", "projects", on_delete: :cascade
add_foreign_key "vulnerability_occurrences", "vulnerability_identifiers", column: "primary_identifier_id", on_delete: :cascade
add_foreign_key "vulnerability_occurrences", "vulnerability_scanners", column: "scanner_id", on_delete: :cascade
add_foreign_key "vulnerability_scanners", "projects", on_delete: :cascade
add_foreign_key "web_hook_logs", "web_hooks", on_delete: :cascade
......
......@@ -10,8 +10,17 @@ module EE
LICENSE_MANAGEMENT_FILE = 'gl-license-management-report.json'.freeze
PERFORMANCE_FILE = 'performance.json'.freeze
LICENSED_PARSER_FEATURES = {
sast: :sast
}.with_indifferent_access.freeze
prepended do
after_save :stick_build_if_status_changed
scope :with_security_reports, -> do
with_existing_job_artifacts(::Ci::JobArtifact.security_reports)
.eager_load_job_artifacts
end
end
def shared_runners_minutes_limit_enabled?
......@@ -46,6 +55,16 @@ module EE
artifacts_metadata?
end
def collect_security_reports!(security_reports)
each_report(::Ci::JobArtifact::SECURITY_REPORT_FILE_TYPES) do |file_type, blob|
next unless project.feature_available?(LICENSED_PARSER_FEATURES[file_type])
security_reports.get_report(file_type).tap do |security_report|
::Gitlab::Ci::Parsers::Security.fabricate!(file_type).parse!(blob, security_report)
end
end
end
private
def name_in?(names)
......
......@@ -9,8 +9,14 @@ module EE
prepended do
after_destroy :log_geo_deleted_event
SECURITY_REPORT_FILE_TYPES = %w[sast dependency_scanning container_scanning dast].freeze
scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) }
scope :geo_syncable, -> { with_files_stored_locally.not_expired }
scope :security_reports, -> do
with_file_types(SECURITY_REPORT_FILE_TYPES)
end
end
def log_geo_deleted_event
......
......@@ -12,7 +12,10 @@ module EE
has_one :chat_data, class_name: 'Ci::PipelineChatData'
has_many :job_artifacts, through: :builds
has_many :vulnerabilities_occurrence_pipelines, class_name: 'Vulnerabilities::OccurrencePipeline'
has_many :vulnerabilities, source: :occurrence, through: :vulnerabilities_occurrence_pipelines, class_name: 'Vulnerabilities::Occurrence'
# Legacy way to fetch security reports based on job name. This has been replaced by the reports feature.
scope :with_security_reports, -> {
joins(:artifacts).where(ci_builds: { name: %w[sast dependency_scanning sast:container container_scanning dast] })
}
......@@ -53,6 +56,16 @@ module EE
files: %w(gl-dast-report.json)
}
}.freeze
state_machine :status do
after_transition any => ::Ci::Pipeline::COMPLETED_STATUSES.map(&:to_sym) do |pipeline|
next unless pipeline.has_security_reports? && pipeline.default_branch?
pipeline.run_after_commit do
StoreSecurityReportsWorker.perform_async(pipeline.id)
end
end
end
end
def any_report_artifact_for_type(file_type)
......@@ -109,6 +122,18 @@ module EE
has_performance_data?
end
def has_security_reports?
complete? && builds.latest.with_security_reports.any?
end
def security_reports
::Gitlab::Ci::Reports::Security::Reports.new.tap do |security_reports|
builds.latest.with_security_reports.each do |build|
build.collect_security_reports!(security_reports)
end
end
end
private
def available_licensed_report_type?(file_type)
......
......@@ -215,6 +215,14 @@ module EE
checked_file_template_project&.id
end
def store_security_reports_available?
::Feature.enabled?(:store_security_reports, self) && (
feature_available?(:sast) ||
feature_available?(:dependency_scanning) ||
feature_available?(:sast_container) ||
feature_available?(:dast))
end
private
def validate_plan_name
......
......@@ -93,6 +93,8 @@ module EE
default_value_for :packages_enabled, true
default_value_for :only_mirror_protected_branches, true
delegate :store_security_reports_available?, to: :namespace
end
class_methods do
......
......@@ -10,8 +10,8 @@ module Vulnerabilities
has_many :occurrence_identifiers, class_name: 'Vulnerabilities::OccurrenceIdentifier'
has_many :occurrences, through: :occurrence_identifiers, class_name: 'Vulnerabilities::Occurrence'
has_many :primary_occurrences, -> { where(vulnerability_occurrences: { primary_identifier_fingerprint: fingerprint }) },
through: :occurrence_identifiers, class_name: 'Vulnerabilities::Occurrence', source: :occurrence
has_many :primary_occurrences, class_name: 'Vulnerabilities::Occurrence', inverse_of: :primary_identifier
belongs_to :project
......@@ -23,5 +23,7 @@ module Vulnerabilities
# TODO: find out why it fails
# validates :fingerprint, presence: true, uniqueness: { scope: :project_id }
validates :name, presence: true
scope :with_fingerprint, -> (fingerprints) { where(fingerprint: fingerprints) }
end
end
......@@ -22,15 +22,16 @@ module Vulnerabilities
}.with_indifferent_access.freeze
sha_attribute :project_fingerprint
sha_attribute :primary_identifier_fingerprint
sha_attribute :location_fingerprint
belongs_to :project
belongs_to :pipeline, class_name: 'Ci::Pipeline'
belongs_to :scanner, class_name: 'Vulnerabilities::Scanner'
belongs_to :primary_identifier, class_name: 'Vulnerabilities::Identifier', inverse_of: :primary_occurrences
has_many :occurrence_identifiers, class_name: 'Vulnerabilities::OccurrenceIdentifier'
has_many :identifiers, through: :occurrence_identifiers, class_name: 'Vulnerabilities::Identifier'
has_many :occurrence_pipelines, class_name: 'Vulnerabilities::OccurrencePipeline'
has_many :pipelines, through: :occurrence_pipelines, class_name: 'Ci::Pipeline'
REPORT_TYPES = {
sast: 0,
......@@ -43,16 +44,14 @@ module Vulnerabilities
validates :scanner, presence: true
validates :project, presence: true
validates :pipeline, presence: true
validates :uuid, presence: true
validates :ref, presence: true
validates :primary_identifier, presence: true
validates :project_fingerprint, presence: true
validates :primary_identifier_fingerprint, presence: true
validates :location_fingerprint, presence: true
# Uniqueness validation doesn't work with binary columns, so save this useless query. It is enforce by DB constraint anyway.
# TODO: find out why it fails
# validates :location_fingerprint, presence: true, uniqueness: { scope: [:primary_identifier_fingerprint, :scanner_id, :ref, :pipeline_id, :project_id] }
# validates :location_fingerprint, presence: true, uniqueness: { scope: [:primary_identifier_id, :scanner_id, :ref, :pipeline_id, :project_id] }
validates :name, presence: true
validates :report_type, presence: true
validates :severity, presence: true, inclusion: { in: LEVELS.keys }
......@@ -61,6 +60,7 @@ module Vulnerabilities
validates :metadata_version, presence: true
validates :raw_metadata, presence: true
scope :report_type, -> (type) { where(report_type: self.report_types[type]) }
scope :ordered, -> { order("severity desc", :id) }
scope :counted_by_report_and_severity, -> { group(:report_type, :severity).count }
......
# frozen_string_literal: true
module Vulnerabilities
class OccurrencePipeline < ActiveRecord::Base
self.table_name = "vulnerability_occurrence_pipelines"
belongs_to :occurrence, class_name: 'Vulnerabilities::Occurrence'
belongs_to :pipeline, class_name: '::Ci::Pipeline'
validates :occurrence, presence: true
validates :pipeline, presence: true
validates :pipeline_id, uniqueness: { scope: [:occurrence_id] }
end
end
......@@ -11,5 +11,7 @@ module Vulnerabilities
validates :project, presence: true
validates :external_id, presence: true, uniqueness: { scope: :project_id }
validates :name, presence: true
scope :with_external_id, -> (external_ids) { where(external_id: external_ids) }
end
end
# frozen_string_literal: true
module Security
# Service for storing a given security report into the database.
#
class StoreReportService < ::BaseService
include Gitlab::Utils::StrongMemoize
attr_reader :pipeline, :report, :project
def initialize(pipeline, report)
@pipeline = pipeline
@report = report
@project = @pipeline.project
end
def execute
# Ensure we're not trying to insert data twice for this report
return error("#{@report.type} report already stored for this pipeline, skipping...") if executed?
create_all_vulnerabilities!
success
end
private
def executed?
pipeline.vulnerabilities.report_type(@report.type).any?
end
def create_all_vulnerabilities!
@report.occurrences.each do |occurrence|
create_vulnerability(occurrence)
end
end
def create_vulnerability(occurrence)
vulnerability = create_or_find_vulnerability_object(occurrence)
occurrence[:identifiers].map do |identifier|
create_vulnerability_identifier_object(vulnerability, identifier)
end
create_vulnerability_pipeline_object(vulnerability, pipeline)
end
# rubocop: disable CodeReuse/ActiveRecord
def create_or_find_vulnerability_object(occurrence)
find_params = {
scanner: scanners_objects[occurrence[:scanner]],
primary_identifier: identifiers_objects[occurrence[:primary_identifier]],
location_fingerprint: occurrence[:location_fingerprint]
}
create_params = occurrence.except(
:scanner, :primary_identifier,
:location_fingerprint, :identifiers)
begin
project.vulnerabilities
.create_with(create_params)
.find_or_create_by!(find_params)
rescue ActiveRecord::RecordNotUnique
project.vulnerabilities.find_by!(find_params)
end
end
# rubocop: enable CodeReuse/ActiveRecord
def create_vulnerability_identifier_object(vulnerability, identifier)
vulnerability.occurrence_identifiers.find_or_create_by!( # rubocop: disable CodeReuse/ActiveRecord
identifier: identifiers_objects[identifier])
rescue ActiveRecord::RecordNotUnique
end
def create_vulnerability_pipeline_object(vulnerability, pipeline)
vulnerability.occurrence_pipelines.find_or_create_by!(pipeline: pipeline) # rubocop: disable CodeReuse/ActiveRecord
rescue ActiveRecord::RecordNotUnique
end
def scanners_objects
strong_memoize(:scanners_objects) do
@report.scanners.map do |key, scanner|
[key, existing_scanner_objects[key] || project.vulnerability_scanners.build(scanner)]
end.to_h
end
end
def all_scanners_external_ids
@report.scanners.values.map { |scanner| scanner[:external_id] }
end
def existing_scanner_objects
strong_memoize(:existing_scanner_objects) do
project.vulnerability_scanners.with_external_id(all_scanners_external_ids).map do |scanner|
[scanner.external_id, scanner]
end.to_h
end
end
def identifiers_objects
strong_memoize(:identifiers_objects) do
@report.identifiers.map do |key, identifier|
[key, existing_identifiers_objects[key] || project.vulnerability_identifiers.build(identifier)]
end.to_h
end
end
def all_identifiers_fingerprints
@report.identifiers.values.map { |identifier| identifier[:fingerprint] }
end
def existing_identifiers_objects
strong_memoize(:existing_identifiers_objects) do
project.vulnerability_identifiers.with_fingerprint(all_identifiers_fingerprints).map do |identifier|
[identifier.fingerprint, identifier]
end.to_h
end
end
end
end
# frozen_string_literal: true
module Security
# Service for storing security reports into the database.
#
class StoreReportsService < ::BaseService
def initialize(pipeline)
@pipeline = pipeline
end
def execute
errors = []
@pipeline.security_reports.reports.each do |report_type, report|
result = StoreReportService.new(@pipeline, report).execute
errors << result[:message] if result[:status] == :error
end
if errors.any?
error(errors.join(", "))
else
success
end
end
end
end
# frozen_string_literal: true
# Worker for storing security reports into the database.
#
class StoreSecurityReportsWorker
include ApplicationWorker
include PipelineQueue
def perform(pipeline_id)
Ci::Pipeline.find(pipeline_id).try do |pipeline|
break unless pipeline.project.store_security_reports_available?
::Security::StoreReportsService.new(pipeline).execute
end
end
end
---
title: Parse SAST reports and store vulnerabilities in database
merge_request: 7578
author:
type: added
class ChangeVulnOccurrenceColumns < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
drop_table :vulnerability_occurrence_identifiers
drop_table :vulnerability_occurrences
create_table :vulnerability_occurrences, id: :bigserial do |t|
t.timestamps_with_timezone null: false
t.integer :severity, null: false, limit: 2
t.integer :confidence, null: false, limit: 2
t.integer :report_type, null: false, limit: 2
t.references :project, null: false, foreign_key: { on_delete: :cascade }
t.bigint :scanner_id, null: false
t.foreign_key :vulnerability_scanners, column: :scanner_id, on_delete: :cascade
t.bigint :primary_identifier_id, null: false
t.foreign_key :vulnerability_identifiers, column: :primary_identifier_id, on_delete: :cascade
t.binary :project_fingerprint, null: false, limit: 20
t.binary :location_fingerprint, null: false, limit: 20
t.string :uuid, null: false, limit: 36
t.string :name, null: false
t.string :metadata_version, null: false
t.text :raw_metadata, null: false
t.index :primary_identifier_id
t.index :scanner_id
t.index :uuid, unique: true
t.index [:project_id, :primary_identifier_id, :location_fingerprint, :scanner_id],
unique: true,
name: 'index_vulnerability_occurrences_on_unique_keys',
length: { location_fingerprint: 20, primary_identifier_fingerprint: 20 }
end
create_table :vulnerability_occurrence_identifiers, id: :bigserial do |t|
t.timestamps_with_timezone null: false
t.bigint :occurrence_id, null: false
t.foreign_key :vulnerability_occurrences, column: :occurrence_id, on_delete: :cascade
t.bigint :identifier_id, null: false
t.foreign_key :vulnerability_identifiers, column: :identifier_id, on_delete: :cascade
t.index :identifier_id
t.index [:occurrence_id, :identifier_id],
unique: true,
name: 'index_vulnerability_occurrence_identifiers_on_unique_keys'
end
end
def down
drop_table :vulnerability_occurrence_identifiers
drop_table :vulnerability_occurrences
create_table :vulnerability_occurrences, id: :bigserial do |t|
t.timestamps_with_timezone null: false
t.integer :severity, null: false, limit: 2
t.integer :confidence, null: false, limit: 2
t.integer :report_type, null: false, limit: 2
t.integer :pipeline_id, null: false
t.foreign_key :ci_pipelines, column: :pipeline_id, on_delete: :cascade
t.references :project, null: false, foreign_key: { on_delete: :cascade }
t.bigint :scanner_id, null: false
t.foreign_key :vulnerability_scanners, column: :scanner_id, on_delete: :cascade
t.binary :project_fingerprint, null: false, limit: 20
t.binary :location_fingerprint, null: false, limit: 20
t.binary :primary_identifier_fingerprint, null: false, limit: 20
t.string :uuid, null: false, limit: 36
t.string :ref, null: false
t.string :name, null: false
t.string :metadata_version, null: false
t.text :raw_metadata, null: false
t.index :pipeline_id
t.index :scanner_id
t.index :uuid, unique: true
t.index [:project_id, :ref, :primary_identifier_fingerprint, :location_fingerprint, :pipeline_id, :scanner_id],
unique: true,
name: 'index_vulnerability_occurrences_on_unique_keys',
length: { location_fingerprint: 20, primary_identifier_fingerprint: 20 }
end
create_table :vulnerability_occurrence_identifiers, id: :bigserial do |t|
t.timestamps_with_timezone null: false
t.bigint :occurrence_id, null: false
t.foreign_key :vulnerability_occurrences, column: :occurrence_id, on_delete: :cascade
t.bigint :identifier_id, null: false
t.foreign_key :vulnerability_identifiers, column: :identifier_id, on_delete: :cascade
t.index :identifier_id
t.index [:occurrence_id, :identifier_id],
unique: true,
name: 'index_vulnerability_occurrence_identifiers_on_unique_keys'
end
end
end
class AddVulnOccurrencePipelines < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
create_table :vulnerability_occurrence_pipelines, id: :bigserial do |t|
t.timestamps_with_timezone null: false
t.bigint :occurrence_id, null: false
t.foreign_key :vulnerability_occurrences, column: :occurrence_id, on_delete: :cascade
t.integer :pipeline_id, null: false
t.foreign_key :ci_pipelines, column: :pipeline_id, on_delete: :cascade
t.index :pipeline_id
t.index [:occurrence_id, :pipeline_id],
unique: true,
name: 'vulnerability_occurrence_pipelines_on_unique_keys'
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Parsers
module Security
ParserNotFoundError = Class.new(StandardError)
PARSERS = {
sast: ::Gitlab::Ci::Parsers::Security::Sast
}.freeze
def self.fabricate!(file_type)
PARSERS.fetch(file_type.to_sym).new
rescue KeyError
raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'"
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Parsers
module Security
class Sast
SastParserError = Class.new(StandardError)
METADATA_VERSION = 'sast:1.3'
def parse!(json_data, report)
vulnerabilities = JSON.parse!(json_data)
vulnerabilities.each do |vulnerability|
create_vulnerability(report, vulnerability)
end
rescue JSON::ParserError
raise SastParserError, 'JSON parsing failed'
rescue
raise SastParserError, 'SAST report parsing failed'
end
protected
def create_vulnerability(report, data)
scanner = create_scanner(report, data['scanner'] || mutate_scanner_tool(data['tool']))
identifiers = create_identifiers(report, data['identifiers'])
report.add_occurrence(
uuid: SecureRandom.uuid,
report_type: report.type,
name: data['message'],
primary_identifier: identifiers.first,
project_fingerprint: generate_project_fingerprint(data['cve']),
location_fingerprint: generate_location_fingerprint(data['location']),
severity: parse_level(data['severity']),
confidence: parse_level(data['confidence']),
scanner: scanner,
identifiers: identifiers,
raw_metadata: data.to_json,
metadata_version: METADATA_VERSION # hardcoded untill provided in the report
)
end
def create_scanner(report, scanner)
return unless scanner.is_a?(Hash)
report.add_scanner(
external_id: scanner['id'],
name: scanner['name'])
end
def create_identifiers(report, identifiers)
return [] unless identifiers.is_a?(Array)
identifiers.map do |identifier|
create_identifier(report, identifier)
end.compact
end
def create_identifier(report, identifier)
return unless identifier.is_a?(Hash)
report.add_identifier(
external_type: identifier['type'],
external_id: identifier['value'],
name: identifier['name'],
fingerprint: generate_identifier_fingerprint(identifier),
url: identifier['url'])
end
def mutate_scanner_tool(tool)
{ 'id' => tool, 'name' => tool.capitalize } if tool
end
def parse_level(input)
input.blank? ? 'undefined' : input.downcase
end
def generate_location_fingerprint(location)
Digest::SHA1.hexdigest("#{location['file']}:#{location['start_line']}:#{location['end_line']}")
end
def generate_project_fingerprint(compare_key)
Digest::SHA1.hexdigest(compare_key)
end
def generate_identifier_fingerprint(identifier)
Digest::SHA1.hexdigest("#{identifier['type']}:#{identifier['value']}")
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Reports
module Security
class Report
attr_reader :type
attr_reader :occurrences
attr_reader :scanners
attr_reader :identifiers
def initialize(type)
@type = type
@occurrences = []
@scanners = {}
@identifiers = {}
end
def add_scanner(params)
scanner_key(params).tap do |key|
scanners[key] ||= params
end
end
def add_identifier(params)
identifier_key(params).tap do |key|
identifiers[key] ||= params
end
end
def add_occurrence(params)
params.tap do |occurrence|
occurrences << occurrence
end
end
private
def scanner_key(params)
params.fetch(:external_id)
end
def identifier_key(params)
params.fetch(:fingerprint)
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Reports
module Security
class Reports
attr_reader :reports
def initialize
@reports = {}
end
def get_report(report_type)
reports[report_type] ||= Report.new(report_type)
end
end
end
end
end
end
......@@ -5,5 +5,11 @@ FactoryBot.define do
failed
failure_reason { Ci::Build.failure_reasons[:protected_environment_failure] }
end
trait :security_reports do
after(:build) do |build|
build.job_artifacts << create(:ee_ci_job_artifact, :sast, job: build)
end
end
end
end
# frozen_string_literal: true
FactoryBot.define do
factory :ee_ci_job_artifact, class: ::Ci::JobArtifact, parent: :ci_job_artifact do
trait :sast do
file_type :sast
file_format :raw
after(:build) do |artifact, evaluator|
artifact.file = fixture_file_upload(
Rails.root.join('ee/spec/fixtures/reports/security/sast.json'), 'application/json')
end
end
trait :sast_with_corrupted_data do
file_type :sast
file_format :raw
after(:build) do |artifact, evaluator|
artifact.file = fixture_file_upload(
Rails.root.join('ee/spec/fixtures/reports/security/sast_with_corrupted_data.json'), 'application/json')
end
end
end
end
# frozen_string_literal: true
FactoryBot.define do
factory :vulnerabilities_occurrence_pipeline, class: Vulnerabilities::OccurrencePipeline do
occurrence factory: :vulnerabilities_occurrence
pipeline factory: :ci_pipeline
end
end
......@@ -8,11 +8,9 @@ FactoryBot.define do
factory :vulnerabilities_occurrence, class: Vulnerabilities::Occurrence do
name 'Cipher with no integrity'
project
pipeline factory: :ci_pipeline
ref 'master'
sequence(:uuid) { generate(:vulnerability_occurrence_uuid) }
project_fingerprint { generate(:project_fingerprint) }
primary_identifier_fingerprint '4e5b6966dd100170b4b1ad599c7058cce91b57b4'
primary_identifier factory: :vulnerabilities_identifier
location_fingerprint '4e5b6966dd100170b4b1ad599c7058cce91b57b4'
report_type :sast
severity :high
......
[
{
"category": "sast",
"name": "Cipher with no integrity",
"message": "Cipher with no integrity",
"description": "The cipher does not provide data integrity",
"cve": "e6449b89335daf53c0db4c0219bc1634:CIPHER_INTEGRITY",
"severity": "Medium",
"confidence": "High",
"location": {
"file": "maven/src/main/java/com/gitlab/security_products/tests/App.java",
"start_line": 29,
"end_line": 29,
"class": "com.gitlab.security_products.tests.App",
"method": "insecureCypher"
},
"scanner": {
"id": "find_sec_bugs",
"name": "Find Security Bugs"
},
"identifiers": [
{
"type": "find_sec_bugs_type",
"name": "Find Security Bugs-CIPHER_INTEGRITY",
"value": "CIPHER_INTEGRITY",
"url": "https://find-sec-bugs.github.io/bugs.htm#CIPHER_INTEGRITY"
},
{
"type": "cwe",
"name": "CWE-353",
"value": "353",
"url": "https://cwe.mitre.org/data/definitions/353.html"
}
],
"priority": "Medium",
"file": "maven/src/main/java/com/gitlab/security_products/tests/App.java",
"line": 29,
"url": "https://find-sec-bugs.github.io/bugs.htm#CIPHER_INTEGRITY",
"tool": "find_sec_bugs"
},
{
"category": "sast",
"message": "Use of insecure MD2, MD4, or MD5 hash function.",
"cve": "python/imports/imports-aliases.py:cb203b465dffb0cb3a8e8bd8910b84b93b0a5995a938e4b903dbb0cd6ffa1254:B303",
"severity": "Medium",
"confidence": "High",
"location": {
"file": "python/imports/imports-aliases.py",
"start_line": 11,
"end_line": 11
},
"scanner": {
"id": "bandit",
"name": "Bandit"
},
"identifiers": [
{
"type": "bandit_test_id",
"name": "Bandit Test ID B303",
"value": "B303"
},
{
"type": "cwe",
"name": "CWE-353",
"value": "353",
"url": "https://cwe.mitre.org/data/definitions/353.html"
}
],
"priority": "Medium",
"file": "python/imports/imports-aliases.py",
"line": 11,
"tool": "bandit"
},
{
"category": "sast",
"name": "Unescaped parameter value",
"message": "Unescaped parameter value",
"cve": "26b8b0ad586712d41ac6877e2292c6da7aa4760078add7fd23edf5b7a1bcb699",
"confidence": "High",
"location": {
"file": "rails5/app/views/widget/show.html.erb",
"start_line": 1
},
"scanner": {
"id": "brakeman",
"name": "Brakeman"
},
"identifiers": [
{
"type": "brakeman_warning_code",
"name": "Brakeman Warning Code 2",
"value": "2",
"url": "https://brakemanscanner.org/docs/warning_types/cross-site_scripting/"
}
],
"file": "rails5/app/views/widget/show.html.erb",
"line": 1,
"url": "https://brakemanscanner.org/docs/warning_types/cross-site_scripting/",
"tool": "brakeman"
}
]
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Ci::Parsers::Security::Sast do
describe '#parse!' do
let(:artifact) { create(:ee_ci_job_artifact, :sast) }
let(:project) { artifact.project }
let(:pipeline) { artifact.job.pipeline }
let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type) }
let(:sast) { described_class.new }
before do
artifact.each_blob do |blob|
sast.parse!(blob, report)
end
end
it "parses all identifiers and occurences" do
expect(report.occurrences.length).to eq(3)
expect(report.identifiers.length).to eq(4)
expect(report.scanners.length).to eq(3)
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Ci::Reports::Security::Report do
let(:pipeline) { create(:ci_pipeline) }
let(:report) { described_class.new('sast') }
it { expect(report.type).to eq('sast') }
describe '#add_scanner' do
let(:scanner) { { external_id: 'find_sec_bugs' } }
subject { report.add_scanner(scanner) }
it 'stores given scanner params in the map' do
subject
expect(report.scanners).to eq({ 'find_sec_bugs' => scanner })
end
it 'returns the map keyap' do
expect(subject).to eq('find_sec_bugs')
end
end
describe '#add_identifier' do
let(:identifier) { { fingerprint: '4e5b6966dd100170b4b1ad599c7058cce91b57b4' } }
subject { report.add_identifier(identifier) }
it 'stores given identifier params in the map' do
subject
expect(report.identifiers).to eq({ '4e5b6966dd100170b4b1ad599c7058cce91b57b4' => identifier })
end
it 'returns the map keyap' do
expect(subject).to eq('4e5b6966dd100170b4b1ad599c7058cce91b57b4')
end
end
describe '#add_occurrence' do
let(:occurrence) { { foo: :bar } }
it 'enriches given occurrence and stores it in the collection' do
report.add_occurrence(occurrence)
expect(report.occurrences).to eq([occurrence])
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Ci::Reports::Security::Reports do
let(:security_reports) { described_class.new }
describe '#get_report' do
subject { security_reports.get_report(report_type) }
context 'when report type is sast' do
let(:report_type) { 'sast' }
it { expect(subject.type).to eq('sast') }
it 'initializes a new report and returns it' do
expect(Gitlab::Ci::Reports::Security::Report).to receive(:new)
.with('sast').and_call_original
is_expected.to be_a(Gitlab::Ci::Reports::Security::Report)
end
context 'when report type is already allocated' do
before do
subject
end
it 'does not initialize a new report' do
expect(Gitlab::Ci::Reports::Security::Report).not_to receive(:new)
is_expected.to be_a(Gitlab::Ci::Reports::Security::Report)
end
end
end
end
end
......@@ -17,6 +17,8 @@ pipelines:
- triggered_pipelines
- chat_data
- job_artifacts
- vulnerabilities_occurrence_pipelines
- vulnerabilities
protected_branches:
- unprotect_access_levels
protected_environments:
......
......@@ -183,4 +183,72 @@ describe Ci::Build do
end
end
end
describe '.with_security_reports' do
subject { described_class.with_security_reports }
context 'when build has a security report' do
let!(:build) { create(:ee_ci_build, :success, :security_reports) }
it 'selects the build' do
is_expected.to eq([build])
end
end
context 'when build does not have security reports' do
let!(:build) { create(:ci_build, :success, :trace_artifact) }
it 'does not select the build' do
is_expected.to be_empty
end
end
context 'when there are multiple builds with security reports' do
let!(:builds) { create_list(:ee_ci_build, 5, :success, :security_reports) }
it 'does not execute a query for selecting job artifacts one by one' do
recorded = ActiveRecord::QueryRecorder.new do
subject.each do |build|
build.job_artifacts.map { |a| a.file.exists? }
end
end
expect(recorded.count).to eq(2)
end
end
end
describe '#collect_security_reports!' do
let(:security_reports) { ::Gitlab::Ci::Reports::Security::Reports.new }
subject { job.collect_security_reports!(security_reports) }
before do
stub_licensed_features(sast: true)
end
context 'when build has a security report' do
context 'when there is a sast report' do
before do
create(:ee_ci_job_artifact, :sast, job: job, project: job.project)
end
it 'parses blobs and add the results to the report' do
expect { subject }.not_to raise_error
expect(security_reports.get_report('sast').occurrences.size).to eq(3)
end
end
context 'when there is a corrupted sast report' do
before do
create(:ee_ci_job_artifact, :sast_with_corrupted_data, job: job, project: job.project)
end
it 'raises an error' do
expect { subject }.to raise_error(::Gitlab::Ci::Parsers::Security::Sast::SastParserError)
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe EE::Ci::JobArtifact do
describe Ci::JobArtifact do
describe '#destroy' do
set(:primary) { create(:geo_node, :primary) }
set(:secondary) { create(:geo_node) }
......@@ -13,4 +15,20 @@ describe EE::Ci::JobArtifact do
end.to change { Geo::JobArtifactDeletedEvent.count }.by(1)
end
end
describe '.security_reports' do
subject { described_class.security_reports }
context 'when there is a security report' do
let!(:artifact) { create(:ee_ci_job_artifact, :sast) }
it { is_expected.to eq([artifact]) }
end
context 'when there are no security reports' do
let!(:artifact) { create(:ci_job_artifact, :archive) }
it { is_expected.to be_empty }
end
end
end
......@@ -10,6 +10,8 @@ describe Ci::Pipeline do
it { is_expected.to have_one(:chat_data) }
it { is_expected.to have_many(:job_artifacts).through(:builds) }
it { is_expected.to have_many(:vulnerabilities).through(:vulnerabilities_occurrence_pipelines).class_name('Vulnerabilities::Occurrence') }
it { is_expected.to have_many(:vulnerabilities_occurrence_pipelines).class_name('Vulnerabilities::OccurrencePipeline') }
describe '.failure_reasons' do
it 'contains failure reasons about exceeded limits' do
......@@ -299,4 +301,141 @@ describe Ci::Pipeline do
expect { pipeline.license_management_artifact }.not_to exceed_query_limit(0)
end
end
describe '#has_security_reports?' do
subject { pipeline.has_security_reports? }
context 'when pipeline has builds with security reports' do
before do
create(:ee_ci_build, :security_reports, pipeline: pipeline, project: project)
end
context 'when pipeline status is running' do
let(:pipeline) { create(:ci_pipeline, :running, project: project) }
it { is_expected.to be_falsey }
end
context 'when pipeline status is success' do
let(:pipeline) { create(:ci_pipeline, :success, project: project) }
it { is_expected.to be_truthy }
end
end
context 'when pipeline does not have builds with security reports' do
before do
create(:ci_build, :artifacts, pipeline: pipeline, project: project)
end
let(:pipeline) { create(:ci_pipeline, :success, project: project) }
it { is_expected.to be_falsey }
end
context 'when retried build has security reports' do
before do
create(:ee_ci_build, :retried, :security_reports, pipeline: pipeline, project: project)
end
let(:pipeline) { create(:ci_pipeline, :success, project: project) }
it { is_expected.to be_falsey }
end
end
describe '#security_reports' do
subject { pipeline.security_reports }
before do
stub_licensed_features(sast: true)
end
context 'when pipeline has multiple builds with security reports' do
let!(:build_sast_1) { create(:ci_build, :success, name: 'sast_1', pipeline: pipeline, project: project) }
let!(:build_sast_2) { create(:ci_build, :success, name: 'sast_2', pipeline: pipeline, project: project) }
before do
create(:ee_ci_job_artifact, :sast, job: build_sast_1, project: project)
create(:ee_ci_job_artifact, :sast, job: build_sast_2, project: project)
end
it 'returns security reports with collected data grouped as expected' do
expect(subject.reports.keys).to eq(%w(sast))
expect(subject.get_report('sast').occurrences.size).to eq(6)
end
context 'when builds are retried' do
let!(:build_sast_1) { create(:ci_build, :retried, name: 'sast_1', pipeline: pipeline, project: project) }
let!(:build_sast_2) { create(:ci_build, :retried, name: 'sast_2', pipeline: pipeline, project: project) }
it 'does not take retried builds into account' do
expect(subject.reports).to eq({})
end
end
end
context 'when pipeline does not have any builds with security reports' do
it 'returns empty security reports' do
expect(subject.reports).to eq({})
end
end
end
describe 'Store security reports worker' do
using RSpec::Parameterized::TableSyntax
where(:state, :transition) do
:success | :succeed
:failed | :drop
:skipped | :skip
:cancelled | :cancel
end
with_them do
context 'when pipeline has security reports and ref is the default branch of project' do
let(:default_branch) { pipeline.ref }
before do
create(:ee_ci_build, :security_reports, pipeline: pipeline, project: project)
allow(project).to receive(:default_branch) { default_branch }
end
context "when transitioning to #{params[:state]}" do
it 'schedules store security report worker' do
expect(StoreSecurityReportsWorker).to receive(:perform_async).with(pipeline.id)
pipeline.update!(status_event: transition)
end
end
end
context 'when pipeline does NOT have security reports' do
context "when transitioning to #{params[:state]}" do
it 'does NOT schedule store security report worker' do
expect(StoreSecurityReportsWorker).not_to receive(:perform_async).with(pipeline.id)
pipeline.update!(status_event: transition)
end
end
end
context "when pipeline ref is not the project's default branch" do
let(:default_branch) { 'another_branch' }
before do
stub_licensed_features(sast: true)
allow(project).to receive(:default_branch) { default_branch }
end
context "when transitioning to #{params[:state]}" do
it 'does NOT schedule store security report worker' do
expect(StoreSecurityReportsWorker).not_to receive(:perform_async).with(pipeline.id)
pipeline.update!(status_event: transition)
end
end
end
end
end
end
......@@ -596,4 +596,40 @@ describe Namespace do
expect(namespace.checked_file_template_project_id).to be_nil
end
end
describe '#store_security_reports_available?' do
subject { namespace.store_security_reports_available? }
context 'when store_security_reports feature is enabled' do
before do
stub_feature_flags(store_security_reports: true)
stub_licensed_features(sast: true)
end
it 'returns true' do
expect(subject).to be_truthy
end
end
context 'when store_security_reports feature is disabled' do
before do
stub_feature_flags(store_security_reports: false)
stub_licensed_features(sast: true)
end
it 'returns false' do
expect(subject).to be_falsey
end
end
context 'when no security report feature is available' do
before do
stub_feature_flags(store_security_reports: true)
end
it 'returns false' do
expect(subject).to be_falsey
end
end
end
end
......@@ -1742,4 +1742,16 @@ describe Project do
end
end
end
describe '#store_security_reports_available?' do
let(:project) { create(:project) }
subject { project.store_security_reports_available? }
it 'delegates to namespace' do
expect(project.namespace).to receive(:store_security_reports_available?).once.and_call_original
subject
end
end
end
......@@ -21,4 +21,26 @@ describe Vulnerabilities::Identifier do
# Uniqueness validation doesn't work with binary columns. See TODO in class file
# it { is_expected.to validate_uniqueness_of(:fingerprint).scoped_to(:project_id) }
end
describe '.with_fingerprint' do
let(:fingerprint) { 'f5724386167705667ae25a1390c0a516020690ba' }
subject { described_class.with_fingerprint(fingerprint) }
context 'when identifier has the corresponding fingerprint' do
let!(:identifier) { create(:vulnerabilities_identifier, fingerprint: fingerprint) }
it 'selects the identifier' do
is_expected.to eq([identifier])
end
end
context 'when identifier does not have the corresponding fingerprint' do
let!(:identifier) { create(:vulnerabilities_identifier) }
it 'does not select the identifier' do
is_expected.to be_empty
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Vulnerabilities::OccurrencePipeline do
describe 'associations' do
it { is_expected.to belong_to(:pipeline).class_name('Ci::Pipeline') }
it { is_expected.to belong_to(:occurrence).class_name('Vulnerabilities::Occurrence') }
end
describe 'validations' do
let!(:occurrence_pipeline) { create(:vulnerabilities_occurrence_pipeline) }
it { is_expected.to validate_presence_of(:occurrence) }
it { is_expected.to validate_presence_of(:pipeline) }
it { is_expected.to validate_uniqueness_of(:pipeline_id).scoped_to(:occurrence_id) }
end
end
......@@ -7,8 +7,10 @@ describe Vulnerabilities::Occurrence do
describe 'associations' do
it { is_expected.to belong_to(:project) }
it { is_expected.to belong_to(:pipeline) }
it { is_expected.to belong_to(:primary_identifier).class_name('Vulnerabilities::Identifier') }
it { is_expected.to belong_to(:scanner).class_name('Vulnerabilities::Scanner') }
it { is_expected.to have_many(:pipelines).class_name('Ci::Pipeline') }
it { is_expected.to have_many(:occurrence_pipelines).class_name('Vulnerabilities::OccurrencePipeline') }
it { is_expected.to have_many(:identifiers).class_name('Vulnerabilities::Identifier') }
it { is_expected.to have_many(:occurrence_identifiers).class_name('Vulnerabilities::OccurrenceIdentifier') }
end
......@@ -18,11 +20,9 @@ describe Vulnerabilities::Occurrence do
it { is_expected.to validate_presence_of(:scanner) }
it { is_expected.to validate_presence_of(:project) }
it { is_expected.to validate_presence_of(:pipeline) }
it { is_expected.to validate_presence_of(:ref) }
it { is_expected.to validate_presence_of(:uuid) }
it { is_expected.to validate_presence_of(:project_fingerprint) }
it { is_expected.to validate_presence_of(:primary_identifier_fingerprint) }
it { is_expected.to validate_presence_of(:primary_identifier) }
it { is_expected.to validate_presence_of(:location_fingerprint) }
it { is_expected.to validate_presence_of(:name) }
it { is_expected.to validate_presence_of(:report_type) }
......@@ -47,10 +47,8 @@ describe Vulnerabilities::Occurrence do
# we use block to delay object creations
where(:key, :value_block) do
:primary_identifier_fingerprint | -> { '005b6966dd100170b4b1ad599c7058cce91b57b4' }
:ref | -> { 'another_ref' }
:primary_identifier | -> { create(:vulnerabilities_identifier) }
:scanner | -> { create(:vulnerabilities_scanner) }
:pipeline | -> { create(:ci_pipeline) }
:project | -> { create(:project) }
end
......@@ -61,4 +59,26 @@ describe Vulnerabilities::Occurrence do
end
end
end
describe '.report_type' do
let(:report_type) { :sast }
subject { described_class.report_type(report_type) }
context 'when occurrence has the corresponding report type' do
let!(:occurrence) { create(:vulnerabilities_occurrence, report_type: report_type) }
it 'selects the occurrence' do
is_expected.to eq([occurrence])
end
end
context 'when occurrence does not have security reports' do
let!(:occurrence) { create(:vulnerabilities_occurrence, report_type: :dependency_scanning) }
it 'does not select the occurrence' do
is_expected.to be_empty
end
end
end
end
......@@ -16,4 +16,26 @@ describe Vulnerabilities::Scanner do
it { is_expected.to validate_presence_of(:project) }
it { is_expected.to validate_uniqueness_of(:external_id).scoped_to(:project_id) }
end
describe '.with_external_id' do
let(:external_id) { 'bandit' }
subject { described_class.with_external_id(external_id) }
context 'when scanner has the corresponding external_id' do
let!(:scanner) { create(:vulnerabilities_scanner, external_id: external_id) }
it 'selects the scanner' do
is_expected.to eq([scanner])
end
end
context 'when scanner does not have the corresponding external_id' do
let!(:scanner) { create(:vulnerabilities_scanner) }
it 'does not select the scanner' do
is_expected.to be_empty
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Security::StoreReportService, '#execute' do
let(:artifact) { create(:ee_ci_job_artifact, :sast) }
let(:project) { artifact.project }
let(:pipeline) { artifact.job.pipeline }
let(:report) { pipeline.security_reports.get_report('sast') }
before do
stub_licensed_features(sast: true)
end
subject { described_class.new(pipeline, report).execute }
context 'without existing data' do
it 'inserts all scanners' do
expect { subject }.to change { Vulnerabilities::Scanner.count }.by(3)
end
it 'inserts all identifiers' do
expect { subject }.to change { Vulnerabilities::Identifier.count }.by(4)
end
it 'inserts all occurrences' do
expect { subject }.to change { Vulnerabilities::Occurrence.count }.by(3)
end
it 'inserts all occurrence identifiers (join model)' do
expect { subject }.to change { Vulnerabilities::OccurrenceIdentifier.count }.by(5)
end
it 'inserts all occurrence pipelines (join model)' do
expect { subject }.to change { Vulnerabilities::OccurrencePipeline.count }.by(3)
end
end
context 'with existing data from previous pipeline' do
let(:scanner) { create(:vulnerabilities_scanner, project: project, external_id: 'find_sec_bugs', name: 'existing_name') }
let(:identifier) { create(:vulnerabilities_identifier, project: project, fingerprint: 'f5724386167705667ae25a1390c0a516020690ba') }
let!(:new_artifact) { create(:ee_ci_job_artifact, :sast, job: new_build) }
let(:new_build) { create(:ci_build, pipeline: new_pipeline) }
let(:new_pipeline) { create(:ci_pipeline, project: project) }
let(:new_report) { new_pipeline.security_reports.get_report('sast') }
let!(:occurrence) do
create(:vulnerabilities_occurrence,
pipelines: [pipeline],
identifiers: [identifier],
primary_identifier: identifier,
scanner: scanner,
project: project,
location_fingerprint: '6b6bb283d43cc510d7d1e73e2882b3652cb34bd5')
end
subject { described_class.new(new_pipeline, new_report).execute }
it 'inserts only new scanners and reuse existing ones' do
expect { subject }.to change { Vulnerabilities::Scanner.count }.by(2)
end
it 'inserts only new identifiers and reuse existing ones' do
expect { subject }.to change { Vulnerabilities::Identifier.count }.by(3)
end
it 'inserts only new occurrences and reuse existing ones' do
expect { subject }.to change { Vulnerabilities::Occurrence.count }.by(2)
end
it 'inserts all occurrence pipelines (join model) for this new pipeline' do
expect { subject }.to change { Vulnerabilities::OccurrencePipeline.where(pipeline: new_pipeline).count }.by(3)
end
end
context 'with existing data from same pipeline' do
let!(:occurrence) { create(:vulnerabilities_occurrence, project: project, pipelines: [pipeline]) }
it 'skips report' do
expect(subject).to eq({
status: :error,
message: "sast report already stored for this pipeline, skipping..."
})
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Security::StoreReportsService, '#execute' do
let(:pipeline) { create(:ci_pipeline) }
subject { described_class.new(pipeline).execute }
context 'when there are reports' do
before do
stub_licensed_features(sast: true)
create(:ee_ci_build, :security_reports, pipeline: pipeline)
end
it 'initializes a new StoreReportService and execute it' do
expect(Security::StoreReportService).to receive(:new)
.with(pipeline, instance_of(::Gitlab::Ci::Reports::Security::Report)).and_call_original
expect_any_instance_of(Security::StoreReportService).to receive(:execute)
.once.and_call_original
subject
end
context 'when StoreReportService returns an error for a report' do
let(:reports) { Gitlab::Ci::Reports::Security::Reports.new }
let(:sast_report) { reports.get_report('sast') }
let(:dast_report) { reports.get_report('dast') }
let(:success) { { status: :success } }
let(:error) { { status: :error, message: "something went wrong" } }
before do
allow(pipeline).to receive(:security_reports).and_return(reports)
end
it 'returns the errors after having processed all reports' do
expect_next_instance_of(Security::StoreReportService, pipeline, sast_report) do |store_service|
expect(store_service).to receive(:execute).and_return(error)
end
expect_next_instance_of(Security::StoreReportService, pipeline, dast_report) do |store_service|
expect(store_service).to receive(:execute).and_return(success)
end
is_expected.to eq(error)
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe StoreSecurityReportsWorker do
describe '#perform' do
let(:pipeline) { create(:ci_pipeline, ref: 'master') }
let(:project) { pipeline.project }
before do
allow(Ci::Pipeline).to receive(:find).with(pipeline.id) { pipeline }
end
context 'when all conditions are met' do
before do
stub_licensed_features(sast: true)
stub_feature_flags(store_security_reports: true)
end
it 'executes StoreReportsService for given pipeline' do
expect(Security::StoreReportsService).to receive(:new)
.with(pipeline).once.and_call_original
described_class.new.perform(pipeline.id)
end
end
context "when security reports feature is not available" do
let(:default_branch) { pipeline.ref }
it 'does not execute StoreReportsService' do
expect(Security::StoreReportsService).not_to receive(:new)
described_class.new.perform(pipeline.id)
end
end
context "when store security reports feature is not enabled" do
let(:default_branch) { pipeline.ref }
before do
stub_licensed_features(sast: true)
stub_feature_flags(store_security_reports: false)
end
it 'does not execute StoreReportsService' do
expect(Security::StoreReportsService).not_to receive(:new)
described_class.new.perform(pipeline.id)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment