Skip to content

G
gitlab-ce
Commit 857a07c5 authored by Arturo Herrero's avatar Arturo Herrero
Browse files
Options

Merge branch 'sy-refactor-alert-processing-specs' into 'master' 

Refactor alert processing specs for uniformity

See merge request gitlab-org/gitlab!58519
parents ad2cb86b df5a8953
Hide whitespace changes
Inline Side-by-side
Showing with 903 additions and 775 deletions
ee/spec/services/alert_management/network_alert_service_spec.rb
View file @ 857a07c5
......@@ -6,37 +6,23 @@ RSpec.describe AlertManagement::NetworkAlertService do
let_it_be(:project, reload: true) { create(:project, :repository) }
let_it_be(:environment) { create(:environment, project: project) }
describe '#execute' do
let(:service) { described_class.new(project, payload) }
let(:tool) { Gitlab::AlertManagement::Payload::MONITORING_TOOLS[:cilium] }
let(:starts_at) { Time.current.change(usec: 0) }
let(:ended_at) { nil }
let(:fingerprint) { 'test' }
let(:domain) { 'threat_monitoring' }
let(:incident_management_setting) { double(auto_close_incident?: auto_close_enabled) }
let(:payload_raw) { build(:network_alert_payload) }
let(:payload) { ActionController::Parameters.new(payload_raw).permit! }
let(:auto_close_enabled) { true }
let(:service) { described_class.new(project, payload) }
before do
allow(service).to receive(:incident_management_setting).and_return(
incident_management_setting
)
end
describe '#execute' do
include_context 'incident management settings enabled'
subject(:execute) { service.execute }
context 'with valid payload' do
let(:payload_raw) { build(:network_alert_payload) }
let(:payload) { ActionController::Parameters.new(payload_raw).permit! }
let(:last_alert_attributes) do
AlertManagement::Alert.last.attributes.except('id', 'iid', 'created_at', 'updated_at')
.with_indifferent_access
end
shared_examples 'never-before-seen network alert' do
it_behaves_like 'creates an alert management alert or errors'
it_behaves_like 'creates expected system notes for alert', :new_alert
it_behaves_like 'does not send alert notification emails'
it_behaves_like 'does not process incident issues'
it 'create alert and assigns properties' do
it 'assigns the correct properties' do
subject
expect(last_alert_attributes).to match(a_hash_including({
......@@ -57,125 +43,67 @@ RSpec.describe AlertManagement::NetworkAlertService do
title: 'Cilium Alert'
}))
end
end
it 'creates a system note corresponding to alert creation' do
expect { subject }.to change(Note, :count).by(1)
expect(Note.last.note).to include('Cilium')
end
context 'when alert exists' do
let!(:alert) do
create(
:alert_management_alert,
project: project, domain: :threat_monitoring, fingerprint: '89269ffa3902af37f036a77bc9ea57cdee3a52c2'
)
end
shared_examples 'existing network alert' do
it_behaves_like 'adds an alert management alert event'
it_behaves_like 'does not create a system note for alert'
it_behaves_like 'does not send alert notification emails'
it_behaves_like 'does not process incident issues'
end
it_behaves_like 'does not an create alert management alert'
context 'with valid payload' do
let(:source) { Gitlab::AlertManagement::Payload::MONITORING_TOOLS[:cilium] }
let(:last_alert_attributes) do
AlertManagement::Alert.last.attributes.except('id', 'iid', 'created_at', 'updated_at')
.with_indifferent_access
end
context 'existing alert with same fingerprint' do
let(:fingerprint_sha) { '89269ffa3902af37f036a77bc9ea57cdee3a52c2' }
let!(:alert) do
create(:alert_management_alert, domain: :threat_monitoring, project: project, fingerprint: fingerprint_sha)
end
it_behaves_like 'adds an alert management alert event'
it_behaves_like 'never-before-seen network alert'
context 'end time given' do
let(:ended_at) { Time.current.change(nsec: 0) }
context 'for an existing alert with the same fingerprint' do
let_it_be(:fingerprint_sha) { '89269ffa3902af37f036a77bc9ea57cdee3a52c2' }
context 'auto_close disabled' do
let(:auto_close_enabled) { false }
it 'does not resolve the alert' do
expect { subject }.not_to change { alert.reload.status }
end
it 'does not set the ended at' do
subject
expect(alert.reload.ended_at).to be_nil
end
it_behaves_like 'does not an create alert management alert'
end
end
context 'existing alert is resolved' do
let!(:alert) do
create(
:alert_management_alert,
:resolved,
project: project, domain: :threat_monitoring, fingerprint: fingerprint_sha
)
context 'which is triggered' do
let_it_be(:alert) do
create(:alert_management_alert, :triggered, domain: :threat_monitoring, project: project, fingerprint: fingerprint_sha)
end
it_behaves_like 'creates an alert management alert'
end
it_behaves_like 'existing network alert'
context 'with an additional existing resolved alert' do
before do
create(
:alert_management_alert,
:resolved,
domain: :threat_monitoring,
project: project,
fingerprint: fingerprint_sha
)
end
context 'existing alert is ignored' do
let!(:alert) do
create(
:alert_management_alert,
:ignored,
project: project, domain: :threat_monitoring, fingerprint: fingerprint_sha
)
it_behaves_like 'existing network alert'
end
it_behaves_like 'adds an alert management alert event'
end
context 'two existing alerts, one resolved one open' do
let!(:resolved_existing_alert) do
create(
:alert_management_alert,
:resolved,
project: project, fingerprint: fingerprint_sha
)
end
let!(:alert) do
create(:alert_management_alert, domain: :threat_monitoring, project: project, fingerprint: fingerprint_sha)
context 'which is resolved' do
let_it_be(:alert) do
create(:alert_management_alert, :resolved, domain: :threat_monitoring, project: project, fingerprint: fingerprint_sha)
end
it_behaves_like 'adds an alert management alert event'
it_behaves_like 'never-before-seen network alert'
end
end
end
context 'with overlong payload' do
let(:deep_size_object) { instance_double(Gitlab::Utils::DeepSize, valid?: false) }
let(:payload) { ActionController::Parameters.new({}).permit! }
before do
allow(Gitlab::Utils::DeepSize).to receive(:new).and_return(deep_size_object)
end
it_behaves_like 'does not process incident issues due to error', http_status: :bad_request
it_behaves_like 'does not an create alert management alert'
end
context 'error duing save' do
let(:payload_raw) { build(:network_alert_payload) }
let(:logger) { double(warn: {}) }
let(:payload) { ActionController::Parameters.new(payload_raw).permit! }
it 'logs warning' do
expect_any_instance_of(AlertManagement::Alert).to receive(:save).and_return(false)
expect_any_instance_of(described_class).to receive(:logger).and_return(logger)
subject
expect(logger).to have_received(:warn).with(
hash_including(
message: "Unable to create AlertManagement::Alert from #{tool}",
project_id: project.id,
alert_errors: {}
)
)
end
it_behaves_like 'alerts service responds with an error and takes no actions', :bad_request
end
end
end
ee/spec/services/alert_management/process_prometheus_alert_service_spec.rb
View file @ 857a07c5
......@@ -5,25 +5,23 @@ require 'spec_helper'
RSpec.describe AlertManagement::ProcessPrometheusAlertService do
let_it_be(:project, refind: true) { create(:project) }
before do
allow(ProjectServiceWorker).to receive(:perform_async)
end
describe '#execute' do
let(:service) { described_class.new(project, payload) }
subject(:execute) { service.execute }
context 'when alert payload is valid' do
let(:parsed_payload) { Gitlab::AlertManagement::Payload.parse(project, payload, monitoring_tool: 'Prometheus') }
let(:fingerprint) { parsed_payload.gitlab_fingerprint }
let_it_be(:starts_at) { '2020-04-27T10:10:22.265949279Z' }
let_it_be(:title) { 'Alert title' }
let_it_be(:gitlab_fingerprint) { Digest::SHA1.hexdigest([starts_at, title, 'vector(1)'].join('/')) }
let(:payload) { raw_payload }
let(:raw_payload) do
{
'status' => 'firing',
'labels' => { 'alertname' => 'GitalyFileServerDown' },
'annotations' => { 'title' => 'Alert title' },
'startsAt' => '2020-04-27T10:10:22.265949279Z',
'annotations' => { 'title' => title },
'startsAt' => starts_at,
'endsAt' => '2020-04-27T10:20:22.265949279Z',
'generatorURL' => 'http://8d467bd4607a:9090/graph?g0.expr=vector%281%29&g0.tab=1'
}
......@@ -33,10 +31,19 @@ RSpec.describe AlertManagement::ProcessPrometheusAlertService do
let_it_be(:schedule) { create(:incident_management_oncall_schedule, project: project) }
let_it_be(:rotation) { create(:incident_management_oncall_rotation, schedule: schedule) }
let_it_be(:participant) { create(:incident_management_oncall_participant, :with_developer_access, rotation: rotation) }
let(:resolving_payload) { raw_payload.merge('status' => 'resolved') }
let(:users) { [participant.user] }
it_behaves_like 'oncall users are correctly notified'
before do
stub_licensed_features(oncall_schedules: project)
end
include_examples 'oncall users are correctly notified of firing alert'
context 'with resolving payload' do
let(:payload) { raw_payload.merge('status' => 'resolved') }
include_examples 'oncall users are correctly notified of Prometheus recovery alert'
end
end
end
end
......
ee/spec/services/projects/alerting/notify_service_spec.rb
View file @ 857a07c5
......@@ -69,13 +69,28 @@ RSpec.describe Projects::Alerting::NotifyService do
let_it_be(:schedule) { create(:incident_management_oncall_schedule, project: project) }
let_it_be(:rotation) { create(:incident_management_oncall_rotation, schedule: schedule) }
let_it_be(:participant) { create(:incident_management_oncall_participant, :with_developer_access, rotation: rotation) }
let_it_be(:fingerprint) { 'fingerprint' }
let_it_be(:gitlab_fingerprint) { Digest::SHA1.hexdigest(fingerprint) }
let(:payload) { { 'fingerprint' => 'fingerprint' } }
let(:resolving_payload) { { 'fingerprint' => 'fingerprint', "end_time": Time.current.iso8601 } }
let(:payload) { { 'fingerprint' => fingerprint } }
let(:users) { [participant.user] }
let(:fingerprint) { Digest::SHA1.hexdigest('fingerprint') }
it_behaves_like 'oncall users are correctly notified'
before do
stub_licensed_features(oncall_schedules: project)
end
include_examples 'oncall users are correctly notified of firing alert'
context 'with resolving payload' do
let(:payload) do
{
'fingerprint' => fingerprint,
'end_time' => Time.current.iso8601
}
end
include_examples 'oncall users are correctly notified of recovery alert'
end
end
end
end
ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb
View file @ 857a07c5
......@@ -10,7 +10,7 @@ RSpec.describe Projects::Prometheus::Alerts::NotifyService do
let(:service) { described_class.new(project, payload) }
let(:token_input) { 'token' }
let!(:setting) do
let_it_be(:setting) do
create(:project_incident_management_setting, project: project, send_email: true, create_issue: true)
end
......@@ -23,6 +23,7 @@ RSpec.describe Projects::Prometheus::Alerts::NotifyService do
let(:payload) { ActionController::Parameters.new(payload_raw).permit! }
let(:payload_alert_firing) { payload_raw['alerts'].first }
let(:token) { 'token' }
let(:source) { 'Prometheus' }
context 'with environment specific clusters' do
let(:prd_cluster) do
......@@ -51,11 +52,11 @@ RSpec.describe Projects::Prometheus::Alerts::NotifyService do
context 'without token' do
let(:token_input) { nil }
it_behaves_like 'Alert Notification Service sends notification email'
include_examples 'processes one firing and one resolved prometheus alerts'
end
context 'with token' do
it_behaves_like 'Alert Notification Service sends no notifications', http_status: :unauthorized
it_behaves_like 'alerts service responds with an error', :unauthorized
end
end
end
......
ee/spec/support/shared_examples/alert_notification_service_shared_examples.rb deleted 100644 → 0
View file @ ad2cb86b
# frozen_string_literal: true
# Requires `users` and `fingerprint` to be defined
RSpec.shared_examples 'Alert Notification Service sends notification email to on-call users' do
let(:notification_service) { instance_double(NotificationService) }
it 'sends a notification' do
expect(NotificationService).to receive(:new).and_return(notification_service)
expect(notification_service)
.to receive_message_chain(:async, :notify_oncall_users_of_alert)
.with(
users,
having_attributes(class: AlertManagement::Alert, fingerprint: fingerprint)
)
expect(subject).to be_success
end
end
ee/spec/support/shared_examples/incident_management/oncall_notifications_shared_examples.rb deleted 100644 → 0
View file @ ad2cb86b
# frozen_string_literal: true
# Requires `project`, `users`, `fingerprint`, and `resolving_payload`
RSpec.shared_examples 'oncall users are correctly notified' do
context 'with feature enabled' do
before do
stub_licensed_features(oncall_schedules: project)
end
it_behaves_like 'Alert Notification Service sends notification email to on-call users'
context 'when alert with the same fingerprint already exists' do
let!(:alert) { create(:alert_management_alert, status, fingerprint: fingerprint, project: project) }
it_behaves_like 'Alert Notification Service sends notification email to on-call users' do
let(:status) { :triggered }
end
it_behaves_like 'Alert Notification Service sends no notifications' do
let(:status) { :acknowledged }
end
it_behaves_like 'Alert Notification Service sends notification email to on-call users' do
let(:status) { :resolved }
end
it_behaves_like 'Alert Notification Service sends no notifications' do
let(:status) { :ignored }
end
context 'with resolving payload' do
let(:status) { :triggered }
let(:payload) { resolving_payload }
it_behaves_like 'Alert Notification Service sends notification email to on-call users'
end
end
end
context 'with feature disabled' do
it_behaves_like 'Alert Notification Service sends no notifications'
end
end
ee/spec/support/shared_examples/services/alert_management/alert_processing/alert_firing_shared_examples.rb 0 → 100644
View file @ 857a07c5
# frozen_string_literal: true
# This shared_example requires the following variables:
# - `project`, expected project for an incoming alert
# - `users`, users expected to receive on-call notifications
# - `gitlab_fingerprint`, sha which is used to uniquely identify the alert
RSpec.shared_examples 'oncall users are correctly notified of firing alert' do
it_behaves_like 'sends on-call notification if enabled'
context 'when alert with the same fingerprint already exists' do
context 'which is triggered' do
let_it_be(:alert) { create(:alert_management_alert, :triggered, fingerprint: gitlab_fingerprint, project: project) }
it_behaves_like 'sends on-call notification if enabled'
end
context 'which is acknowledged' do
let_it_be(:alert) { create(:alert_management_alert, :acknowledged, fingerprint: gitlab_fingerprint, project: project) }
it_behaves_like 'does not send on-call notification'
end
context 'which is resolved' do
let_it_be(:alert) { create(:alert_management_alert, :resolved, fingerprint: gitlab_fingerprint, project: project) }
it_behaves_like 'sends on-call notification if enabled'
end
context 'which is ignored' do
let_it_be(:alert) { create(:alert_management_alert, :ignored, fingerprint: gitlab_fingerprint, project: project) }
it_behaves_like 'does not send on-call notification'
end
end
end
ee/spec/support/shared_examples/services/alert_management/alert_processing/alert_recovery_shared_examples.rb 0 → 100644
View file @ 857a07c5
# frozen_string_literal: true
# This shared_example requires the following variables:
# - `project`, expected project for an incoming alert
# - `users`, users expected to receive on-call notifications
# - `gitlab_fingerprint`, sha which is used to uniquely identify the alert
RSpec.shared_examples 'oncall users are correctly notified of recovery alert' do
it_behaves_like 'sends on-call notification if enabled'
context 'when alert with the same fingerprint already exists' do
context 'which is triggered' do
let_it_be(:alert) { create(:alert_management_alert, :triggered, fingerprint: gitlab_fingerprint, project: project) }
it_behaves_like 'sends on-call notification if enabled'
end
context 'which is acknowledged' do
let_it_be(:alert) { create(:alert_management_alert, :acknowledged, fingerprint: gitlab_fingerprint, project: project) }
it_behaves_like 'sends on-call notification if enabled'
end
context 'which is resolved' do
let_it_be(:alert) { create(:alert_management_alert, :resolved, fingerprint: gitlab_fingerprint, project: project) }
it_behaves_like 'sends on-call notification if enabled'
end
context 'which is ignored' do
let_it_be(:alert) { create(:alert_management_alert, :ignored, fingerprint: gitlab_fingerprint, project: project) }
it_behaves_like 'sends on-call notification if enabled'
end
end
end
RSpec.shared_examples 'oncall users are correctly notified of Prometheus recovery alert' do
it_behaves_like 'does not send on-call notification'
context 'when alert with the same fingerprint already exists' do
context 'which is triggered' do
let_it_be(:alert) { create(:alert_management_alert, :triggered, fingerprint: gitlab_fingerprint, project: project) }
it_behaves_like 'sends on-call notification if enabled'
end
context 'which is acknowledged' do
let_it_be(:alert) { create(:alert_management_alert, :acknowledged, fingerprint: gitlab_fingerprint, project: project) }
it_behaves_like 'sends on-call notification if enabled'
end
context 'which is resolved' do
let_it_be(:alert) { create(:alert_management_alert, :resolved, fingerprint: gitlab_fingerprint, project: project) }
it_behaves_like 'does not send on-call notification'
end
context 'which is ignored' do
let_it_be(:alert) { create(:alert_management_alert, :ignored, fingerprint: gitlab_fingerprint, project: project) }
it_behaves_like 'sends on-call notification if enabled'
end
end
end
ee/spec/support/shared_examples/services/alert_management/alert_processing/oncall_notifications_shared_examples.rb 0 → 100644
View file @ 857a07c5
# frozen_string_literal: true
# This shared_example requires the following variables:
# - `users`, users expected to receive on-call notifications
# - `gitlab_fingerprint`, SHA which is used to uniquely identify the alert
RSpec.shared_examples 'sends on-call notification if enabled' do
context 'with on-call schedules enabled' do
let(:notification_async) { double(NotificationService::Async) }
it 'sends on-call notification' do
allow(NotificationService).to receive_message_chain(:new, :async).and_return(notification_async)
expect(notification_async).to receive(:notify_oncall_users_of_alert).with(
users,
having_attributes(class: AlertManagement::Alert, fingerprint: gitlab_fingerprint)
)
subject
end
end
context 'with on-call schedules disabled' do
before do
stub_licensed_features(oncall_schedules: false)
end
it_behaves_like 'does not send on-call notification'
end
end
RSpec.shared_examples 'does not send on-call notification' do
specify do
expect(NotificationService).not_to receive(:new)
subject
end
end
spec/services/alert_management/process_prometheus_alert_service_spec.rb
View file @ 857a07c5
......@@ -5,38 +5,27 @@ require 'spec_helper'
RSpec.describe AlertManagement::ProcessPrometheusAlertService do
let_it_be(:project, reload: true) { create(:project, :repository) }
before do
allow(ProjectServiceWorker).to receive(:perform_async)
end
let(:service) { described_class.new(project, payload) }
describe '#execute' do
let(:service) { described_class.new(project, payload) }
let(:source) { 'Prometheus' }
let(:auto_close_incident) { true }
let(:create_issue) { true }
let(:send_email) { true }
let(:incident_management_setting) do
double(
auto_close_incident?: auto_close_incident,
create_issue?: create_issue,
send_email?: send_email
)
end
include_context 'incident management settings enabled'
subject(:execute) { service.execute }
before do
allow(service)
.to receive(:incident_management_setting)
.and_return(incident_management_setting)
stub_licensed_features(oncall_schedules: false, generic_alert_fingerprinting: false)
end
subject(:execute) { service.execute }
context 'when alert payload is valid' do
let(:parsed_payload) { Gitlab::AlertManagement::Payload.parse(project, payload, monitoring_tool: source) }
let(:fingerprint) { parsed_payload.gitlab_fingerprint }
let_it_be(:starts_at) { '2020-04-27T10:10:22.265949279Z' }
let_it_be(:title) { 'Alert title' }
let_it_be(:fingerprint) { [starts_at, title, 'vector(1)'].join('/') }
let_it_be(:source) { 'Prometheus' }
let(:prometheus_status) { 'firing' }
let(:payload) do
{
'status' => status,
'status' => prometheus_status,
'labels' => {
'alertname' => 'GitalyFileServerDown',
'channel' => 'gitaly',
......@@ -46,196 +35,32 @@ RSpec.describe AlertManagement::ProcessPrometheusAlertService do
'annotations' => {
'description' => 'Alert description',
'runbook' => 'troubleshooting/gitaly-down.md',
'title' => 'Alert title'
'title' => title
},
'startsAt' => '2020-04-27T10:10:22.265949279Z',
'startsAt' => starts_at,
'endsAt' => '2020-04-27T10:20:22.265949279Z',
'generatorURL' => 'http://8d467bd4607a:9090/graph?g0.expr=vector%281%29&g0.tab=1',
'fingerprint' => 'b6ac4d42057c43c1'
'generatorURL' => 'http://8d467bd4607a:9090/graph?g0.expr=vector%281%29&g0.tab=1'
}
end
let(:status) { 'firing' }
context 'when Prometheus alert status is firing' do
context 'when alert with the same fingerprint already exists' do
let!(:alert) { create(:alert_management_alert, project: project, fingerprint: fingerprint) }
it_behaves_like 'adds an alert management alert event'
it_behaves_like 'processes incident issues'
it_behaves_like 'Alert Notification Service sends notification email'
context 'existing alert is resolved' do
let!(:alert) { create(:alert_management_alert, :resolved, project: project, fingerprint: fingerprint) }
it_behaves_like 'creates an alert management alert'
it_behaves_like 'Alert Notification Service sends notification email'
end
context 'existing alert is ignored' do
let!(:alert) { create(:alert_management_alert, :ignored, project: project, fingerprint: fingerprint) }
it_behaves_like 'adds an alert management alert event'
it_behaves_like 'Alert Notification Service sends no notifications'
end
context 'existing alert is acknowledged' do
let!(:alert) { create(:alert_management_alert, :acknowledged, project: project, fingerprint: fingerprint) }
it_behaves_like 'adds an alert management alert event'
it_behaves_like 'Alert Notification Service sends no notifications'
end
context 'two existing alerts, one resolved one open' do
let!(:resolved_alert) { create(:alert_management_alert, :resolved, project: project, fingerprint: fingerprint) }
let!(:alert) { create(:alert_management_alert, project: project, fingerprint: fingerprint) }
it_behaves_like 'adds an alert management alert event'
it_behaves_like 'Alert Notification Service sends notification email'
end
context 'when auto-creation of issues is disabled' do
let(:create_issue) { false }
it_behaves_like 'does not process incident issues'
end
context 'when emails are disabled' do
let(:send_email) { false }
it_behaves_like 'Alert Notification Service sends no notifications'
end
end
context 'when alert does not exist' do
context 'when alert can be created' do
it_behaves_like 'creates an alert management alert'
it_behaves_like 'Alert Notification Service sends notification email'
it_behaves_like 'processes incident issues'
it_behaves_like 'creates single system note based on the source of the alert'
context 'when auto-alert creation is disabled' do
let(:create_issue) { false }
it_behaves_like 'does not process incident issues'
end
context 'when emails are disabled' do
let(:send_email) { false }
it_behaves_like 'Alert Notification Service sends no notifications'
end
end
context 'when alert cannot be created' do
let(:errors) { double(messages: { hosts: ['hosts array is over 255 chars'] })}
before do
allow(service).to receive(:alert).and_call_original
allow(service).to receive_message_chain(:alert, :save).and_return(false)
allow(service).to receive_message_chain(:alert, :errors).and_return(errors)
end
it_behaves_like 'Alert Notification Service sends no notifications', http_status: :bad_request
it_behaves_like 'does not process incident issues due to error', http_status: :bad_request
it 'writes a warning to the log' do
expect(Gitlab::AppLogger).to receive(:warn).with(
message: 'Unable to create AlertManagement::Alert from Prometheus',
project_id: project.id,
alert_errors: { hosts: ['hosts array is over 255 chars'] }
)
execute
end
end
it { is_expected.to be_success }
end
end
context 'when Prometheus alert status is resolved' do
let(:status) { 'resolved' }
let!(:alert) { create(:alert_management_alert, project: project, fingerprint: fingerprint, monitoring_tool: source) }
context 'when auto_resolve_incident set to true' do
context 'when status can be changed' do
it_behaves_like 'Alert Notification Service sends notification email'
it_behaves_like 'does not process incident issues'
it 'resolves an existing alert without error' do
expect(Gitlab::AppLogger).not_to receive(:warn)
expect { execute }.to change { alert.reload.resolved? }.to(true)
end
it_behaves_like 'creates status-change system note for an auto-resolved alert'
context 'existing issue' do
let!(:alert) { create(:alert_management_alert, :with_issue, project: project, fingerprint: fingerprint) }
it 'closes the issue' do
issue = alert.issue
expect { execute }
.to change { issue.reload.state }
.from('opened')
.to('closed')
end
it 'creates a resource state event' do
expect { execute }.to change(ResourceStateEvent, :count).by(1)
end
end
end
context 'when status change did not succeed' do
before do
allow(AlertManagement::Alert).to receive(:for_fingerprint).and_return([alert])
allow(alert).to receive(:resolve).and_return(false)
end
it 'writes a warning to the log' do
expect(Gitlab::AppLogger).to receive(:warn).with(
message: 'Unable to update AlertManagement::Alert status to resolved',
project_id: project.id,
alert_id: alert.id
)
execute
end
it_behaves_like 'Alert Notification Service sends notification email'
end
it { is_expected.to be_success }
end
it_behaves_like 'processes new firing alert'
context 'when auto_resolve_incident set to false' do
let(:auto_close_incident) { false }
context 'with resolving payload' do
let(:prometheus_status) { 'resolved' }
it 'does not resolve an existing alert' do
expect { execute }.not_to change { alert.reload.resolved? }
end
it_behaves_like 'creates single system note based on the source of the alert'
end
context 'when emails are disabled' do
let(:send_email) { false }
it_behaves_like 'Alert Notification Service sends no notifications'
end
it_behaves_like 'processes prometheus recovery alert'
end
context 'environment given' do
let(:environment) { create(:environment, project: project) }
let(:alert) { project.alert_management_alerts.last }
it 'sets the environment' do
before do
payload['labels']['gitlab_environment_name'] = environment.name
execute
end
alert = project.alert_management_alerts.last
it 'sets the environment' do
execute
expect(alert.environment).to eq(environment)
end
......@@ -243,12 +68,14 @@ RSpec.describe AlertManagement::ProcessPrometheusAlertService do
context 'prometheus alert given' do
let(:prometheus_alert) { create(:prometheus_alert, project: project) }
let(:alert) { project.alert_management_alerts.last }
it 'sets the prometheus alert and environment' do
before do
payload['labels']['gitlab_alert_id'] = prometheus_alert.prometheus_metric_id
execute
end
alert = project.alert_management_alerts.last
it 'sets the prometheus alert and environment' do
execute
expect(alert.prometheus_alert).to eq(prometheus_alert)
expect(alert.environment).to eq(prometheus_alert.environment)
......@@ -259,10 +86,7 @@ RSpec.describe AlertManagement::ProcessPrometheusAlertService do
context 'when alert payload is invalid' do
let(:payload) { {} }
it 'responds with bad_request' do
expect(execute).to be_error
expect(execute.http_status).to eq(:bad_request)
end
it_behaves_like 'alerts service responds with an error and takes no actions', :bad_request
end
end
end
spec/services/projects/alerting/notify_service_spec.rb
View file @ 857a07c5
......@@ -3,77 +3,49 @@
require 'spec_helper'
RSpec.describe Projects::Alerting::NotifyService do
let_it_be_with_reload(:project) { create(:project, :repository) }
let_it_be_with_reload(:project) { create(:project) }
let(:payload) { ActionController::Parameters.new(payload_raw).permit! }
let(:payload_raw) { {} }
let(:service) { described_class.new(project, payload) }
before do
allow(ProjectServiceWorker).to receive(:perform_async)
stub_licensed_features(oncall_schedules: false, generic_alert_fingerprinting: false)
end
describe '#execute' do
let(:token) { 'invalid-token' }
let(:starts_at) { Time.current.change(usec: 0) }
let(:fingerprint) { 'testing' }
let(:service) { described_class.new(project, payload) }
let_it_be(:environment) { create(:environment, project: project) }
let(:environment) { create(:environment, project: project) }
let(:ended_at) { nil }
let(:payload_raw) do
{
title: 'alert title',
start_time: starts_at.rfc3339,
end_time: ended_at&.rfc3339,
severity: 'low',
monitoring_tool: 'GitLab RSpec',
service: 'GitLab Test Suite',
description: 'Very detailed description',
hosts: ['1.1.1.1', '2.2.2.2'],
fingerprint: fingerprint,
gitlab_environment_name: environment.name
}.with_indifferent_access
end
include_context 'incident management settings enabled'
let(:payload) { ActionController::Parameters.new(payload_raw).permit! }
subject { service.execute(token, integration) }
subject { service.execute(token, nil) }
context 'with HTTP integration' do
let_it_be_with_reload(:integration) { create(:alert_management_http_integration, project: project) }
shared_examples 'notifications are handled correctly' do
context 'with valid token' do
let(:token) { integration.token }
let(:incident_management_setting) { double(send_email?: email_enabled, create_issue?: issue_enabled, auto_close_incident?: auto_close_enabled) }
let(:email_enabled) { false }
let(:issue_enabled) { false }
let(:auto_close_enabled) { false }
before do
allow(service)
.to receive(:incident_management_setting)
.and_return(incident_management_setting)
end
context 'with valid payload' do
shared_examples 'assigns the alert properties' do
it 'ensure that created alert has all data properly assigned' do
subject
expect(last_alert_attributes).to match(
project_id: project.id,
title: payload_raw.fetch(:title),
started_at: Time.zone.parse(payload_raw.fetch(:start_time)),
severity: payload_raw.fetch(:severity),
status: AlertManagement::Alert.status_value(:triggered),
events: 1,
domain: 'operations',
hosts: payload_raw.fetch(:hosts),
payload: payload_raw.with_indifferent_access,
issue_id: nil,
description: payload_raw.fetch(:description),
monitoring_tool: payload_raw.fetch(:monitoring_tool),
service: payload_raw.fetch(:service),
fingerprint: Digest::SHA1.hexdigest(fingerprint),
environment_id: environment.id,
ended_at: nil,
prometheus_alert_id: nil
)
end
let_it_be(:environment) { create(:environment, project: project) }
let_it_be(:fingerprint) { 'testing' }
let_it_be(:source) { 'GitLab RSpec' }
let_it_be(:starts_at) { Time.current.change(usec: 0) }
let(:ended_at) { nil }
let(:domain) { 'operations' }
let(:payload_raw) do
{
title: 'alert title',
start_time: starts_at.rfc3339,
end_time: ended_at&.rfc3339,
severity: 'low',
monitoring_tool: source,
service: 'GitLab Test Suite',
description: 'Very detailed description',
hosts: ['1.1.1.1', '2.2.2.2'],
fingerprint: fingerprint,
gitlab_environment_name: environment.name
}.with_indifferent_access
end
let(:last_alert_attributes) do
......@@ -82,8 +54,8 @@ RSpec.describe Projects::Alerting::NotifyService do
.with_indifferent_access
end
it_behaves_like 'creates an alert management alert'
it_behaves_like 'assigns the alert properties'
it_behaves_like 'processes new firing alert'
it_behaves_like 'properly assigns the alert properties'
it 'passes the integration to alert processing' do
expect(Gitlab::AlertManagement::Payload)
......@@ -94,101 +66,18 @@ RSpec.describe Projects::Alerting::NotifyService do
subject
end
it 'creates a system note corresponding to alert creation' do
expect { subject }.to change(Note, :count).by(1)
expect(Note.last.note).to include(payload_raw.fetch(:monitoring_tool))
end
context 'existing alert with same fingerprint' do
let(:fingerprint_sha) { Digest::SHA1.hexdigest(fingerprint) }
let!(:alert) { create(:alert_management_alert, project: project, fingerprint: fingerprint_sha) }
it_behaves_like 'adds an alert management alert event'
context 'end time given' do
let(:ended_at) { Time.current.change(nsec: 0) }
it 'does not resolve the alert' do
expect { subject }.not_to change { alert.reload.status }
end
it 'does not set the ended at' do
subject
expect(alert.reload.ended_at).to be_nil
end
it_behaves_like 'does not an create alert management alert'
it_behaves_like 'creates single system note based on the source of the alert'
context 'auto_close_enabled setting enabled' do
let(:auto_close_enabled) { true }
it 'resolves the alert and sets the end time', :aggregate_failures do
subject
alert.reload
expect(alert.resolved?).to eq(true)
expect(alert.ended_at).to eql(ended_at)
end
it_behaves_like 'creates status-change system note for an auto-resolved alert'
context 'related issue exists' do
let(:alert) { create(:alert_management_alert, :with_issue, project: project, fingerprint: fingerprint_sha) }
let(:issue) { alert.issue }
it { expect { subject }.to change { issue.reload.state }.from('opened').to('closed') }
it { expect { subject }.to change(ResourceStateEvent, :count).by(1) }
end
context 'with issue enabled' do
let(:issue_enabled) { true }
it_behaves_like 'does not process incident issues'
end
end
end
context 'existing alert is resolved' do
let!(:alert) { create(:alert_management_alert, :resolved, project: project, fingerprint: fingerprint_sha) }
it_behaves_like 'creates an alert management alert'
it_behaves_like 'assigns the alert properties'
end
context 'existing alert is ignored' do
let!(:alert) { create(:alert_management_alert, :ignored, project: project, fingerprint: fingerprint_sha) }
it_behaves_like 'adds an alert management alert event'
end
context 'two existing alerts, one resolved one open' do
let!(:resolved_existing_alert) { create(:alert_management_alert, :resolved, project: project, fingerprint: fingerprint_sha) }
let!(:alert) { create(:alert_management_alert, project: project, fingerprint: fingerprint_sha) }
it_behaves_like 'adds an alert management alert event'
end
end
context 'end time given' do
let(:ended_at) { Time.current }
it_behaves_like 'creates an alert management alert'
it_behaves_like 'assigns the alert properties'
end
context 'with a minimal payload' do
let(:payload_raw) do
context 'with partial payload' do
let_it_be(:source) { integration.name }
let_it_be(:payload_raw) do
{
title: 'alert title',
start_time: starts_at.rfc3339
}
end
it_behaves_like 'creates an alert management alert'
include_examples 'processes never-before-seen alert'
it 'created alert has all data properly assigned' do
it 'assigns the alert properties' do
subject
expect(last_alert_attributes).to match(
......@@ -212,7 +101,19 @@ RSpec.describe Projects::Alerting::NotifyService do
)
end
it_behaves_like 'creates single system note based on the source of the alert'
context 'with existing alert with matching payload' do
let_it_be(:fingerprint) { payload_raw.except(:start_time).stringify_keys }
let_it_be(:gitlab_fingerprint) { Gitlab::AlertManagement::Fingerprint.generate(fingerprint) }
let_it_be(:alert) { create(:alert_management_alert, project: project, fingerprint: gitlab_fingerprint) }
include_examples 'processes never-before-seen alert'
end
end
context 'with resolving payload' do
let(:ended_at) { Time.current.change(usec: 0) }
it_behaves_like 'processes recovery alert'
end
end
......@@ -223,63 +124,30 @@ RSpec.describe Projects::Alerting::NotifyService do
allow(Gitlab::Utils::DeepSize).to receive(:new).and_return(deep_size_object)
end
it_behaves_like 'does not process incident issues due to error', http_status: :bad_request
it_behaves_like 'does not an create alert management alert'
it_behaves_like 'alerts service responds with an error and takes no actions', :bad_request
end
it_behaves_like 'does not process incident issues'
context 'issue enabled' do
let(:issue_enabled) { true }
it_behaves_like 'processes incident issues'
context 'when alert already exists' do
let(:fingerprint_sha) { Digest::SHA1.hexdigest(fingerprint) }
let!(:alert) { create(:alert_management_alert, project: project, fingerprint: fingerprint_sha) }
context 'when existing alert does not have an associated issue' do
it_behaves_like 'processes incident issues'
end
context 'when existing alert has an associated issue' do
let!(:alert) { create(:alert_management_alert, :with_issue, project: project, fingerprint: fingerprint_sha) }
it_behaves_like 'does not process incident issues'
end
context 'with inactive integration' do
before do
integration.update!(active: false)
end
end
context 'with emails turned on' do
let(:email_enabled) { true }
it_behaves_like 'Alert Notification Service sends notification email'
it_behaves_like 'alerts service responds with an error and takes no actions', :forbidden
end
end
context 'with invalid token' do
it_behaves_like 'does not process incident issues due to error', http_status: :unauthorized
it_behaves_like 'does not an create alert management alert'
end
end
context 'with an HTTP Integration' do
let_it_be_with_reload(:integration) { create(:alert_management_http_integration, project: project) }
let(:token) { 'invalid-token' }
subject { service.execute(token, integration) }
it_behaves_like 'notifications are handled correctly' do
let(:source) { integration.name }
it_behaves_like 'alerts service responds with an error and takes no actions', :unauthorized
end
end
context 'with deactivated HTTP Integration' do
before do
integration.update!(active: false)
end
context 'without HTTP integration' do
let(:integration) { nil }
let(:token) { nil }
it_behaves_like 'does not process incident issues due to error', http_status: :forbidden
it_behaves_like 'does not an create alert management alert'
end
it_behaves_like 'alerts service responds with an error and takes no actions', :forbidden
end
end
end
spec/services/projects/prometheus/alerts/notify_service_spec.rb
View file @ 857a07c5
......@@ -6,25 +6,26 @@ RSpec.describe Projects::Prometheus::Alerts::NotifyService do
include PrometheusHelpers
using RSpec::Parameterized::TableSyntax
let_it_be(:project, reload: true) { create(:project) }
let_it_be_with_reload(:project) { create(:project) }
let_it_be_with_reload(:setting) do
create(:project_incident_management_setting, project: project, send_email: true, create_issue: true)
end
let(:service) { described_class.new(project, payload) }
let(:token_input) { 'token' }
let!(:setting) do
create(:project_incident_management_setting, project: project, send_email: true, create_issue: true)
end
let(:subject) { service.execute(token_input) }
subject { service.execute(token_input) }
context 'with valid payload' do
let_it_be(:alert_firing) { create(:prometheus_alert, project: project) }
let_it_be(:alert_resolved) { create(:prometheus_alert, project: project) }
let_it_be(:cluster) { create(:cluster, :provided_by_user, projects: [project]) }
let_it_be(:cluster, reload: true) { create(:cluster, :provided_by_user, projects: [project]) }
let(:payload_raw) { prometheus_alert_payload(firing: [alert_firing], resolved: [alert_resolved]) }
let(:payload) { ActionController::Parameters.new(payload_raw).permit! }
let(:payload_alert_firing) { payload_raw['alerts'].first }
let(:token) { 'token' }
let(:source) { 'Prometheus' }
context 'with environment specific clusters' do
let(:prd_cluster) do
......@@ -53,11 +54,11 @@ RSpec.describe Projects::Prometheus::Alerts::NotifyService do
context 'without token' do
let(:token_input) { nil }
it_behaves_like 'Alert Notification Service sends notification email'
include_examples 'processes one firing and one resolved prometheus alerts'
end
context 'with token' do
it_behaves_like 'Alert Notification Service sends no notifications', http_status: :unauthorized
it_behaves_like 'alerts service responds with an error and takes no actions', :unauthorized
end
end
......@@ -87,9 +88,9 @@ RSpec.describe Projects::Prometheus::Alerts::NotifyService do
case result = params[:result]
when :success
it_behaves_like 'Alert Notification Service sends notification email'
include_examples 'processes one firing and one resolved prometheus alerts'
when :failure
it_behaves_like 'Alert Notification Service sends no notifications', http_status: :unauthorized
it_behaves_like 'alerts service responds with an error and takes no actions', :unauthorized
else
raise "invalid result: #{result.inspect}"
end
......@@ -97,9 +98,9 @@ RSpec.describe Projects::Prometheus::Alerts::NotifyService do
end
context 'without project specific cluster' do
let!(:cluster) { create(:cluster, enabled: true) }
let_it_be(:cluster) { create(:cluster, enabled: true) }
it_behaves_like 'Alert Notification Service sends no notifications', http_status: :unauthorized
it_behaves_like 'alerts service responds with an error and takes no actions', :unauthorized
end
context 'with manual prometheus installation' do
......@@ -126,9 +127,9 @@ RSpec.describe Projects::Prometheus::Alerts::NotifyService do
case result = params[:result]
when :success
it_behaves_like 'Alert Notification Service sends notification email'
it_behaves_like 'processes one firing and one resolved prometheus alerts'
when :failure
it_behaves_like 'Alert Notification Service sends no notifications', http_status: :unauthorized
it_behaves_like 'alerts service responds with an error and takes no actions', :unauthorized
else
raise "invalid result: #{result.inspect}"
end
......@@ -150,50 +151,53 @@ RSpec.describe Projects::Prometheus::Alerts::NotifyService do
let(:token_input) { public_send(token) if token }
let(:integration) { create(:alert_management_http_integration, active, project: project) if active }
let(:subject) { service.execute(token_input, integration) }
subject { service.execute(token_input, integration) }
case result = params[:result]
when :success
it_behaves_like 'Alert Notification Service sends notification email'
it_behaves_like 'processes one firing and one resolved prometheus alerts'
when :failure
it_behaves_like 'Alert Notification Service sends no notifications', http_status: :unauthorized
it_behaves_like 'alerts service responds with an error and takes no actions', :unauthorized
else
raise "invalid result: #{result.inspect}"
end
end
end
context 'alert emails' do
context 'incident settings' do
before do
create(:prometheus_service, project: project)
create(:project_alerting_setting, project: project, token: token)
end
context 'when incident_management_setting does not exist' do
let!(:setting) { nil }
it 'does not send notification email', :sidekiq_might_not_need_inline do
expect_any_instance_of(NotificationService)
.not_to receive(:async)
it_behaves_like 'processes one firing and one resolved prometheus alerts'
expect(subject).to be_success
context 'when incident_management_setting does not exist' do
before do
setting.destroy!
end
end
context 'when incident_management_setting.send_email is true' do
it_behaves_like 'Alert Notification Service sends notification email'
it { is_expected.to be_success }
include_examples 'does not send alert notification emails'
include_examples 'does not process incident issues'
end
context 'incident_management_setting.send_email is false' do
let!(:setting) do
create(:project_incident_management_setting, send_email: false, project: project)
before do
setting.update!(send_email: false)
end
it 'does not send notification' do
expect(NotificationService).not_to receive(:new)
it { is_expected.to be_success }
include_examples 'does not send alert notification emails'
end
expect(subject).to be_success
context 'incident_management_setting.create_issue is false' do
before do
setting.update!(create_issue: false)
end
it { is_expected.to be_success }
include_examples 'does not process incident issues'
end
end
......@@ -233,7 +237,7 @@ RSpec.describe Projects::Prometheus::Alerts::NotifyService do
.and_return(false)
end
it_behaves_like 'Alert Notification Service sends no notifications', http_status: :unprocessable_entity
it_behaves_like 'alerts service responds with an error and takes no actions', :unprocessable_entity
end
context 'when the payload is too big' do
......@@ -244,14 +248,7 @@ RSpec.describe Projects::Prometheus::Alerts::NotifyService do
allow(Gitlab::Utils::DeepSize).to receive(:new).and_return(deep_size_object)
end
it_behaves_like 'Alert Notification Service sends no notifications', http_status: :bad_request
it 'does not process Prometheus alerts' do
expect(AlertManagement::ProcessPrometheusAlertService)
.not_to receive(:new)
subject
end
it_behaves_like 'alerts service responds with an error and takes no actions', :bad_request
end
end
......
spec/support/shared_examples/alert_notification_service_shared_examples.rb deleted 100644 → 0
View file @ ad2cb86b
# frozen_string_literal: true
RSpec.shared_examples 'Alert Notification Service sends notification email' do
let(:notification_service) { spy }
it 'sends a notification' do
expect(NotificationService)
.to receive(:new)
.and_return(notification_service)
expect(notification_service)
.to receive_message_chain(:async, :prometheus_alerts_fired)
expect(subject).to be_success
end
end
RSpec.shared_examples 'Alert Notification Service sends no notifications' do |http_status: nil|
it 'does not notify' do
expect(NotificationService).not_to receive(:new)
if http_status.present?
expect(subject).to be_error
expect(subject.http_status).to eq(http_status)
else
expect(subject).to be_success
end
end
end
RSpec.shared_examples 'creates status-change system note for an auto-resolved alert' do
it 'has 2 new system notes' do
expect { subject }.to change(Note, :count).by(2)
expect(Note.last.note).to include('Resolved')
end
end
# Requires `source` to be defined
RSpec.shared_examples 'creates single system note based on the source of the alert' do
it 'has one new system note' do
expect { subject }.to change(Note, :count).by(1)
expect(Note.last.note).to include(source)
end
end
spec/support/shared_examples/services/alert_management/alert_processing/alert_firing_shared_examples.rb 0 → 100644
View file @ 857a07c5
# frozen_string_literal: true
# This shared_example requires the following variables:
# - `service`, the service which includes AlertManagement::AlertProcessing
RSpec.shared_examples 'creates an alert management alert or errors' do
it { is_expected.to be_success }
it 'creates AlertManagement::Alert' do
expect(Gitlab::AppLogger).not_to receive(:warn)
expect { subject }.to change(AlertManagement::Alert, :count).by(1)
end
it 'executes the alert service hooks' do
expect_next_instance_of(AlertManagement::Alert) do |alert|
expect(alert).to receive(:execute_services)
end
subject
end
context 'and fails to save' do
let(:errors) { double(messages: { hosts: ['hosts array is over 255 chars'] })}
before do
allow(service).to receive(:alert).and_call_original
allow(service).to receive_message_chain(:alert, :save).and_return(false)
allow(service).to receive_message_chain(:alert, :errors).and_return(errors)
end
it_behaves_like 'alerts service responds with an error', :bad_request
it 'writes a warning to the log' do
expect(Gitlab::AppLogger).to receive(:warn).with(
message: "Unable to create AlertManagement::Alert from #{source}",
project_id: project.id,
alert_errors: { hosts: ['hosts array is over 255 chars'] }
)
subject
end
end
end
# This shared_example requires the following variables:
# - last_alert_attributes, last created alert
# - project, project that alert created
# - payload_raw, hash representation of payload
# - environment, project's environment
# - fingerprint, fingerprint hash
RSpec.shared_examples 'properly assigns the alert properties' do
specify do
subject
expect(last_alert_attributes).to match({
project_id: project.id,
title: payload_raw.fetch(:title),
started_at: Time.zone.parse(payload_raw.fetch(:start_time)),
severity: payload_raw.fetch(:severity, nil),
status: AlertManagement::Alert.status_value(:triggered),
events: 1,
domain: domain,
hosts: payload_raw.fetch(:hosts, nil),
payload: payload_raw.with_indifferent_access,
issue_id: nil,
description: payload_raw.fetch(:description, nil),
monitoring_tool: payload_raw.fetch(:monitoring_tool, nil),
service: payload_raw.fetch(:service, nil),
fingerprint: Digest::SHA1.hexdigest(fingerprint),
environment_id: environment.id,
ended_at: nil,
prometheus_alert_id: nil
}.with_indifferent_access)
end
end
RSpec.shared_examples 'does not create an alert management alert' do
specify do
expect { subject }.not_to change(AlertManagement::Alert, :count)
end
end
# This shared_example requires the following variables:
# - `alert`, the alert for which events should be incremented
RSpec.shared_examples 'adds an alert management alert event' do
specify do
expect(alert).not_to receive(:execute_services)
expect { subject }.to change { alert.reload.events }.by(1)
expect(subject).to be_success
end
it_behaves_like 'does not create an alert management alert'
end
# This shared_example requires the following variables:
# - `alert`, the alert for which events should not be incremented
RSpec.shared_examples 'does not add an alert management alert event' do
specify do
expect { subject }.not_to change { alert.reload.events }
end
end
RSpec.shared_examples 'processes new firing alert' do
include_examples 'processes never-before-seen alert'
context 'for an existing alert with the same fingerprint' do
let_it_be(:gitlab_fingerprint) { Digest::SHA1.hexdigest(fingerprint) }
context 'which is triggered' do
let_it_be(:alert) { create(:alert_management_alert, :triggered, fingerprint: gitlab_fingerprint, project: project) }
it_behaves_like 'adds an alert management alert event'
it_behaves_like 'sends alert notification emails if enabled'
it_behaves_like 'processes incident issues if enabled', with_issue: true
it_behaves_like 'does not create an alert management alert'
it_behaves_like 'does not create a system note for alert'
context 'with an existing resolved alert as well' do
let_it_be(:resolved_alert) { create(:alert_management_alert, :resolved, project: project, fingerprint: gitlab_fingerprint) }
it_behaves_like 'adds an alert management alert event'
it_behaves_like 'sends alert notification emails if enabled'
it_behaves_like 'processes incident issues if enabled', with_issue: true
it_behaves_like 'does not create an alert management alert'
it_behaves_like 'does not create a system note for alert'
end
end
context 'which is acknowledged' do
let_it_be(:alert) { create(:alert_management_alert, :acknowledged, fingerprint: gitlab_fingerprint, project: project) }
it_behaves_like 'adds an alert management alert event'
it_behaves_like 'processes incident issues if enabled', with_issue: true
it_behaves_like 'does not create an alert management alert'
it_behaves_like 'does not create a system note for alert'
it_behaves_like 'does not send alert notification emails'
end
context 'which is ignored' do
let_it_be(:alert) { create(:alert_management_alert, :ignored, fingerprint: gitlab_fingerprint, project: project) }
it_behaves_like 'adds an alert management alert event'
it_behaves_like 'processes incident issues if enabled', with_issue: true
it_behaves_like 'does not create an alert management alert'
it_behaves_like 'does not create a system note for alert'
it_behaves_like 'does not send alert notification emails'
end
context 'which is resolved' do
let_it_be(:alert) { create(:alert_management_alert, :resolved, fingerprint: gitlab_fingerprint, project: project) }
include_examples 'processes never-before-seen alert'
end
end
end
spec/support/shared_examples/services/alert_management/alert_processing/alert_recovery_shared_examples.rb 0 → 100644
View file @ 857a07c5
# frozen_string_literal: true
# This shared_example requires the following variables:
# - `alert`, the alert to be resolved
RSpec.shared_examples 'resolves an existing alert management alert' do
it 'sets the end time and status' do
expect(Gitlab::AppLogger).not_to receive(:warn)
expect { subject }
.to change { alert.reload.resolved? }.to(true)
.and change { alert.ended_at.present? }.to(true)
expect(subject).to be_success
end
end
# This shared_example requires the following variables:
# - `alert`, the alert not to be updated
RSpec.shared_examples 'does not change the alert end time' do
specify do
expect { subject }.not_to change { alert.reload.ended_at }
end
end
# This shared_example requires the following variables:
# - `project`, expected project for an incoming alert
# - `service`, a service which includes AlertManagement::AlertProcessing
# - `alert` (optional), the alert which should fail to resolve. If not
# included, the log is expected to correspond to a new alert
RSpec.shared_examples 'writes a warning to the log for a failed alert status update' do
before do
allow(service).to receive(:alert).and_call_original
allow(service).to receive_message_chain(:alert, :resolve).and_return(false)
end
specify do
expect(Gitlab::AppLogger).to receive(:warn).with(
message: 'Unable to update AlertManagement::Alert status to resolved',
project_id: project.id,
alert_id: alert ? alert.id : (last_alert_id + 1)
)
# Failure to resolve a recovery alert is not a critical failure
expect(subject).to be_success
end
private
def last_alert_id
AlertManagement::Alert.connection
.select_value("SELECT nextval('#{AlertManagement::Alert.sequence_name}')")
end
end
RSpec.shared_examples 'processes recovery alert' do
context 'seen for the first time' do
let(:alert) { AlertManagement::Alert.last }
include_examples 'processes never-before-seen recovery alert'
end
context 'for an existing alert with the same fingerprint' do
let_it_be(:gitlab_fingerprint) { Digest::SHA1.hexdigest(fingerprint) }
context 'which is triggered' do
let_it_be(:alert) { create(:alert_management_alert, :triggered, project: project, fingerprint: gitlab_fingerprint, monitoring_tool: source) }
it_behaves_like 'resolves an existing alert management alert'
it_behaves_like 'creates expected system notes for alert', :recovery_alert, :resolve_alert
it_behaves_like 'sends alert notification emails if enabled'
it_behaves_like 'closes related incident if enabled'
it_behaves_like 'writes a warning to the log for a failed alert status update'
it_behaves_like 'does not create an alert management alert'
it_behaves_like 'does not process incident issues'
it_behaves_like 'does not add an alert management alert event'
end
context 'which is ignored' do
let_it_be(:alert) { create(:alert_management_alert, :ignored, project: project, fingerprint: gitlab_fingerprint, monitoring_tool: source) }
it_behaves_like 'resolves an existing alert management alert'
it_behaves_like 'creates expected system notes for alert', :recovery_alert, :resolve_alert
it_behaves_like 'sends alert notification emails if enabled'
it_behaves_like 'closes related incident if enabled'
it_behaves_like 'writes a warning to the log for a failed alert status update'
it_behaves_like 'does not create an alert management alert'
it_behaves_like 'does not process incident issues'
it_behaves_like 'does not add an alert management alert event'
end
context 'which is acknowledged' do
let_it_be(:alert) { create(:alert_management_alert, :acknowledged, project: project, fingerprint: gitlab_fingerprint, monitoring_tool: source) }
it_behaves_like 'resolves an existing alert management alert'
it_behaves_like 'creates expected system notes for alert', :recovery_alert, :resolve_alert
it_behaves_like 'sends alert notification emails if enabled'
it_behaves_like 'closes related incident if enabled'
it_behaves_like 'writes a warning to the log for a failed alert status update'
it_behaves_like 'does not create an alert management alert'
it_behaves_like 'does not process incident issues'
it_behaves_like 'does not add an alert management alert event'
end
context 'which is resolved' do
let_it_be(:alert) { create(:alert_management_alert, :resolved, project: project, fingerprint: gitlab_fingerprint, monitoring_tool: source) }
include_examples 'processes never-before-seen recovery alert'
end
end
end
RSpec.shared_examples 'processes prometheus recovery alert' do
context 'seen for the first time' do
it_behaves_like 'does not create an alert management alert'
it_behaves_like 'does not send alert notification emails'
it_behaves_like 'does not process incident issues'
end
context 'for an existing alert with the same fingerprint' do
let_it_be(:gitlab_fingerprint) { Digest::SHA1.hexdigest(fingerprint) }
context 'which is triggered' do
let_it_be(:alert) { create(:alert_management_alert, :triggered, project: project, fingerprint: gitlab_fingerprint, monitoring_tool: source) }
it_behaves_like 'resolves an existing alert management alert'
it_behaves_like 'creates expected system notes for alert', :recovery_alert, :resolve_alert
it_behaves_like 'sends alert notification emails if enabled'
it_behaves_like 'closes related incident if enabled'
it_behaves_like 'writes a warning to the log for a failed alert status update'
it_behaves_like 'does not create an alert management alert'
it_behaves_like 'does not process incident issues'
it_behaves_like 'does not add an alert management alert event'
end
context 'which is ignored' do
let_it_be(:alert) { create(:alert_management_alert, :ignored, project: project, fingerprint: gitlab_fingerprint, monitoring_tool: source) }
it_behaves_like 'resolves an existing alert management alert'
it_behaves_like 'creates expected system notes for alert', :recovery_alert, :resolve_alert
it_behaves_like 'sends alert notification emails if enabled'
it_behaves_like 'closes related incident if enabled'
it_behaves_like 'writes a warning to the log for a failed alert status update'
it_behaves_like 'does not create an alert management alert'
it_behaves_like 'does not process incident issues'
it_behaves_like 'does not add an alert management alert event'
end
context 'which is acknowledged' do
let_it_be(:alert) { create(:alert_management_alert, :acknowledged, project: project, fingerprint: gitlab_fingerprint, monitoring_tool: source) }
it_behaves_like 'resolves an existing alert management alert'
it_behaves_like 'creates expected system notes for alert', :recovery_alert, :resolve_alert
it_behaves_like 'sends alert notification emails if enabled'
it_behaves_like 'closes related incident if enabled'
it_behaves_like 'writes a warning to the log for a failed alert status update'
it_behaves_like 'does not create an alert management alert'
it_behaves_like 'does not process incident issues'
it_behaves_like 'does not add an alert management alert event'
end
context 'which is resolved' do
let_it_be(:alert) { create(:alert_management_alert, :resolved, project: project, fingerprint: gitlab_fingerprint, monitoring_tool: source) }
it_behaves_like 'does not create an alert management alert'
it_behaves_like 'does not send alert notification emails'
it_behaves_like 'does not change the alert end time'
it_behaves_like 'does not process incident issues'
it_behaves_like 'does not add an alert management alert event'
end
end
end
spec/support/shared_examples/services/alert_management/alert_processing/incident_creation_shared_examples.rb 0 → 100644
View file @ 857a07c5
# frozen_string_literal: true
# Expects usage of 'incident settings enabled' context.
#
# This shared_example includes the following option:
# - with_issue: includes a test for when the defined `alert` has an associated issue
#
# This shared_example requires the following variables:
# - `alert`, required if :with_issue is true
RSpec.shared_examples 'processes incident issues if enabled' do |with_issue: false|
include_examples 'processes incident issues', with_issue
context 'with incident setting disabled' do
let(:create_issue) { false }
it_behaves_like 'does not process incident issues'
end
end
RSpec.shared_examples 'processes incident issues' do |with_issue: false|
before do
allow_next_instance_of(AlertManagement::Alert) do |alert|
allow(alert).to receive(:execute_services)
end
end
specify do
expect(IncidentManagement::ProcessAlertWorker)
.to receive(:perform_async)
.with(nil, nil, kind_of(Integer))
Sidekiq::Testing.inline! do
expect(subject).to be_success
end
end
context 'with issue', if: with_issue do
before do
alert.update!(issue: create(:issue, project: project))
end
it_behaves_like 'does not process incident issues'
end
end
RSpec.shared_examples 'does not process incident issues' do
specify do
expect(IncidentManagement::ProcessAlertWorker).not_to receive(:perform_async)
subject
end
end
spec/support/shared_examples/services/alert_management/alert_processing/incident_resolution_shared_examples.rb 0 → 100644
View file @ 857a07c5
# frozen_string_literal: true
# Expects usage of 'incident settings enabled' context.
#
# This shared_example requires the following variables:
# - `alert`, alert for which related incidents should be closed
# - `project`, project of the alert
RSpec.shared_examples 'closes related incident if enabled' do
context 'with issue' do
before do
alert.update!(issue: create(:issue, project: project))
end
it { expect { subject }.to change { alert.issue.reload.closed? }.from(false).to(true) }
it { expect { subject }.to change(ResourceStateEvent, :count).by(1) }
end
context 'without issue' do
it { expect { subject }.not_to change { alert.reload.issue } }
it { expect { subject }.not_to change(ResourceStateEvent, :count) }
end
context 'with incident setting disabled' do
let(:auto_close_incident) { false }
it_behaves_like 'does not close related incident'
end
end
RSpec.shared_examples 'does not close related incident' do
context 'with issue' do
before do
alert.update!(issue: create(:issue, project: project))
end
it { expect { subject }.not_to change { alert.issue.reload.state } }
it { expect { subject }.not_to change(ResourceStateEvent, :count) }
end
context 'without issue' do
it { expect { subject }.not_to change { alert.reload.issue } }
it { expect { subject }.not_to change(ResourceStateEvent, :count) }
end
end
spec/support/shared_examples/services/alert_management/alert_processing/notifications_shared_examples.rb 0 → 100644
View file @ 857a07c5
# frozen_string_literal: true
# Expects usage of 'incident settings enabled' context.
#
# This shared_example includes the following option:
# - count: number of notifications expected to be sent
RSpec.shared_examples 'sends alert notification emails if enabled' do |count: 1|
include_examples 'sends alert notification emails', count
context 'with email setting disabled' do
let(:send_email) { false }
it_behaves_like 'does not send alert notification emails'
end
end
RSpec.shared_examples 'sends alert notification emails' do |count: 1|
let(:notification_async) { double(NotificationService::Async) }
specify do
allow(NotificationService).to receive_message_chain(:new, :async).and_return(notification_async)
expect(notification_async).to receive(:prometheus_alerts_fired).exactly(count).times
subject
end
end
RSpec.shared_examples 'does not send alert notification emails' do
specify do
expect(NotificationService).not_to receive(:new)
subject
end
end
spec/support/shared_examples/services/alert_management/alert_processing/system_notes_shared_examples.rb 0 → 100644
View file @ 857a07c5
# frozen_string_literal: true
# This shared_example includes the following option:
# - notes: any of [:new_alert, :recovery_alert, :resolve_alert].
# Represents which notes are expected to be created.
#
# This shared_example requires the following variables:
# - `source` (optional), the monitoring tool or integration name
# expected in the applicable system notes
RSpec.shared_examples 'creates expected system notes for alert' do |*notes|
let(:expected_note_count) { expected_notes.length }
let(:new_notes) { Note.last(expected_note_count).pluck(:note) }
let(:expected_notes) do
{
new_alert: source,
recovery_alert: source,
resolve_alert: 'Resolved'
}.slice(*notes)
end
it "for #{notes.join(', ')}" do
expect { subject }.to change(Note, :count).by(expected_note_count)
expected_notes.each_value.with_index do |value, index|
expect(new_notes[index]).to include(value)
end
end
end
RSpec.shared_examples 'does not create a system note for alert' do
specify do
expect { subject }.not_to change(Note, :count)
end
end
spec/support/shared_examples/services/alert_management_shared_examples.rb
View file @ 857a07c5
# frozen_string_literal: true
RSpec.shared_examples 'creates an alert management alert' do
it { is_expected.to be_success }
RSpec.shared_examples 'alerts service responds with an error and takes no actions' do |http_status|
include_examples 'alerts service responds with an error', http_status
it 'creates AlertManagement::Alert' do
expect { subject }.to change(AlertManagement::Alert, :count).by(1)
end
it 'executes the alert service hooks' do
expect_next_instance_of(AlertManagement::Alert) do |alert|
expect(alert).to receive(:execute_services)
end
it_behaves_like 'does not create an alert management alert'
it_behaves_like 'does not create a system note for alert'
it_behaves_like 'does not process incident issues'
it_behaves_like 'does not send alert notification emails'
end
subject
RSpec.shared_examples 'alerts service responds with an error' do |http_status|
specify do
expect(subject).to be_error
expect(subject.http_status).to eq(http_status)
end
end
# This shared_example requires the following variables:
# - last_alert_attributes, last created alert
# - project, project that alert created
# - payload_raw, hash representation of payload
# - environment, project's environment
# - fingerprint, fingerprint hash
RSpec.shared_examples 'assigns the alert properties' do
it 'ensures that created alert has all data properly assigned' do
subject
expect(last_alert_attributes).to match(
project_id: project.id,
title: payload_raw.fetch(:title),
started_at: Time.zone.parse(payload_raw.fetch(:start_time)),
severity: payload_raw.fetch(:severity),
status: AlertManagement::Alert.status_value(:triggered),
events: 1,
domain: domain,
hosts: payload_raw.fetch(:hosts),
payload: payload_raw.with_indifferent_access,
issue_id: nil,
description: payload_raw.fetch(:description),
monitoring_tool: payload_raw.fetch(:monitoring_tool),
service: payload_raw.fetch(:service),
fingerprint: Digest::SHA1.hexdigest(fingerprint),
environment_id: environment.id,
ended_at: nil,
prometheus_alert_id: nil
# - `service`, a service which includes ::IncidentManagement::Settings
RSpec.shared_context 'incident management settings enabled' do
let(:auto_close_incident) { true }
let(:create_issue) { true }
let(:send_email) { true }
let(:incident_management_setting) do
double(
auto_close_incident?: auto_close_incident,
create_issue?: create_issue,
send_email?: send_email
)
end
end
RSpec.shared_examples 'does not an create alert management alert' do
it 'does not create alert' do
expect { subject }.not_to change(AlertManagement::Alert, :count)
before do
allow(ProjectServiceWorker).to receive(:perform_async)
allow(service)
.to receive(:incident_management_setting)
.and_return(incident_management_setting)
end
end
RSpec.shared_examples 'adds an alert management alert event' do
it { is_expected.to be_success }
it 'does not create an alert' do
expect { subject }.not_to change(AlertManagement::Alert, :count)
end
it 'increases alert events count' do
expect { subject }.to change { alert.reload.events }.by(1)
end
it 'does not executes the alert service hooks' do
expect(alert).not_to receive(:execute_services)
subject
end
RSpec.shared_examples 'processes never-before-seen alert' do
it_behaves_like 'creates an alert management alert or errors'
it_behaves_like 'creates expected system notes for alert', :new_alert
it_behaves_like 'processes incident issues if enabled'
it_behaves_like 'sends alert notification emails if enabled'
end
RSpec.shared_examples 'processes incident issues' do
let(:create_incident_service) { spy }
before do
allow_any_instance_of(AlertManagement::Alert).to receive(:execute_services)
end
it 'processes issues' do
expect(IncidentManagement::ProcessAlertWorker)
.to receive(:perform_async)
.with(nil, nil, kind_of(Integer))
.once
Sidekiq::Testing.inline! do
expect(subject).to be_success
end
end
RSpec.shared_examples 'processes never-before-seen recovery alert' do
it_behaves_like 'creates an alert management alert or errors'
it_behaves_like 'creates expected system notes for alert', :new_alert
it_behaves_like 'sends alert notification emails if enabled'
it_behaves_like 'processes incident issues if enabled'
end
RSpec.shared_examples 'does not process incident issues' do
it 'does not process issues' do
expect(IncidentManagement::ProcessAlertWorker)
.not_to receive(:perform_async)
RSpec.shared_examples 'processes one firing and one resolved prometheus alerts' do
it 'creates AlertManagement::Alert' do
expect(Gitlab::AppLogger).not_to receive(:warn)
expect(subject).to be_success
expect { subject }
.to change(AlertManagement::Alert, :count).by(1)
.and change(Note, :count).by(1)
end
end
RSpec.shared_examples 'does not process incident issues due to error' do |http_status:|
it 'does not process issues' do
expect(IncidentManagement::ProcessAlertWorker)
.not_to receive(:perform_async)
expect(subject).to be_error
expect(subject.http_status).to eq(http_status)
end
it_behaves_like 'processes incident issues'
it_behaves_like 'sends alert notification emails', count: 1
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7