Commit 275dd46f authored by Patrick Bajao's avatar Patrick Bajao

Merge branch 'sarnold-fix-prometheus-issue-creation' into 'master'

Create issue automatically from Prometheus alert

Closes #231497

See merge request gitlab-org/gitlab!37884
parents 3e436c6f b69e3a04
......@@ -34,6 +34,8 @@ module AlertManagement
else
create_alert_management_alert
end
process_incident_alert
end
def reset_alert_management_alert_status
......@@ -47,16 +49,17 @@ module AlertManagement
end
def create_alert_management_alert
am_alert = AlertManagement::Alert.new(am_alert_params.merge(ended_at: nil))
if am_alert.save
am_alert.execute_services
new_alert = AlertManagement::Alert.new(am_alert_params.merge(ended_at: nil))
if new_alert.save
new_alert.execute_services
@am_alert = new_alert
return
end
logger.warn(
message: 'Unable to create AlertManagement::Alert',
project_id: project.id,
alert_errors: am_alert.errors.messages
alert_errors: new_alert.errors.messages
)
end
......@@ -89,12 +92,21 @@ module AlertManagement
SystemNoteService.auto_resolve_prometheus_alert(issue, project, User.alert_bot) if issue.reset.closed?
end
def process_incident_alert
return unless am_alert
return if am_alert.issue
IncidentManagement::ProcessAlertWorker.perform_async(nil, nil, am_alert.id)
end
def logger
@logger ||= Gitlab::AppLogger
end
def am_alert
@am_alert ||= AlertManagement::Alert.not_resolved.for_fingerprint(project, gitlab_fingerprint).first
strong_memoize(:am_alert) do
AlertManagement::Alert.not_resolved.for_fingerprint(project, gitlab_fingerprint).first
end
end
def bad_request
......
......@@ -30,7 +30,11 @@ module IncidentManagement
end
def parsed_payload(alert)
Gitlab::Alerting::NotificationPayloadParser.call(alert.payload.to_h, alert.project)
if alert.prometheus?
alert.payload
else
Gitlab::Alerting::NotificationPayloadParser.call(alert.payload.to_h, alert.project)
end
end
def create_issue_for(alert)
......
---
title: Fix automatic issue creation via Prometheus alerts
merge_request: 37884
author:
type: fixed
......@@ -101,6 +101,16 @@ FactoryBot.define do
trait :prometheus do
monitoring_tool { Gitlab::AlertManagement::AlertParams::MONITORING_TOOLS[:prometheus] }
payload do
{
annotations: {
title: 'This is a prometheus error',
summary: 'Summary of the error',
description: 'Description of the error'
},
startsAt: started_at
}.with_indifferent_access
end
end
trait :all_fields do
......
......@@ -83,6 +83,15 @@ RSpec.describe AlertManagement::ProcessPrometheusAlertService do
context 'when alert does not exist' do
context 'when alert can be created' do
it_behaves_like 'creates an alert management alert'
it 'processes the incident alert' do
expect(IncidentManagement::ProcessAlertWorker)
.to receive(:perform_async)
.with(nil, nil, kind_of(Integer))
.once
expect(subject).to be_success
end
end
context 'when alert cannot be created' do
......@@ -102,6 +111,13 @@ RSpec.describe AlertManagement::ProcessPrometheusAlertService do
execute
end
it 'does not create incident issue' do
expect(IncidentManagement::ProcessAlertWorker)
.not_to receive(:perform_async)
expect(subject).to be_success
end
end
it { is_expected.to be_success }
......
......@@ -16,66 +16,73 @@ RSpec.describe IncidentManagement::ProcessAlertWorker do
subject { described_class.new.perform(nil, nil, alert.id) }
before do
allow(Gitlab::AppLogger).to receive(:warn).and_call_original
allow(IncidentManagement::CreateIssueService)
.to receive(:new).with(alert.project, parsed_payload)
.and_call_original
end
it 'creates an issue' do
expect(IncidentManagement::CreateIssueService)
.to receive(:new).with(alert.project, parsed_payload)
shared_examples 'creates issue successfully' do
it 'creates an issue' do
expect(IncidentManagement::CreateIssueService)
.to receive(:new).with(alert.project, parsed_payload)
expect { subject }.to change { Issue.count }.by(1)
end
expect { subject }.to change { Issue.count }.by(1)
end
context 'with invalid alert' do
let(:invalid_alert_id) { non_existing_record_id }
it 'updates AlertManagement::Alert#issue_id' do
subject
subject { described_class.new.perform(nil, nil, invalid_alert_id) }
expect(alert.reload.issue_id).to eq(created_issue.id)
end
it 'does not create issues' do
expect(IncidentManagement::CreateIssueService).not_to receive(:new)
it 'does not write a warning to log' do
subject
expect { subject }.not_to change { Issue.count }
expect(Gitlab::AppLogger).not_to have_received(:warn)
end
end
context 'with valid alert' do
before do
allow(Gitlab::AppLogger).to receive(:warn).and_call_original
end
it_behaves_like 'creates issue successfully'
context 'when alert can be updated' do
it 'updates AlertManagement::Alert#issue_id' do
subject
context 'when alert cannot be updated' do
let_it_be(:alert) { create(:alert_management_alert, :with_validation_errors, project: project, payload: payload) }
expect(alert.reload.issue_id).to eq(created_issue.id)
it 'updates AlertManagement::Alert#issue_id' do
expect { subject }.not_to change { alert.reload.issue_id }
end
it 'does not write a warning to log' do
it 'logs a warning' do
subject
expect(Gitlab::AppLogger).not_to have_received(:warn)
expect(Gitlab::AppLogger).to have_received(:warn).with(
message: 'Cannot link an Issue with Alert',
issue_id: created_issue.id,
alert_id: alert.id,
alert_errors: { hosts: ['hosts array is over 255 chars'] }
)
end
end
context 'when alert cannot be updated' do
let_it_be(:alert) { create(:alert_management_alert, :with_validation_errors, project: project, payload: payload) }
context 'prometheus alert' do
let_it_be(:alert) { create(:alert_management_alert, :prometheus, project: project, started_at: started_at) }
let_it_be(:parsed_payload) { alert.payload }
it 'updates AlertManagement::Alert#issue_id' do
expect { subject }.not_to change { alert.reload.issue_id }
end
it_behaves_like 'creates issue successfully'
end
end
it 'logs a warning' do
subject
context 'with invalid alert' do
let(:invalid_alert_id) { non_existing_record_id }
expect(Gitlab::AppLogger).to have_received(:warn).with(
message: 'Cannot link an Issue with Alert',
issue_id: created_issue.id,
alert_id: alert.id,
alert_errors: { hosts: ['hosts array is over 255 chars'] }
)
end
end
subject { described_class.new.perform(nil, nil, invalid_alert_id) }
it 'does not create issues' do
expect(IncidentManagement::CreateIssueService).not_to receive(:new)
expect { subject }.not_to change { Issue.count }
end
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment