Commit b1d19581 authored by Sean Arnold's avatar Sean Arnold Committed by Douglas Barbosa Alexandre

Add join table between Issue and PrometheusAlertEvent

Prometheus specific alert handling
parent 13d6cf21
# frozen_string_literal: true
class AddIssuesPrometheusAlertEventJoinTable < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
create_table :issues_prometheus_alert_events, id: false do |t|
t.references :issue, null: false,
index: false, # Uses the index below
foreign_key: { on_delete: :cascade }
t.references :prometheus_alert_event, null: false,
index: { name: 'issue_id_issues_prometheus_alert_events_index' },
foreign_key: { on_delete: :cascade }
t.timestamps_with_timezone
t.index [:issue_id, :prometheus_alert_event_id],
unique: true, name: 'issue_id_prometheus_alert_event_id_index'
end
end
end
......@@ -1877,6 +1877,15 @@ ActiveRecord::Schema.define(version: 2019_09_29_180827) do
t.index ["updated_by_id"], name: "index_issues_on_updated_by_id", where: "(updated_by_id IS NOT NULL)"
end
create_table "issues_prometheus_alert_events", id: false, force: :cascade do |t|
t.bigint "issue_id", null: false
t.bigint "prometheus_alert_event_id", null: false
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.index ["issue_id", "prometheus_alert_event_id"], name: "issue_id_prometheus_alert_event_id_index", unique: true
t.index ["prometheus_alert_event_id"], name: "issue_id_issues_prometheus_alert_events_index"
end
create_table "jira_connect_installations", force: :cascade do |t|
t.string "client_key"
t.string "encrypted_shared_secret"
......@@ -4071,6 +4080,8 @@ ActiveRecord::Schema.define(version: 2019_09_29_180827) do
add_foreign_key "issues", "users", column: "author_id", name: "fk_05f1e72feb", on_delete: :nullify
add_foreign_key "issues", "users", column: "closed_by_id", name: "fk_c63cbf6c25", on_delete: :nullify
add_foreign_key "issues", "users", column: "updated_by_id", name: "fk_ffed080f01", on_delete: :nullify
add_foreign_key "issues_prometheus_alert_events", "issues", on_delete: :cascade
add_foreign_key "issues_prometheus_alert_events", "prometheus_alert_events", on_delete: :cascade
add_foreign_key "jira_connect_subscriptions", "jira_connect_installations", on_delete: :cascade
add_foreign_key "jira_connect_subscriptions", "namespaces", on_delete: :cascade
add_foreign_key "jira_tracker_data", "services", on_delete: :cascade
......
......@@ -27,6 +27,9 @@ module EE
end
end
has_and_belongs_to_many :prometheus_alert_events, join_table: :issues_prometheus_alert_events
has_many :prometheus_alerts, through: :prometheus_alert_events
validates :weight, allow_nil: true, numericality: { greater_than_or_equal_to: 0 }
end
......
......@@ -14,6 +14,7 @@ class PrometheusAlert < ApplicationRecord
belongs_to :prometheus_metric, required: true, validate: true, inverse_of: :prometheus_alerts
has_many :prometheus_alert_events, inverse_of: :prometheus_alert
has_many :related_issues, through: :prometheus_alert_events
after_save :clear_prometheus_adapter_cache!
after_destroy :clear_prometheus_adapter_cache!
......
......@@ -3,6 +3,7 @@
class PrometheusAlertEvent < ApplicationRecord
belongs_to :project, required: true, validate: true, inverse_of: :prometheus_alert_events
belongs_to :prometheus_alert, required: true, validate: true, inverse_of: :prometheus_alert_events
has_and_belongs_to_many :related_issues, class_name: 'Issue', join_table: :issues_prometheus_alert_events
validates :payload_key, uniqueness: { scope: :prometheus_alert_id }
......@@ -63,6 +64,10 @@ class PrometheusAlertEvent < ApplicationRecord
find_or_initialize_by(project: project, prometheus_alert: alert, payload_key: payload_key)
end
def self.find_by_payload_key(payload_key)
find_by(payload_key: payload_key)
end
def self.status_value_for(name)
state_machines[:status].states[name].value
end
......
......@@ -10,9 +10,9 @@ module Projects
return false unless valid_version?
return false unless valid_alert_manager_token?(token)
persist_events
send_alert_email if send_email?
process_incident_issues if create_issue?
persist_events
true
end
......@@ -129,7 +129,7 @@ module Projects
def process_incident_issues
firings.each do |alert|
IncidentManagement::ProcessAlertWorker
IncidentManagement::ProcessPrometheusAlertWorker
.perform_async(project.id, alert.to_h)
end
end
......
......@@ -52,6 +52,7 @@
- pipeline_default:ci_pipeline_bridge_status
- incident_management:incident_management_process_alert
- incident_management:incident_management_process_prometheus_alert
- jira_connect:jira_connect_sync_branch
- jira_connect:jira_connect_sync_merge_request
......
# frozen_string_literal: true
module IncidentManagement
class ProcessPrometheusAlertWorker
include ApplicationWorker
queue_namespace :incident_management
def perform(project_id, alert_hash)
project = find_project(project_id)
return unless project
event = find_prometheus_alert_event(alert_hash)
issue = create_issue(project, alert_hash)
relate_issue_to_event(event, issue)
end
private
def find_project(project_id)
Project.find_by_id(project_id)
end
def find_prometheus_alert_event(alert_hash)
started_at = alert_hash.dig('startsAt')
gitlab_alert_id = alert_hash.dig('labels', 'gitlab_alert_id')
payload_key = PrometheusAlertEvent.payload_key_for(gitlab_alert_id, started_at)
PrometheusAlertEvent.find_by_payload_key(payload_key)
end
def create_issue(project, alert)
IncidentManagement::CreateIssueService
.new(project, alert)
.execute
.dig(:issue)
end
def relate_issue_to_event(event, issue)
return unless event && issue
if event.related_issues.exclude?(issue)
event.related_issues << issue
end
end
end
end
---
title: Link Gitlab managed Prometheus alerts and issues
merge_request: 17477
author:
type: added
......@@ -45,6 +45,8 @@ describe Issue do
describe 'relations' do
it { is_expected.to have_many(:designs) }
it { is_expected.to have_many(:design_versions) }
it { is_expected.to have_and_belong_to_many(:prometheus_alert_events) }
it { is_expected.to have_many(:prometheus_alerts) }
describe 'versions.most_recent' do
it 'returns the most recent version' do
......
......@@ -33,7 +33,7 @@ describe Projects::Prometheus::Alerts::NotifyService do
let(:create_incident_service) { spy }
it 'processes issues', :sidekiq do
expect(IncidentManagement::ProcessAlertWorker)
expect(IncidentManagement::ProcessPrometheusAlertWorker)
.to receive(:perform_async)
.with(project.id, kind_of(Hash))
.exactly(amount).times
......@@ -46,7 +46,7 @@ describe Projects::Prometheus::Alerts::NotifyService do
shared_examples 'does not process incident issues' do
it 'does not process issues' do
expect(IncidentManagement::ProcessAlertWorker)
expect(IncidentManagement::ProcessPrometheusAlertWorker)
.not_to receive(:perform_async)
expect(subject).to eq(true)
......
# frozen_string_literal: true
require 'spec_helper'
describe IncidentManagement::ProcessPrometheusAlertWorker do
describe '#perform' do
let_it_be(:project) { create(:project) }
let_it_be(:prometheus_alert) { create(:prometheus_alert, project: project) }
before_all do
payload_key = PrometheusAlertEvent.payload_key_for(prometheus_alert.id, prometheus_alert.created_at.rfc3339)
create(:prometheus_alert_event, prometheus_alert: prometheus_alert, payload_key: payload_key)
end
let(:alert_params) do
{
startsAt: prometheus_alert.created_at.rfc3339,
labels: {
gitlab_alert_id: prometheus_alert.id
}
}.with_indifferent_access
end
it 'creates an issue' do
expect { subject.perform(project.id, alert_params) }
.to change(Issue, :count)
.by(1)
end
it 'relates issue to an event' do
expect { subject.perform(project.id, alert_params) }
.to change(prometheus_alert.related_issues, :count).from(0).to(1)
end
context 'when project could not be found' do
it 'does not create an issue' do
expect { subject.perform('1234', alert_params) }
.not_to change(Issue, :count)
end
it 'does not relate issue to an event' do
expect { subject.perform('1234', alert_params) }
.not_to change(prometheus_alert.related_issues, :count)
end
end
context 'when event could not be found' do
before do
alert_params[:labels][:gitlab_alert_id] = '1234'
end
it 'does not create an issue' do
expect { subject.perform(project.id, alert_params) }
.not_to change(Issue, :count)
end
it 'does not relate issue to an event' do
expect { subject.perform(project.id, alert_params) }
.not_to change(Issue, :count)
end
end
context 'when issue could not be created' do
before do
allow_any_instance_of(IncidentManagement::CreateIssueService)
.to receive(:execute)
.and_return( { error: true } )
end
it 'does not relate issue to an event' do
expect { subject.perform(project.id, alert_params) }
.not_to change(prometheus_alert.related_issues, :count)
end
end
end
end
......@@ -25,6 +25,8 @@ issues:
- epic
- designs
- design_versions
- prometheus_alerts
- prometheus_alert_events
events:
- author
- project
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment