Commit 1bdfb664 authored by Jan Provaznik's avatar Jan Provaznik

Merge branch 'sy-auto-embed-alert' into 'master'

Auto-embed non-gitlab-managed prometheus alerts

See merge request gitlab-org/gitlab!28622
parents f3660831 4dbf0e69
......@@ -3,7 +3,7 @@
module Projects
module Prometheus
class AlertPresenter < Gitlab::View::Presenter::Delegated
RESERVED_ANNOTATIONS = %w(gitlab_incident_markdown title).freeze
RESERVED_ANNOTATIONS = %w(gitlab_incident_markdown gitlab_y_label title).freeze
GENERIC_ALERT_SUMMARY_ANNOTATIONS = %w(monitoring_tool service hosts).freeze
MARKDOWN_LINE_BREAK = " \n".freeze
INCIDENT_LABEL_NAME = IncidentManagement::CreateIssueService::INCIDENT_LABEL[:title].freeze
......
......@@ -138,6 +138,4 @@ Incident Management features can be easily enabled & disabled via the Project se
#### Auto-creation
GitLab Issues can automatically be created as a result of an Alert notification. An Issue created this way will contain error information to help you further debug the error.
For [GitLab-managed alerting rules](../project/integrations/prometheus.md#setting-up-alerts-for-prometheus-metrics-ultimate), the issue will include an embedded chart for the query corresponding to the alert. The chart will show an hour of data surrounding the starting point of the incident, 30 minutes before and after.
You can automatically create GitLab issues from an Alert notification. Issues created this way contain error information to help you debug the error. Appropriately configured alerts include an [embedded chart](../project/integrations/prometheus.md#embedding-metrics-based-on-alerts-in-incident-issues) for the query corresponding to the alert.
......@@ -802,6 +802,22 @@ It is also possible to embed either the default dashboard metrics or individual
![Embedded Metrics in issue templates](img/embed_metrics_issue_template.png)
### Embedding metrics based on alerts in incident issues
For [GitLab-managed alerting rules](#setting-up-alerts-for-prometheus-metrics-ultimate), the issue will include an embedded chart for the query corresponding to the alert. The chart displays an hour of data surrounding the starting point of the incident, 30 minutes before and after.
For [manually configured Prometheus instances](#manual-configuration-of-prometheus), a chart corresponding to the query can be included if these requirements are met:
- The alert corresponds to an environment managed through GitLab.
- The alert corresponds to a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries).
- The alert contains the required attributes listed in the chart below.
| Attributes | Required | Description |
| ---------- | -------- | ----------- |
| `annotations/gitlab_environment_name` | Yes | Name of the GitLab-managed environment corresponding to the alert |
| One of `annotations/title`, `annotations/summary`, `labels/alertname` | Yes | Will be used as the chart title |
| `annotations/gitlab_y_label` | No | Will be used as the chart's y-axis label |
### Embedding Cluster Health Charts **(ULTIMATE)**
> [Introduced](<https://gitlab.com/gitlab-org/gitlab/issues/40997>) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9.
......
......@@ -10,22 +10,63 @@ module EE
override :metric_embed_for_alert
def metric_embed_for_alert
url = embed_url_for_gitlab_alert || embed_url_for_self_managed_alert
"\n[](#{url})" if url
end
private
def embed_url_for_gitlab_alert
return unless gitlab_alert
time = starts_at ? Time.rfc3339(starts_at) : Time.current
url = metrics_dashboard_project_prometheus_alert_url(
metrics_dashboard_project_prometheus_alert_url(
project,
gitlab_alert.prometheus_metric_id,
environment_id: environment.id,
**alert_embed_window_params(embed_time)
)
end
def embed_url_for_self_managed_alert
return unless environment && full_query && title
metrics_dashboard_project_environment_url(
project,
environment,
embed_json: dashboard_for_self_managed_alert.to_json,
**alert_embed_window_params(embed_time)
)
end
def embed_time
starts_at ? Time.rfc3339(starts_at) : Time.current
end
def alert_embed_window_params(time)
{
start: format_embed_timestamp(time - METRIC_TIME_WINDOW),
end: format_embed_timestamp(time + METRIC_TIME_WINDOW)
)
}
end
"\n[](#{url})"
def format_embed_timestamp(time)
time.utc.strftime('%FT%TZ')
end
def format_embed_timestamp(timestamp)
timestamp.utc.strftime('%FT%TZ')
def dashboard_for_self_managed_alert
{
panel_groups: [{
panels: [{
type: 'line-graph',
title: title,
y_label: y_label,
metrics: [{
query_range: full_query
}]
}]
}]
}
end
end
end
......
---
title: Automatically embed metrics in issues for alerts from manually configured Prometheus instances
merge_request: 28622
author:
type: added
......@@ -8,18 +8,11 @@ describe Projects::Prometheus::AlertPresenter do
let(:presenter) { described_class.new(alert) }
let(:payload) { {} }
let(:alert) { create(:alerting_alert, project: project, payload: payload) }
let(:markdown_line_break) { ' ' }
let(:starts_at) { '2018-03-12T09:06:00Z' }
describe '#issue_summary_markdown' do
let(:markdown_line_break) { ' ' }
subject { presenter.issue_summary_markdown }
context 'with gitlab alert' do
let(:gitlab_alert) { create(:prometheus_alert, project: project) }
let(:metric_id) { gitlab_alert.prometheus_metric_id }
let(:env_id) { gitlab_alert.environment_id }
let(:starts_at) { '2018-03-12T09:06:00Z' }
shared_examples_for 'markdown with metrics embed' do
let(:expected_markdown) do
<<~MARKDOWN.chomp
#### Summary
......@@ -27,14 +20,10 @@ describe Projects::Prometheus::AlertPresenter do
**Start time:** #{presenter.starts_at}#{markdown_line_break}
**full_query:** `avg(metric) > 1.0`
[](http://localhost/#{project.full_path}/prometheus/alerts/#{metric_id}/metrics_dashboard?end=2018-03-12T09%3A36%3A00Z&environment_id=#{env_id}&start=2018-03-12T08%3A36%3A00Z)
[](#{url})
MARKDOWN
end
before do
payload['labels'] = { 'gitlab_alert_id' => metric_id }
end
context 'without a starting time available' do
around do |example|
Timecop.freeze(starts_at) { example.run }
......@@ -51,5 +40,116 @@ describe Projects::Prometheus::AlertPresenter do
it { is_expected.to eq(expected_markdown) }
end
end
subject { presenter.issue_summary_markdown }
context 'for gitlab-managed prometheus alerts' do
let(:gitlab_alert) { create(:prometheus_alert, project: project) }
let(:metric_id) { gitlab_alert.prometheus_metric_id }
let(:env_id) { gitlab_alert.environment_id }
before do
payload['labels'] = { 'gitlab_alert_id' => metric_id }
end
let(:url) { "http://localhost/#{project.full_path}/prometheus/alerts/#{metric_id}/metrics_dashboard?end=2018-03-12T09%3A36%3A00Z&environment_id=#{env_id}&start=2018-03-12T08%3A36%3A00Z" }
it_behaves_like 'markdown with metrics embed'
end
context 'for alerts from a self-managed prometheus' do
let!(:environment) { create(:environment, project: project, name: 'production') }
let(:url) { "http://localhost/#{project.full_path}/-/environments/#{environment.id}/metrics_dashboard?embed_json=#{CGI.escape(embed_content.to_json)}&end=2018-03-12T09%3A36%3A00Z&start=2018-03-12T08%3A36%3A00Z" }
let(:title) { 'title' }
let(:y_label) { 'y_label' }
let(:query) { 'avg(metric) > 1.0' }
let(:embed_content) do
{
panel_groups: [{
panels: [{
type: 'line-graph',
title: title,
y_label: y_label,
metrics: [{ query_range: query }]
}]
}]
}
end
before do
# Setup embed time range
payload['startsAt'] = starts_at
# Setup query
payload['generatorURL'] = "http://host?g0.expr=#{CGI.escape(query)}"
# Setup environment
payload['labels'] ||= {}
payload['labels']['gitlab_environment_name'] = 'production'
# Setup chart title & axis labels
payload['annotations'] ||= {}
payload['annotations']['title'] = 'title'
payload['annotations']['gitlab_y_label'] = 'y_label'
end
it_behaves_like 'markdown with metrics embed'
context 'without y_label' do
let(:y_label) { title }
before do
payload['annotations'].delete('gitlab_y_label')
end
it_behaves_like 'markdown with metrics embed'
end
context 'when not enough information is present for an embed' do
let(:expected_markdown) do
<<~MARKDOWN.chomp
#### Summary
**Start time:** #{presenter.starts_at}#{markdown_line_break}
**full_query:** `avg(metric) > 1.0`
MARKDOWN
end
context 'without title' do
before do
payload['annotations'].delete('title')
end
it { is_expected.to eq(expected_markdown) }
end
context 'without environment' do
before do
payload['labels'].delete('gitlab_environment_name')
end
it { is_expected.to eq(expected_markdown) }
end
context 'without full_query' do
let(:expected_markdown) do
<<~MARKDOWN.chomp
#### Summary
**Start time:** #{presenter.starts_at}
MARKDOWN
end
before do
payload.delete('generatorURL')
end
it { is_expected.to eq(expected_markdown) }
end
end
end
end
end
......@@ -69,6 +69,12 @@ module Gitlab
end
end
def y_label
strong_memoize(:y_label) do
parse_y_label_from_payload || title
end
end
def alert_markdown
strong_memoize(:alert_markdown) do
parse_alert_markdown_from_payload
......@@ -162,6 +168,10 @@ module Gitlab
def parse_alert_markdown_from_payload
payload&.dig('annotations', 'gitlab_incident_markdown')
end
def parse_y_label_from_payload
payload&.dig('annotations', 'gitlab_y_label')
end
end
end
end
......@@ -192,6 +192,16 @@ describe Gitlab::Alerting::Alert do
end
end
describe '#y_label' do
subject { alert.y_label }
it_behaves_like 'parse payload', 'annotations/gitlab_y_label'
context 'when y_label is not included in the payload' do
it_behaves_like 'parse payload', 'annotations/title'
end
end
describe '#alert_markdown' do
subject { alert.alert_markdown }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment