Implement a worker processing alerts

Currently, the worker calls a service to create an issue
based on the incoming alert

config/sidekiq_queues.yml

@@ -103,6 +103,7 @@
   - [elastic_indexer, 1]
   - [elastic_commit_indexer, 1]
   - [export_csv, 1]
+  - [incident_management, 2]
 
   # Deprecated queues: Remove after 10.7
   - geo_base_scheduler

ee/app/models/incident_management/project_incident_management_setting.rb

@@ -2,6 +2,8 @@
 
 module IncidentManagement
   class ProjectIncidentManagementSetting < ApplicationRecord
+    include Gitlab::Utils::StrongMemoize
+
     belongs_to :project
 
     validate :issue_template_exists, if: :create_issue?
@@ -10,14 +12,23 @@ module IncidentManagement
       Gitlab::Template::IssueTemplate.all(project)
     end
 
+    def issue_template_content
+      strong_memoize(:issue_template_content) do
+        issue_template&.content if issue_template_key.present?
+      end
+    end
+
     private
 
     def issue_template_exists
       return unless issue_template_key.present?
 
+      errors.add(:issue_template_key, 'not found') unless issue_template
+    end
+
+    def issue_template
       Gitlab::Template::IssueTemplate.find(issue_template_key, project)
     rescue Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError
-      errors.add(:issue_template_key, 'not found')
     end
   end
 end

ee/app/services/incident_management/create_issue_service.rb

+# frozen_string_literal: true
+
+module IncidentManagement
+  class CreateIssueService < BaseService
+    include Gitlab::Utils::StrongMemoize
+
+    def execute
+      return error_with('setting disabled') unless incident_management_setting.create_issue?
+      return error_with('invalid alert') unless alert.valid?
+
+      success(issue: create_issue)
+    end
+
+    private
+
+    def create_issue
+      Issues::CreateService.new(
+        project,
+        issue_author,
+        title: issue_title,
+        description: issue_description
+      ).execute
+    end
+
+    def issue_author
+      strong_memoize(:issue_author) do
+        # This is a temporary solution before we've implemented User.alert_bot
+        # https://gitlab.com/gitlab-org/gitlab-ee/issues/10159
+        User.ghost
+      end
+    end
+
+    def issue_title
+      alert.title
+    end
+
+    def issue_description
+      return alert_summary unless issue_template_content
+
+      horizontal_line = "\n---\n\n"
+
+      alert_summary + horizontal_line + issue_template_content
+    end
+
+    def alert_summary
+      <<~MARKDOWN
+        ## Summary
+
+        #{annotation_list}
+      MARKDOWN
+    end
+
+    def annotation_list
+      strong_memoize(:annotation_list) do
+        alert.annotations
+          .map { |annotation| "* #{annotation.label}: #{annotation.value}" }
+          .join("\n")
+      end
+    end
+
+    def alert
+      strong_memoize(:alert) do
+        Gitlab::Alerting::Alert.new(project: project, payload: params)
+      end
+    end
+
+    def issue_template_content
+      incident_management_setting.issue_template_content
+    end
+
+    def incident_management_setting
+      strong_memoize(:incident_management_setting) do
+        project.incident_management_setting ||
+          project.build_incident_management_setting
+      end
+    end
+
+    def error_with(message)
+      log_error(%{Cannot create incident issue for "#{project.full_name}": #{message}})
+
+      error(message)
+    end
+  end
+end

ee/app/services/projects/prometheus/alerts/notify_service.rb

@@ -4,12 +4,15 @@ module Projects
   module Prometheus
     module Alerts
       class NotifyService < BaseService
+        include Gitlab::Utils::StrongMemoize
+
         def execute(token)
           return false unless valid_version?
           return false unless valid_alert_manager_token?(token)
 
           send_alert_email if send_email?
-          persist_events(project, params)
+          process_incident_issues if create_issue?
+          persist_events
 
           true
         end
@@ -24,13 +27,28 @@ module Projects
           Feature.enabled?(:incident_management)
         end
 
+        def incident_management_available?
+          has_incident_management_license? && incident_management_feature_enabled?
+        end
+
+        def incident_management_setting
+          strong_memoize(:incident_management_setting) do
+            project.incident_management_setting ||
+              project.build_incident_management_setting
+          end
+        end
+
         def send_email?
-          return firings.any? unless incident_management_feature_enabled? &&
-              has_incident_management_license?
+          return firings.any? unless incident_management_available?
 
-          setting = project.incident_management_setting || project.build_incident_management_setting
+          incident_management_setting.send_email && firings.any?
+        end
+
+        def create_issue?
+          return unless firings.any?
+          return unless incident_management_available?
 
-          setting.send_email && firings.any?
+          incident_management_setting.create_issue?
         end
 
         def firings
@@ -112,7 +130,14 @@ module Projects
             .prometheus_alerts_fired(project, firings)
         end
 
-        def persist_events(project, params)
+        def process_incident_issues
+          firings.each do |alert|
+            IncidentManagement::ProcessAlertWorker
+              .perform_async(project.id, alert)
+          end
+        end
+
+        def persist_events
           CreateEventsService.new(project, nil, params).execute
         end
       end

ee/app/workers/all_queues.yml

@@ -45,6 +45,8 @@
 - pipeline_default:store_security_reports
 - pipeline_default:ci_create_cross_project_pipeline
 
+- incident_management:incident_management_process_alert
+
 - admin_emails
 - create_github_webhook
 - elastic_batch_project_indexer

ee/app/workers/incident_management/process_alert_worker.rb

+# frozen_string_literal: true
+
+module IncidentManagement
+  class ProcessAlertWorker
+    include ApplicationWorker
+
+    queue_namespace :incident_management
+
+    def perform(project_id, alert)
+      project = find_project(project_id)
+      return unless project
+
+      create_issue(project, alert)
+    end
+
+    private
+
+    def find_project(project_id)
+      Project.find_by_id(project_id)
+    end
+
+    def create_issue(project, alert)
+      IncidentManagement::CreateIssueService
+        .new(project, nil, alert)
+        .execute
+    end
+  end
+end

ee/lib/gitlab/alerting/alert.rb

@@ -31,6 +31,12 @@ module Gitlab
         gitlab_alert&.environment
       end
 
+      def annotations
+        strong_memoize(:annotations) do
+          parse_annotations_from_payload || []
+        end
+      end
+
       def valid?
         project && title
       end
@@ -60,6 +66,12 @@ module Gitlab
       def parse_description_from_payload
         payload&.dig('annotations', 'description')
       end
+
+      def parse_annotations_from_payload
+        payload&.dig('annotations')&.map do |label, value|
+          Alerting::AlertAnnotation.new(label: label, value: value)
+        end
+      end
     end
   end
 end

ee/lib/gitlab/alerting/alert_annotation.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Alerting
+    class AlertAnnotation
+      include ActiveModel::Model
+
+      attr_accessor :label, :value
+    end
+  end
+end

ee/spec/lib/gitlab/alerting/alert_spec.rb

@@ -52,6 +52,27 @@ describe Gitlab::Alerting::Alert do
     end
   end
 
+  context 'with annotations' do
+    before do
+      payload['annotations'] = {
+        'label' => 'value',
+        'another' => 'value2'
+      }
+    end
+
+    it 'parses annotations' do
+      expect(alert.annotations.size).to eq(2)
+      expect(alert.annotations.map(&:label)).to eq(%w(label another))
+      expect(alert.annotations.map(&:value)).to eq(%w(value value2))
+    end
+  end
+
+  context 'without annotations' do
+    it 'has no annotations' do
+      expect(alert.annotations).to be_empty
+    end
+  end
+
   context 'with empty payload' do
     it 'cannot load gitlab_alert' do
       expect(alert.gitlab_alert).to be_nil

ee/spec/models/incident_management/project_incident_management_setting_spec.rb

@@ -72,4 +72,40 @@ describe IncidentManagement::ProjectIncidentManagementSetting do
       end
     end
   end
+
+  describe '#issue_template_content' do
+    subject { build(:project_incident_management_setting, project: project) }
+
+    shared_examples 'no content' do
+      it 'returns no content' do
+        expect(subject.issue_template_content).to be_nil
+      end
+    end
+
+    context 'with valid issue_template_key' do
+      before do
+        subject.issue_template_key = 'bug'
+      end
+
+      it 'returns issue content' do
+        expect(subject.issue_template_content).to eq('something valid')
+      end
+    end
+
+    context 'with unknown issue_template_key' do
+      before do
+        subject.issue_template_key = 'unknown'
+      end
+
+      it_behaves_like 'no content'
+    end
+
+    context 'without issue_template_key' do
+      before do
+        subject.issue_template_key = nil
+      end
+
+      it_behaves_like 'no content'
+    end
+  end
 end

ee/spec/services/incident_management/create_issue_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe IncidentManagement::CreateIssueService do
+  set(:project) { create(:project, :repository, create_templates: :issue) }
+  let(:service) { described_class.new(project, nil, alert_payload) }
+  let(:alert_title) { 'TITLE' }
+  let(:alert_payload) do
+    build_alert_payload(annotations: { title: alert_title })
+  end
+
+  let!(:setting) do
+    create(:project_incident_management_setting, project: project)
+  end
+
+  subject { service.execute }
+
+  context 'when create_issue enabled' do
+    let(:user) { create(:user) }
+
+    before do
+      setting.update!(create_issue: true)
+    end
+
+    context 'without issue_template_content' do
+      it 'creates an issue with alert summary only' do
+        expect(subject).to include(status: :success)
+
+        issue = subject[:issue]
+        expect(issue.author).to eq(User.ghost)
+        expect(issue.title).to eq(alert_title)
+        expect(issue.description).to include('Summary')
+        expect(issue.description).to include(alert_title)
+        expect(issue.description).not_to include("---\n\n")
+      end
+    end
+
+    context 'with issue_template_content' do
+      before do
+        setting.update!(issue_template_key: 'bug')
+      end
+
+      it 'creates an issue appending issue template' do
+        expect(subject).to include(status: :success)
+
+        issue = subject[:issue]
+        expect(issue.description).to include("---\n\n")
+        expect(issue.description).to include(setting.issue_template_content)
+      end
+    end
+
+    context 'with an invalid alert payload' do
+      let(:alert_payload) { build_alert_payload(annotations: {}) }
+
+      it 'does not create an issue' do
+        expect(service)
+          .to receive(:log_error)
+          .with(error_message('invalid alert'))
+
+        expect(subject).to eq(status: :error, message: 'invalid alert')
+      end
+    end
+  end
+
+  context 'when create_issue disabled' do
+    before do
+      setting.update!(create_issue: false)
+    end
+
+    it 'returns an error' do
+      expect(service)
+        .to receive(:log_error)
+        .with(error_message('setting disabled'))
+
+      expect(subject).to eq(status: :error, message: 'setting disabled')
+    end
+  end
+
+  private
+
+  def build_alert_payload(annotations: {})
+    { 'annotations' => annotations.stringify_keys }
+  end
+
+  def error_message(message)
+    %{Cannot create incident issue for "#{project.full_name}": #{message}}
+  end
+end

ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb

@@ -29,6 +29,28 @@ describe Projects::Prometheus::Alerts::NotifyService do
     end
   end
 
+  shared_examples 'processes incident issues' do |amount|
+    let(:create_incident_service) { spy }
+
+    it 'processes issues' do
+      expect(IncidentManagement::ProcessAlertWorker)
+        .to receive(:perform_async)
+        .with(project.id, anything)
+        .exactly(amount).times
+
+      expect(subject).to eq(true)
+    end
+  end
+
+  shared_examples 'does not process incident issues' do
+    it 'does not process issues' do
+      expect(IncidentManagement::ProcessAlertWorker)
+        .not_to receive(:perform_async)
+
+      expect(subject).to eq(true)
+    end
+  end
+
   shared_examples 'persists events' do
     let(:create_events_service) { spy }
 
@@ -250,6 +272,59 @@ describe Projects::Prometheus::Alerts::NotifyService do
         end
       end
     end
+
+    context 'process incident issues' do
+      let!(:setting) do
+        create(
+          :project_incident_management_setting,
+          project: project,
+          create_issue: true
+        )
+      end
+
+      before do
+        create(:prometheus_service, project: project)
+        create(:project_alerting_setting, project: project, token: token)
+      end
+
+      context 'with license' do
+        before do
+          stub_licensed_features(incident_management: true)
+        end
+
+        context 'with create_issue setting enabled' do
+          before do
+            setting.update!(create_issue: true)
+          end
+
+          it_behaves_like 'processes incident issues', 1
+
+          context 'without firing alerts' do
+            let(:payload_raw) do
+              payload_for(firing: [], resolved: [alert_resolved])
+            end
+
+            it_behaves_like 'does not process incident issues'
+          end
+        end
+
+        context 'with create_issue setting disabled' do
+          before do
+            setting.update!(create_issue: false)
+          end
+
+          it_behaves_like 'does not process incident issues'
+        end
+      end
+
+      context 'without license' do
+        before do
+          stub_licensed_features(incident_management: false)
+        end
+
+        it_behaves_like 'does not process incident issues'
+      end
+    end
   end
 
   context 'with invalid payload' do

ee/spec/workers/incident_management/process_alert_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe IncidentManagement::ProcessAlertWorker do
+  set(:project) { create(:project) }
+
+  describe '#perform' do
+    let(:alert) { :alert }
+    let(:create_issue_service) { spy(:create_issue_service) }
+
+    subject { described_class.new.perform(project.id, alert) }
+
+    it 'calls create issue service' do
+      expect(Project).to receive(:find_by_id).and_call_original
+
+      expect(IncidentManagement::CreateIssueService)
+        .to receive(:new).with(project, nil, :alert)
+        .and_return(create_issue_service)
+
+      expect(create_issue_service).to receive(:execute)
+
+      subject
+    end
+
+    context 'with invalid project' do
+      let(:invalid_project_id) { 0 }
+
+      subject { described_class.new.perform(invalid_project_id, alert) }
+
+      it 'does not create issues' do
+        expect(Project).to receive(:find_by_id).and_call_original
+        expect(IncidentManagement::CreateIssueService).not_to receive(:new)
+
+        subject
+      end
+    end
+  end
+end