Merge branch 'execute-auto-roll-back-worker-when-alert-is-raised' into 'master'

Auto Rollback When a critical alert is raised

See merge request gitlab-org/gitlab!48222

app/models/alert_management/alert.rb

@@ -263,3 +263,5 @@ module AlertManagement
     end
   end
 end
+
+AlertManagement::Alert.prepend_if_ee('EE::AlertManagement::Alert')

doc/ci/environments/index.md

@@ -894,6 +894,32 @@ longer visible on the environment page.
 If the alert requires a [rollback](#retrying-and-rolling-back), you can select the
 deployment tab from the environment page and select which deployment to roll back to.
 
+#### Auto Rollback **(ULTIMATE)**
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35404) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.7.
+
+In a typical Continuous Deployment workflow, the CI pipeline tests every commit before deploying to
+production. However, problematic code can still make it to production. For example, inefficient code
+that is logically correct can pass tests even though it causes severe performance degradation.
+Operators and SREs monitor the system to catch such problems as soon as possible. If they find a
+problematic deployment, they can roll back to a previous stable version.
+
+GitLab Auto Rollback eases this workflow by automatically triggering a rollback when a
+[critical alert](../../operations/incident_management/alerts.md)
+is detected. GitLab selects and redeploys the most recent successful deployment.
+
+Limitations of GitLab Auto Rollback:
+
+- The rollback is skipped if a deployment is running when the alert is detected.
+- A rollback can happen only once in three minutes. If multiple alerts are detected at once, only
+  one rollback is performed.
+
+GitLab Auto Rollback is turned off by default. To turn it on:
+
+1. Visit **Project > Settings > CI/CD > Automatic deployment rollbacks**.
+1. Select the checkbox for **Enable automatic rollbacks**.
+1. Click **Save changes**.
+
 ### Monitoring environments
 
 If you have enabled [Prometheus for monitoring system and response metrics](../../user/project/integrations/prometheus.md),

ee/app/models/ee/alert_management/alert.rb

+# frozen_string_literal: true
+
+module EE
+  module AlertManagement
+    module Alert
+      extend ActiveSupport::Concern
+
+      prepended do
+        include AfterCommitQueue
+
+        after_create do |alert|
+          run_after_commit { alert.trigger_auto_rollback }
+        end
+      end
+
+      def trigger_auto_rollback
+        return unless triggered? && critical? && environment&.auto_rollback_enabled?
+
+        ::Deployments::AutoRollbackWorker.perform_async(environment.id)
+      end
+    end
+  end
+end

ee/app/views/projects/settings/ci_cd/_auto_rollback.html.haml

@@ -23,7 +23,6 @@
                         = s_('AutoRollback|Enable automatic rollbacks')
                       %small.form-text.text-gl-muted
                         = s_('AutoRollback|Automatic rollbacks start when a critical alert is triggered. If the last successful deployment fails to roll back automatically, it can still be done manually.')
-                        -# This will be added once the documentation page has been created
-                        -# = link_to _('More information'), help_page_path('topics/auto_rollback/index.md'), target: '_blank'
+                        = link_to _('More information'), help_page_path('ci/environments/index.md', anchor: 'auto-rollback'), target: '_blank'
 
               = f.submit _('Save changes'), class: "gl-button btn btn-success gl-mt-5", data: { qa_selector: 'save_changes_button' }

ee/spec/models/ee/alert_management/alert_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe EE::AlertManagement::Alert do
+  let_it_be(:project, refind: true) { create(:project) }
+  let_it_be(:environment, refind: true) { create(:environment, project: project) }
+
+  describe 'after_create' do
+    it 'attempts to trigger auto rollback' do
+      alert = build(:alert_management_alert, :triggered, :critical)
+
+      expect(alert).to receive(:trigger_auto_rollback)
+
+      alert.save!
+    end
+  end
+
+  describe '#trigger_auto_rollback' do
+    subject { alert.trigger_auto_rollback }
+
+    let!(:alert) { create(:alert_management_alert, :triggered, :critical, project: project, environment: environment) }
+
+    before do
+      stub_licensed_features(auto_rollback: true)
+      environment.project.auto_rollback_enabled = true
+    end
+
+    it 'executes AutoRollbackWorker' do
+      expect(Deployments::AutoRollbackWorker).to receive(:perform_async).with(environment.id)
+
+      subject
+    end
+
+    context 'when status is not triggered' do
+      let!(:alert) { create(:alert_management_alert, :acknowledged, :critical, project: project, environment: environment) }
+
+      it 'does not execute AutoRollbackWorker' do
+        expect(Deployments::AutoRollbackWorker).not_to receive(:perform_async)
+
+        subject
+      end
+    end
+
+    context 'when severity is not critical' do
+      let!(:alert) { create(:alert_management_alert, :triggered, :high, project: project, environment: environment) }
+
+      it 'does not execute AutoRollbackWorker' do
+        expect(Deployments::AutoRollbackWorker).not_to receive(:perform_async)
+
+        subject
+      end
+    end
+
+    context 'when project does not enable auto rollback' do
+      before do
+        environment.project.auto_rollback_enabled = false
+      end
+
+      it 'does not execute AutoRollbackWorker' do
+        expect(Deployments::AutoRollbackWorker).not_to receive(:perform_async)
+
+        subject
+      end
+    end
+
+    context 'when project does not have a license for auto rollback' do
+      before do
+        stub_licensed_features(auto_rollback: false)
+      end
+
+      it 'does not execute AutoRollbackWorker' do
+        expect(Deployments::AutoRollbackWorker).not_to receive(:perform_async)
+
+        subject
+      end
+    end
+
+    context 'when feature flag is disabled' do
+      before do
+        stub_feature_flags(cd_auto_rollback: false)
+      end
+
+      it 'does not execute AutoRollbackWorker' do
+        expect(Deployments::AutoRollbackWorker).not_to receive(:perform_async)
+
+        subject
+      end
+    end
+  end
+end

spec/models/alert_management/alert_spec.rb

@@ -99,7 +99,8 @@ RSpec.describe AlertManagement::Alert do
 
     describe 'fingerprint' do
       let_it_be(:fingerprint) { 'fingerprint' }
-      let(:new_alert) { build(:alert_management_alert, fingerprint: fingerprint, project: project) }
+      let_it_be(:project3, refind: true) { create(:project) }
+      let(:new_alert) { build(:alert_management_alert, fingerprint: fingerprint, project: project3) }
 
       subject { new_alert }
 
@@ -107,7 +108,7 @@ RSpec.describe AlertManagement::Alert do
         context 'same project, various states' do
           using RSpec::Parameterized::TableSyntax
 
-          let_it_be(:existing_alert) { create(:alert_management_alert, fingerprint: fingerprint, project: project) }
+          let_it_be(:existing_alert, refind: true) { create(:alert_management_alert, fingerprint: fingerprint, project: project3) }
 
           # We are only validating uniqueness for non-resolved alerts
           where(:existing_status, :new_status, :valid) do
@@ -130,7 +131,7 @@ RSpec.describe AlertManagement::Alert do
           end
 
           with_them do
-            let(:new_alert) { build(:alert_management_alert, new_status, fingerprint: fingerprint, project: project) }
+            let(:new_alert) { build(:alert_management_alert, new_status, fingerprint: fingerprint, project: project3) }
 
             before do
               existing_alert.change_status_to(existing_status)