Track CI minutes notifications for new monthly tracking

On new monthly tracking of CI minutes we will track on
Ci::Minutes::NamespaceMonthlyUsage which notifications
have been sent out to the namespace owners.

This is introduced side-by-side to the legacy attributes
in Namespace, with the plan to deprecate those.

Changelog: added

db/migrate/20210824160459_add_notification_level_to_ci_namespace_monthly_usages.rb

+# frozen_string_literal: true
+
+class AddNotificationLevelToCiNamespaceMonthlyUsages < Gitlab::Database::Migration[1.0]
+  def change
+    add_column :ci_namespace_monthly_usages, :notification_level, :integer, limit: 2, default: 100, null: false
+  end
+end

db/schema_migrations/20210824160459

+39924743a04ba01cb85eed5ef88762a6a3e29c56f397a59632ba43e0ccec40b3
\ No newline at end of file

db/structure.sql

@@ -11633,6 +11633,7 @@ CREATE TABLE ci_namespace_monthly_usages (
     date date NOT NULL,
     additional_amount_available integer DEFAULT 0 NOT NULL,
     amount_used numeric(18,2) DEFAULT 0.0 NOT NULL,
+    notification_level smallint DEFAULT 100 NOT NULL,
     CONSTRAINT ci_namespace_monthly_usages_year_month_constraint CHECK ((date = date_trunc('month'::text, (date)::timestamp with time zone)))
 );
 

ee/app/models/ci/minutes/namespace_monthly_usage.rb

@@ -35,6 +35,28 @@ module Ci
         # This is better for concurrent updates.
         update_counters(usage, amount_used: amount)
       end
+
+      def total_usage_notified?
+        usage_notified?(0)
+      end
+
+      # Notification_level is set to 100 (meaning 100% remaining minutes) by default.
+      # It is reduced to 30 when the quota available drops below 30%
+      # It is reduced to 5 when the quota available drops below 5%
+      # It is reduced to 0 when the there are no more minutes available.
+      #
+      # Legacy tracking of CI minutes (in `namespaces` table) uses 2 attributes instead.
+      # We are condensing both into `notification_level` in the new monthly tracking.
+      #
+      # Until we retire the legacy CI minutes tracking:
+      #   * notification_level == 0 is equivalent to last_ci_minutes_notification_at being set
+      #   * notification_level between 100 and 0 is equivalent to last_ci_minutes_usage_notification_level
+      #     being set
+      #   * notification_level == 100 is equivalent to neither of the legacy attributes being set,
+      #     meaning that the quota used is still in the bucket 100%-to-30% used.
+      def usage_notified?(remaining_percentage)
+        notification_level == remaining_percentage
+      end
     end
   end
 end

ee/app/services/ci/minutes/email_notification_service.rb

@@ -24,17 +24,31 @@ module Ci
       end
 
       def notify_total_usage
-        return if namespace.last_ci_minutes_notification_at
+        # TODO: Enable the FF on the month after this is released.
+        # https://gitlab.com/gitlab-org/gitlab/-/issues/339324
+        if Feature.enabled?(:ci_minutes_use_notification_level, namespace, default_enabled: :yaml)
+          return if namespace_usage.total_usage_notified?
+        else
+          return if namespace.last_ci_minutes_notification_at
+        end
 
-        namespace.update_columns(last_ci_minutes_notification_at: Time.current)
+        legacy_track_total_usage
+        namespace_usage.update!(notification_level: current_alert_percentage)
 
         CiMinutesUsageMailer.notify(namespace, recipients).deliver_later
       end
 
       def notify_partial_usage
-        return if already_notified_running_out
+        # TODO: Enable the FF on the month after this is released.
+        # https://gitlab.com/gitlab-org/gitlab/-/issues/339324
+        if Feature.enabled?(:ci_minutes_use_notification_level, namespace, default_enabled: :yaml)
+          return if namespace_usage.usage_notified?(current_alert_percentage)
+        else
+          return if already_notified_running_out
+        end
 
-        namespace.update_columns(last_ci_minutes_usage_notification_level: current_alert_percentage)
+        legacy_track_partial_usage
+        namespace_usage.update!(notification_level: current_alert_percentage)
 
         CiMinutesUsageMailer.notify_limit(namespace, recipients, current_alert_percentage).deliver_later
       end
@@ -51,9 +65,24 @@ module Ci
         @namespace ||= project.shared_runners_limit_namespace
       end
 
+      def namespace_usage
+        @namespace_usage ||= Ci::Minutes::NamespaceMonthlyUsage
+          .find_or_create_current(namespace_id: namespace.id)
+      end
+
       def current_alert_percentage
         notification.stage_percentage
       end
+
+      # TODO: delete this method after full rollout of ci_minutes_use_notification_level Feature Flag
+      def legacy_track_total_usage
+        namespace.update_columns(last_ci_minutes_notification_at: Time.current)
+      end
+
+      # TODO: delete this method after full rollout of ci_minutes_use_notification_level Feature Flag
+      def legacy_track_partial_usage
+        namespace.update_columns(last_ci_minutes_usage_notification_level: current_alert_percentage)
+      end
     end
   end
 end

ee/config/feature_flags/development/ci_minutes_use_notification_level.yml

+---
+name: ci_minutes_use_notification_level
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69059
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/339324
+milestone: '14.3'
+type: development
+group: group::pipeline execution
+default_enabled: false

ee/spec/models/ci/minutes/namespace_monthly_usage_spec.rb

@@ -101,4 +101,41 @@ RSpec.describe Ci::Minutes::NamespaceMonthlyUsage do
       expect(usages).to contain_exactly(matching_usage)
     end
   end
+
+  describe '#usage_notified?' do
+    let(:usage) { build(:ci_namespace_monthly_usage, notification_level: notification_level) }
+    let(:notification_level) { 100 }
+
+    subject { usage.usage_notified?(remaining_percentage) }
+
+    context 'when parameter is different than notification level' do
+      let(:remaining_percentage) { 30 }
+
+      it { is_expected.to be_falsey }
+    end
+
+    context 'when parameter is same as the notification level' do
+      let(:remaining_percentage) { notification_level }
+
+      it { is_expected.to be_truthy }
+    end
+  end
+
+  describe '#total_usage_notified?' do
+    let(:usage) { build(:ci_namespace_monthly_usage, notification_level: notification_level) }
+
+    subject { usage.total_usage_notified? }
+
+    context 'notification level is higher than zero' do
+      let(:notification_level) { 30 }
+
+      it { is_expected.to be_falsey }
+    end
+
+    context 'when notification level is zero' do
+      let(:notification_level) { 0 }
+
+      it { is_expected.to be_truthy }
+    end
+  end
 end

ee/spec/services/ci/minutes/email_notification_service_spec.rb

@@ -26,20 +26,134 @@ RSpec.describe Ci::Minutes::EmailNotificationService do
   let(:project) { create(:project, namespace: namespace) }
   let(:user) { create(:user) }
   let(:user_2) { create(:user) }
-
   let(:ci_minutes_used) { 0 }
+
   let!(:namespace_statistics) do
     create(:namespace_statistics, namespace: namespace, shared_runners_seconds: ci_minutes_used * 60)
   end
 
+  let(:namespace_usage) do
+    Ci::Minutes::NamespaceMonthlyUsage.find_or_create_current(namespace_id: namespace.id)
+  end
+
   describe '#execute' do
+    using RSpec::Parameterized::TableSyntax
+
+    subject { described_class.new(project).execute }
+
+    def expect_warning_usage_notification(new_notification_level)
+      expect(CiMinutesUsageMailer)
+        .to receive(:notify_limit)
+        .with(namespace, match_array([user.email, user_2.email]), new_notification_level)
+        .and_call_original
+
+      subject
+
+      expect(namespace_usage.reload.notification_level).to eq(new_notification_level)
+      expect(namespace.reload.last_ci_minutes_usage_notification_level).to eq(new_notification_level)
+    end
+
+    def expect_quota_exceeded_notification
+      expect(CiMinutesUsageMailer)
+        .to receive(:notify)
+        .with(namespace, match_array([user.email, user_2.email]))
+        .and_call_original
+
+      subject
+
+      expect(namespace_usage.reload.notification_level).to eq(0)
+      expect(namespace.reload.last_ci_minutes_notification_at).to be_present
+    end
+
+    def expect_no_notification(current_notification_level)
+      expect(CiMinutesUsageMailer)
+        .not_to receive(:notify_limit)
+      expect(CiMinutesUsageMailer)
+        .not_to receive(:notify)
+
+      subject
+
+      # notification level remains the same
+      expect(namespace_usage.reload.notification_level).to eq(current_notification_level)
+    end
+
+    where(:monthly_minutes_limit, :minutes_used, :current_notification_level, :result) do
+      1000 | 500  | 100 | [:not_notified]
+      1000 | 800  | 100 | [:notified, 30]
+      1000 | 800  | 30  | [:not_notified]
+      1000 | 950  | 100 | [:notified, 5]
+      1000 | 950  | 30  | [:notified, 5]
+      1000 | 950  | 5   | [:not_notified]
+      1000 | 1000 | 100 | [:notified, 0]
+      1000 | 1000 | 30  | [:notified, 0]
+      1000 | 1000 | 5   | [:notified, 0]
+      1000 | 1001 | 5   | [:notified, 0]
+      1000 | 1000 | 0   | [:not_notified]
+      0    | 1000 | 100 | [:not_notified]
+    end
+
+    with_them do
+      shared_examples 'matches the expectation' do
+        it 'matches the expectation' do
+          expectation, new_notification_level = result
+
+          if expectation == :notified && new_notification_level > 0
+            expect_warning_usage_notification(new_notification_level)
+
+          elsif expectation == :notified && new_notification_level == 0
+            expect_quota_exceeded_notification
+
+          elsif expectation == :not_notified
+            expect_no_notification(current_notification_level)
+
+          else
+            raise 'unexpected test scenario'
+          end
+        end
+      end
+
+      let_it_be(:user) { create(:user) }
+      let_it_be(:user_2) { create(:user) }
+      let_it_be_with_reload(:namespace) { create(:group) }
+      let_it_be_with_reload(:project) { create(:project, namespace: namespace) }
+
+      let!(:namespace_statistics) do
+        create(:namespace_statistics, namespace: namespace, shared_runners_seconds: minutes_used * 60)
+      end
+
+      let(:namespace_usage) do
+        Ci::Minutes::NamespaceMonthlyUsage.find_or_create_current(namespace_id: namespace.id)
+      end
+
+      before do
+        namespace_usage.update_column(:notification_level, current_notification_level)
+        namespace.update_column(:shared_runners_minutes_limit, monthly_minutes_limit)
+        namespace.add_owner(user)
+        namespace.add_owner(user_2)
+
+        if current_notification_level == 0
+          namespace.update_column(:last_ci_minutes_notification_at, Time.current)
+        elsif current_notification_level != 100
+          namespace.update_column(:last_ci_minutes_usage_notification_level, current_notification_level)
+        end
+      end
+
+      it_behaves_like 'matches the expectation'
+
+      context 'when feature flag ci_minutes_use_notification_level is disabled' do
+        before do
+          stub_feature_flags(ci_minutes_use_notification_level: false)
+        end
+
+        it_behaves_like 'matches the expectation'
+      end
+    end
+
     let(:extra_ci_minutes) { 0 }
     let(:namespace) do
       create(:namespace, shared_runners_minutes_limit: 2000, extra_shared_runners_minutes_limit: extra_ci_minutes)
     end
 
-    subject { described_class.new(project).execute }
-
     context 'with a personal namespace' do
       before do
         namespace.update!(owner_id: user.id)
@@ -106,9 +220,9 @@ RSpec.describe Ci::Minutes::EmailNotificationService do
             subject
           end
 
-          context 'when last_ci_minutes_notification_at has a value' do
+          context 'when we have already notified the user that their quota is used up' do
             before do
-              namespace.update_attribute(:last_ci_minutes_notification_at, Time.current)
+              namespace_usage.update_column(:notification_level, 0)
             end
 
             it 'does not notify owners' do
@@ -117,6 +231,24 @@ RSpec.describe Ci::Minutes::EmailNotificationService do
               subject
             end
           end
+
+          context 'when ci_minutes_use_notification_level feature flag is disabled' do
+            before do
+              stub_feature_flags(ci_minutes_use_notification_level: false)
+            end
+
+            context 'when last_ci_minutes_notification_at has a value' do
+              before do
+                namespace.update_column(:last_ci_minutes_notification_at, Time.current)
+              end
+
+              it 'does not notify owners' do
+                expect(CiMinutesUsageMailer).not_to receive(:notify)
+
+                subject
+              end
+            end
+          end
         end
       end
     end
@@ -139,7 +271,7 @@ RSpec.describe Ci::Minutes::EmailNotificationService do
 
     shared_examples 'notification for custom level is sent' do |minutes_used, expected_level|
       before do
-        namespace_statistics.update_attribute(:shared_runners_seconds, minutes_used * 60)
+        namespace_statistics.update_column(:shared_runners_seconds, minutes_used * 60)
       end
 
       it 'notifies the the owners about it' do
@@ -167,7 +299,7 @@ RSpec.describe Ci::Minutes::EmailNotificationService do
         let(:ci_minutes_used) { 1500 }
 
         before do
-          namespace.update_attribute(:shared_runners_minutes_limit, 0)
+          namespace.update_column(:shared_runners_minutes_limit, 0)
         end
 
         it_behaves_like 'no notification is sent'
@@ -177,10 +309,10 @@ RSpec.describe Ci::Minutes::EmailNotificationService do
 
       context 'when other Pipeline has finished but second level of alert has not been reached' do
         before do
-          namespace_statistics.update_attribute(:shared_runners_seconds, 1500 * 60)
+          namespace_statistics.update_column(:shared_runners_seconds, 1500 * 60)
           notify_owners
 
-          namespace_statistics.update_attribute(:shared_runners_seconds, 1600 * 60)
+          namespace_statistics.update_column(:shared_runners_seconds, 1600 * 60)
         end
 
         it_behaves_like 'no notification is sent'
@@ -194,7 +326,16 @@ RSpec.describe Ci::Minutes::EmailNotificationService do
     end
 
     context 'when there are not available minutes to use' do
-      include_examples 'no notification is sent'
+      let(:ci_minutes_used) { 2001 }
+
+      it 'notifies owners' do
+        expect(CiMinutesUsageMailer)
+          .to receive(:notify)
+          .with(namespace, array_including(user_2.email, user.email))
+          .and_call_original
+
+        notify_owners
+      end
     end
   end
 end