Schedule devops adoption calculation only for pending namespaces

We schedule checks for calculations more frequently but only
calculate those namespaces which were not calculated yet

Changelog: changed
EE: true

config/initializers/1_settings.rb

@@ -582,7 +582,7 @@ Settings.cron_jobs['batched_background_migrations_worker']['job_class'] = 'Datab
 
 Gitlab.ee do
   Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker'] ||= Settingslogic.new({})
-  Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['cron'] ||= '0 0 1 * *'
+  Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['cron'] ||= '0 1 * * *'
   Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['job_class'] = 'Analytics::DevopsAdoption::CreateAllSnapshotsWorker'
   Settings.cron_jobs['active_user_count_threshold_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['active_user_count_threshold_worker']['cron'] ||= '0 12 * * *'

db/migrate/20210706084713_add_devops_adoption_snapshots_index.rb

+# frozen_string_literal: true
+
+class AddDevopsAdoptionSnapshotsIndex < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'idx_analytics_devops_adoption_snapshots_finalized'
+
+  def up
+    add_concurrent_index :analytics_devops_adoption_snapshots, [:namespace_id, :end_time], where: "recorded_at >= end_time", name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :analytics_devops_adoption_snapshots, INDEX_NAME
+  end
+end

db/schema_migrations/20210706084713

+f69e3f50e4e5642a59e157a3c4a133090ec843b563e47198d560a54328176e56
\ No newline at end of file

db/structure.sql

@@ -22470,6 +22470,8 @@ CREATE UNIQUE INDEX i_ci_job_token_project_scope_links_on_source_and_target_proj
 
 CREATE INDEX idx_analytics_devops_adoption_segments_on_namespace_id ON analytics_devops_adoption_segments USING btree (namespace_id);
 
+CREATE INDEX idx_analytics_devops_adoption_snapshots_finalized ON analytics_devops_adoption_snapshots USING btree (namespace_id, end_time) WHERE (recorded_at >= end_time);
+
 CREATE INDEX idx_audit_events_part_on_entity_id_desc_author_id_created_at ON ONLY audit_events USING btree (entity_id, entity_type, id DESC, author_id, created_at);
 
 CREATE INDEX idx_award_emoji_on_user_emoji_name_awardable_type_awardable_id ON award_emoji USING btree (user_id, name, awardable_type, awardable_id);

ee/app/models/analytics/devops_adoption/enabled_namespace.rb

@@ -3,6 +3,7 @@
 class Analytics::DevopsAdoption::EnabledNamespace < ApplicationRecord
   self.table_name = 'analytics_devops_adoption_segments'
 
+  include EachBatch
   include IgnorableColumns
 
   belongs_to :namespace
@@ -16,6 +17,13 @@ class Analytics::DevopsAdoption::EnabledNamespace < ApplicationRecord
   scope :for_display_namespaces, -> (namespaces) { where(display_namespace_id: namespaces) }
   scope :for_namespaces, -> (namespaces) { where(namespace_id: namespaces) }
   scope :for_parent, -> (namespace) { for_namespaces(namespace.self_and_descendants) }
+  scope :pending_calculation, -> {
+    subquery = Analytics::DevopsAdoption::Snapshot.finalized.for_month(1.month.before(Time.zone.now))
+      .where(Analytics::DevopsAdoption::Snapshot.arel_table[:namespace_id].eq(arel_table[:namespace_id]))
+      .arel.exists
+
+    where.not(subquery)
+  }
 
   ignore_column :last_recorded_at, remove_with: '14.2', remove_after: '2021-07-22'
 

ee/app/workers/analytics/devops_adoption/create_all_snapshots_worker.rb

@@ -16,13 +16,23 @@ module Analytics
 
       WORKERS_GAP = 5.seconds
 
-      # rubocop: disable CodeReuse/ActiveRecord
       def perform
-        ::Analytics::DevopsAdoption::EnabledNamespace.all.pluck(:id).each.with_index do |enabled_namespace_id, i|
-          CreateSnapshotWorker.perform_in(i * WORKERS_GAP, enabled_namespace_id)
+        each_pending_namespace_id do |enabled_namespace_id, index|
+          CreateSnapshotWorker.perform_in(index * WORKERS_GAP, enabled_namespace_id)
+        end
+      end
+
+      private
+
+      def each_pending_namespace_id
+        index = 0
+        ::Analytics::DevopsAdoption::EnabledNamespace.pending_calculation.each_batch(of: 100) do |batch|
+          batch.pluck_primary_key.each do |id|
+            yield(id, index)
+            index += 1
+          end
         end
       end
-      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end

ee/spec/models/analytics/devops_adoption/enabled_namespace_spec.rb

@@ -81,4 +81,24 @@ RSpec.describe Analytics::DevopsAdoption::EnabledNamespace, type: :model do
       expect(enabled_namespace.latest_snapshot).to eq(latest_snapshot)
     end
   end
+
+  describe '.pending_calculation' do
+    let!(:enabled_namespace_without_snasphot) { create :devops_adoption_enabled_namespace }
+
+    let!(:enabled_namespace_with_not_finalized) do
+      create(:devops_adoption_enabled_namespace).tap do |enabled_namespace|
+        create(:devops_adoption_snapshot, namespace: enabled_namespace.namespace, recorded_at: 1.year.ago)
+      end
+    end
+
+    let!(:enabled_namespace_with_finalized) do
+      create(:devops_adoption_enabled_namespace).tap do |enabled_namespace|
+        create(:devops_adoption_snapshot, namespace: enabled_namespace.namespace)
+      end
+    end
+
+    it 'returns all namespaces without finalized snapshot for previous month' do
+      expect(described_class.pending_calculation).to match_array([enabled_namespace_without_snasphot, enabled_namespace_with_not_finalized])
+    end
+  end
 end

ee/spec/workers/analytics/devops_adoption/create_all_snapshots_worker_spec.rb

@@ -8,11 +8,17 @@ RSpec.describe Analytics::DevopsAdoption::CreateAllSnapshotsWorker do
   describe "#perform" do
     let!(:enabled_namespace1) { create :devops_adoption_enabled_namespace }
     let!(:enabled_namespace2) { create :devops_adoption_enabled_namespace }
+    let!(:enabled_namespace3) do
+      create(:devops_adoption_enabled_namespace).tap do |enabled_namespace|
+        create(:devops_adoption_snapshot, namespace: enabled_namespace.namespace)
+      end
+    end
 
-    it 'schedules workers for each individual enabled_namespace' do
+    it 'schedules workers individually for all pending enabled_namespaces' do
       freeze_time do
         expect(Analytics::DevopsAdoption::CreateSnapshotWorker).to receive(:perform_in).with(0, enabled_namespace1.id)
         expect(Analytics::DevopsAdoption::CreateSnapshotWorker).to receive(:perform_in).with(5, enabled_namespace2.id)
+        expect(Analytics::DevopsAdoption::CreateSnapshotWorker).not_to receive(:perform_in).with(anything, enabled_namespace3.id)
 
         worker.perform
       end