Commit 1b6c33fa authored by Mark Chao's avatar Mark Chao

Merge branch '334153-daily-checks-for-pending-devops-adoption-calculations' into 'master'

Schedule devops adoption calculation only for pending namespaces

See merge request gitlab-org/gitlab!65443
parents e378ebea 9193e19e
......@@ -582,7 +582,7 @@ Settings.cron_jobs['batched_background_migrations_worker']['job_class'] = 'Datab
Gitlab.ee do
Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['cron'] ||= '0 0 1 * *'
Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['cron'] ||= '0 1 * * *'
Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['job_class'] = 'Analytics::DevopsAdoption::CreateAllSnapshotsWorker'
Settings.cron_jobs['active_user_count_threshold_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['active_user_count_threshold_worker']['cron'] ||= '0 12 * * *'
......
# frozen_string_literal: true
class AddDevopsAdoptionSnapshotsIndex < ActiveRecord::Migration[6.1]
include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
INDEX_NAME = 'idx_analytics_devops_adoption_snapshots_finalized'
def up
add_concurrent_index :analytics_devops_adoption_snapshots, [:namespace_id, :end_time], where: "recorded_at >= end_time", name: INDEX_NAME
end
def down
remove_concurrent_index_by_name :analytics_devops_adoption_snapshots, INDEX_NAME
end
end
f69e3f50e4e5642a59e157a3c4a133090ec843b563e47198d560a54328176e56
\ No newline at end of file
......@@ -22497,6 +22497,8 @@ CREATE UNIQUE INDEX i_ci_job_token_project_scope_links_on_source_and_target_proj
CREATE INDEX idx_analytics_devops_adoption_segments_on_namespace_id ON analytics_devops_adoption_segments USING btree (namespace_id);
CREATE INDEX idx_analytics_devops_adoption_snapshots_finalized ON analytics_devops_adoption_snapshots USING btree (namespace_id, end_time) WHERE (recorded_at >= end_time);
CREATE INDEX idx_audit_events_part_on_entity_id_desc_author_id_created_at ON ONLY audit_events USING btree (entity_id, entity_type, id DESC, author_id, created_at);
CREATE INDEX idx_award_emoji_on_user_emoji_name_awardable_type_awardable_id ON award_emoji USING btree (user_id, name, awardable_type, awardable_id);
......@@ -3,6 +3,7 @@
class Analytics::DevopsAdoption::EnabledNamespace < ApplicationRecord
self.table_name = 'analytics_devops_adoption_segments'
include EachBatch
include IgnorableColumns
belongs_to :namespace
......@@ -16,6 +17,13 @@ class Analytics::DevopsAdoption::EnabledNamespace < ApplicationRecord
scope :for_display_namespaces, -> (namespaces) { where(display_namespace_id: namespaces) }
scope :for_namespaces, -> (namespaces) { where(namespace_id: namespaces) }
scope :for_parent, -> (namespace) { for_namespaces(namespace.self_and_descendants) }
scope :pending_calculation, -> {
subquery = Analytics::DevopsAdoption::Snapshot.finalized.for_month(1.month.before(Time.zone.now))
.where(Analytics::DevopsAdoption::Snapshot.arel_table[:namespace_id].eq(arel_table[:namespace_id]))
.arel.exists
where.not(subquery)
}
ignore_column :last_recorded_at, remove_with: '14.2', remove_after: '2021-07-22'
......
......@@ -16,13 +16,23 @@ module Analytics
WORKERS_GAP = 5.seconds
# rubocop: disable CodeReuse/ActiveRecord
def perform
::Analytics::DevopsAdoption::EnabledNamespace.all.pluck(:id).each.with_index do |enabled_namespace_id, i|
CreateSnapshotWorker.perform_in(i * WORKERS_GAP, enabled_namespace_id)
each_pending_namespace_id do |enabled_namespace_id, index|
CreateSnapshotWorker.perform_in(index * WORKERS_GAP, enabled_namespace_id)
end
end
private
def each_pending_namespace_id
index = 0
::Analytics::DevopsAdoption::EnabledNamespace.pending_calculation.each_batch(of: 100) do |batch|
batch.pluck_primary_key.each do |id|
yield(id, index)
index += 1
end
end
end
# rubocop: enable CodeReuse/ActiveRecord
end
end
end
......@@ -81,4 +81,24 @@ RSpec.describe Analytics::DevopsAdoption::EnabledNamespace, type: :model do
expect(enabled_namespace.latest_snapshot).to eq(latest_snapshot)
end
end
describe '.pending_calculation' do
let!(:enabled_namespace_without_snasphot) { create :devops_adoption_enabled_namespace }
let!(:enabled_namespace_with_not_finalized) do
create(:devops_adoption_enabled_namespace).tap do |enabled_namespace|
create(:devops_adoption_snapshot, namespace: enabled_namespace.namespace, recorded_at: 1.year.ago)
end
end
let!(:enabled_namespace_with_finalized) do
create(:devops_adoption_enabled_namespace).tap do |enabled_namespace|
create(:devops_adoption_snapshot, namespace: enabled_namespace.namespace)
end
end
it 'returns all namespaces without finalized snapshot for previous month' do
expect(described_class.pending_calculation).to match_array([enabled_namespace_without_snasphot, enabled_namespace_with_not_finalized])
end
end
end
......@@ -8,11 +8,17 @@ RSpec.describe Analytics::DevopsAdoption::CreateAllSnapshotsWorker do
describe "#perform" do
let!(:enabled_namespace1) { create :devops_adoption_enabled_namespace }
let!(:enabled_namespace2) { create :devops_adoption_enabled_namespace }
let!(:enabled_namespace3) do
create(:devops_adoption_enabled_namespace).tap do |enabled_namespace|
create(:devops_adoption_snapshot, namespace: enabled_namespace.namespace)
end
end
it 'schedules workers for each individual enabled_namespace' do
it 'schedules workers individually for all pending enabled_namespaces' do
freeze_time do
expect(Analytics::DevopsAdoption::CreateSnapshotWorker).to receive(:perform_in).with(0, enabled_namespace1.id)
expect(Analytics::DevopsAdoption::CreateSnapshotWorker).to receive(:perform_in).with(5, enabled_namespace2.id)
expect(Analytics::DevopsAdoption::CreateSnapshotWorker).not_to receive(:perform_in).with(anything, enabled_namespace3.id)
worker.perform
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment