Merge branch '299997-define-ssot-for-time-periods-available-in-usage-ping' into 'master'

Define SSOT for time periods available in Usage Ping

See merge request gitlab-org/gitlab!60658

lib/gitlab/usage/metrics/aggregates/aggregate.rb

@@ -22,9 +22,7 @@ module Gitlab
         }.freeze
 
         class Aggregate
-          delegate :weekly_time_range,
-                   :monthly_time_range,
-                   to: Gitlab::UsageDataCounters::HLLRedisCounter
+          include Gitlab::Usage::TimeFrame
 
           def initialize(recorded_at)
             @aggregated_metrics = load_metrics(AGGREGATED_METRICS_PATH)
@@ -32,15 +30,15 @@ module Gitlab
           end
 
           def all_time_data
-            aggregated_metrics_data(start_date: nil, end_date: nil, time_frame: Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME)
+            aggregated_metrics_data(start_date: nil, end_date: nil, time_frame: Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME)
           end
 
           def monthly_data
-            aggregated_metrics_data(**monthly_time_range.merge(time_frame: Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME))
+            aggregated_metrics_data(**monthly_time_range.merge(time_frame: Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME))
           end
 
           def weekly_data
-            aggregated_metrics_data(**weekly_time_range.merge(time_frame: Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME))
+            aggregated_metrics_data(**weekly_time_range.merge(time_frame: Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME))
           end
 
           private
@@ -54,7 +52,7 @@ module Gitlab
 
               case aggregation[:source]
               when REDIS_SOURCE
-                if time_frame == Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME
+                if time_frame == Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME
                   data[aggregation[:name]] = Gitlab::Utils::UsageData::FALLBACK
                   Gitlab::ErrorTracking
                     .track_and_raise_for_dev_exception(

lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb

@@ -56,15 +56,15 @@ module Gitlab
               end
 
               def time_period_to_human_name(time_period)
-                return Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME if time_period.blank?
+                return Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME if time_period.blank?
 
                 start_date = time_period.first.to_date
                 end_date = time_period.last.to_date
 
                 if (end_date - start_date).to_i > 7
-                  Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME
+                  Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME
                 else
-                  Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME
+                  Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME
                 end
               end
             end

lib/gitlab/usage/metrics/instrumentations/base_metric.rb

@@ -6,6 +6,7 @@ module Gitlab
       module Instrumentations
         class BaseMetric
           include Gitlab::Utils::UsageData
+          include Gitlab::Usage::TimeFrame
 
           attr_reader :time_frame
           attr_reader :options

lib/gitlab/usage/metrics/instrumentations/database_metric.rb

@@ -52,7 +52,7 @@ module Gitlab
           def time_constraints
             case time_frame
             when '28d'
-              { created_at: 30.days.ago..2.days.ago }
+              monthly_time_range_db_params
             when 'all'
               {}
             when 'none'

lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb

@@ -35,9 +35,9 @@ module Gitlab
           def time_constraints
             case time_frame
             when '28d'
-              { start_date: 4.weeks.ago.to_date, end_date: Date.current }
+              monthly_time_range
             when '7d'
-              { start_date: 7.days.ago.to_date, end_date: Date.current }
+              weekly_time_range
             else
               raise "Unknown time frame: #{time_frame} for RedisHLLMetric"
             end

lib/gitlab/usage/time_frame.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module TimeFrame
+      ALL_TIME_TIME_FRAME_NAME = "all"
+      SEVEN_DAYS_TIME_FRAME_NAME = "7d"
+      TWENTY_EIGHT_DAYS_TIME_FRAME_NAME = "28d"
+
+      def weekly_time_range
+        { start_date: 7.days.ago.to_date, end_date: Date.current }
+      end
+
+      def monthly_time_range
+        { start_date: 4.weeks.ago.to_date, end_date: Date.current }
+      end
+
+      # This time range is skewed for batch counter performance.
+      # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42972
+      def monthly_time_range_db_params(column: :created_at)
+        { column => 30.days.ago..2.days.ago }
+      end
+    end
+  end
+end

lib/gitlab/usage_data.rb

@@ -33,6 +33,7 @@ module Gitlab
     class << self
       include Gitlab::Utils::UsageData
       include Gitlab::Utils::StrongMemoize
+      include Gitlab::Usage::TimeFrame
 
       def data(force_refresh: false)
         Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do
@@ -55,7 +56,7 @@ module Gitlab
             .merge(object_store_usage_data)
             .merge(topology_usage_data)
             .merge(usage_activity_by_stage)
-            .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, last_28_days_time_period))
+            .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, monthly_time_range_db_params))
             .merge(analytics_unique_visits_data)
             .merge(compliance_unique_visits_data)
             .merge(search_unique_visits_data)
@@ -228,17 +229,17 @@ module Gitlab
         {
           counts_monthly: {
             # rubocop: disable UsageData/LargeTable:
-            deployments: deployment_count(Deployment.where(last_28_days_time_period)),
-            successful_deployments: deployment_count(Deployment.success.where(last_28_days_time_period)),
-            failed_deployments: deployment_count(Deployment.failed.where(last_28_days_time_period)),
+            deployments: deployment_count(Deployment.where(monthly_time_range_db_params)),
+            successful_deployments: deployment_count(Deployment.success.where(monthly_time_range_db_params)),
+            failed_deployments: deployment_count(Deployment.failed.where(monthly_time_range_db_params)),
             # rubocop: enable UsageData/LargeTable:
-            projects: count(Project.where(last_28_days_time_period), start: minimum_id(Project), finish: maximum_id(Project)),
-            packages: count(::Packages::Package.where(last_28_days_time_period)),
-            personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)),
-            project_snippets: count(ProjectSnippet.where(last_28_days_time_period)),
-            projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(last_28_days_time_period), :project_id)
+            projects: count(Project.where(monthly_time_range_db_params), start: minimum_id(Project), finish: maximum_id(Project)),
+            packages: count(::Packages::Package.where(monthly_time_range_db_params)),
+            personal_snippets: count(PersonalSnippet.where(monthly_time_range_db_params)),
+            project_snippets: count(ProjectSnippet.where(monthly_time_range_db_params)),
+            projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(monthly_time_range_db_params), :project_id)
           }.merge(
-            snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp))
+            snowplow_event_counts(monthly_time_range_db_params(column: :collector_tstamp))
           ).tap do |data|
             data[:snippets] = add(data[:personal_snippets], data[:project_snippets])
           end
@@ -522,10 +523,6 @@ module Gitlab
         "#{platform}-#{ohai_data['platform_version']}"
       end
 
-      def last_28_days_time_period(column: :created_at)
-        { column => 30.days.ago..2.days.ago }
-      end
-
       # Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv
       def usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {})
         {
@@ -743,7 +740,7 @@ module Gitlab
           hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) }
         end
         results['analytics_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics) }
-        results['analytics_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics, start_date: 4.weeks.ago.to_date, end_date: Date.current) }
+        results['analytics_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics, **monthly_time_range) }
 
         { analytics_unique_visits: results }
       end
@@ -753,7 +750,7 @@ module Gitlab
           hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) }
         end
         results['compliance_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance) }
-        results['compliance_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance, start_date: 4.weeks.ago.to_date, end_date: Date.current) }
+        results['compliance_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance, **monthly_time_range) }
 
         { compliance_unique_visits: results }
       end
@@ -761,11 +758,11 @@ module Gitlab
       def search_unique_visits_data
         events = ::Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('search')
         results = events.each_with_object({}) do |event, hash|
-          hash[event] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) }
+          hash[event] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event, **weekly_time_range) }
         end
 
-        results['search_unique_visits_for_any_target_weekly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: 7.days.ago.to_date, end_date: Date.current) }
-        results['search_unique_visits_for_any_target_monthly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: 4.weeks.ago.to_date, end_date: Date.current) }
+        results['search_unique_visits_for_any_target_weekly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, **weekly_time_range) }
+        results['search_unique_visits_for_any_target_monthly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, **monthly_time_range) }
 
         { search_unique_visits: results }
       end

lib/gitlab/usage_data_counters/hll_redis_counter.rb

@@ -38,6 +38,7 @@ module Gitlab
       # * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current)
       class << self
         include Gitlab::Utils::UsageData
+        include Gitlab::Usage::TimeFrame
 
         # Track unique events
         #
@@ -98,14 +99,6 @@ module Gitlab
           end
         end
 
-        def weekly_time_range
-          { start_date: 7.days.ago.to_date, end_date: Date.current }
-        end
-
-        def monthly_time_range
-          { start_date: 4.weeks.ago.to_date, end_date: Date.current }
-        end
-
         def known_event?(event_name)
           event_for(event_name).present?
         end

lib/gitlab/utils/usage_data.rb

@@ -42,9 +42,6 @@ module Gitlab
       FALLBACK = -1
       HISTOGRAM_FALLBACK = { '-1' => -1 }.freeze
       DISTRIBUTED_HLL_FALLBACK = -2
-      ALL_TIME_TIME_FRAME_NAME = "all"
-      SEVEN_DAYS_TIME_FRAME_NAME = "7d"
-      TWENTY_EIGHT_DAYS_TIME_FRAME_NAME = "28d"
       MAX_BUCKET_SIZE = 100
 
       def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)

spec/config/metrics/aggregates/aggregated_metrics_spec.rb

@@ -25,9 +25,9 @@ RSpec.describe 'aggregated metrics' do
 
   RSpec::Matchers.define :have_known_time_frame do
     allowed_time_frames = [
-      Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME,
-      Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME,
-      Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME
+      Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME,
+      Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME,
+      Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME
     ]
 
     match do |aggregate|
@@ -63,7 +63,7 @@ RSpec.describe 'aggregated metrics' do
         let_it_be(:events_records) { known_events.select { |event| aggregate[:events].include?(event[:name]) } }
 
         it "does not include 'all' time frame for Redis sourced aggregate" do
-          expect(aggregate[:time_frame]).not_to include(Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME)
+          expect(aggregate[:time_frame]).not_to include(Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME)
         end
 
         it "only refers to known events" do

spec/lib/gitlab/usage_data_spec.rb

@@ -91,7 +91,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
       expect(described_class.usage_activity_by_stage_package({})).to eq(
         projects_with_packages: 2
       )
-      expect(described_class.usage_activity_by_stage_package(described_class.last_28_days_time_period)).to eq(
+      expect(described_class.usage_activity_by_stage_package(described_class.monthly_time_range_db_params)).to eq(
         projects_with_packages: 1
       )
     end
@@ -135,7 +135,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
         project_clusters_disabled: 2,
         project_clusters_enabled: 10
       )
-      expect(described_class.usage_activity_by_stage_configure(described_class.last_28_days_time_period)).to include(
+      expect(described_class.usage_activity_by_stage_configure(described_class.monthly_time_range_db_params)).to include(
         clusters_applications_cert_managers: 1,
         clusters_applications_helm: 1,
         clusters_applications_ingress: 1,
@@ -185,7 +185,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
         snippets: 2,
         suggestions: 2
       )
-      expect(described_class.usage_activity_by_stage_create(described_class.last_28_days_time_period)).to include(
+      expect(described_class.usage_activity_by_stage_create(described_class.monthly_time_range_db_params)).to include(
         deploy_keys: 1,
         keys: 1,
         merge_requests: 1,
@@ -225,7 +225,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
         omniauth_providers: ['google_oauth2'],
         user_auth_by_provider: { 'group_saml' => 2, 'ldap' => 4, 'standard' => 0, 'two-factor' => 0, 'two-factor-via-u2f-device' => 0, "two-factor-via-webauthn-device" => 0 }
       )
-      expect(described_class.usage_activity_by_stage_manage(described_class.last_28_days_time_period)).to include(
+      expect(described_class.usage_activity_by_stage_manage(described_class.monthly_time_range_db_params)).to include(
         events: 1,
         groups: 1,
         users_created: 3,
@@ -252,7 +252,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
         unique_users_all_imports: 10
       )
 
-      expect(described_class.usage_activity_by_stage_manage(described_class.last_28_days_time_period)).to include(
+      expect(described_class.usage_activity_by_stage_manage(described_class.monthly_time_range_db_params)).to include(
         unique_users_all_imports: 5
       )
     end
@@ -327,7 +327,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
           groups_imported: Gitlab::UsageData::DEPRECATED_VALUE
         }
       )
-      expect(described_class.usage_activity_by_stage_manage(described_class.last_28_days_time_period)).to include(
+      expect(described_class.usage_activity_by_stage_manage(described_class.monthly_time_range_db_params)).to include(
         {
           bulk_imports: {
             gitlab_v1: 1,
@@ -411,7 +411,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
         projects_with_enabled_alert_integrations_histogram: { '1' => 2 }
       )
 
-      data_28_days = described_class.usage_activity_by_stage_monitor(described_class.last_28_days_time_period)
+      data_28_days = described_class.usage_activity_by_stage_monitor(described_class.monthly_time_range_db_params)
       expect(data_28_days).to include(
         clusters: 1,
         clusters_applications_prometheus: 1,
@@ -450,7 +450,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
         projects_jira_dvcs_cloud_active: 2,
         projects_jira_dvcs_server_active: 2
       )
-      expect(described_class.usage_activity_by_stage_plan(described_class.last_28_days_time_period)).to include(
+      expect(described_class.usage_activity_by_stage_plan(described_class.monthly_time_range_db_params)).to include(
         issues: 2,
         notes: 1,
         projects: 1,
@@ -479,7 +479,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
         releases: 2,
         successful_deployments: 2
       )
-      expect(described_class.usage_activity_by_stage_release(described_class.last_28_days_time_period)).to include(
+      expect(described_class.usage_activity_by_stage_release(described_class.monthly_time_range_db_params)).to include(
         deployments: 1,
         failed_deployments: 1,
         releases: 1,
@@ -513,7 +513,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
         ci_triggers: 2,
         clusters_applications_runner: 2
       )
-      expect(described_class.usage_activity_by_stage_verify(described_class.last_28_days_time_period)).to include(
+      expect(described_class.usage_activity_by_stage_verify(described_class.monthly_time_range_db_params)).to include(
         ci_builds: 1,
         ci_external_pipelines: 1,
         ci_internal_pipelines: 1,