Commit ad5736b1 authored by Imre Farkas's avatar Imre Farkas

Merge branch '299997-define-ssot-for-time-periods-available-in-usage-ping' into 'master'

Define SSOT for time periods available in Usage Ping

See merge request gitlab-org/gitlab!60658
parents e6aa7d8b e51d2609
...@@ -270,7 +270,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -270,7 +270,7 @@ RSpec.describe Gitlab::UsageData do
projects_slack_slash_active: 2, projects_slack_slash_active: 2,
projects_with_prometheus_alerts: 2 projects_with_prometheus_alerts: 2
) )
expect(described_class.usage_activity_by_stage_configure(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_configure(described_class.monthly_time_range_db_params)).to include(
projects_slack_notifications_active: 1, projects_slack_notifications_active: 1,
projects_slack_slash_active: 1, projects_slack_slash_active: 1,
projects_with_prometheus_alerts: 1 projects_with_prometheus_alerts: 1
...@@ -341,7 +341,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -341,7 +341,7 @@ RSpec.describe Gitlab::UsageData do
total_number_of_path_locks: 20, total_number_of_path_locks: 20,
total_number_of_locked_files: 14 total_number_of_locked_files: 14
) )
expect(described_class.usage_activity_by_stage_create(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_create(described_class.monthly_time_range_db_params)).to include(
approval_project_rules: 10, approval_project_rules: 10,
approval_project_rules_with_target_branch: 2, approval_project_rules_with_target_branch: 2,
approval_project_rules_with_more_approvers_than_required: 2, approval_project_rules_with_more_approvers_than_required: 2,
...@@ -385,7 +385,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -385,7 +385,7 @@ RSpec.describe Gitlab::UsageData do
end end
subject do subject do
described_class.usage_activity_by_stage_enablement(described_class.last_28_days_time_period) described_class.usage_activity_by_stage_enablement(described_class.monthly_time_range_db_params)
end end
it 'excludes data outside of the date range' do it 'excludes data outside of the date range' do
...@@ -431,7 +431,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -431,7 +431,7 @@ RSpec.describe Gitlab::UsageData do
ldap_admin_sync_enabled: true, ldap_admin_sync_enabled: true,
group_saml_enabled: true group_saml_enabled: true
) )
expect(described_class.usage_activity_by_stage_manage(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_manage(described_class.monthly_time_range_db_params)).to include(
ldap_keys: 1, ldap_keys: 1,
ldap_users: 1, ldap_users: 1,
value_stream_management_customized_group_stages: 2, value_stream_management_customized_group_stages: 2,
...@@ -476,7 +476,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -476,7 +476,7 @@ RSpec.describe Gitlab::UsageData do
operations_dashboard_users_with_projects_added: 2, operations_dashboard_users_with_projects_added: 2,
projects_incident_sla_enabled: 2 projects_incident_sla_enabled: 2
) )
expect(described_class.usage_activity_by_stage_monitor(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_monitor(described_class.monthly_time_range_db_params)).to include(
operations_dashboard_users_with_projects_added: 1, operations_dashboard_users_with_projects_added: 1,
projects_incident_sla_enabled: 2 projects_incident_sla_enabled: 2
) )
...@@ -503,7 +503,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -503,7 +503,7 @@ RSpec.describe Gitlab::UsageData do
label_lists: 2, label_lists: 2,
milestone_lists: 2 milestone_lists: 2
) )
expect(described_class.usage_activity_by_stage_plan(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_plan(described_class.monthly_time_range_db_params)).to include(
assignee_lists: 1, assignee_lists: 1,
epics: 1, epics: 1,
label_lists: 1, label_lists: 1,
...@@ -521,7 +521,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -521,7 +521,7 @@ RSpec.describe Gitlab::UsageData do
expect(described_class.usage_activity_by_stage_release({})).to include( expect(described_class.usage_activity_by_stage_release({})).to include(
projects_mirrored_with_pipelines_enabled: 2 projects_mirrored_with_pipelines_enabled: 2
) )
expect(described_class.usage_activity_by_stage_release(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_release(described_class.monthly_time_range_db_params)).to include(
projects_mirrored_with_pipelines_enabled: 1 projects_mirrored_with_pipelines_enabled: 1
) )
end end
...@@ -549,7 +549,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -549,7 +549,7 @@ RSpec.describe Gitlab::UsageData do
end end
it 'includes accurate usage_activity_by_stage data' do it 'includes accurate usage_activity_by_stage data' do
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to include(
user_preferences_group_overview_security_dashboard: 3, user_preferences_group_overview_security_dashboard: 3,
user_container_scanning_jobs: 1, user_container_scanning_jobs: 1,
user_api_fuzzing_jobs: 1, user_api_fuzzing_jobs: 1,
...@@ -611,7 +611,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -611,7 +611,7 @@ RSpec.describe Gitlab::UsageData do
coverage_fuzzing_scans: 0 coverage_fuzzing_scans: 0
) )
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to include(
user_preferences_group_overview_security_dashboard: 3, user_preferences_group_overview_security_dashboard: 3,
user_api_fuzzing_jobs: 1, user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1, user_api_fuzzing_dnd_jobs: 1,
...@@ -647,7 +647,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -647,7 +647,7 @@ RSpec.describe Gitlab::UsageData do
create(:ci_build, name: 'dast', user: user3) create(:ci_build, name: 'dast', user: user3)
end end
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to include(
user_preferences_group_overview_security_dashboard: 3, user_preferences_group_overview_security_dashboard: 3,
user_api_fuzzing_jobs: 1, user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1, user_api_fuzzing_dnd_jobs: 1,
...@@ -681,7 +681,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -681,7 +681,7 @@ RSpec.describe Gitlab::UsageData do
create(:ci_build, name: 'license_scanning', user: user) create(:ci_build, name: 'license_scanning', user: user)
end end
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to include(
user_preferences_group_overview_security_dashboard: 3, user_preferences_group_overview_security_dashboard: 3,
user_api_fuzzing_jobs: 1, user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1, user_api_fuzzing_dnd_jobs: 1,
...@@ -720,7 +720,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -720,7 +720,7 @@ RSpec.describe Gitlab::UsageData do
allow(Gitlab::Database::PostgresHll::BatchDistinctCounter).to receive(:new).and_raise(ActiveRecord::StatementInvalid) allow(Gitlab::Database::PostgresHll::BatchDistinctCounter).to receive(:new).and_raise(ActiveRecord::StatementInvalid)
allow(::Ci::Build).to receive(:distinct_count_by).and_raise(ActiveRecord::StatementInvalid) allow(::Ci::Build).to receive(:distinct_count_by).and_raise(ActiveRecord::StatementInvalid)
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to include(
user_preferences_group_overview_security_dashboard: -1, user_preferences_group_overview_security_dashboard: -1,
user_api_fuzzing_jobs: -1, user_api_fuzzing_jobs: -1,
user_api_fuzzing_dnd_jobs: -1, user_api_fuzzing_dnd_jobs: -1,
...@@ -765,7 +765,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -765,7 +765,7 @@ RSpec.describe Gitlab::UsageData do
create(:ee_ci_build, :secret_detection, :failed, user: user2) create(:ee_ci_build, :secret_detection, :failed, user: user2)
end end
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to include(
user_api_fuzzing_scans: be_within(error_rate).percent_of(1), user_api_fuzzing_scans: be_within(error_rate).percent_of(1),
user_container_scanning_scans: be_within(error_rate).percent_of(1), user_container_scanning_scans: be_within(error_rate).percent_of(1),
user_coverage_fuzzing_scans: be_within(error_rate).percent_of(1), user_coverage_fuzzing_scans: be_within(error_rate).percent_of(1),
...@@ -794,7 +794,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -794,7 +794,7 @@ RSpec.describe Gitlab::UsageData do
create(:ee_ci_build, :secret_detection, :failed, user: user3) create(:ee_ci_build, :secret_detection, :failed, user: user3)
end end
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).not_to include( expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).not_to include(
:user_api_fuzzing_scans, :user_api_fuzzing_scans,
:user_container_scanning_scans, :user_container_scanning_scans,
:user_coverage_fuzzing_scans, :user_coverage_fuzzing_scans,
...@@ -806,7 +806,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -806,7 +806,7 @@ RSpec.describe Gitlab::UsageData do
end end
it 'includes accurate usage_activity_by_stage data' do it 'includes accurate usage_activity_by_stage data' do
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to include(
user_preferences_group_overview_security_dashboard: 3, user_preferences_group_overview_security_dashboard: 3,
user_container_scanning_jobs: 1, user_container_scanning_jobs: 1,
user_api_fuzzing_jobs: 1, user_api_fuzzing_jobs: 1,
...@@ -868,7 +868,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -868,7 +868,7 @@ RSpec.describe Gitlab::UsageData do
coverage_fuzzing_scans: 0 coverage_fuzzing_scans: 0
) )
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to include(
user_preferences_group_overview_security_dashboard: 3, user_preferences_group_overview_security_dashboard: 3,
user_api_fuzzing_jobs: 1, user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1, user_api_fuzzing_dnd_jobs: 1,
...@@ -904,7 +904,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -904,7 +904,7 @@ RSpec.describe Gitlab::UsageData do
create(:ci_build, name: 'dast', user: user3) create(:ci_build, name: 'dast', user: user3)
end end
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to include(
user_preferences_group_overview_security_dashboard: 3, user_preferences_group_overview_security_dashboard: 3,
user_api_fuzzing_jobs: 1, user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1, user_api_fuzzing_dnd_jobs: 1,
...@@ -938,7 +938,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -938,7 +938,7 @@ RSpec.describe Gitlab::UsageData do
create(:ci_build, name: 'license_scanning', user: user) create(:ci_build, name: 'license_scanning', user: user)
end end
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to include(
user_preferences_group_overview_security_dashboard: 3, user_preferences_group_overview_security_dashboard: 3,
user_api_fuzzing_jobs: 1, user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1, user_api_fuzzing_dnd_jobs: 1,
...@@ -972,7 +972,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -972,7 +972,7 @@ RSpec.describe Gitlab::UsageData do
allow(Gitlab::Database::BatchCount).to receive(:batch_count).and_raise(ActiveRecord::StatementInvalid) allow(Gitlab::Database::BatchCount).to receive(:batch_count).and_raise(ActiveRecord::StatementInvalid)
allow(::Ci::Build).to receive(:distinct_count_by).and_raise(ActiveRecord::StatementInvalid) allow(::Ci::Build).to receive(:distinct_count_by).and_raise(ActiveRecord::StatementInvalid)
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to eq( expect(described_class.usage_activity_by_stage_secure(described_class.monthly_time_range_db_params)).to eq(
user_preferences_group_overview_security_dashboard: -1, user_preferences_group_overview_security_dashboard: -1,
user_api_fuzzing_jobs: -1, user_api_fuzzing_jobs: -1,
user_api_fuzzing_dnd_jobs: -1, user_api_fuzzing_dnd_jobs: -1,
...@@ -1012,7 +1012,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -1012,7 +1012,7 @@ RSpec.describe Gitlab::UsageData do
expect(described_class.usage_activity_by_stage_verify({})).to include( expect(described_class.usage_activity_by_stage_verify({})).to include(
projects_reporting_ci_cd_back_to_github: 2 projects_reporting_ci_cd_back_to_github: 2
) )
expect(described_class.usage_activity_by_stage_verify(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_verify(described_class.monthly_time_range_db_params)).to include(
projects_reporting_ci_cd_back_to_github: 1 projects_reporting_ci_cd_back_to_github: 1
) )
end end
......
...@@ -22,9 +22,7 @@ module Gitlab ...@@ -22,9 +22,7 @@ module Gitlab
}.freeze }.freeze
class Aggregate class Aggregate
delegate :weekly_time_range, include Gitlab::Usage::TimeFrame
:monthly_time_range,
to: Gitlab::UsageDataCounters::HLLRedisCounter
def initialize(recorded_at) def initialize(recorded_at)
@aggregated_metrics = load_metrics(AGGREGATED_METRICS_PATH) @aggregated_metrics = load_metrics(AGGREGATED_METRICS_PATH)
...@@ -32,15 +30,15 @@ module Gitlab ...@@ -32,15 +30,15 @@ module Gitlab
end end
def all_time_data def all_time_data
aggregated_metrics_data(start_date: nil, end_date: nil, time_frame: Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME) aggregated_metrics_data(start_date: nil, end_date: nil, time_frame: Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME)
end end
def monthly_data def monthly_data
aggregated_metrics_data(**monthly_time_range.merge(time_frame: Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME)) aggregated_metrics_data(**monthly_time_range.merge(time_frame: Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME))
end end
def weekly_data def weekly_data
aggregated_metrics_data(**weekly_time_range.merge(time_frame: Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME)) aggregated_metrics_data(**weekly_time_range.merge(time_frame: Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME))
end end
private private
...@@ -54,7 +52,7 @@ module Gitlab ...@@ -54,7 +52,7 @@ module Gitlab
case aggregation[:source] case aggregation[:source]
when REDIS_SOURCE when REDIS_SOURCE
if time_frame == Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME if time_frame == Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME
data[aggregation[:name]] = Gitlab::Utils::UsageData::FALLBACK data[aggregation[:name]] = Gitlab::Utils::UsageData::FALLBACK
Gitlab::ErrorTracking Gitlab::ErrorTracking
.track_and_raise_for_dev_exception( .track_and_raise_for_dev_exception(
......
...@@ -56,15 +56,15 @@ module Gitlab ...@@ -56,15 +56,15 @@ module Gitlab
end end
def time_period_to_human_name(time_period) def time_period_to_human_name(time_period)
return Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME if time_period.blank? return Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME if time_period.blank?
start_date = time_period.first.to_date start_date = time_period.first.to_date
end_date = time_period.last.to_date end_date = time_period.last.to_date
if (end_date - start_date).to_i > 7 if (end_date - start_date).to_i > 7
Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME
else else
Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME
end end
end end
end end
......
...@@ -6,6 +6,7 @@ module Gitlab ...@@ -6,6 +6,7 @@ module Gitlab
module Instrumentations module Instrumentations
class BaseMetric class BaseMetric
include Gitlab::Utils::UsageData include Gitlab::Utils::UsageData
include Gitlab::Usage::TimeFrame
attr_reader :time_frame attr_reader :time_frame
attr_reader :options attr_reader :options
......
...@@ -52,7 +52,7 @@ module Gitlab ...@@ -52,7 +52,7 @@ module Gitlab
def time_constraints def time_constraints
case time_frame case time_frame
when '28d' when '28d'
{ created_at: 30.days.ago..2.days.ago } monthly_time_range_db_params
when 'all' when 'all'
{} {}
when 'none' when 'none'
......
...@@ -35,9 +35,9 @@ module Gitlab ...@@ -35,9 +35,9 @@ module Gitlab
def time_constraints def time_constraints
case time_frame case time_frame
when '28d' when '28d'
{ start_date: 4.weeks.ago.to_date, end_date: Date.current } monthly_time_range
when '7d' when '7d'
{ start_date: 7.days.ago.to_date, end_date: Date.current } weekly_time_range
else else
raise "Unknown time frame: #{time_frame} for RedisHLLMetric" raise "Unknown time frame: #{time_frame} for RedisHLLMetric"
end end
......
# frozen_string_literal: true
module Gitlab
module Usage
module TimeFrame
ALL_TIME_TIME_FRAME_NAME = "all"
SEVEN_DAYS_TIME_FRAME_NAME = "7d"
TWENTY_EIGHT_DAYS_TIME_FRAME_NAME = "28d"
def weekly_time_range
{ start_date: 7.days.ago.to_date, end_date: Date.current }
end
def monthly_time_range
{ start_date: 4.weeks.ago.to_date, end_date: Date.current }
end
# This time range is skewed for batch counter performance.
# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42972
def monthly_time_range_db_params(column: :created_at)
{ column => 30.days.ago..2.days.ago }
end
end
end
end
...@@ -33,6 +33,7 @@ module Gitlab ...@@ -33,6 +33,7 @@ module Gitlab
class << self class << self
include Gitlab::Utils::UsageData include Gitlab::Utils::UsageData
include Gitlab::Utils::StrongMemoize include Gitlab::Utils::StrongMemoize
include Gitlab::Usage::TimeFrame
def data(force_refresh: false) def data(force_refresh: false)
Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do
...@@ -55,7 +56,7 @@ module Gitlab ...@@ -55,7 +56,7 @@ module Gitlab
.merge(object_store_usage_data) .merge(object_store_usage_data)
.merge(topology_usage_data) .merge(topology_usage_data)
.merge(usage_activity_by_stage) .merge(usage_activity_by_stage)
.merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, last_28_days_time_period)) .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, monthly_time_range_db_params))
.merge(analytics_unique_visits_data) .merge(analytics_unique_visits_data)
.merge(compliance_unique_visits_data) .merge(compliance_unique_visits_data)
.merge(search_unique_visits_data) .merge(search_unique_visits_data)
...@@ -228,17 +229,17 @@ module Gitlab ...@@ -228,17 +229,17 @@ module Gitlab
{ {
counts_monthly: { counts_monthly: {
# rubocop: disable UsageData/LargeTable: # rubocop: disable UsageData/LargeTable:
deployments: deployment_count(Deployment.where(last_28_days_time_period)), deployments: deployment_count(Deployment.where(monthly_time_range_db_params)),
successful_deployments: deployment_count(Deployment.success.where(last_28_days_time_period)), successful_deployments: deployment_count(Deployment.success.where(monthly_time_range_db_params)),
failed_deployments: deployment_count(Deployment.failed.where(last_28_days_time_period)), failed_deployments: deployment_count(Deployment.failed.where(monthly_time_range_db_params)),
# rubocop: enable UsageData/LargeTable: # rubocop: enable UsageData/LargeTable:
projects: count(Project.where(last_28_days_time_period), start: minimum_id(Project), finish: maximum_id(Project)), projects: count(Project.where(monthly_time_range_db_params), start: minimum_id(Project), finish: maximum_id(Project)),
packages: count(::Packages::Package.where(last_28_days_time_period)), packages: count(::Packages::Package.where(monthly_time_range_db_params)),
personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)), personal_snippets: count(PersonalSnippet.where(monthly_time_range_db_params)),
project_snippets: count(ProjectSnippet.where(last_28_days_time_period)), project_snippets: count(ProjectSnippet.where(monthly_time_range_db_params)),
projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(last_28_days_time_period), :project_id) projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(monthly_time_range_db_params), :project_id)
}.merge( }.merge(
snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp)) snowplow_event_counts(monthly_time_range_db_params(column: :collector_tstamp))
).tap do |data| ).tap do |data|
data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) data[:snippets] = add(data[:personal_snippets], data[:project_snippets])
end end
...@@ -522,10 +523,6 @@ module Gitlab ...@@ -522,10 +523,6 @@ module Gitlab
"#{platform}-#{ohai_data['platform_version']}" "#{platform}-#{ohai_data['platform_version']}"
end end
def last_28_days_time_period(column: :created_at)
{ column => 30.days.ago..2.days.ago }
end
# Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv # Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv
def usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {}) def usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {})
{ {
...@@ -743,7 +740,7 @@ module Gitlab ...@@ -743,7 +740,7 @@ module Gitlab
hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) } hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) }
end end
results['analytics_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics) } results['analytics_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics) }
results['analytics_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics, start_date: 4.weeks.ago.to_date, end_date: Date.current) } results['analytics_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics, **monthly_time_range) }
{ analytics_unique_visits: results } { analytics_unique_visits: results }
end end
...@@ -753,7 +750,7 @@ module Gitlab ...@@ -753,7 +750,7 @@ module Gitlab
hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) } hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) }
end end
results['compliance_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance) } results['compliance_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance) }
results['compliance_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance, start_date: 4.weeks.ago.to_date, end_date: Date.current) } results['compliance_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance, **monthly_time_range) }
{ compliance_unique_visits: results } { compliance_unique_visits: results }
end end
...@@ -761,11 +758,11 @@ module Gitlab ...@@ -761,11 +758,11 @@ module Gitlab
def search_unique_visits_data def search_unique_visits_data
events = ::Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('search') events = ::Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('search')
results = events.each_with_object({}) do |event, hash| results = events.each_with_object({}) do |event, hash|
hash[event] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) } hash[event] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event, **weekly_time_range) }
end end
results['search_unique_visits_for_any_target_weekly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: 7.days.ago.to_date, end_date: Date.current) } results['search_unique_visits_for_any_target_weekly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, **weekly_time_range) }
results['search_unique_visits_for_any_target_monthly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: 4.weeks.ago.to_date, end_date: Date.current) } results['search_unique_visits_for_any_target_monthly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, **monthly_time_range) }
{ search_unique_visits: results } { search_unique_visits: results }
end end
......
...@@ -38,6 +38,7 @@ module Gitlab ...@@ -38,6 +38,7 @@ module Gitlab
# * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current) # * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current)
class << self class << self
include Gitlab::Utils::UsageData include Gitlab::Utils::UsageData
include Gitlab::Usage::TimeFrame
# Track unique events # Track unique events
# #
...@@ -98,14 +99,6 @@ module Gitlab ...@@ -98,14 +99,6 @@ module Gitlab
end end
end end
def weekly_time_range
{ start_date: 7.days.ago.to_date, end_date: Date.current }
end
def monthly_time_range
{ start_date: 4.weeks.ago.to_date, end_date: Date.current }
end
def known_event?(event_name) def known_event?(event_name)
event_for(event_name).present? event_for(event_name).present?
end end
......
...@@ -42,9 +42,6 @@ module Gitlab ...@@ -42,9 +42,6 @@ module Gitlab
FALLBACK = -1 FALLBACK = -1
HISTOGRAM_FALLBACK = { '-1' => -1 }.freeze HISTOGRAM_FALLBACK = { '-1' => -1 }.freeze
DISTRIBUTED_HLL_FALLBACK = -2 DISTRIBUTED_HLL_FALLBACK = -2
ALL_TIME_TIME_FRAME_NAME = "all"
SEVEN_DAYS_TIME_FRAME_NAME = "7d"
TWENTY_EIGHT_DAYS_TIME_FRAME_NAME = "28d"
MAX_BUCKET_SIZE = 100 MAX_BUCKET_SIZE = 100
def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
......
...@@ -25,9 +25,9 @@ RSpec.describe 'aggregated metrics' do ...@@ -25,9 +25,9 @@ RSpec.describe 'aggregated metrics' do
RSpec::Matchers.define :have_known_time_frame do RSpec::Matchers.define :have_known_time_frame do
allowed_time_frames = [ allowed_time_frames = [
Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME, Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME,
Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME, Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME,
Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME
] ]
match do |aggregate| match do |aggregate|
...@@ -63,7 +63,7 @@ RSpec.describe 'aggregated metrics' do ...@@ -63,7 +63,7 @@ RSpec.describe 'aggregated metrics' do
let_it_be(:events_records) { known_events.select { |event| aggregate[:events].include?(event[:name]) } } let_it_be(:events_records) { known_events.select { |event| aggregate[:events].include?(event[:name]) } }
it "does not include 'all' time frame for Redis sourced aggregate" do it "does not include 'all' time frame for Redis sourced aggregate" do
expect(aggregate[:time_frame]).not_to include(Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME) expect(aggregate[:time_frame]).not_to include(Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME)
end end
it "only refers to known events" do it "only refers to known events" do
......
...@@ -91,7 +91,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do ...@@ -91,7 +91,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
expect(described_class.usage_activity_by_stage_package({})).to eq( expect(described_class.usage_activity_by_stage_package({})).to eq(
projects_with_packages: 2 projects_with_packages: 2
) )
expect(described_class.usage_activity_by_stage_package(described_class.last_28_days_time_period)).to eq( expect(described_class.usage_activity_by_stage_package(described_class.monthly_time_range_db_params)).to eq(
projects_with_packages: 1 projects_with_packages: 1
) )
end end
...@@ -135,7 +135,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do ...@@ -135,7 +135,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
project_clusters_disabled: 2, project_clusters_disabled: 2,
project_clusters_enabled: 10 project_clusters_enabled: 10
) )
expect(described_class.usage_activity_by_stage_configure(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_configure(described_class.monthly_time_range_db_params)).to include(
clusters_applications_cert_managers: 1, clusters_applications_cert_managers: 1,
clusters_applications_helm: 1, clusters_applications_helm: 1,
clusters_applications_ingress: 1, clusters_applications_ingress: 1,
...@@ -185,7 +185,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do ...@@ -185,7 +185,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
snippets: 2, snippets: 2,
suggestions: 2 suggestions: 2
) )
expect(described_class.usage_activity_by_stage_create(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_create(described_class.monthly_time_range_db_params)).to include(
deploy_keys: 1, deploy_keys: 1,
keys: 1, keys: 1,
merge_requests: 1, merge_requests: 1,
...@@ -225,7 +225,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do ...@@ -225,7 +225,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
omniauth_providers: ['google_oauth2'], omniauth_providers: ['google_oauth2'],
user_auth_by_provider: { 'group_saml' => 2, 'ldap' => 4, 'standard' => 0, 'two-factor' => 0, 'two-factor-via-u2f-device' => 0, "two-factor-via-webauthn-device" => 0 } user_auth_by_provider: { 'group_saml' => 2, 'ldap' => 4, 'standard' => 0, 'two-factor' => 0, 'two-factor-via-u2f-device' => 0, "two-factor-via-webauthn-device" => 0 }
) )
expect(described_class.usage_activity_by_stage_manage(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_manage(described_class.monthly_time_range_db_params)).to include(
events: 1, events: 1,
groups: 1, groups: 1,
users_created: 3, users_created: 3,
...@@ -252,7 +252,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do ...@@ -252,7 +252,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
unique_users_all_imports: 10 unique_users_all_imports: 10
) )
expect(described_class.usage_activity_by_stage_manage(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_manage(described_class.monthly_time_range_db_params)).to include(
unique_users_all_imports: 5 unique_users_all_imports: 5
) )
end end
...@@ -327,7 +327,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do ...@@ -327,7 +327,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
groups_imported: Gitlab::UsageData::DEPRECATED_VALUE groups_imported: Gitlab::UsageData::DEPRECATED_VALUE
} }
) )
expect(described_class.usage_activity_by_stage_manage(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_manage(described_class.monthly_time_range_db_params)).to include(
{ {
bulk_imports: { bulk_imports: {
gitlab_v1: 1, gitlab_v1: 1,
...@@ -411,7 +411,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do ...@@ -411,7 +411,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
projects_with_enabled_alert_integrations_histogram: { '1' => 2 } projects_with_enabled_alert_integrations_histogram: { '1' => 2 }
) )
data_28_days = described_class.usage_activity_by_stage_monitor(described_class.last_28_days_time_period) data_28_days = described_class.usage_activity_by_stage_monitor(described_class.monthly_time_range_db_params)
expect(data_28_days).to include( expect(data_28_days).to include(
clusters: 1, clusters: 1,
clusters_applications_prometheus: 1, clusters_applications_prometheus: 1,
...@@ -450,7 +450,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do ...@@ -450,7 +450,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
projects_jira_dvcs_cloud_active: 2, projects_jira_dvcs_cloud_active: 2,
projects_jira_dvcs_server_active: 2 projects_jira_dvcs_server_active: 2
) )
expect(described_class.usage_activity_by_stage_plan(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_plan(described_class.monthly_time_range_db_params)).to include(
issues: 2, issues: 2,
notes: 1, notes: 1,
projects: 1, projects: 1,
...@@ -479,7 +479,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do ...@@ -479,7 +479,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
releases: 2, releases: 2,
successful_deployments: 2 successful_deployments: 2
) )
expect(described_class.usage_activity_by_stage_release(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_release(described_class.monthly_time_range_db_params)).to include(
deployments: 1, deployments: 1,
failed_deployments: 1, failed_deployments: 1,
releases: 1, releases: 1,
...@@ -513,7 +513,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do ...@@ -513,7 +513,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
ci_triggers: 2, ci_triggers: 2,
clusters_applications_runner: 2 clusters_applications_runner: 2
) )
expect(described_class.usage_activity_by_stage_verify(described_class.last_28_days_time_period)).to include( expect(described_class.usage_activity_by_stage_verify(described_class.monthly_time_range_db_params)).to include(
ci_builds: 1, ci_builds: 1,
ci_external_pipelines: 1, ci_external_pipelines: 1,
ci_internal_pipelines: 1, ci_internal_pipelines: 1,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment