Skip to content

G
gitlab-ce
Commit 7d68d72f authored by Alper Akgun's avatar Alper Akgun
Browse files
Options

Fix usage ping timeouts with batch counters 

Usage pings time out on large instances with large tables. This change
adds batch counters to fix the issue
parent 2ab97d54
Hide whitespace changes
Inline Side-by-side
Showing with 889 additions and 653 deletions
changelogs/unreleased/193075-usage-ping-timing-out-for-larger-instances-batch.yml 0 → 100644
View file @ 7d68d72f
---
title: Fix usage ping timeouts with batch counters
merge_request: 22705
author:
type: performance
ee/lib/ee/gitlab/usage_data.rb
View file @ 7d68d72f
......@@ -19,7 +19,8 @@ module EE
# to time out on GitLab.com. Since we are mostly interested in gathering these statistics for
# self hosted instances, prevent them from running on GitLab.com and allow instance maintainers
# to disable them via a feature flag.
return super if ::Gitlab.com? || ::Feature.disabled?(:usage_activity_by_stage, default_enabled: true)
return super if (::Feature.disabled?(:usage_ping_batch_counter) && ::Gitlab.com?) ||
::Feature.disabled?(:usage_activity_by_stage, default_enabled: true)
super.merge(usage_activity_by_stage)
end
......@@ -94,7 +95,7 @@ module EE
sast: :sast_jobs
}
results = count(::Ci::Build.where(name: types.keys).group(:name), fallback: Hash.new(-1))
results = count(::Ci::Build.where(name: types.keys).group(:name), fallback: Hash.new(-1), batch: false)
license_scan_count = results.delete("license_scanning")
if license_scan_count && results["license_management"]
......@@ -119,7 +120,7 @@ module EE
def operations_dashboard_usage
users_with_ops_dashboard_as_default = count(::User.active.with_dashboard('operations'))
users_with_projects_added = count(UsersOpsDashboardProject.distinct_users(::User.active))
users_with_projects_added = count(UsersOpsDashboardProject.distinct_users(::User.active), batch: false)
{
operations_dashboard_default_dashboard: users_with_ops_dashboard_as_default,
......@@ -137,15 +138,15 @@ module EE
geo_nodes: count(::GeoNode),
ldap_group_links: count(::LdapGroupLink),
ldap_keys: count(::LDAPKey),
ldap_users: count(::User.ldap),
ldap_users: count(::User.ldap, 'users.id'),
pod_logs_usages_total: ::Gitlab::UsageCounters::PodLogs.usage_totals[:total],
projects_enforcing_code_owner_approval: count(::Project.without_deleted.non_archived.requiring_code_owner_approval),
projects_mirrored_with_pipelines_enabled: count(::Project.mirrored_with_enabled_pipelines),
projects_reporting_ci_cd_back_to_github: count(::GithubService.without_defaults.active),
projects_with_packages: count(::Packages::Package.select('distinct project_id')),
projects_with_prometheus_alerts: count(PrometheusAlert.distinct_projects),
projects_with_tracing_enabled: count(ProjectTracingSetting),
template_repositories: count(::Project.with_repos_templates) + count(::Project.with_groups_level_repos_templates)
projects_enforcing_code_owner_approval: count(::Project.without_deleted.non_archived.requiring_code_owner_approval, batch: false),
projects_mirrored_with_pipelines_enabled: count(::Project.mirrored_with_enabled_pipelines, batch: false),
projects_reporting_ci_cd_back_to_github: count(::GithubService.without_defaults.active, batch: false),
projects_with_packages: count(::Packages::Package.select('distinct project_id'), batch: false),
projects_with_prometheus_alerts: count(PrometheusAlert.distinct_projects, batch: false),
projects_with_tracing_enabled: count(ProjectTracingSetting, batch: false),
template_repositories: count(::Project.with_repos_templates, batch: false) + count(::Project.with_groups_level_repos_templates, batch: false)
},
service_desk_counts,
security_products_usage,
......@@ -191,62 +192,62 @@ module EE
clusters_applications_helm: ::Clusters::Applications::Helm.distinct_by_user,
clusters_applications_ingress: ::Clusters::Applications::Ingress.distinct_by_user,
clusters_applications_knative: ::Clusters::Applications::Knative.distinct_by_user,
clusters_disabled: ::Clusters::Cluster.disabled.distinct_count_by(:user_id),
clusters_enabled: ::Clusters::Cluster.enabled.distinct_count_by(:user_id),
clusters_platforms_gke: ::Clusters::Cluster.gcp_installed.enabled.distinct_count_by(:user_id),
clusters_platforms_eks: ::Clusters::Cluster.aws_installed.enabled.distinct_count_by(:user_id),
clusters_platforms_user: ::Clusters::Cluster.user_provided.enabled.distinct_count_by(:user_id),
group_clusters_disabled: ::Clusters::Cluster.disabled.group_type.distinct_count_by(:user_id),
group_clusters_enabled: ::Clusters::Cluster.enabled.group_type.distinct_count_by(:user_id),
project_clusters_disabled: ::Clusters::Cluster.disabled.project_type.distinct_count_by(:user_id),
project_clusters_enabled: ::Clusters::Cluster.enabled.project_type.distinct_count_by(:user_id),
projects_slack_notifications_active: ::Project.with_slack_service.distinct_count_by(:creator_id),
projects_slack_slash_active: ::Project.with_slack_slash_commands_service.distinct_count_by(:creator_id),
projects_with_prometheus_alerts: ::Project.with_prometheus_service.distinct_count_by(:creator_id)
clusters_disabled: distinct_count(::Clusters::Cluster.disabled, :user_id),
clusters_enabled: distinct_count(::Clusters::Cluster.enabled, :user_id),
clusters_platforms_gke: distinct_count(::Clusters::Cluster.gcp_installed.enabled, :user_id),
clusters_platforms_eks: distinct_count(::Clusters::Cluster.aws_installed.enabled, :user_id),
clusters_platforms_user: distinct_count(::Clusters::Cluster.user_provided.enabled, :user_id),
group_clusters_disabled: distinct_count(::Clusters::Cluster.disabled.group_type, :user_id),
group_clusters_enabled: distinct_count(::Clusters::Cluster.enabled.group_type, :user_id),
project_clusters_disabled: distinct_count(::Clusters::Cluster.disabled.project_type, :user_id),
project_clusters_enabled: distinct_count(::Clusters::Cluster.enabled.project_type, :user_id),
projects_slack_notifications_active: distinct_count(::Project.with_slack_service, :creator_id),
projects_slack_slash_active: distinct_count(::Project.with_slack_slash_commands_service, :creator_id),
projects_with_prometheus_alerts: distinct_count(::Project.with_prometheus_service, :creator_id)
}
end
# Omitted because no user, creator or author associated: `lfs_objects`, `pool_repositories`, `web_hooks`
def usage_activity_by_stage_create
{
deploy_keys: ::DeployKey.distinct_count_by(:user_id),
keys: ::Key.regular_keys.distinct_count_by(:user_id),
merge_requests: ::MergeRequest.distinct_count_by(:author_id),
projects_enforcing_code_owner_approval: ::Project.requiring_code_owner_approval.distinct_count_by(:creator_id),
projects_imported_from_github: ::Project.github_imported.distinct_count_by(:creator_id),
projects_with_repositories_enabled: ::Project.with_repositories_enabled.distinct_count_by(:creator_id),
protected_branches: ::Project.with_protected_branches.distinct_count_by(:creator_id),
remote_mirrors: ::Project.with_remote_mirrors.distinct_count_by(:creator_id),
snippets: ::Snippet.distinct_count_by(:author_id),
suggestions: ::Note.with_suggestions.distinct_count_by(:author_id)
deploy_keys: distinct_count(::DeployKey, :user_id),
keys: distinct_count(::Key.regular_keys, :user_id),
merge_requests: distinct_count(::MergeRequest, :author_id),
projects_enforcing_code_owner_approval: distinct_count(::Project.requiring_code_owner_approval, :creator_id),
projects_imported_from_github: distinct_count(::Project.github_imported, :creator_id),
projects_with_repositories_enabled: distinct_count(::Project.with_repositories_enabled, :creator_id),
protected_branches: distinct_count(::Project.with_protected_branches, :creator_id),
remote_mirrors: distinct_count(::Project.with_remote_mirrors, :creator_id),
snippets: distinct_count(::Snippet, :author_id),
suggestions: distinct_count(::Note.with_suggestions, :author_id)
}
end
# Omitted because no user, creator or author associated: `campaigns_imported_from_github`, `ldap_group_links`
def usage_activity_by_stage_manage
{
events: ::Event.distinct_count_by(:author_id),
groups: ::GroupMember.distinct_count_by(:user_id),
ldap_keys: ::LDAPKey.distinct_count_by(:user_id),
ldap_users: ::GroupMember.of_ldap_type.distinct_count_by(:user_id)
events: distinct_count(::Event, :author_id),
groups: distinct_count(::GroupMember, :user_id),
ldap_keys: distinct_count(::LDAPKey, :user_id),
ldap_users: distinct_count(::GroupMember.of_ldap_type, :user_id)
}
end
def usage_activity_by_stage_monitor
{
clusters: ::Clusters::Cluster.distinct_count_by(:user_id),
clusters: distinct_count(::Clusters::Cluster, :user_id),
clusters_applications_prometheus: ::Clusters::Applications::Prometheus.distinct_by_user,
operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations')),
operations_dashboard_users_with_projects_added: count(UsersOpsDashboardProject.distinct_users(::User.active)),
projects_prometheus_active: ::Project.with_active_prometheus_service.distinct_count_by(:creator_id),
projects_with_error_tracking_enabled: ::Project.with_enabled_error_tracking.distinct_count_by(:creator_id),
projects_with_tracing_enabled: ::Project.with_tracing_enabled.distinct_count_by(:creator_id)
operations_dashboard_users_with_projects_added: count(UsersOpsDashboardProject.distinct_users(::User.active), batch: false),
projects_prometheus_active: distinct_count(::Project.with_active_prometheus_service, :creator_id),
projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking, :creator_id),
projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled, :creator_id)
}
end
def usage_activity_by_stage_package
{
projects_with_packages: ::Project.with_packages.distinct_count_by(:creator_id)
projects_with_packages: distinct_count(::Project.with_packages, :creator_id)
}
end
......@@ -255,46 +256,46 @@ module EE
# Omitted because of encrypted properties: `projects_jira_cloud_active`, `projects_jira_server_active`
def usage_activity_by_stage_plan
{
assignee_lists: ::List.assignee.distinct_count_by(:user_id),
epics: ::Epic.distinct_count_by(:author_id),
issues: ::Issue.distinct_count_by(:author_id),
label_lists: ::List.label.distinct_count_by(:user_id),
milestone_lists: ::List.milestone.distinct_count_by(:user_id),
notes: ::Note.distinct_count_by(:author_id),
projects: ::Project.distinct_count_by(:creator_id),
projects_jira_active: ::Project.with_active_jira_services.distinct_count_by(:creator_id),
projects_jira_dvcs_cloud_active: ::Project.with_active_jira_services.with_jira_dvcs_cloud.distinct_count_by(:creator_id),
projects_jira_dvcs_server_active: ::Project.with_active_jira_services.with_jira_dvcs_server.distinct_count_by(:creator_id),
service_desk_enabled_projects: ::Project.with_active_services.service_desk_enabled.distinct_count_by(:creator_id),
service_desk_issues: ::Issue.service_desk.distinct_count_by,
todos: ::Todo.distinct_count_by(:author_id)
assignee_lists: distinct_count(::List.assignee, :user_id),
epics: distinct_count(::Epic, :author_id),
issues: distinct_count(::Issue, :author_id),
label_lists: distinct_count(::List.label, :user_id),
milestone_lists: distinct_count(::List.milestone, :user_id),
notes: distinct_count(::Note, :author_id),
projects: distinct_count(::Project, :creator_id),
projects_jira_active: distinct_count(::Project.with_active_jira_services, :creator_id),
projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_jira_services.with_jira_dvcs_cloud, :creator_id),
projects_jira_dvcs_server_active: distinct_count(::Project.with_active_jira_services.with_jira_dvcs_server, :creator_id),
service_desk_enabled_projects: distinct_count(::Project.with_active_services.service_desk_enabled, :creator_id),
service_desk_issues: distinct_count(::Issue.service_desk),
todos: distinct_count(::Todo, :author_id)
}
end
# Omitted because no user, creator or author associated: `environments`, `feature_flags`, `in_review_folder`, `pages_domains`
def usage_activity_by_stage_release
{
deployments: ::Deployment.distinct_count_by(:user_id),
failed_deployments: ::Deployment.failed.distinct_count_by(:user_id),
projects_mirrored_with_pipelines_enabled: ::Project.mirrored_with_enabled_pipelines.distinct_count_by(:creator_id),
releases: ::Release.distinct_count_by(:author_id),
successful_deployments: ::Deployment.success.distinct_count_by(:user_id)
deployments: distinct_count(::Deployment, :user_id),
failed_deployments: distinct_count(::Deployment.failed, :user_id),
projects_mirrored_with_pipelines_enabled: distinct_count(::Project.mirrored_with_enabled_pipelines, :creator_id),
releases: distinct_count(::Release, :author_id),
successful_deployments: distinct_count(::Deployment.success, :user_id)
}
end
# Omitted because no user, creator or author associated: `ci_runners`
def usage_activity_by_stage_verify
{
ci_builds: ::Ci::Build.distinct_count_by(:user_id),
ci_external_pipelines: ::Ci::Pipeline.external.distinct_count_by(:user_id),
ci_internal_pipelines: ::Ci::Pipeline.internal.distinct_count_by(:user_id),
ci_pipeline_config_auto_devops: ::Ci::Pipeline.auto_devops_source.distinct_count_by(:user_id),
ci_pipeline_config_repository: ::Ci::Pipeline.repository_source.distinct_count_by(:user_id),
ci_pipeline_schedules: ::Ci::PipelineSchedule.distinct_count_by(:owner_id),
ci_pipelines: ::Ci::Pipeline.distinct_count_by(:user_id),
ci_triggers: ::Ci::Trigger.distinct_count_by(:owner_id),
ci_builds: distinct_count(::Ci::Build, :user_id),
ci_external_pipelines: distinct_count(::Ci::Pipeline.external, :user_id),
ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal, :user_id),
ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source, :user_id),
ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source, :user_id),
ci_pipeline_schedules: distinct_count(::Ci::PipelineSchedule, :owner_id),
ci_pipelines: distinct_count(::Ci::Pipeline, :user_id),
ci_triggers: distinct_count(::Ci::Trigger, :owner_id),
clusters_applications_runner: ::Clusters::Applications::Runner.distinct_by_user,
projects_reporting_ci_cd_back_to_github: ::Project.with_github_service_pipeline_events.distinct_count_by(:creator_id)
projects_reporting_ci_cd_back_to_github: distinct_count(::Project.with_github_service_pipeline_events, :creator_id)
}
end
......
ee/spec/controllers/admin/instance_review_controller_spec.rb
View file @ 7d68d72f
......@@ -17,6 +17,7 @@ describe Admin::InstanceReviewController do
context 'with usage ping enabled' do
before do
allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false)
stub_application_setting(usage_ping_enabled: true)
::Gitlab::UsageData.data(force_refresh: true)
subject
......
ee/spec/lib/ee/gitlab/usage_data_spec.rb
View file @ 7d68d72f
......@@ -3,250 +3,262 @@
require 'spec_helper'
describe Gitlab::UsageData do
describe '.uncached_data' do
context 'when on Gitlab.com' do
before do
allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false)
end
[true, false].each do |usage_ping_batch_counter_on|
describe "when the feature flag usage_ping_batch_counter is set to #{usage_ping_batch_counter_on}" do
before do
allow(Gitlab).to receive(:com?).and_return(true)
stub_feature_flags(usage_ping_batch_counter: usage_ping_batch_counter_on)
end
it 'does not include usage_activity_by_stage data' do
expect(described_class.uncached_data).not_to include(:usage_activity_by_stage)
end
describe '.uncached_data' do
context 'when on Gitlab.com' do
before do
allow(Gitlab).to receive(:com?).and_return(true)
end
it "does #{'not' unless usage_ping_batch_counter_on} include usage_activity_by_stage data" do
expect(described_class.uncached_data.include?(:usage_activity_by_stage)).to be(usage_ping_batch_counter_on)
end
context 'when feature is enabled' do
before do
stub_feature_flags(usage_activity_by_stage: true)
end
context 'when feature is enabled' do
before do
stub_feature_flags(usage_activity_by_stage: true)
it "does #{'not' unless usage_ping_batch_counter_on} include usage_activity_by_stage data" do
expect(described_class.uncached_data.include?(:usage_activity_by_stage)).to be(usage_ping_batch_counter_on)
end
end
end
it 'does not include usage_activity_by_stage data' do
expect(described_class.uncached_data).not_to include(:usage_activity_by_stage)
context 'when the :usage_activity_by_stage feature is not enabled' do
before do
stub_feature_flags(usage_activity_by_stage: false)
end
it "does not include usage_activity_by_stage data" do
expect(described_class.uncached_data).not_to include(:usage_activity_by_stage)
end
end
end
end
context 'when the :usage_activity_by_stage feature is not enabled' do
before do
stub_feature_flags(usage_activity_by_stage: false)
end
context 'when not on Gitlab.com' do
it 'includes usage_activity_by_stage data' do
expect(described_class.uncached_data).to include(:usage_activity_by_stage)
end
it 'does not include usage_activity_by_stage data' do
expect(described_class.uncached_data).not_to include(:usage_activity_by_stage)
end
end
context 'for configure' do
it 'includes accurate usage_activity_by_stage data' do
user = create(:user)
cluster = create(:cluster, user: user)
project = create(:project, creator: user)
create(:clusters_applications_cert_manager, :installed, cluster: cluster)
create(:clusters_applications_helm, :installed, cluster: cluster)
create(:clusters_applications_ingress, :installed, cluster: cluster)
create(:clusters_applications_knative, :installed, cluster: cluster)
create(:cluster, :disabled, user: user)
create(:cluster_provider_gcp, :created)
create(:cluster_provider_aws, :created)
create(:cluster_platform_kubernetes)
create(:cluster, :group, :disabled, user: user)
create(:cluster, :group, user: user)
create(:slack_service, project: project)
create(:slack_slash_commands_service, project: project)
create(:prometheus_service, project: project)
context 'when not on Gitlab.com' do
it 'includes usage_activity_by_stage data' do
expect(described_class.uncached_data).to include(:usage_activity_by_stage)
end
expect(described_class.uncached_data[:usage_activity_by_stage][:configure]).to eq(
clusters_applications_cert_managers: 1,
clusters_applications_helm: 1,
clusters_applications_ingress: 1,
clusters_applications_knative: 1,
clusters_disabled: 1,
clusters_enabled: 4,
clusters_platforms_gke: 1,
clusters_platforms_eks: 1,
clusters_platforms_user: 1,
group_clusters_disabled: 1,
group_clusters_enabled: 1,
project_clusters_disabled: 1,
project_clusters_enabled: 4,
projects_slack_notifications_active: 1,
projects_slack_slash_active: 1,
projects_with_prometheus_alerts: 1
)
end
end
context 'for configure' do
it 'includes accurate usage_activity_by_stage data' do
user = create(:user)
cluster = create(:cluster, user: user)
project = create(:project, creator: user)
create(:clusters_applications_cert_manager, :installed, cluster: cluster)
create(:clusters_applications_helm, :installed, cluster: cluster)
create(:clusters_applications_ingress, :installed, cluster: cluster)
create(:clusters_applications_knative, :installed, cluster: cluster)
create(:cluster, :disabled, user: user)
create(:cluster_provider_gcp, :created)
create(:cluster_provider_aws, :created)
create(:cluster_platform_kubernetes)
create(:cluster, :group, :disabled, user: user)
create(:cluster, :group, user: user)
create(:slack_service, project: project)
create(:slack_slash_commands_service, project: project)
create(:prometheus_service, project: project)
expect(described_class.uncached_data[:usage_activity_by_stage][:configure]).to eq(
clusters_applications_cert_managers: 1,
clusters_applications_helm: 1,
clusters_applications_ingress: 1,
clusters_applications_knative: 1,
clusters_disabled: 1,
clusters_enabled: 4,
clusters_platforms_gke: 1,
clusters_platforms_eks: 1,
clusters_platforms_user: 1,
group_clusters_disabled: 1,
group_clusters_enabled: 1,
project_clusters_disabled: 1,
project_clusters_enabled: 4,
projects_slack_notifications_active: 1,
projects_slack_slash_active: 1,
projects_with_prometheus_alerts: 1
)
end
end
context 'for create' do
it 'includes accurate usage_activity_by_stage data' do
user = create(:user)
project = create(:project, :repository_private, :github_imported,
:test_repo, :remote_mirror, creator: user)
create(:deploy_key, user: user)
create(:key, user: user)
create(:merge_request, source_project: project)
create(:project, creator: user)
create(:protected_branch, project: project)
create(:remote_mirror, project: project)
create(:snippet, author: user)
create(:suggestion, note: create(:note, project: project))
context 'for create' do
it 'includes accurate usage_activity_by_stage data' do
user = create(:user)
project = create(:project, :repository_private, :github_imported,
:test_repo, :remote_mirror, creator: user)
create(:deploy_key, user: user)
create(:key, user: user)
create(:merge_request, source_project: project)
create(:project, creator: user)
create(:protected_branch, project: project)
create(:remote_mirror, project: project)
create(:snippet, author: user)
create(:suggestion, note: create(:note, project: project))
expect(described_class.uncached_data[:usage_activity_by_stage][:create]).to eq(
deploy_keys: 1,
keys: 1,
merge_requests: 1,
projects_enforcing_code_owner_approval: 0,
projects_imported_from_github: 1,
projects_with_repositories_enabled: 1,
protected_branches: 1,
remote_mirrors: 1,
snippets: 1,
suggestions: 1
)
end
end
expect(described_class.uncached_data[:usage_activity_by_stage][:create]).to eq(
deploy_keys: 1,
keys: 1,
merge_requests: 1,
projects_enforcing_code_owner_approval: 0,
projects_imported_from_github: 1,
projects_with_repositories_enabled: 1,
protected_branches: 1,
remote_mirrors: 1,
snippets: 1,
suggestions: 1
)
end
end
context 'for manage' do
it 'includes accurate usage_activity_by_stage data' do
user = create(:user)
create(:event, author: user)
create(:group_member, user: user)
create(:key, type: 'LDAPKey', user: user)
create(:group_member, ldap: true, user: user)
expect(described_class.uncached_data[:usage_activity_by_stage][:manage]).to eq(
events: 1,
groups: 1,
ldap_keys: 1,
ldap_users: 1
)
end
end
context 'for manage' do
it 'includes accurate usage_activity_by_stage data' do
user = create(:user)
create(:event, author: user)
create(:group_member, user: user)
create(:key, type: 'LDAPKey', user: user)
create(:group_member, ldap: true, user: user)
context 'for monitor' do
it 'includes accurate usage_activity_by_stage data' do
user = create(:user, dashboard: 'operations')
cluster = create(:cluster, user: user)
project = create(:project, creator: user)
create(:clusters_applications_prometheus, :installed, cluster: cluster)
create(:users_ops_dashboard_project, user: user)
create(:prometheus_service, project: project)
create(:project_error_tracking_setting, project: project)
create(:project_tracing_setting, project: project)
expect(described_class.uncached_data[:usage_activity_by_stage][:monitor]).to eq(
clusters: 1,
clusters_applications_prometheus: 1,
operations_dashboard_default_dashboard: 1,
operations_dashboard_users_with_projects_added: 1,
projects_prometheus_active: 1,
projects_with_error_tracking_enabled: 1,
projects_with_tracing_enabled: 1
)
end
end
expect(described_class.uncached_data[:usage_activity_by_stage][:manage]).to eq(
events: 1,
groups: 1,
ldap_keys: 1,
ldap_users: 1
)
end
end
context 'for package' do
it 'includes accurate usage_activity_by_stage data' do
create(:project, packages: [create(:package)] )
context 'for monitor' do
it 'includes accurate usage_activity_by_stage data' do
user = create(:user, dashboard: 'operations')
cluster = create(:cluster, user: user)
project = create(:project, creator: user)
expect(described_class.uncached_data[:usage_activity_by_stage][:package]).to eq(
projects_with_packages: 1
)
end
end
create(:clusters_applications_prometheus, :installed, cluster: cluster)
create(:users_ops_dashboard_project, user: user)
create(:prometheus_service, project: project)
create(:project_error_tracking_setting, project: project)
create(:project_tracing_setting, project: project)
context 'for plan' do
it 'includes accurate usage_activity_by_stage data' do
stub_licensed_features(board_assignee_lists: true, board_milestone_lists: true)
user = create(:user)
project = create(:project, creator: user)
issue = create(:issue, project: project, author: User.support_bot)
board = create(:board, project: project)
create(:user_list, board: board, user: user)
create(:milestone_list, board: board, milestone: create(:milestone, project: project), user: user)
create(:list, board: board, label: create(:label, project: project), user: user)
create(:note, project: project, noteable: issue, author: user)
create(:epic, author: user)
create(:todo, project: project, target: issue, author: user)
create(:jira_service, :jira_cloud_service, active: true, project: create(:project, :jira_dvcs_cloud, creator: user))
create(:jira_service, active: true, project: create(:project, :jira_dvcs_server, creator: user))
expect(described_class.uncached_data[:usage_activity_by_stage][:plan]).to eq(
assignee_lists: 1,
epics: 1,
issues: 1,
label_lists: 1,
milestone_lists: 1,
notes: 1,
projects: 1,
projects_jira_active: 1,
projects_jira_dvcs_cloud_active: 1,
projects_jira_dvcs_server_active: 1,
service_desk_enabled_projects: 1,
service_desk_issues: 1,
todos: 1
)
end
end
expect(described_class.uncached_data[:usage_activity_by_stage][:monitor]).to eq(
clusters: 1,
clusters_applications_prometheus: 1,
operations_dashboard_default_dashboard: 1,
operations_dashboard_users_with_projects_added: 1,
projects_prometheus_active: 1,
projects_with_error_tracking_enabled: 1,
projects_with_tracing_enabled: 1
)
end
end
context 'for release' do
it 'includes accurate usage_activity_by_stage data' do
user = create(:user)
create(:deployment, :failed, user: user)
create(:project, :mirror, mirror_trigger_builds: true)
create(:release, author: user)
create(:deployment, :success, user: user)
expect(described_class.uncached_data[:usage_activity_by_stage][:release]).to eq(
deployments: 1,
failed_deployments: 1,
projects_mirrored_with_pipelines_enabled: 1,
releases: 1,
successful_deployments: 1
)
end
end
context 'for package' do
it 'includes accurate usage_activity_by_stage data' do
create(:project, packages: [create(:package)] )
context 'for secure' do
it 'includes accurate usage_activity_by_stage data' do
create(:user, group_view: :security_dashboard)
expect(described_class.uncached_data[:usage_activity_by_stage][:package]).to eq(
projects_with_packages: 1
)
end
end
expect(described_class.uncached_data[:usage_activity_by_stage][:secure]).to eq(
user_preferences_group_overview_security_dashboard: 1
)
end
end
context 'for plan' do
it 'includes accurate usage_activity_by_stage data' do
stub_licensed_features(board_assignee_lists: true, board_milestone_lists: true)
user = create(:user)
project = create(:project, creator: user)
issue = create(:issue, project: project, author: User.support_bot)
board = create(:board, project: project)
create(:user_list, board: board, user: user)
create(:milestone_list, board: board, milestone: create(:milestone, project: project), user: user)
create(:list, board: board, label: create(:label, project: project), user: user)
create(:note, project: project, noteable: issue, author: user)
create(:epic, author: user)
create(:todo, project: project, target: issue, author: user)
create(:jira_service, :jira_cloud_service, active: true, project: create(:project, :jira_dvcs_cloud, creator: user))
create(:jira_service, active: true, project: create(:project, :jira_dvcs_server, creator: user))
expect(described_class.uncached_data[:usage_activity_by_stage][:plan]).to eq(
assignee_lists: 1,
epics: 1,
issues: 1,
label_lists: 1,
milestone_lists: 1,
notes: 1,
projects: 1,
projects_jira_active: 1,
projects_jira_dvcs_cloud_active: 1,
projects_jira_dvcs_server_active: 1,
service_desk_enabled_projects: 1,
service_desk_issues: 1,
todos: 1
)
end
end
context 'for release' do
it 'includes accurate usage_activity_by_stage data' do
user = create(:user)
create(:deployment, :failed, user: user)
create(:project, :mirror, mirror_trigger_builds: true)
create(:release, author: user)
create(:deployment, :success, user: user)
expect(described_class.uncached_data[:usage_activity_by_stage][:release]).to eq(
deployments: 1,
failed_deployments: 1,
projects_mirrored_with_pipelines_enabled: 1,
releases: 1,
successful_deployments: 1
)
end
end
context 'for secure' do
it 'includes accurate usage_activity_by_stage data' do
create(:user, group_view: :security_dashboard)
expect(described_class.uncached_data[:usage_activity_by_stage][:secure]).to eq(
user_preferences_group_overview_security_dashboard: 1
)
end
end
context 'for verify' do
it 'includes accurate usage_activity_by_stage data' do
user = create(:user)
create(:ci_build, user: user)
create(:ci_empty_pipeline, source: :external, user: user)
create(:ci_empty_pipeline, user: user)
create(:ci_pipeline, :auto_devops_source, user: user)
create(:ci_pipeline, :repository_source, user: user)
create(:ci_pipeline_schedule, owner: user)
create(:ci_trigger, owner: user)
create(:clusters_applications_runner, :installed)
create(:github_service)
context 'for verify' do
it 'includes accurate usage_activity_by_stage data' do
user = create(:user)
create(:ci_build, user: user)
create(:ci_empty_pipeline, source: :external, user: user)
create(:ci_empty_pipeline, user: user)
create(:ci_pipeline, :auto_devops_source, user: user)
create(:ci_pipeline, :repository_source, user: user)
create(:ci_pipeline_schedule, owner: user)
create(:ci_trigger, owner: user)
create(:clusters_applications_runner, :installed)
create(:github_service)
expect(described_class.uncached_data[:usage_activity_by_stage][:verify]).to eq(
ci_builds: 1,
ci_external_pipelines: 1,
ci_internal_pipelines: 1,
ci_pipeline_config_auto_devops: 1,
ci_pipeline_config_repository: 1,
ci_pipeline_schedules: 1,
ci_pipelines: 1,
ci_triggers: 1,
clusters_applications_runner: 1,
projects_reporting_ci_cd_back_to_github: 1
)
expect(described_class.uncached_data[:usage_activity_by_stage][:verify]).to eq(
ci_builds: 1,
ci_external_pipelines: 1,
ci_internal_pipelines: 1,
ci_pipeline_config_auto_devops: 1,
ci_pipeline_config_repository: 1,
ci_pipeline_schedules: 1,
ci_pipelines: 1,
ci_triggers: 1,
clusters_applications_runner: 1,
projects_reporting_ci_cd_back_to_github: 1
)
end
end
end
end
end
......
ee/spec/lib/gitlab/usage_data_spec.rb
View file @ 7d68d72f
......@@ -3,6 +3,10 @@
require 'spec_helper'
describe Gitlab::UsageData do
before do
allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false)
end
describe '#data' do
# using Array.new to create a different creator User for each of the projects
let_it_be(:projects) { Array.new(3) { create(:project, :repository, creator: create(:user, group_view: :security_dashboard)) } }
......
lib/gitlab/database/batch_count.rb 0 → 100644
View file @ 7d68d72f
# frozen_string_literal: true
# For large tables, PostgreSQL can take a long time to count rows due to MVCC.
# Implements a distinct and ordinary batch counter
# Needs indexes on the column below to calculate max, min and range queries
# For larger tables just set use higher batch_size with index optimization
# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705
# Examples:
# extend ::Gitlab::Database::BatchCount
# batch_count(User.active)
# batch_count(::Clusters::Cluster.aws_installed.enabled, :cluster_id)
# batch_distinct_count(::Project, :creator_id)
module Gitlab
module Database
module BatchCount
def batch_count(relation, column = nil, batch_size: nil)
BatchCounter.new(relation, column: column).count(batch_size: batch_size)
end
def batch_distinct_count(relation, column = nil, batch_size: nil)
BatchCounter.new(relation, column: column).count(mode: :distinct, batch_size: batch_size)
end
class << self
include BatchCount
end
end
class BatchCounter
FALLBACK = -1
MIN_REQUIRED_BATCH_SIZE = 2_000
MAX_ALLOWED_LOOPS = 10_000
SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep
# Each query should take <<500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705
DEFAULT_DISTINCT_BATCH_SIZE = 100_000
DEFAULT_BATCH_SIZE = 10_000
def initialize(relation, column: nil)
@relation = relation
@column = column || relation.primary_key
end
def unwanted_configuration?(finish, batch_size, start)
batch_size <= MIN_REQUIRED_BATCH_SIZE ||
(finish - start) / batch_size >= MAX_ALLOWED_LOOPS ||
start > finish
end
def count(batch_size: nil, mode: :itself)
raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open?
raise "The mode #{mode.inspect} is not supported" unless [:itself, :distinct].include?(mode)
# non-distinct have better performance
batch_size ||= mode == :distinct ? DEFAULT_BATCH_SIZE : DEFAULT_DISTINCT_BATCH_SIZE
start = @relation.minimum(@column) || 0
finish = @relation.maximum(@column) || 0
raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0
return FALLBACK if unwanted_configuration?(finish, batch_size, start)
counter = 0
batch_start = start
while batch_start <= finish
begin
counter += batch_fetch(batch_start, batch_start + batch_size, mode)
batch_start += batch_size
rescue ActiveRecord::QueryCanceled
# retry with a safe batch size & warmer cache
if batch_size >= 2 * MIN_REQUIRED_BATCH_SIZE
batch_size /= 2
else
return FALLBACK
end
end
sleep(SLEEP_TIME_IN_SECONDS)
end
counter
end
def batch_fetch(start, finish, mode)
# rubocop:disable GitlabSecurity/PublicSend
@relation.select(@column).public_send(mode).where(@column => start..(finish - 1)).count
end
end
end
end
lib/gitlab/usage_data.rb
View file @ 7d68d72f
......@@ -67,8 +67,8 @@ module Gitlab
clusters_disabled: count(::Clusters::Cluster.disabled),
project_clusters_disabled: count(::Clusters::Cluster.disabled.project_type),
group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type),
clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled),
clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled),
clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled, batch: false),
clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled, batch: false),
clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled),
clusters_applications_helm: count(::Clusters::Applications::Helm.available),
clusters_applications_ingress: count(::Clusters::Applications::Ingress.available),
......@@ -85,7 +85,7 @@ module Gitlab
issues: count(Issue),
issues_created_from_gitlab_error_tracking_ui: count(SentryIssue),
issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue),
issues_using_zoom_quick_actions: count(ZoomMeeting.select(:issue_id).distinct),
issues_using_zoom_quick_actions: count(ZoomMeeting.select(:issue_id).distinct, batch: false),
issues_with_embedded_grafana_charts_approx: ::Gitlab::GrafanaEmbedUsageData.issue_count,
incident_issues: count(::Issue.authored(::User.alert_bot)),
keys: count(Key),
......@@ -99,7 +99,7 @@ module Gitlab
projects_imported_from_github: count(Project.where(import_type: 'github')),
projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)),
projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)),
projects_with_alerts_service_enabled: count(AlertsService.active),
projects_with_alerts_service_enabled: count(AlertsService.active, batch: false),
protected_branches: count(ProtectedBranch),
releases: count(Release),
remote_mirrors: count(RemoteMirror),
......@@ -181,7 +181,7 @@ module Gitlab
# rubocop: disable CodeReuse/ActiveRecord
def services_usage
service_counts = count(Service.active.where(template: false).where.not(type: 'JiraService').group(:type), fallback: Hash.new(-1))
service_counts = count(Service.active.where(template: false).where.not(type: 'JiraService').group(:type), fallback: Hash.new(-1), batch: false)
results = Service.available_services_names.each_with_object({}) do |service_name, response|
response["projects_#{service_name}_active".to_sym] = service_counts["#{service_name}_service".camelize] || 0
......@@ -217,9 +217,9 @@ module Gitlab
results[:projects_jira_server_active] += counts[:server].count if counts[:server]
results[:projects_jira_cloud_active] += counts[:cloud].count if counts[:cloud]
if results[:projects_jira_active] == -1
results[:projects_jira_active] = count(services)
results[:projects_jira_active] = count(services, batch: false)
else
results[:projects_jira_active] += count(services)
results[:projects_jira_active] += count(services, batch: false)
end
end
......@@ -231,8 +231,22 @@ module Gitlab
{} # augmented in EE
end
def count(relation, fallback: -1)
relation.count
def count(relation, column = nil, fallback: -1, batch: true)
if batch && Feature.enabled?(:usage_ping_batch_counter)
Gitlab::Database::BatchCount.batch_count(relation, column)
else
relation.count
end
rescue ActiveRecord::StatementInvalid
fallback
end
def distinct_count(relation, column = nil, fallback: -1, batch: true)
if batch && Feature.enabled?(:usage_ping_batch_counter)
Gitlab::Database::BatchCount.batch_distinct_count(relation, column)
else
relation.distinct_count_by(column)
end
rescue ActiveRecord::StatementInvalid
fallback
end
......
spec/controllers/admin/application_settings_controller_spec.rb
View file @ 7d68d72f
......@@ -16,6 +16,7 @@ describe Admin::ApplicationSettingsController do
describe 'GET #usage_data with no access' do
before do
allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false)
sign_in(user)
end
......@@ -28,6 +29,7 @@ describe Admin::ApplicationSettingsController do
describe 'GET #usage_data' do
before do
allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false)
sign_in(admin)
end
......
spec/features/admin/admin_settings_spec.rb
View file @ 7d68d72f
......@@ -326,6 +326,8 @@ describe 'Admin updates settings', :clean_gitlab_redis_shared_state, :do_not_moc
end
it 'loads usage ping payload on click', :js do
allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false)
expect(page).to have_button 'Preview payload'
find('.js-usage-ping-payload-trigger').click
......
spec/lib/gitlab/database/batch_count_spec.rb 0 → 100644
View file @ 7d68d72f
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Database::BatchCount do
let(:model) { Issue }
let(:column) { :author_id }
let(:in_transaction) { false }
let(:user) { create(:user) }
let(:another_user) { create(:user) }
before do
create_list(:issue, 3, author: user )
create_list(:issue, 2, author: another_user )
allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(in_transaction)
end
describe '#batch_count' do
it 'counts table' do
expect(described_class.batch_count(model)).to eq(5)
end
it 'counts with :id field' do
expect(described_class.batch_count(model, :id)).to eq(5)
end
it 'counts with "id" field' do
expect(described_class.batch_count(model, 'id')).to eq(5)
end
it 'counts with table.id field' do
expect(described_class.batch_count(model, "#{model.table_name}.id")).to eq(5)
end
it 'counts table with batch_size 50K' do
expect(described_class.batch_count(model, batch_size: 50_000)).to eq(5)
end
it 'will not count table with batch_size 1K' do
fallback = ::Gitlab::Database::BatchCounter::FALLBACK
expect(described_class.batch_count(model, batch_size: fallback / 2)).to eq(fallback)
end
it 'counts with a small edge case batch_sizes than result' do
stub_const('Gitlab::Database::BatchCounter::MIN_REQUIRED_BATCH_SIZE', 0)
[1, 2, 4, 5, 6].each { |i| expect(described_class.batch_count(model, batch_size: i)).to eq(5) }
end
context 'in a transaction' do
let(:in_transaction) { true }
it 'cannot count' do
expect do
described_class.batch_count(model)
end.to raise_error 'BatchCount can not be run inside a transaction'
end
end
end
describe '#batch_distinct_count' do
it 'counts with :id field' do
expect(described_class.batch_distinct_count(model, :id)).to eq(5)
end
it 'counts with column field' do
expect(described_class.batch_distinct_count(model, column)).to eq(2)
end
it 'counts with "id" field' do
expect(described_class.batch_distinct_count(model, "#{column}")).to eq(2)
end
it 'counts with table.column field' do
expect(described_class.batch_distinct_count(model, "#{model.table_name}.#{column}")).to eq(2)
end
it 'counts with :column field with batch_size of 50K' do
expect(described_class.batch_distinct_count(model, column, batch_size: 50_000)).to eq(2)
end
it 'will not count table with batch_size 1K' do
fallback = ::Gitlab::Database::BatchCounter::FALLBACK
expect(described_class.batch_distinct_count(model, column, batch_size: fallback / 2)).to eq(fallback)
end
it 'counts with a small edge case batch_sizes than result' do
stub_const('Gitlab::Database::BatchCounter::MIN_REQUIRED_BATCH_SIZE', 0)
[1, 2, 4, 5, 6].each { |i| expect(described_class.batch_distinct_count(model, column, batch_size: i)).to eq(2) }
end
end
end
spec/lib/gitlab/usage_data_spec.rb
View file @ 7d68d72f
......@@ -6,381 +6,392 @@ describe Gitlab::UsageData do
let(:projects) { create_list(:project, 4) }
let!(:board) { create(:board, project: projects[0]) }
describe '#data' do
before do
create(:jira_service, project: projects[0])
create(:jira_service, :without_properties_callback, project: projects[1])
create(:jira_service, :jira_cloud_service, project: projects[2])
create(:jira_service, :without_properties_callback, project: projects[3],
properties: { url: 'https://mysite.atlassian.net' })
create(:prometheus_service, project: projects[1])
create(:service, project: projects[0], type: 'SlackSlashCommandsService', active: true)
create(:service, project: projects[1], type: 'SlackService', active: true)
create(:service, project: projects[2], type: 'SlackService', active: true)
create(:service, project: projects[2], type: 'MattermostService', active: false)
create(:service, project: projects[2], type: 'MattermostService', active: true, template: true)
create(:service, project: projects[2], type: 'CustomIssueTrackerService', active: true)
create(:project_error_tracking_setting, project: projects[0])
create(:project_error_tracking_setting, project: projects[1], enabled: false)
create(:alerts_service, project: projects[0])
create(:alerts_service, :inactive, project: projects[1])
create_list(:issue, 2, project: projects[0], author: User.alert_bot)
create_list(:issue, 2, project: projects[1], author: User.alert_bot)
create_list(:issue, 4, project: projects[0])
create(:zoom_meeting, project: projects[0], issue: projects[0].issues[0], issue_status: :added)
create_list(:zoom_meeting, 2, project: projects[0], issue: projects[0].issues[1], issue_status: :removed)
create(:zoom_meeting, project: projects[0], issue: projects[0].issues[2], issue_status: :added)
create_list(:zoom_meeting, 2, project: projects[0], issue: projects[0].issues[2], issue_status: :removed)
create(:sentry_issue, issue: projects[0].issues[0])
# Enabled clusters
gcp_cluster = create(:cluster_provider_gcp, :created).cluster
create(:cluster_provider_aws, :created)
create(:cluster_platform_kubernetes)
create(:cluster, :group)
# Disabled clusters
create(:cluster, :disabled)
create(:cluster, :group, :disabled)
create(:cluster, :group, :disabled)
# Applications
create(:clusters_applications_helm, :installed, cluster: gcp_cluster)
create(:clusters_applications_ingress, :installed, cluster: gcp_cluster)
create(:clusters_applications_cert_manager, :installed, cluster: gcp_cluster)
create(:clusters_applications_prometheus, :installed, cluster: gcp_cluster)
create(:clusters_applications_crossplane, :installed, cluster: gcp_cluster)
create(:clusters_applications_runner, :installed, cluster: gcp_cluster)
create(:clusters_applications_knative, :installed, cluster: gcp_cluster)
create(:clusters_applications_elastic_stack, :installed, cluster: gcp_cluster)
create(:clusters_applications_jupyter, :installed, cluster: gcp_cluster)
create(:grafana_integration, project: projects[0], enabled: true)
create(:grafana_integration, project: projects[1], enabled: true)
create(:grafana_integration, project: projects[2], enabled: false)
allow(Gitlab::GrafanaEmbedUsageData).to receive(:issue_count).and_return(2)
ProjectFeature.first.update_attribute('repository_access_level', 0)
end
subject { described_class.data }
it 'gathers usage data', :aggregate_failures do
expect(subject.keys).to include(*%i(
active_user_count
counts
recorded_at
edition
version
installation_type
uuid
hostname
mattermost_enabled
signup_enabled
ldap_enabled
gravatar_enabled
omniauth_enabled
reply_by_email_enabled
container_registry_enabled
dependency_proxy_enabled
gitlab_shared_runners_enabled
gitlab_pages
git
gitaly
database
avg_cycle_analytics
influxdb_metrics_enabled
prometheus_metrics_enabled
web_ide_clientside_preview_enabled
ingress_modsecurity_enabled
))
end
it 'gathers usage counts' do
smau_keys = %i(
snippet_create
snippet_update
snippet_comment
merge_request_comment
merge_request_create
commit_comment
wiki_pages_create
wiki_pages_update
wiki_pages_delete
web_ide_views
web_ide_commits
web_ide_merge_requests
web_ide_previews
navbar_searches
cycle_analytics_views
productivity_analytics_views
source_code_pushes
)
expected_keys = %i(
assignee_lists
boards
ci_builds
ci_internal_pipelines
ci_external_pipelines
ci_pipeline_config_auto_devops
ci_pipeline_config_repository
ci_runners
ci_triggers
ci_pipeline_schedules
auto_devops_enabled
auto_devops_disabled
deploy_keys
deployments
successful_deployments
failed_deployments
environments
clusters
clusters_enabled
project_clusters_enabled
group_clusters_enabled
clusters_disabled
project_clusters_disabled
group_clusters_disabled
clusters_platforms_eks
clusters_platforms_gke
clusters_platforms_user
clusters_applications_helm
clusters_applications_ingress
clusters_applications_cert_managers
clusters_applications_prometheus
clusters_applications_crossplane
clusters_applications_runner
clusters_applications_knative
clusters_applications_elastic_stack
clusters_applications_jupyter
in_review_folder
grafana_integrated_projects
groups
issues
issues_created_from_gitlab_error_tracking_ui
issues_with_associated_zoom_link
issues_using_zoom_quick_actions
issues_with_embedded_grafana_charts_approx
incident_issues
keys
label_lists
labels
lfs_objects
merge_requests
milestone_lists
milestones
notes
pool_repositories
projects
projects_imported_from_github
projects_asana_active
projects_jira_active
projects_jira_server_active
projects_jira_cloud_active
projects_slack_notifications_active
projects_slack_slash_active
projects_slack_active
projects_slack_slash_commands_active
projects_custom_issue_tracker_active
projects_mattermost_active
projects_prometheus_active
projects_with_repositories_enabled
projects_with_error_tracking_enabled
projects_with_alerts_service_enabled
pages_domains
protected_branches
releases
remote_mirrors
snippets
suggestions
todos
uploads
web_hooks
).push(*smau_keys)
count_data = subject[:counts]
expect(count_data[:boards]).to eq(1)
expect(count_data[:projects]).to eq(4)
expect(count_data.values_at(*smau_keys)).to all(be_an(Integer))
expect(count_data.keys).to include(*expected_keys)
expect(expected_keys - count_data.keys).to be_empty
end
it 'gathers projects data correctly', :aggregate_failures do
count_data = subject[:counts]
expect(count_data[:projects]).to eq(4)
expect(count_data[:projects_asana_active]).to eq(0)
expect(count_data[:projects_prometheus_active]).to eq(1)
expect(count_data[:projects_jira_active]).to eq(4)
expect(count_data[:projects_jira_server_active]).to eq(2)
expect(count_data[:projects_jira_cloud_active]).to eq(2)
expect(count_data[:projects_slack_notifications_active]).to eq(2)
expect(count_data[:projects_slack_slash_active]).to eq(1)
expect(count_data[:projects_slack_active]).to eq(2)
expect(count_data[:projects_slack_slash_commands_active]).to eq(1)
expect(count_data[:projects_custom_issue_tracker_active]).to eq(1)
expect(count_data[:projects_mattermost_active]).to eq(0)
expect(count_data[:projects_with_repositories_enabled]).to eq(3)
expect(count_data[:projects_with_error_tracking_enabled]).to eq(1)
expect(count_data[:projects_with_alerts_service_enabled]).to eq(1)
expect(count_data[:issues_created_from_gitlab_error_tracking_ui]).to eq(1)
expect(count_data[:issues_with_associated_zoom_link]).to eq(2)
expect(count_data[:issues_using_zoom_quick_actions]).to eq(3)
expect(count_data[:issues_with_embedded_grafana_charts_approx]).to eq(2)
expect(count_data[:incident_issues]).to eq(4)
expect(count_data[:clusters_enabled]).to eq(4)
expect(count_data[:project_clusters_enabled]).to eq(3)
expect(count_data[:group_clusters_enabled]).to eq(1)
expect(count_data[:clusters_disabled]).to eq(3)
expect(count_data[:project_clusters_disabled]).to eq(1)
expect(count_data[:group_clusters_disabled]).to eq(2)
expect(count_data[:group_clusters_enabled]).to eq(1)
expect(count_data[:clusters_platforms_eks]).to eq(1)
expect(count_data[:clusters_platforms_gke]).to eq(1)
expect(count_data[:clusters_platforms_user]).to eq(1)
expect(count_data[:clusters_applications_helm]).to eq(1)
expect(count_data[:clusters_applications_ingress]).to eq(1)
expect(count_data[:clusters_applications_cert_managers]).to eq(1)
expect(count_data[:clusters_applications_crossplane]).to eq(1)
expect(count_data[:clusters_applications_prometheus]).to eq(1)
expect(count_data[:clusters_applications_runner]).to eq(1)
expect(count_data[:clusters_applications_knative]).to eq(1)
expect(count_data[:clusters_applications_elastic_stack]).to eq(1)
expect(count_data[:grafana_integrated_projects]).to eq(2)
expect(count_data[:clusters_applications_jupyter]).to eq(1)
end
before do
allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false)
end
[true, false].each do |usage_ping_batch_counter_on|
describe "when the feature flag usage_ping_batch_counter is set to #{usage_ping_batch_counter_on}" do
before do
stub_feature_flags(usage_ping_batch_counter: usage_ping_batch_counter_on)
end
it 'works when queries time out' do
allow_any_instance_of(ActiveRecord::Relation)
.to receive(:count).and_raise(ActiveRecord::StatementInvalid.new(''))
describe '#data' do
before do
create(:jira_service, project: projects[0])
create(:jira_service, :without_properties_callback, project: projects[1])
create(:jira_service, :jira_cloud_service, project: projects[2])
create(:jira_service, :without_properties_callback, project: projects[3],
properties: { url: 'https://mysite.atlassian.net' })
create(:prometheus_service, project: projects[1])
create(:service, project: projects[0], type: 'SlackSlashCommandsService', active: true)
create(:service, project: projects[1], type: 'SlackService', active: true)
create(:service, project: projects[2], type: 'SlackService', active: true)
create(:service, project: projects[2], type: 'MattermostService', active: false)
create(:service, project: projects[2], type: 'MattermostService', active: true, template: true)
create(:service, project: projects[2], type: 'CustomIssueTrackerService', active: true)
create(:project_error_tracking_setting, project: projects[0])
create(:project_error_tracking_setting, project: projects[1], enabled: false)
create(:alerts_service, project: projects[0])
create(:alerts_service, :inactive, project: projects[1])
create_list(:issue, 2, project: projects[0], author: User.alert_bot)
create_list(:issue, 2, project: projects[1], author: User.alert_bot)
create_list(:issue, 4, project: projects[0])
create(:zoom_meeting, project: projects[0], issue: projects[0].issues[0], issue_status: :added)
create_list(:zoom_meeting, 2, project: projects[0], issue: projects[0].issues[1], issue_status: :removed)
create(:zoom_meeting, project: projects[0], issue: projects[0].issues[2], issue_status: :added)
create_list(:zoom_meeting, 2, project: projects[0], issue: projects[0].issues[2], issue_status: :removed)
create(:sentry_issue, issue: projects[0].issues[0])
# Enabled clusters
gcp_cluster = create(:cluster_provider_gcp, :created).cluster
create(:cluster_provider_aws, :created)
create(:cluster_platform_kubernetes)
create(:cluster, :group)
# Disabled clusters
create(:cluster, :disabled)
create(:cluster, :group, :disabled)
create(:cluster, :group, :disabled)
# Applications
create(:clusters_applications_helm, :installed, cluster: gcp_cluster)
create(:clusters_applications_ingress, :installed, cluster: gcp_cluster)
create(:clusters_applications_cert_manager, :installed, cluster: gcp_cluster)
create(:clusters_applications_prometheus, :installed, cluster: gcp_cluster)
create(:clusters_applications_crossplane, :installed, cluster: gcp_cluster)
create(:clusters_applications_runner, :installed, cluster: gcp_cluster)
create(:clusters_applications_knative, :installed, cluster: gcp_cluster)
create(:clusters_applications_elastic_stack, :installed, cluster: gcp_cluster)
create(:clusters_applications_jupyter, :installed, cluster: gcp_cluster)
create(:grafana_integration, project: projects[0], enabled: true)
create(:grafana_integration, project: projects[1], enabled: true)
create(:grafana_integration, project: projects[2], enabled: false)
allow(Gitlab::GrafanaEmbedUsageData).to receive(:issue_count).and_return(2)
ProjectFeature.first.update_attribute('repository_access_level', 0)
end
subject { described_class.data }
it 'gathers usage data', :aggregate_failures do
expect(subject.keys).to include(*%i(
active_user_count
counts
recorded_at
edition
version
installation_type
uuid
hostname
mattermost_enabled
signup_enabled
ldap_enabled
gravatar_enabled
omniauth_enabled
reply_by_email_enabled
container_registry_enabled
dependency_proxy_enabled
gitlab_shared_runners_enabled
gitlab_pages
git
gitaly
database
avg_cycle_analytics
influxdb_metrics_enabled
prometheus_metrics_enabled
web_ide_clientside_preview_enabled
ingress_modsecurity_enabled
))
end
it 'gathers usage counts' do
smau_keys = %i(
snippet_create
snippet_update
snippet_comment
merge_request_comment
merge_request_create
commit_comment
wiki_pages_create
wiki_pages_update
wiki_pages_delete
web_ide_views
web_ide_commits
web_ide_merge_requests
web_ide_previews
navbar_searches
cycle_analytics_views
productivity_analytics_views
source_code_pushes
)
expected_keys = %i(
assignee_lists
boards
ci_builds
ci_internal_pipelines
ci_external_pipelines
ci_pipeline_config_auto_devops
ci_pipeline_config_repository
ci_runners
ci_triggers
ci_pipeline_schedules
auto_devops_enabled
auto_devops_disabled
deploy_keys
deployments
successful_deployments
failed_deployments
environments
clusters
clusters_enabled
project_clusters_enabled
group_clusters_enabled
clusters_disabled
project_clusters_disabled
group_clusters_disabled
clusters_platforms_eks
clusters_platforms_gke
clusters_platforms_user
clusters_applications_helm
clusters_applications_ingress
clusters_applications_cert_managers
clusters_applications_prometheus
clusters_applications_crossplane
clusters_applications_runner
clusters_applications_knative
clusters_applications_elastic_stack
clusters_applications_jupyter
in_review_folder
grafana_integrated_projects
groups
issues
issues_created_from_gitlab_error_tracking_ui
issues_with_associated_zoom_link
issues_using_zoom_quick_actions
issues_with_embedded_grafana_charts_approx
incident_issues
keys
label_lists
labels
lfs_objects
merge_requests
milestone_lists
milestones
notes
pool_repositories
projects
projects_imported_from_github
projects_asana_active
projects_jira_active
projects_jira_server_active
projects_jira_cloud_active
projects_slack_notifications_active
projects_slack_slash_active
projects_slack_active
projects_slack_slash_commands_active
projects_custom_issue_tracker_active
projects_mattermost_active
projects_prometheus_active
projects_with_repositories_enabled
projects_with_error_tracking_enabled
projects_with_alerts_service_enabled
pages_domains
protected_branches
releases
remote_mirrors
snippets
suggestions
todos
uploads
web_hooks
).push(*smau_keys)
count_data = subject[:counts]
expect(count_data[:boards]).to eq(1)
expect(count_data[:projects]).to eq(4)
expect(count_data.values_at(*smau_keys)).to all(be_an(Integer))
expect(count_data.keys).to include(*expected_keys)
expect(expected_keys - count_data.keys).to be_empty
end
it 'gathers projects data correctly', :aggregate_failures do
count_data = subject[:counts]
expect(count_data[:projects]).to eq(4)
expect(count_data[:projects_asana_active]).to eq(0)
expect(count_data[:projects_prometheus_active]).to eq(1)
expect(count_data[:projects_jira_active]).to eq(4)
expect(count_data[:projects_jira_server_active]).to eq(2)
expect(count_data[:projects_jira_cloud_active]).to eq(2)
expect(count_data[:projects_slack_notifications_active]).to eq(2)
expect(count_data[:projects_slack_slash_active]).to eq(1)
expect(count_data[:projects_slack_active]).to eq(2)
expect(count_data[:projects_slack_slash_commands_active]).to eq(1)
expect(count_data[:projects_custom_issue_tracker_active]).to eq(1)
expect(count_data[:projects_mattermost_active]).to eq(0)
expect(count_data[:projects_with_repositories_enabled]).to eq(3)
expect(count_data[:projects_with_error_tracking_enabled]).to eq(1)
expect(count_data[:projects_with_alerts_service_enabled]).to eq(1)
expect(count_data[:issues_created_from_gitlab_error_tracking_ui]).to eq(1)
expect(count_data[:issues_with_associated_zoom_link]).to eq(2)
expect(count_data[:issues_using_zoom_quick_actions]).to eq(3)
expect(count_data[:issues_with_embedded_grafana_charts_approx]).to eq(2)
expect(count_data[:incident_issues]).to eq(4)
expect(count_data[:clusters_enabled]).to eq(4)
expect(count_data[:project_clusters_enabled]).to eq(3)
expect(count_data[:group_clusters_enabled]).to eq(1)
expect(count_data[:clusters_disabled]).to eq(3)
expect(count_data[:project_clusters_disabled]).to eq(1)
expect(count_data[:group_clusters_disabled]).to eq(2)
expect(count_data[:group_clusters_enabled]).to eq(1)
expect(count_data[:clusters_platforms_eks]).to eq(1)
expect(count_data[:clusters_platforms_gke]).to eq(1)
expect(count_data[:clusters_platforms_user]).to eq(1)
expect(count_data[:clusters_applications_helm]).to eq(1)
expect(count_data[:clusters_applications_ingress]).to eq(1)
expect(count_data[:clusters_applications_cert_managers]).to eq(1)
expect(count_data[:clusters_applications_crossplane]).to eq(1)
expect(count_data[:clusters_applications_prometheus]).to eq(1)
expect(count_data[:clusters_applications_runner]).to eq(1)
expect(count_data[:clusters_applications_knative]).to eq(1)
expect(count_data[:clusters_applications_elastic_stack]).to eq(1)
expect(count_data[:grafana_integrated_projects]).to eq(2)
expect(count_data[:clusters_applications_jupyter]).to eq(1)
end
it 'works when queries time out' do
allow_any_instance_of(ActiveRecord::Relation)
.to receive(:count).and_raise(ActiveRecord::StatementInvalid.new(''))
expect { subject }.not_to raise_error
end
end
expect { subject }.not_to raise_error
end
end
describe '#usage_data_counters' do
subject { described_class.usage_data_counters }
describe '#usage_data_counters' do
subject { described_class.usage_data_counters }
it { is_expected.to all(respond_to :totals) }
it { is_expected.to all(respond_to :totals) }
describe 'the results of calling #totals on all objects in the array' do
subject { described_class.usage_data_counters.map(&:totals) }
describe 'the results of calling #totals on all objects in the array' do
subject { described_class.usage_data_counters.map(&:totals) }
it { is_expected.to all(be_a Hash) }
it { is_expected.to all(have_attributes(keys: all(be_a Symbol), values: all(be_a Integer))) }
end
it { is_expected.to all(be_a Hash) }
it { is_expected.to all(have_attributes(keys: all(be_a Symbol), values: all(be_a Integer))) }
end
it 'does not have any conflicts' do
all_keys = subject.flat_map { |counter| counter.totals.keys }
it 'does not have any conflicts' do
all_keys = subject.flat_map { |counter| counter.totals.keys }
expect(all_keys.size).to eq all_keys.to_set.size
end
end
expect(all_keys.size).to eq all_keys.to_set.size
end
end
describe '#features_usage_data_ce' do
subject { described_class.features_usage_data_ce }
it 'gathers feature usage data', :aggregate_failures do
expect(subject[:mattermost_enabled]).to eq(Gitlab.config.mattermost.enabled)
expect(subject[:signup_enabled]).to eq(Gitlab::CurrentSettings.allow_signup?)
expect(subject[:ldap_enabled]).to eq(Gitlab.config.ldap.enabled)
expect(subject[:gravatar_enabled]).to eq(Gitlab::CurrentSettings.gravatar_enabled?)
expect(subject[:omniauth_enabled]).to eq(Gitlab::Auth.omniauth_enabled?)
expect(subject[:reply_by_email_enabled]).to eq(Gitlab::IncomingEmail.enabled?)
expect(subject[:container_registry_enabled]).to eq(Gitlab.config.registry.enabled)
expect(subject[:dependency_proxy_enabled]).to eq(Gitlab.config.dependency_proxy.enabled)
expect(subject[:gitlab_shared_runners_enabled]).to eq(Gitlab.config.gitlab_ci.shared_runners_enabled)
expect(subject[:web_ide_clientside_preview_enabled]).to eq(Gitlab::CurrentSettings.web_ide_clientside_preview_enabled?)
end
end
describe '#features_usage_data_ce' do
subject { described_class.features_usage_data_ce }
it 'gathers feature usage data', :aggregate_failures do
expect(subject[:mattermost_enabled]).to eq(Gitlab.config.mattermost.enabled)
expect(subject[:signup_enabled]).to eq(Gitlab::CurrentSettings.allow_signup?)
expect(subject[:ldap_enabled]).to eq(Gitlab.config.ldap.enabled)
expect(subject[:gravatar_enabled]).to eq(Gitlab::CurrentSettings.gravatar_enabled?)
expect(subject[:omniauth_enabled]).to eq(Gitlab::Auth.omniauth_enabled?)
expect(subject[:reply_by_email_enabled]).to eq(Gitlab::IncomingEmail.enabled?)
expect(subject[:container_registry_enabled]).to eq(Gitlab.config.registry.enabled)
expect(subject[:dependency_proxy_enabled]).to eq(Gitlab.config.dependency_proxy.enabled)
expect(subject[:gitlab_shared_runners_enabled]).to eq(Gitlab.config.gitlab_ci.shared_runners_enabled)
expect(subject[:web_ide_clientside_preview_enabled]).to eq(Gitlab::CurrentSettings.web_ide_clientside_preview_enabled?)
end
end
describe '#components_usage_data' do
subject { described_class.components_usage_data }
it 'gathers components usage data', :aggregate_failures do
expect(subject[:gitlab_pages][:enabled]).to eq(Gitlab.config.pages.enabled)
expect(subject[:gitlab_pages][:version]).to eq(Gitlab::Pages::VERSION)
expect(subject[:git][:version]).to eq(Gitlab::Git.version)
expect(subject[:database][:adapter]).to eq(Gitlab::Database.adapter_name)
expect(subject[:database][:version]).to eq(Gitlab::Database.version)
expect(subject[:gitaly][:version]).to be_present
expect(subject[:gitaly][:servers]).to be >= 1
expect(subject[:gitaly][:filesystems]).to be_an(Array)
expect(subject[:gitaly][:filesystems].first).to be_a(String)
end
end
describe '#components_usage_data' do
subject { described_class.components_usage_data }
it 'gathers components usage data', :aggregate_failures do
expect(subject[:gitlab_pages][:enabled]).to eq(Gitlab.config.pages.enabled)
expect(subject[:gitlab_pages][:version]).to eq(Gitlab::Pages::VERSION)
expect(subject[:git][:version]).to eq(Gitlab::Git.version)
expect(subject[:database][:adapter]).to eq(Gitlab::Database.adapter_name)
expect(subject[:database][:version]).to eq(Gitlab::Database.version)
expect(subject[:gitaly][:version]).to be_present
expect(subject[:gitaly][:servers]).to be >= 1
expect(subject[:gitaly][:filesystems]).to be_an(Array)
expect(subject[:gitaly][:filesystems].first).to be_a(String)
end
end
describe '#ingress_modsecurity_usage' do
subject { described_class.ingress_modsecurity_usage }
it 'gathers variable data' do
allow_any_instance_of(
::Clusters::Applications::IngressModsecurityUsageService
).to receive(:execute).and_return(
{
ingress_modsecurity_blocking: 1,
ingress_modsecurity_disabled: 2
}
)
expect(subject[:ingress_modsecurity_blocking]).to eq(1)
expect(subject[:ingress_modsecurity_disabled]).to eq(2)
end
end
describe '#ingress_modsecurity_usage' do
subject { described_class.ingress_modsecurity_usage }
it 'gathers variable data' do
allow_any_instance_of(
::Clusters::Applications::IngressModsecurityUsageService
).to receive(:execute).and_return(
{
ingress_modsecurity_blocking: 1,
ingress_modsecurity_disabled: 2
}
)
expect(subject[:ingress_modsecurity_blocking]).to eq(1)
expect(subject[:ingress_modsecurity_disabled]).to eq(2)
end
end
describe '#license_usage_data' do
subject { described_class.license_usage_data }
describe '#license_usage_data' do
subject { described_class.license_usage_data }
it 'gathers license data', :aggregate_failures do
expect(subject[:uuid]).to eq(Gitlab::CurrentSettings.uuid)
expect(subject[:version]).to eq(Gitlab::VERSION)
expect(subject[:installation_type]).to eq('gitlab-development-kit')
expect(subject[:active_user_count]).to eq(User.active.count)
expect(subject[:recorded_at]).to be_a(Time)
end
end
it 'gathers license data', :aggregate_failures do
expect(subject[:uuid]).to eq(Gitlab::CurrentSettings.uuid)
expect(subject[:version]).to eq(Gitlab::VERSION)
expect(subject[:installation_type]).to eq('gitlab-development-kit')
expect(subject[:active_user_count]).to eq(User.active.count)
expect(subject[:recorded_at]).to be_a(Time)
end
end
describe '#count' do
let(:relation) { double(:relation) }
describe '#count' do
let(:relation) { double(:relation) }
it 'returns the count when counting succeeds' do
allow(relation).to receive(:count).and_return(1)
it 'returns the count when counting succeeds' do
allow(relation).to receive(:count).and_return(1)
expect(described_class.count(relation, batch: false)).to eq(1)
end
expect(described_class.count(relation)).to eq(1)
end
it 'returns the fallback value when counting fails' do
allow(relation).to receive(:count).and_raise(ActiveRecord::StatementInvalid.new(''))
it 'returns the fallback value when counting fails' do
allow(relation).to receive(:count).and_raise(ActiveRecord::StatementInvalid.new(''))
expect(described_class.count(relation, fallback: 15, batch: false)).to eq(15)
end
end
expect(described_class.count(relation, fallback: 15)).to eq(15)
end
end
describe '#approximate_counts' do
it 'gets approximate counts for selected models', :aggregate_failures do
create(:label)
describe '#approximate_counts' do
it 'gets approximate counts for selected models', :aggregate_failures do
create(:label)
expect(Gitlab::Database::Count).to receive(:approximate_counts)
.with(described_class::APPROXIMATE_COUNT_MODELS).once.and_call_original
expect(Gitlab::Database::Count).to receive(:approximate_counts)
.with(described_class::APPROXIMATE_COUNT_MODELS).once.and_call_original
counts = described_class.approximate_counts.values
counts = described_class.approximate_counts.values
expect(counts.count).to eq(described_class::APPROXIMATE_COUNT_MODELS.count)
expect(counts.any? { |count| count < 0 }).to be_falsey
end
expect(counts.count).to eq(described_class::APPROXIMATE_COUNT_MODELS.count)
expect(counts.any? { |count| count < 0 }).to be_falsey
end
it 'returns default values if counts can not be retrieved', :aggregate_failures do
described_class::APPROXIMATE_COUNT_MODELS.map do |model|
model.name.underscore.pluralize.to_sym
end
it 'returns default values if counts can not be retrieved', :aggregate_failures do
described_class::APPROXIMATE_COUNT_MODELS.map do |model|
model.name.underscore.pluralize.to_sym
expect(Gitlab::Database::Count).to receive(:approximate_counts).and_return({})
expect(described_class.approximate_counts.values.uniq).to eq([-1])
end
end
expect(Gitlab::Database::Count).to receive(:approximate_counts).and_return({})
expect(described_class.approximate_counts.values.uniq).to eq([-1])
end
end
end
spec/services/submit_usage_ping_service_spec.rb
View file @ 7d68d72f
......@@ -76,6 +76,7 @@ describe SubmitUsagePingService do
context 'when usage ping is enabled' do
before do
allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false)
stub_application_setting(usage_ping_enabled: true)
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7