Commit 86775529 authored by Aleksei Lipniagov's avatar Aleksei Lipniagov

Merge branch 'remove-cluster-applications-usage-data' into 'master'

Remove cluster applications usage data

See merge request gitlab-org/gitlab!70341
parents 2fe79453 341f9735
......@@ -14,6 +14,8 @@ module Clusters
validates :cluster, presence: true
validates :enabled, inclusion: { in: [true, false] }
scope :enabled, -> { where(enabled: true) }
def available?
enabled
end
......
......@@ -21,6 +21,8 @@ module Clusters
default_value_for(:alert_manager_token) { SecureRandom.hex }
scope :enabled, -> { where(enabled: true) }
after_destroy do
run_after_commit do
deactivate_project_integrations
......
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: 28d
data_source: database
distribution:
......@@ -20,3 +20,4 @@ tier:
name: 'count_distinct_user_id_from_clusters_applications_cert_managers'
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: 28d
data_source: database
distribution:
......@@ -20,3 +20,4 @@ tier:
name: 'count_distinct_user_id_from_clusters_applications_helm'
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: 28d
data_source: database
distribution:
......@@ -20,3 +20,4 @@ tier:
name: 'count_distinct_user_id_from_clusters_applications_ingress'
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: 28d
data_source: database
distribution:
......@@ -20,3 +20,4 @@ tier:
name: 'count_distinct_user_id_from_clusters_applications_knative'
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: monitor
product_group: group::monitor
product_category: metrics
value_type: number
status: active
status: removed
time_frame: 28d
data_source: database
distribution:
......@@ -19,3 +19,4 @@ tier:
- ultimate
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: verify
product_group: group::runner
product_category: runner
value_type: number
status: active
status: removed
time_frame: 28d
data_source: database
distribution:
......@@ -19,3 +19,4 @@ tier:
- ultimate
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
---
data_category: optional
key_path: usage_activity_by_stage_monthly.monitor.clusters_integrations_prometheus
description: Users creating clusters with Prometheus integration enabled in last 28 days.
product_section: ops
product_stage: monitor
product_group: group::monitor
product_category: metrics
value_type: number
status: active
time_frame: 28d
data_source: database
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
performance_indicator_type: []
milestone: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -19,3 +19,4 @@ tier:
- ultimate
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -19,3 +19,4 @@ tier:
- ultimate
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -19,3 +19,4 @@ tier:
- ultimate
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -19,3 +19,4 @@ tier:
- ultimate
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -19,3 +19,4 @@ tier:
- ultimate
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -19,3 +19,4 @@ tier:
- ultimate
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -19,3 +19,4 @@ tier:
- ultimate
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -19,3 +19,4 @@ tier:
- ultimate
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -19,3 +19,4 @@ tier:
- ultimate
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -19,3 +19,4 @@ tier:
- ultimate
performance_indicator_type: []
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -18,3 +18,4 @@ tier:
- premium
- ultimate
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -18,3 +18,4 @@ tier:
- premium
- ultimate
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -18,3 +18,4 @@ tier:
- premium
- ultimate
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -18,3 +18,4 @@ tier:
- premium
- ultimate
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: monitor
product_group: group::monitor
product_category: metrics
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -18,3 +18,4 @@ tier:
- premium
- ultimate
milestone: "<13.9"
milestone_removed: "14.4"
......@@ -7,7 +7,7 @@ product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
status: removed
time_frame: all
data_source: database
distribution:
......@@ -18,3 +18,4 @@ tier:
- premium
- ultimate
milestone: "<13.9"
milestone_removed: "14.4"
---
data_category: optional
key_path: counts.clusters_integrations_prometheus
description: Total clusters with Clusters::Integrations::Prometheus enabled
product_section: ops
product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
time_frame: all
data_source: database
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
performance_indicator_type: []
milestone: "14.4"
---
data_category: optional
key_path: counts.clusters_integrations_elastic_stack
description: Total clusters with Clusters::Integrations::ElasticStack enabled
product_section: ops
product_stage: configure
product_group: group::configure
product_category: kubernetes_management
value_type: number
status: active
time_frame: all
data_source: database
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
performance_indicator_type: []
milestone: "14.4"
---
data_category: optional
key_path: usage_activity_by_stage.monitor.clusters_integrations_prometheus
description: Users creating clusters with Prometheus integration enabled in last 28 days.
product_section: ops
product_stage: monitor
product_group: group::monitor
product_category: metrics
value_type: number
status: active
time_frame: all
data_source: database
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
performance_indicator_type: []
milestone: "14.4"
# frozen_string_literal: true
class AddIndexOnClustersIntegrationPrometheusEnabled < Gitlab::Database::Migration[1.0]
disable_ddl_transaction!
INDEX_NAME = 'index_clusters_integration_prometheus_enabled'
def up
add_concurrent_index(:clusters_integration_prometheus, [:enabled, :created_at, :cluster_id], name: INDEX_NAME)
end
def down
remove_concurrent_index_by_name(:clusters_integration_prometheus, INDEX_NAME)
end
end
# frozen_string_literal: true
class AddIndexOnClustersIntegrationElasticStackEnabled < Gitlab::Database::Migration[1.0]
disable_ddl_transaction!
INDEX_NAME = 'index_clusters_integration_elasticstack_enabled'
def up
add_concurrent_index(:clusters_integration_elasticstack, [:enabled, :created_at, :cluster_id], name: INDEX_NAME)
end
def down
remove_concurrent_index_by_name(:clusters_integration_elasticstack, INDEX_NAME)
end
end
ffb4e066420e1492550255e5866df6dc9f83d3d6cec9033284c4c3194d53b827
\ No newline at end of file
fc8f86f794d77902fd86acaec6046b65821ff685f841d28158dc05fb55773c16
\ No newline at end of file
......@@ -24767,6 +24767,10 @@ CREATE UNIQUE INDEX index_clusters_applications_runners_on_cluster_id ON cluster
CREATE INDEX index_clusters_applications_runners_on_runner_id ON clusters_applications_runners USING btree (runner_id);
CREATE INDEX index_clusters_integration_elasticstack_enabled ON clusters_integration_elasticstack USING btree (enabled, created_at, cluster_id);
CREATE INDEX index_clusters_integration_prometheus_enabled ON clusters_integration_prometheus USING btree (enabled, created_at, cluster_id);
CREATE INDEX index_clusters_kubernetes_namespaces_on_cluster_id ON clusters_kubernetes_namespaces USING btree (cluster_id);
CREATE INDEX index_clusters_kubernetes_namespaces_on_cluster_project_id ON clusters_kubernetes_namespaces USING btree (cluster_project_id);
......@@ -123,17 +123,9 @@ module Gitlab
clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled),
clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled),
clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled),
clusters_applications_helm: count(::Clusters::Applications::Helm.available),
clusters_applications_ingress: count(::Clusters::Applications::Ingress.available),
clusters_applications_cert_managers: count(::Clusters::Applications::CertManager.available),
clusters_applications_crossplane: count(::Clusters::Applications::Crossplane.available),
clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.available),
clusters_applications_runner: count(::Clusters::Applications::Runner.available),
clusters_applications_knative: count(::Clusters::Applications::Knative.available),
clusters_applications_elastic_stack: count(::Clusters::Applications::ElasticStack.available),
clusters_applications_jupyter: count(::Clusters::Applications::Jupyter.available),
clusters_applications_cilium: count(::Clusters::Applications::Cilium.available),
clusters_management_project: count(::Clusters::Cluster.with_management_project),
clusters_integrations_elastic_stack: count(::Clusters::Integrations::ElasticStack.enabled),
clusters_integrations_prometheus: count(::Clusters::Integrations::Prometheus.enabled),
kubernetes_agents: count(::Clusters::Agent),
kubernetes_agents_with_token: distinct_count(::Clusters::AgentToken, :agent_id),
in_review_folder: count(::Environment.in_review_folder),
......@@ -523,10 +515,6 @@ module Gitlab
# rubocop: disable UsageData/LargeTable
def usage_activity_by_stage_configure(time_period)
{
clusters_applications_cert_managers: cluster_applications_user_distinct_count(::Clusters::Applications::CertManager, time_period),
clusters_applications_helm: cluster_applications_user_distinct_count(::Clusters::Applications::Helm, time_period),
clusters_applications_ingress: cluster_applications_user_distinct_count(::Clusters::Applications::Ingress, time_period),
clusters_applications_knative: cluster_applications_user_distinct_count(::Clusters::Applications::Knative, time_period),
clusters_management_project: clusters_user_distinct_count(::Clusters::Cluster.with_management_project, time_period),
clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled, time_period),
clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled, time_period),
......@@ -621,7 +609,7 @@ module Gitlab
{
clusters: distinct_count(::Clusters::Cluster.where(time_period), :user_id),
clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period),
clusters_integrations_prometheus: cluster_integrations_user_distinct_count(::Clusters::Integrations::Prometheus, time_period),
operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period),
start: minimum_id(User),
finish: maximum_id(User)),
......@@ -687,8 +675,7 @@ module Gitlab
ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
ci_pipeline_schedules: distinct_count(::Ci::PipelineSchedule.where(time_period), :owner_id),
ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id),
clusters_applications_runner: cluster_applications_user_distinct_count(::Clusters::Applications::Runner, time_period)
ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id)
}
end
# rubocop: enable CodeReuse/ActiveRecord
......@@ -873,8 +860,8 @@ module Gitlab
end
# rubocop: disable CodeReuse/ActiveRecord
def cluster_applications_user_distinct_count(applications, time_period)
distinct_count(applications.where(time_period).available.joins(:cluster), 'clusters.user_id')
def cluster_integrations_user_distinct_count(integrations, time_period)
distinct_count(integrations.where(time_period).enabled.joins(:cluster), 'clusters.user_id')
end
def clusters_user_distinct_count(clusters, time_period)
......
......@@ -88,17 +88,9 @@ FactoryBot.define do
create(:cluster, :group, :disabled)
create(:cluster, :instance, :disabled)
# Applications
create(:clusters_applications_helm, :installed, cluster: gcp_cluster)
create(:clusters_applications_ingress, :installed, cluster: gcp_cluster)
create(:clusters_applications_cert_manager, :installed, cluster: gcp_cluster)
create(:clusters_applications_prometheus, :installed, cluster: gcp_cluster)
create(:clusters_applications_crossplane, :installed, cluster: gcp_cluster)
create(:clusters_applications_runner, :installed, cluster: gcp_cluster)
create(:clusters_applications_knative, :installed, cluster: gcp_cluster)
create(:clusters_applications_elastic_stack, :installed, cluster: gcp_cluster)
create(:clusters_applications_jupyter, :installed, cluster: gcp_cluster)
create(:clusters_applications_cilium, :installed, cluster: gcp_cluster)
# Cluster Integrations
create(:clusters_integrations_prometheus, cluster: gcp_cluster)
create(:clusters_integrations_elastic_stack, cluster: gcp_cluster)
create(:grafana_integration, project: projects[0], enabled: true)
create(:grafana_integration, project: projects[1], enabled: true)
......
......@@ -101,11 +101,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
it 'includes accurate usage_activity_by_stage data' do
for_defined_days_back do
user = create(:user)
cluster = create(:cluster, user: user)
create(:clusters_applications_cert_manager, :installed, cluster: cluster)
create(:clusters_applications_helm, :installed, cluster: cluster)
create(:clusters_applications_ingress, :installed, cluster: cluster)
create(:clusters_applications_knative, :installed, cluster: cluster)
create(:cluster, user: user)
create(:cluster, :disabled, user: user)
create(:cluster_provider_gcp, :created)
create(:cluster_provider_aws, :created)
......@@ -118,10 +114,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
end
expect(described_class.usage_activity_by_stage_configure({})).to include(
clusters_applications_cert_managers: 2,
clusters_applications_helm: 2,
clusters_applications_ingress: 2,
clusters_applications_knative: 2,
clusters_management_project: 2,
clusters_disabled: 4,
clusters_enabled: 12,
......@@ -136,10 +128,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
project_clusters_enabled: 10
)
expect(described_class.usage_activity_by_stage_configure(described_class.monthly_time_range_db_params)).to include(
clusters_applications_cert_managers: 1,
clusters_applications_helm: 1,
clusters_applications_ingress: 1,
clusters_applications_knative: 1,
clusters_management_project: 1,
clusters_disabled: 2,
clusters_enabled: 6,
......@@ -392,7 +380,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
user = create(:user, dashboard: 'operations')
cluster = create(:cluster, user: user)
project = create(:project, creator: user)
create(:clusters_applications_prometheus, :installed, cluster: cluster)
create(:clusters_integrations_prometheus, cluster: cluster)
create(:project_tracing_setting)
create(:project_error_tracking_setting)
create(:incident)
......@@ -402,7 +390,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
expect(described_class.usage_activity_by_stage_monitor({})).to include(
clusters: 2,
clusters_applications_prometheus: 2,
clusters_integrations_prometheus: 2,
operations_dashboard_default_dashboard: 2,
projects_with_tracing_enabled: 2,
projects_with_error_tracking_enabled: 2,
......@@ -414,7 +402,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
data_28_days = described_class.usage_activity_by_stage_monitor(described_class.monthly_time_range_db_params)
expect(data_28_days).to include(
clusters: 1,
clusters_applications_prometheus: 1,
clusters_integrations_prometheus: 1,
operations_dashboard_default_dashboard: 1,
projects_with_tracing_enabled: 1,
projects_with_error_tracking_enabled: 1,
......@@ -502,7 +490,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
create(:ci_pipeline, :repository_source, user: user)
create(:ci_pipeline_schedule, owner: user)
create(:ci_trigger, owner: user)
create(:clusters_applications_runner, :installed)
end
expect(described_class.usage_activity_by_stage_verify({})).to include(
......@@ -513,8 +500,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
ci_pipeline_config_repository: 2,
ci_pipeline_schedules: 2,
ci_pipelines: 2,
ci_triggers: 2,
clusters_applications_runner: 2
ci_triggers: 2
)
expect(described_class.usage_activity_by_stage_verify(described_class.monthly_time_range_db_params)).to include(
ci_builds: 1,
......@@ -524,8 +510,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
ci_pipeline_config_repository: 1,
ci_pipeline_schedules: 1,
ci_pipelines: 1,
ci_triggers: 1,
clusters_applications_runner: 1
ci_triggers: 1
)
end
end
......@@ -607,17 +592,9 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
expect(count_data[:clusters_platforms_eks]).to eq(1)
expect(count_data[:clusters_platforms_gke]).to eq(1)
expect(count_data[:clusters_platforms_user]).to eq(1)
expect(count_data[:clusters_applications_helm]).to eq(1)
expect(count_data[:clusters_applications_ingress]).to eq(1)
expect(count_data[:clusters_applications_cert_managers]).to eq(1)
expect(count_data[:clusters_applications_crossplane]).to eq(1)
expect(count_data[:clusters_applications_prometheus]).to eq(1)
expect(count_data[:clusters_applications_runner]).to eq(1)
expect(count_data[:clusters_applications_knative]).to eq(1)
expect(count_data[:clusters_applications_elastic_stack]).to eq(1)
expect(count_data[:clusters_integrations_elastic_stack]).to eq(1)
expect(count_data[:clusters_integrations_prometheus]).to eq(1)
expect(count_data[:grafana_integrated_projects]).to eq(2)
expect(count_data[:clusters_applications_jupyter]).to eq(1)
expect(count_data[:clusters_applications_cilium]).to eq(1)
expect(count_data[:clusters_management_project]).to eq(1)
expect(count_data[:kubernetes_agents]).to eq(2)
expect(count_data[:kubernetes_agents_with_token]).to eq(1)
......
......@@ -54,15 +54,8 @@ module UsageDataHelpers
clusters_platforms_eks
clusters_platforms_gke
clusters_platforms_user
clusters_applications_helm
clusters_applications_ingress
clusters_applications_cert_managers
clusters_applications_prometheus
clusters_applications_crossplane
clusters_applications_runner
clusters_applications_knative
clusters_applications_elastic_stack
clusters_applications_jupyter
clusters_integrations_elastic_stack
clusters_integrations_prometheus
clusters_management_project
in_review_folder
grafana_integrated_projects
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment