Merge branch '217362-restructure-usage-ping-add-usage-activity-to-ce' into 'master'

Move Usage activity by stage for Configure to Core

Closes #217362

See merge request gitlab-org/gitlab!33672

changelogs/unreleased/217362-restructure-usage-ping-add-usage-activity-to-ce.yml

+---
+title: Move Usage activity by stage for Configure to Core
+merge_request: 33672
+author:
+type: changed

ee/spec/lib/ee/gitlab/usage_data_spec.rb

@@ -302,64 +302,18 @@ RSpec.describe Gitlab::UsageData do
         it 'includes accurate usage_activity_by_stage data' do
           for_defined_days_back do
             user = create(:user)
-            cluster = create(:cluster, user: user)
             project = create(:project, creator: user)
-            create(:clusters_applications_cert_manager, :installed, cluster: cluster)
-            create(:clusters_applications_helm, :installed, cluster: cluster)
-            create(:clusters_applications_ingress, :installed, cluster: cluster)
-            create(:clusters_applications_knative, :installed, cluster: cluster)
-            create(:cluster, :disabled, user: user)
-            create(:cluster_provider_gcp, :created)
-            create(:cluster_provider_aws, :created)
-            create(:cluster_platform_kubernetes)
-            create(:cluster, :group, :disabled, user: user)
-            create(:cluster, :group, user: user)
-            create(:cluster, :instance, :disabled, :production_environment)
-            create(:cluster, :instance, :production_environment)
-            create(:cluster, :management_project)
             create(:slack_service, project: project)
             create(:slack_slash_commands_service, project: project)
             create(:prometheus_service, project: project)
           end
 
-          expect(described_class.uncached_data[:usage_activity_by_stage][:configure]).to eq(
-            clusters_applications_cert_managers: 2,
-            clusters_applications_helm: 2,
-            clusters_applications_ingress: 2,
-            clusters_applications_knative: 2,
-            clusters_management_project: 2,
-            clusters_disabled: 4,
-            clusters_enabled: 12,
-            clusters_platforms_gke: 2,
-            clusters_platforms_eks: 2,
-            clusters_platforms_user: 2,
-            instance_clusters_disabled: 2,
-            instance_clusters_enabled: 2,
-            group_clusters_disabled: 2,
-            group_clusters_enabled: 2,
-            project_clusters_disabled: 2,
-            project_clusters_enabled: 10,
+          expect(described_class.uncached_data[:usage_activity_by_stage][:configure]).to include(
             projects_slack_notifications_active: 2,
             projects_slack_slash_active: 2,
             projects_with_prometheus_alerts: 2
           )
-          expect(described_class.uncached_data[:usage_activity_by_stage_monthly][:configure]).to eq(
-            clusters_applications_cert_managers: 1,
-            clusters_applications_helm: 1,
-            clusters_applications_ingress: 1,
-            clusters_applications_knative: 1,
-            clusters_management_project: 1,
-            clusters_disabled: 2,
-            clusters_enabled: 6,
-            clusters_platforms_gke: 1,
-            clusters_platforms_eks: 1,
-            clusters_platforms_user: 1,
-            instance_clusters_disabled: 1,
-            instance_clusters_enabled: 1,
-            group_clusters_disabled: 1,
-            group_clusters_enabled: 1,
-            project_clusters_disabled: 1,
-            project_clusters_enabled: 5,
+          expect(described_class.uncached_data[:usage_activity_by_stage_monthly][:configure]).to include(
             projects_slack_notifications_active: 1,
             projects_slack_slash_active: 1,
             projects_with_prometheus_alerts: 1

lib/gitlab/usage_data.rb

@@ -37,6 +37,8 @@ module Gitlab
             .merge(cycle_analytics_usage_data)
             .merge(object_store_usage_data)
             .merge(topology_usage_data)
+            .merge(usage_activity_by_stage)
+            .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, default_time_period))
         end
       end
 
@@ -427,6 +429,88 @@ module Gitlab
         { created_at: 28.days.ago..Time.current }
       end
 
+      # Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv
+      def usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {})
+        {
+          key => {
+            configure: usage_activity_by_stage_configure(time_period),
+            create: usage_activity_by_stage_create(time_period),
+            manage: usage_activity_by_stage_manage(time_period),
+            monitor: usage_activity_by_stage_monitor(time_period),
+            package: usage_activity_by_stage_package(time_period),
+            plan: usage_activity_by_stage_plan(time_period),
+            release: usage_activity_by_stage_release(time_period),
+            secure: usage_activity_by_stage_secure(time_period),
+            verify: usage_activity_by_stage_verify(time_period)
+          }
+        }
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def usage_activity_by_stage_configure(time_period)
+        {
+          clusters_applications_cert_managers: cluster_applications_user_distinct_count(::Clusters::Applications::CertManager, time_period),
+          clusters_applications_helm: cluster_applications_user_distinct_count(::Clusters::Applications::Helm, time_period),
+          clusters_applications_ingress: cluster_applications_user_distinct_count(::Clusters::Applications::Ingress, time_period),
+          clusters_applications_knative: cluster_applications_user_distinct_count(::Clusters::Applications::Knative, time_period),
+          clusters_management_project: clusters_user_distinct_count(::Clusters::Cluster.with_management_project, time_period),
+          clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled, time_period),
+          clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled, time_period),
+          clusters_platforms_gke: clusters_user_distinct_count(::Clusters::Cluster.gcp_installed.enabled, time_period),
+          clusters_platforms_eks: clusters_user_distinct_count(::Clusters::Cluster.aws_installed.enabled, time_period),
+          clusters_platforms_user: clusters_user_distinct_count(::Clusters::Cluster.user_provided.enabled, time_period),
+          instance_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.instance_type, time_period),
+          instance_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.instance_type, time_period),
+          group_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.group_type, time_period),
+          group_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.group_type, time_period),
+          project_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.project_type, time_period),
+          project_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.project_type, time_period)
+        }
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      # Omitted because no user, creator or author associated: `lfs_objects`, `pool_repositories`, `web_hooks`
+      def usage_activity_by_stage_create(time_period)
+        {}
+      end
+
+      # Omitted because no user, creator or author associated: `campaigns_imported_from_github`, `ldap_group_links`
+      def usage_activity_by_stage_manage(time_period)
+        {}
+      end
+
+      def usage_activity_by_stage_monitor(time_period)
+        {}
+      end
+
+      def usage_activity_by_stage_package(time_period)
+        {}
+      end
+
+      # Omitted because no user, creator or author associated: `boards`, `labels`, `milestones`, `uploads`
+      # Omitted because too expensive: `epics_deepest_relationship_level`
+      # Omitted because of encrypted properties: `projects_jira_cloud_active`, `projects_jira_server_active`
+      def usage_activity_by_stage_plan(time_period)
+        {}
+      end
+
+      # Omitted because no user, creator or author associated: `environments`, `feature_flags`, `in_review_folder`, `pages_domains`
+      def usage_activity_by_stage_release(time_period)
+        {}
+      end
+
+      # Omitted because no user, creator or author associated: `ci_runners`
+      def usage_activity_by_stage_verify(time_period)
+        {}
+      end
+
+      # Currently too complicated and to get reliable counts for these stats:
+      # container_scanning_jobs, dast_jobs, dependency_scanning_jobs, license_management_jobs, sast_jobs, secret_detection_jobs
+      # Once https://gitlab.com/gitlab-org/gitlab/merge_requests/17568 is merged, this might be doable
+      def usage_activity_by_stage_secure(time_period)
+        {}
+      end
+
       private
 
       def total_alert_issues
@@ -455,6 +539,16 @@ module Gitlab
         clear_memoization(:user_minimum_id)
         clear_memoization(:user_maximum_id)
       end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def cluster_applications_user_distinct_count(applications, time_period)
+        distinct_count(applications.where(time_period).available.joins(:cluster), 'clusters.user_id')
+      end
+
+      def clusters_user_distinct_count(clusters, time_period)
+        distinct_count(clusters.where(time_period), :user_id)
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end

spec/lib/gitlab/usage_data_spec.rb

@@ -10,7 +10,73 @@ describe Gitlab::UsageData, :aggregate_failures do
     stub_object_store_settings
   end
 
-  describe '#uncached_data' do
+  describe '.uncached_data' do
+    describe '.usage_activity_by_stage' do
+      it 'includes usage_activity_by_stage data' do
+        expect(described_class.uncached_data).to include(:usage_activity_by_stage)
+        expect(described_class.uncached_data).to include(:usage_activity_by_stage_monthly)
+      end
+
+      context 'for configure' do
+        it 'includes accurate usage_activity_by_stage data' do
+          for_defined_days_back do
+            user = create(:user)
+            cluster = create(:cluster, user: user)
+            create(:clusters_applications_cert_manager, :installed, cluster: cluster)
+            create(:clusters_applications_helm, :installed, cluster: cluster)
+            create(:clusters_applications_ingress, :installed, cluster: cluster)
+            create(:clusters_applications_knative, :installed, cluster: cluster)
+            create(:cluster, :disabled, user: user)
+            create(:cluster_provider_gcp, :created)
+            create(:cluster_provider_aws, :created)
+            create(:cluster_platform_kubernetes)
+            create(:cluster, :group, :disabled, user: user)
+            create(:cluster, :group, user: user)
+            create(:cluster, :instance, :disabled, :production_environment)
+            create(:cluster, :instance, :production_environment)
+            create(:cluster, :management_project)
+          end
+
+          expect(described_class.uncached_data[:usage_activity_by_stage][:configure]).to include(
+            clusters_applications_cert_managers: 2,
+            clusters_applications_helm: 2,
+            clusters_applications_ingress: 2,
+            clusters_applications_knative: 2,
+            clusters_management_project: 2,
+            clusters_disabled: 4,
+            clusters_enabled: 12,
+            clusters_platforms_gke: 2,
+            clusters_platforms_eks: 2,
+            clusters_platforms_user: 2,
+            instance_clusters_disabled: 2,
+            instance_clusters_enabled: 2,
+            group_clusters_disabled: 2,
+            group_clusters_enabled: 2,
+            project_clusters_disabled: 2,
+            project_clusters_enabled: 10
+          )
+          expect(described_class.uncached_data[:usage_activity_by_stage_monthly][:configure]).to include(
+            clusters_applications_cert_managers: 1,
+            clusters_applications_helm: 1,
+            clusters_applications_ingress: 1,
+            clusters_applications_knative: 1,
+            clusters_management_project: 1,
+            clusters_disabled: 2,
+            clusters_enabled: 6,
+            clusters_platforms_gke: 1,
+            clusters_platforms_eks: 1,
+            clusters_platforms_user: 1,
+            instance_clusters_disabled: 1,
+            instance_clusters_enabled: 1,
+            group_clusters_disabled: 1,
+            group_clusters_enabled: 1,
+            project_clusters_disabled: 1,
+            project_clusters_enabled: 5
+          )
+        end
+      end
+    end
+
     it 'ensures recorded_at is set before any other usage data calculation' do
       %i(alt_usage_data redis_usage_data distinct_count count).each do |method|
         expect(described_class).not_to receive(method)
@@ -598,4 +664,12 @@ describe Gitlab::UsageData, :aggregate_failures do
       )
     end
   end
+
+  def for_defined_days_back(days: [29, 2])
+    days.each do |n|
+      Timecop.travel(n.days.ago) do
+        yield
+      end
+    end
+  end
 end