Merge branch 'mwaw/333680-add-collected_data_categories-to-usage-ping-payload' into 'master'

[RUN ALL RSPEC] [RUN AS-IF-FOSS] Add collected_data_categories to usage ping payload

See merge request gitlab-org/gitlab!65336

app/services/service_ping/permit_data_categories_service.rb

+# frozen_string_literal: true
+
+module ServicePing
+  class PermitDataCategoriesService
+    STANDARD_CATEGORY = 'Standard'
+    SUBSCRIPTION_CATEGORY = 'Subscription'
+    OPERATIONAL_CATEGORY = 'Operational'
+    OPTIONAL_CATEGORY = 'Optional'
+    CATEGORIES = [
+      STANDARD_CATEGORY,
+      SUBSCRIPTION_CATEGORY,
+      OPERATIONAL_CATEGORY,
+      OPTIONAL_CATEGORY
+    ].to_set.freeze
+
+    def execute
+      return [] unless product_intelligence_enabled?
+
+      CATEGORIES
+    end
+
+    private
+
+    def product_intelligence_enabled?
+      pings_enabled? && !User.single_user&.requires_usage_stats_consent?
+    end
+
+    def pings_enabled?
+      ::Gitlab::CurrentSettings.usage_ping_enabled?
+    end
+  end
+end
+
+ServicePing::PermitDataCategoriesService.prepend_mod_with('ServicePing::PermitDataCategoriesService')

config/metrics/objects_schemas/collected_data_categories_schema.json

+{
+  "type": "array",
+  "items": {
+    "type": ["string", "null"],
+    "enum": ["Standard", "Subscription", "Operational", "Optional"]
+  }
+}

config/metrics/settings/20210702140138_collected_data_categories.yml

+---
+key_path: settings.collected_data_categories
+name: collected_data_categories
+description: List of collected data categories corresponding to instance settings
+product_section: growth
+product_stage: growth
+product_group: group::product intelligence
+product_category: collection
+value_type: object
+status: implemented
+milestone: "14.1"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65336
+time_frame: none
+data_source: system
+data_category: Standard
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+value_json_schema: 'config/metrics/objects_schemas/collected_data_categories_schema.json'

doc/development/usage_ping/dictionary.md

@@ -18316,6 +18316,22 @@ Status: `data_available`
 
 Tiers: `free`, `premium`, `ultimate`
 
+### `settings.collected_data_categories`
+
+List of collected data categories corresponding to instance settings
+
+[Object JSON schema](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/objects_schemas/collected_data_categories_schema.json)
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/settings/20210702140138_collected_data_categories.yml)
+
+Group: `group::product intelligence`
+
+Data Category: `Standard`
+
+Status: `implemented`
+
+Tiers: `free`, `premium`, `ultimate`
+
 ### `settings.gitaly_apdex`
 
 Gitaly application performance

ee/app/services/ee/service_ping/build_payload_service.rb

@@ -5,11 +5,6 @@ module EE
     module BuildPayloadService
       extend ::Gitlab::Utils::Override
 
-      STANDARD_CATEGORY = 'Standard'
-      SUBSCRIPTION_CATEGORY = 'Subscription'
-      OPTIONAL_CATEGORY = 'Optional'
-      OPERATIONAL_CATEGORY = 'Operational'
-
       override :execute
       def execute
         return super unless ::License.current.present?
@@ -35,19 +30,14 @@ module EE
       end
 
       def permitted_categories
-        @permitted_categories ||= collect_permitted_categories
-      end
-
-      def collect_permitted_categories
-        categories = [STANDARD_CATEGORY, SUBSCRIPTION_CATEGORY]
-        categories << OPTIONAL_CATEGORY if ::Gitlab::CurrentSettings.usage_ping_enabled?
-        categories << OPERATIONAL_CATEGORY if ::License.current.usage_ping?
-        categories
+        @permitted_categories ||= ::ServicePing::PermitDataCategoriesService.new.execute
       end
 
       def metric_category(key, parent_keys)
         key_path = parent_keys.dup.append(key).join('.')
-        metric_definitions[key_path]&.attributes&.fetch(:data_category, OPTIONAL_CATEGORY)
+        metric_definitions[key_path]
+          &.attributes
+          &.fetch(:data_category, ::ServicePing::PermitDataCategoriesService::OPTIONAL_CATEGORY)
       end
 
       def metric_definitions

ee/app/services/ee/service_ping/permit_data_categories_service.rb

+# frozen_string_literal: true
+
+module EE
+  module ServicePing
+    module PermitDataCategoriesService
+      extend ::Gitlab::Utils::Override
+
+      STANDARD_CATEGORY = ::ServicePing::PermitDataCategoriesService::STANDARD_CATEGORY
+      SUBSCRIPTION_CATEGORY = ::ServicePing::PermitDataCategoriesService::SUBSCRIPTION_CATEGORY
+      OPTIONAL_CATEGORY = ::ServicePing::PermitDataCategoriesService::OPTIONAL_CATEGORY
+      OPERATIONAL_CATEGORY = ::ServicePing::PermitDataCategoriesService::OPERATIONAL_CATEGORY
+
+      override :execute
+      def execute
+        return super unless ::License.current.present?
+        return [] unless product_intelligence_enabled?
+
+        optional_enabled = ::Gitlab::CurrentSettings.usage_ping_enabled?
+        customer_service_enabled = ::License.current.usage_ping?
+
+        [STANDARD_CATEGORY, SUBSCRIPTION_CATEGORY].tap do |categories|
+          categories << OPERATIONAL_CATEGORY << OPTIONAL_CATEGORY if optional_enabled
+          categories << OPERATIONAL_CATEGORY if customer_service_enabled
+        end.to_set
+      end
+
+      private
+
+      override :pings_enabled?
+      def pings_enabled?
+        ::License.current&.usage_ping? || super
+      end
+    end
+  end
+end

ee/spec/config/metrics/every_metric_definition_spec.rb

@@ -29,6 +29,7 @@ RSpec.describe 'Every metric definition' do
       geo_node_usage
       mock_ci
       mock_monitoring
+      projects_with_enabled_alert_integrations_histogram
       user_auth_by_provider
       user_dast_scans
       user_sast_scans
@@ -37,13 +38,14 @@ RSpec.describe 'Every metric definition' do
       user_secret_detection_scans
       user_coverage_fuzzing_scans
       user_api_fuzzing_scans
+      topology
     ).freeze
   end
 
   let(:metric_files_key_paths) do
     Gitlab::Usage::MetricDefinition
       .definitions
-      .reject { |k, v| v.status == 'removed' || v.value_type == 'object' || v.key_path =~ Regexp.union(ignored_metric_files_key_patterns)}
+      .reject { |k, v| v.status == 'removed' || v.key_path =~ Regexp.union(ignored_metric_files_key_patterns)}
       .keys
       .sort
   end

ee/spec/lib/ee/gitlab/usage_data_non_sql_metrics_spec.rb

@@ -15,7 +15,7 @@ RSpec.describe Gitlab::UsageDataNonSqlMetrics do
         described_class.uncached_data
       end
 
-      expect(recorder.count).to eq(54)
+      expect(recorder.count).to eq(55)
     end
   end
 end

ee/spec/services/ee/service_ping/build_payload_service_spec.rb

@@ -42,11 +42,7 @@ RSpec.describe ServicePing::BuildPayloadService do
           end
 
           it_behaves_like 'service ping payload with all expected metrics' do
-            let(:expected_metrics) { standard_metrics + subscription_metrics + optional_metrics }
-          end
-
-          it_behaves_like 'service ping payload without restricted metrics' do
-            let(:restricted_metrics) { operational_metrics }
+            let(:expected_metrics) { standard_metrics + subscription_metrics + optional_metrics + operational_metrics }
           end
         end
       end

ee/spec/services/ee/service_ping/permit_data_categories_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ServicePing::PermitDataCategoriesService do
+  describe '#execute' do
+    subject(:permitted_categories) { described_class.new.execute }
+
+    context 'with out current license', :without_license do
+      context 'when usage ping setting is set to true' do
+        before do
+          stub_config_setting(usage_ping_enabled: true)
+        end
+
+        it 'returns all categories' do
+          expect(permitted_categories).to match_array(%w[Standard Subscription Operational Optional])
+        end
+      end
+
+      context 'when usage ping setting is set to false' do
+        before do
+          stub_config_setting(usage_ping_enabled: false)
+        end
+
+        it 'returns no categories' do
+          expect(permitted_categories).to match_array([])
+        end
+      end
+    end
+
+    context 'with current license' do
+      context 'when usage ping setting is set to true' do
+        before do
+          stub_config_setting(usage_ping_enabled: true)
+        end
+
+        context 'and license has usage_ping_required_metrics_enabled set to true' do
+          before do
+            # License.current.usage_ping? == true
+            create_current_license(usage_ping_required_metrics_enabled: true)
+          end
+
+          it 'returns all categories' do
+            expect(permitted_categories).to match_array(%w[Standard Subscription Operational Optional])
+          end
+
+          context 'when User.single_user&.requires_usage_stats_consent? is required' do
+            before do
+              allow(User).to receive(:single_user).and_return(double(:user, requires_usage_stats_consent?: true))
+            end
+
+            it 'returns no categories' do
+              expect(permitted_categories).to match_array([])
+            end
+          end
+        end
+
+        context 'and license has usage_ping_required_metrics_enabled set to false' do
+          before do
+            # License.current.usage_ping? == true
+            create_current_license(usage_ping_required_metrics_enabled: false)
+          end
+
+          it 'returns all categories' do
+            expect(permitted_categories).to match_array(%w[Standard Subscription Operational Optional])
+          end
+        end
+      end
+
+      context 'when usage ping setting is set to false' do
+        before do
+          stub_config_setting(usage_ping_enabled: false)
+        end
+
+        context 'and license has usage_ping_required_metrics_enabled set to true' do
+          before do
+            # License.current.usage_ping? == true
+            create_current_license(usage_ping_required_metrics_enabled: true)
+          end
+
+          it 'returns all categories' do
+            expect(permitted_categories).to match_array(%w[Standard Subscription Operational])
+          end
+        end
+
+        context 'and license has usage_ping_required_metrics_enabled set to false' do
+          before do
+            # License.current.usage_ping? == true
+            create_current_license(usage_ping_required_metrics_enabled: false)
+          end
+
+          it 'returns all categories' do
+            expect(permitted_categories).to match_array(%w[])
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module Instrumentations
+        class CollectedDataCategoriesMetric < GenericMetric
+          def value
+            ::ServicePing::PermitDataCategoriesService.new.execute
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/usage_data.rb

@@ -256,7 +256,8 @@ module Gitlab
           settings: {
             ldap_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth::Ldap::Config.encrypted_secrets.active? },
             operating_system: alt_usage_data(fallback: nil) { operating_system },
-            gitaly_apdex: alt_usage_data { gitaly_apdex }
+            gitaly_apdex: alt_usage_data { gitaly_apdex },
+            collected_data_categories: alt_usage_data(fallback: []) { Gitlab::Usage::Metrics::Instrumentations::CollectedDataCategoriesMetric.new(time_frame: 'none').value }
           }
         }
       end

spec/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CollectedDataCategoriesMetric do
+  it_behaves_like 'a correct instrumented metric value', {} do
+    let(:expected_value) { %w[Standard Subscription Operational Optional] }
+
+    before do
+      allow_next_instance_of(ServicePing::PermitDataCategoriesService) do |instance|
+        expect(instance).to receive(:execute).and_return(expected_value)
+      end
+    end
+  end
+end

spec/lib/gitlab/usage_data_spec.rb

@@ -1078,6 +1078,16 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
       it 'gathers gitaly apdex', :aggregate_failures do
         expect(subject[:settings][:gitaly_apdex]).to be_within(0.001).of(0.95)
       end
+
+      it 'reports collected data categories' do
+        expected_value = %w[Standard Subscription Operational Optional]
+
+        allow_next_instance_of(ServicePing::PermitDataCategoriesService) do |instance|
+          expect(instance).to receive(:execute).and_return(expected_value)
+        end
+
+        expect(subject[:settings][:collected_data_categories]).to eq(expected_value)
+      end
     end
   end
 

spec/services/service_ping/permit_data_categories_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ServicePing::PermitDataCategoriesService do
+  describe '#execute', :without_license do
+    subject(:permitted_categories) { described_class.new.execute }
+
+    context 'when usage ping setting is set to true' do
+      before do
+        allow(User).to receive(:single_user).and_return(double(:user, requires_usage_stats_consent?: false))
+        stub_config_setting(usage_ping_enabled: true)
+      end
+
+      it 'returns all categories' do
+        expect(permitted_categories).to match_array(%w[Standard Subscription Operational Optional])
+      end
+    end
+
+    context 'when usage ping setting is set to false' do
+      before do
+        allow(User).to receive(:single_user).and_return(double(:user, requires_usage_stats_consent?: false))
+        stub_config_setting(usage_ping_enabled: false)
+      end
+
+      it 'returns no categories' do
+        expect(permitted_categories).to match_array([])
+      end
+    end
+
+    context 'when User.single_user&.requires_usage_stats_consent? is required' do
+      before do
+        allow(User).to receive(:single_user).and_return(double(:user, requires_usage_stats_consent?: true))
+        stub_config_setting(usage_ping_enabled: true)
+      end
+
+      it 'returns no categories' do
+        expect(permitted_categories).to match_array([])
+      end
+    end
+  end
+end