Merge branch 'mwaw/quarantine_broken_ci_build_jobs_metrics' into 'master'

Quarantine broken security_products_usage metrics

See merge request gitlab-org/gitlab!71722

config/metrics/counts_all/20210216182112_sast_jobs.yml

@@ -7,7 +7,8 @@ product_stage: secure
 product_group: group::static analysis
 product_category: static_application_security_testing
 value_type: number
-status: active
+status: broken
+repair_issue_url: tbd
 time_frame: all
 data_source: database
 distribution:

config/metrics/counts_all/20210216182114_secret_detection_jobs.yml

@@ -7,7 +7,8 @@ product_stage: secure
 product_group: group::static analysis
 product_category: secret_detection
 value_type: number
-status: active
+status: broken
+repair_issue_url: tbd
 time_frame: all
 data_source: database
 distribution:

ee/config/feature_flags/ops/quarantine_security_products_usage_metrics.yml

+---
+name: quarantine_security_products_usage_metrics
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71722
+rollout_issue_url:
+milestone: '14.4'
+type: ops
+group: group::product intelligence
+default_enabled: false

ee/config/metrics/counts_all/20210204124854_license_management_jobs.yml

@@ -6,7 +6,8 @@ product_stage: secure
 product_group: group::composition analysis
 product_category: license_compliance
 value_type: number
-status: active
+status: broken
+repair_issue_url: tbd
 time_frame: all
 data_source: database
 data_category: operational

ee/config/metrics/counts_all/20210216175213_dependency_scanning_jobs.yml

@@ -7,7 +7,8 @@ product_stage: secure
 product_group: group::composition analysis
 product_category: dependency_scanning
 value_type: number
-status: active
+status: broken
+repair_issue_url: tbd
 time_frame: all
 data_source: database
 distribution:

ee/config/metrics/counts_all/20210216175458_container_scanning_jobs.yml

@@ -7,7 +7,8 @@ product_stage: protect
 product_group: group::container security
 product_category: container_scanning
 value_type: number
-status: active
+status: broken
+repair_issue_url: tbd
 time_frame: all
 data_source: database
 distribution:

ee/config/metrics/counts_all/20210216175612_dast_jobs.yml

@@ -6,7 +6,8 @@ product_stage: secure
 product_group: group::dynamic analysis
 product_category: dynamic_application_security_testing
 value_type: number
-status: active
+status: broken
+repair_issue_url: tbd
 time_frame: all
 data_source: database
 data_category: operational

ee/config/metrics/counts_all/20210216180344_api_fuzzing_jobs.yml

@@ -7,7 +7,8 @@ product_stage: secure
 product_group: group::dynamic analysis
 product_category: fuzz-testing
 value_type: number
-status: active
+status: broken
+repair_issue_url: tbd
 time_frame: all
 data_source: database
 distribution:

ee/config/metrics/counts_all/20210216180346_api_fuzzing_dnd_jobs.yml

@@ -7,7 +7,8 @@ product_stage: secure
 product_group: group::dynamic analysis
 product_category: fuzz-testing
 value_type: number
-status: active
+status: broken
+repair_issue_url: tbd
 time_frame: all
 data_source: database
 distribution:

ee/config/metrics/counts_all/20210216183146_coverage_fuzzing_jobs.yml

@@ -7,7 +7,8 @@ product_stage: secure
 product_group: group::dynamic analysis
 product_category: fuzz-testing
 value_type: number
-status: active
+status: broken
+repair_issue_url: tbd
 time_frame: all
 data_source: database
 distribution:

ee/lib/ee/gitlab/usage_data.rb

@@ -143,7 +143,11 @@ module EE
 
         def security_products_usage
           results = SECURE_PRODUCT_TYPES.each_with_object({}) do |(secure_type, attribs), response|
-            response[attribs[:name]] = count(::Ci::Build.where(name: secure_type)) # rubocop:disable CodeReuse/ActiveRecord
+            response[attribs[:name]] = if ::Feature.enabled?(:quarantine_security_products_usage_metrics, type: :ops, default_enabled: :yaml)
+                                         ::Gitlab::Database::BatchCounter::FALLBACK
+                                       else
+                                         count(::Ci::Build.where(name: secure_type)) # rubocop:disable CodeReuse/ActiveRecord
+                                       end
           end
 
           # handle license rename https://gitlab.com/gitlab-org/gitlab/issues/8911

ee/spec/lib/ee/gitlab/usage_data_spec.rb

@@ -128,17 +128,50 @@ RSpec.describe Gitlab::UsageData do
       expect(count_data[:epic_issues]).to eq(2)
     end
 
-    it 'gathers security products usage data' do
-      expect(count_data[:container_scanning_jobs]).to eq(1)
-      expect(count_data[:dast_jobs]).to eq(1)
-      expect(count_data[:dependency_scanning_jobs]).to eq(1)
-      expect(count_data[:license_management_jobs]).to eq(2)
-      expect(count_data[:sast_jobs]).to eq(1)
-      expect(count_data[:secret_detection_jobs]).to eq(1)
-      expect(count_data[:coverage_fuzzing_jobs]).to eq(1)
-      expect(count_data[:api_fuzzing_jobs]).to eq(1)
-      expect(count_data[:api_fuzzing_dnd_jobs]).to eq(1)
-      expect(count_data[:dast_on_demand_pipelines]).to eq(1)
+    context 'with quarantine_security_products_usage_metrics turned on' do
+      it 'returns fallback value for quarantined security products usage data' do
+        expect(count_data[:container_scanning_jobs]).to eq(-1)
+        expect(count_data[:dast_jobs]).to eq(-1)
+        expect(count_data[:dependency_scanning_jobs]).to eq(-1)
+        expect(count_data[:license_management_jobs]).to eq(-1)
+        expect(count_data[:sast_jobs]).to eq(-1)
+        expect(count_data[:secret_detection_jobs]).to eq(-1)
+        expect(count_data[:coverage_fuzzing_jobs]).to eq(-1)
+        expect(count_data[:api_fuzzing_jobs]).to eq(-1)
+        expect(count_data[:api_fuzzing_dnd_jobs]).to eq(-1)
+        expect(count_data[:dast_on_demand_pipelines]).to eq(1)
+      end
+
+      it 'avoid counting metrics' do
+        expect(described_class).not_to receive(:count)
+
+        described_class.security_products_usage
+      end
+    end
+
+    context 'with quarantine_security_products_usage_metrics turned off' do
+      before do
+        stub_feature_flags(quarantine_security_products_usage_metrics: false)
+      end
+
+      it 'counts metrics data' do
+        expect(described_class).to receive(:count).exactly(10).times
+
+        described_class.security_products_usage
+      end
+
+      it 'gathers security products usage data' do
+        expect(count_data[:container_scanning_jobs]).to eq(1)
+        expect(count_data[:dast_jobs]).to eq(1)
+        expect(count_data[:dependency_scanning_jobs]).to eq(1)
+        expect(count_data[:license_management_jobs]).to eq(2)
+        expect(count_data[:sast_jobs]).to eq(1)
+        expect(count_data[:secret_detection_jobs]).to eq(1)
+        expect(count_data[:coverage_fuzzing_jobs]).to eq(1)
+        expect(count_data[:api_fuzzing_jobs]).to eq(1)
+        expect(count_data[:api_fuzzing_dnd_jobs]).to eq(1)
+        expect(count_data[:dast_on_demand_pipelines]).to eq(1)
+      end
     end
 
     it 'gathers group overview preferences usage data', :aggregate_failures do