Merge branch '338010-latest-successful-by-build' into 'master'

Remove `[analyzer]_pipeline` metrics join on ci_builds"

See merge request gitlab-org/gitlab!71210

doc/development/service_ping/implement.md

@@ -1031,9 +1031,9 @@ Example metrics persistence:
 class UsageData
   def count_secure_pipelines(time_period)
     ...
-    relation = ::Security::Scan.latest_successful_by_build.by_scan_types(scan_type).where(security_scans: time_period)
+    relation = ::Security::Scan.by_scan_types(scan_type).where(time_period)
 
-    pipelines_with_secure_jobs['dependency_scanning_pipeline'] = estimate_batch_distinct_count(relation, :commit_id, batch_size: 1000, start: start_id, finish: finish_id) do |result|
+    pipelines_with_secure_jobs['dependency_scanning_pipeline'] = estimate_batch_distinct_count(relation, :pipeline_id, batch_size: 1000, start: start_id, finish: finish_id) do |result|
       ::Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll
         .save_aggregated_metrics(metric_name: 'dependency_scanning_pipeline', recorded_at_timestamp: recorded_at, time_period: time_period, data: result)
     end

ee/app/models/security/scan.rb

@@ -41,6 +41,8 @@ module Security
     end
 
     scope :latest, -> { where(latest: true) }
+    # We are going to deprecate the following scope soon as this requires join between ci and non-ci table
+    # which will not be possible after database decomposition (https://gitlab.com/groups/gitlab-org/-/epics/6373)
     scope :latest_successful_by_build, -> { joins(:build).where(ci_builds: { retried: [nil, false], status: 'success' }) }
     scope :without_errors, -> { where("jsonb_array_length(COALESCE(info->'errors', '[]'::jsonb)) = 0") }
 

ee/lib/ee/gitlab/usage_data.rb

@@ -416,9 +416,8 @@ module EE
 
           ::Security::Scan.scan_types.each do |name, scan_type|
             relation = ::Security::Scan
-                         .latest_successful_by_build
                          .by_scan_types(scan_type)
-                         .where(security_scans: time_period)
+                         .where(time_period)
 
             metric_name = "#{name}_pipeline"
             aggregated_metrics_params = {
@@ -429,7 +428,7 @@ module EE
 
             pipelines_with_secure_jobs[metric_name.to_sym] =
               if start_id && finish_id
-                estimate_batch_distinct_count(relation, :commit_id, batch_size: 1000, start: start_id, finish: finish_id) do |result|
+                estimate_batch_distinct_count(relation, :pipeline_id, batch_size: 1000, start: start_id, finish: finish_id) do |result|
                   ::Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll
                     .save_aggregated_metrics(**aggregated_metrics_params.merge({ data: result }))
                 end

ee/spec/lib/ee/gitlab/usage_data_spec.rb

@@ -617,8 +617,8 @@ RSpec.describe Gitlab::UsageData do
         user_license_management_jobs: 1,
         user_sast_jobs: 1,
         user_secret_detection_jobs: 1,
-        sast_pipeline: be_within(error_rate).percent_of(0),
-        dependency_scanning_pipeline: be_within(error_rate).percent_of(1),
+        sast_pipeline: be_within(error_rate).percent_of(1),
+        dependency_scanning_pipeline: be_within(error_rate).percent_of(2),
         container_scanning_pipeline: be_within(error_rate).percent_of(1),
         dast_pipeline: be_within(error_rate).percent_of(0),
         secret_detection_pipeline: be_within(error_rate).percent_of(1),