Merge branch 'mwaw/add_index_to_security_scans_on_created_at_and_build_id' into 'master'

Improve usage ping metrics calculations with new index added to security scans table

See merge request gitlab-org/gitlab!48671

changelogs/unreleased/mwaw-add_index_to_security_scans_on_created_at_and_build_id.yml

+---
+title: Add database index to improve performance of usage ping metrics calculation
+merge_request: 48671
+author:
+type: performance

db/migrate/20201127104228_add_index_to_security_scans_on_created_at_and_id.rb

+# frozen_string_literal: true
+
+class AddIndexToSecurityScansOnCreatedAtAndId < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'index_security_scans_on_date_created_at_and_id'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :security_scans, "date(timezone('UTC', created_at)), id", name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name(:security_scans, INDEX_NAME)
+  end
+end

db/schema_migrations/20201127104228

+a66f13cf62740c590aa195dd510f1e9f4a36e12d29fd0c824fbd0ace03717706
\ No newline at end of file

db/structure.sql

@@ -22950,6 +22950,8 @@ CREATE INDEX index_security_findings_on_severity ON security_findings USING btre
 
 CREATE UNIQUE INDEX index_security_findings_on_uuid_and_scan_id ON security_findings USING btree (uuid, scan_id);
 
+CREATE INDEX index_security_scans_on_date_created_at_and_id ON security_scans USING btree (date(timezone('UTC'::text, created_at)), id);
+
 CREATE INDEX index_self_managed_prometheus_alert_events_on_environment_id ON self_managed_prometheus_alert_events USING btree (environment_id);
 
 CREATE INDEX index_sent_notifications_on_noteable_type_noteable_id ON sent_notifications USING btree (noteable_id) WHERE ((noteable_type)::text = 'Issue'::text);

ee/lib/ee/gitlab/usage_data.rb

@@ -407,10 +407,28 @@ module EE
           # time outing batch queries, to avoid that
           # different join strategy is used for HLL counter
           if ::Feature.enabled?(:postgres_hll_batch_counting)
-            relation = ::Security::Scan.where(time_period).group(:created_at)
-
-            start = relation.select('MIN(id) as min_id').order('min_id ASC').first&.min_id
-            finish = relation.select('MAX(id) as max_id').order('max_id DESC').first&.max_id
+            scans_table = ::Security::Scan.arel_table
+            inner_relation = ::Security::Scan.select(:id)
+                               .where(
+                                 to_date_arel_node(Arel.sql('date_range_source'))
+                                   .eq(to_date_arel_node(scans_table[time_period.keys[0]]))
+                               )
+
+            outer_relation = ::Security::Scan
+                               .from("generate_series(
+                                '#{time_period.values[0].first.to_time.to_s(:db)}'::timestamp,
+                                '#{time_period.values[0].last.to_time.to_s(:db)}'::timestamp,
+                                '1 day'::interval) date_range_source")
+
+            start_id = outer_relation
+                         .select("(#{inner_relation.order(id: :asc).limit(1).to_sql})")
+                         .order('1 ASC NULLS LAST')
+                         .first&.id
+
+            finish_id = outer_relation
+                          .select("(#{inner_relation.order(id: :desc).limit(1).to_sql})")
+                          .order('1 DESC NULLS LAST')
+                          .first&.id
 
             ::Security::Scan.scan_types.each do |name, scan_type|
               relation = ::Security::Scan.joins(:build)
@@ -419,8 +437,8 @@ module EE
                            .where(security_scans: time_period)
 
               pipelines_with_secure_jobs["#{name}_pipeline".to_sym] =
-                if start && finish
-                  estimate_batch_distinct_count(relation, :commit_id, batch_size: 1000, start: start, finish: finish)
+                if start_id && finish_id
+                  estimate_batch_distinct_count(relation, :commit_id, batch_size: 1000, start: start_id, finish: finish_id)
                 else
                   0
                 end
@@ -442,6 +460,11 @@ module EE
         end
         # rubocop: enable UsageData/LargeTable
 
+        def to_date_arel_node(column)
+          locked_timezone = Arel::Nodes::NamedFunction.new('TIMEZONE', [Arel.sql("'UTC'"), column])
+          Arel::Nodes::NamedFunction.new('DATE', [locked_timezone])
+        end
+
         def approval_merge_request_rule_minimum_id
           strong_memoize(:approval_merge_request_rule_minimum_id) do
             ::ApprovalMergeRequestRule.minimum(:merge_request_id)

rubocop/rubocop-usage-data.yml

@@ -21,6 +21,8 @@ UsageData/LargeTable:
     - :Gitaly::Server
     - :Gitlab::UsageData
     - :Gitlab::UsageDataCounters
+    - :Arel::Nodes::NamedFunction
+    - :Arel
     - :License
     - :Rails
     - :Time