Merge branch '344259-fix-service-ping-failing-metrics-for-manage-stage' into 'master'

Fix Service ping failing metrics for manage stage

See merge request gitlab-org/gitlab!74116

db/post_migrate/20211115151704_add_index_on_projects_import_type_id.rb

+# frozen_string_literal: true
+
+class AddIndexOnProjectsImportTypeId < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_imported_projects_on_import_type_id'
+
+  def up
+    add_concurrent_index(:projects, [:import_type, :id], where: 'import_type IS NOT NULL', name: INDEX_NAME)
+  end
+
+  def down
+    remove_concurrent_index_by_name(:projects, INDEX_NAME)
+  end
+end

db/schema_migrations/20211115151704

+3db18116febc760bcfeab597e0508d5b2835d0135068d79073770d343aa4b09c
\ No newline at end of file

db/structure.sql

@@ -26064,6 +26064,8 @@ CREATE INDEX index_import_failures_on_project_id_not_null ON import_failures USI
 
 CREATE INDEX index_imported_projects_on_import_type_creator_id_created_at ON projects USING btree (import_type, creator_id, created_at) WHERE (import_type IS NOT NULL);
 
+CREATE INDEX index_imported_projects_on_import_type_id ON projects USING btree (import_type, id) WHERE (import_type IS NOT NULL);
+
 CREATE INDEX index_in_product_marketing_emails_on_user_id ON in_product_marketing_emails USING btree (user_id);
 
 CREATE UNIQUE INDEX index_in_product_marketing_emails_on_user_track_series ON in_product_marketing_emails USING btree (user_id, track, series);

ee/spec/lib/ee/gitlab/usage_data_non_sql_metrics_spec.rb

@@ -15,7 +15,7 @@ RSpec.describe Gitlab::UsageDataNonSqlMetrics do
         described_class.uncached_data
       end
 
-      expect(recorder.count).to eq(59)
+      expect(recorder.count).to eq(61)
     end
   end
 end

lib/gitlab/usage_data.rb

@@ -618,9 +618,9 @@ module Gitlab
           todos: distinct_count(::Todo.where(time_period), :author_id),
           service_desk_enabled_projects: distinct_count_service_desk_enabled_projects(time_period),
           service_desk_issues: count(::Issue.service_desk.where(time_period)),
-          projects_jira_active: distinct_count(::Project.with_active_integration(::Integrations::Jira) .where(time_period), :creator_id),
-          projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_integration(::Integrations::Jira) .with_jira_dvcs_cloud.where(time_period), :creator_id),
-          projects_jira_dvcs_server_active: distinct_count(::Project.with_active_integration(::Integrations::Jira) .with_jira_dvcs_server.where(time_period), :creator_id)
+          projects_jira_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).where(time_period), :creator_id),
+          projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).with_jira_dvcs_cloud.where(time_period), :creator_id),
+          projects_jira_dvcs_server_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).with_jira_dvcs_server.where(time_period), :creator_id)
         }
       end
       # rubocop: enable CodeReuse/ActiveRecord
@@ -915,7 +915,30 @@ module Gitlab
       end
 
       def projects_imported_count(from, time_period)
-        count(::Project.imported_from(from).where(time_period).where.not(import_type: nil)) # rubocop: disable CodeReuse/ActiveRecord
+        # rubocop:disable CodeReuse/ActiveRecord
+        relation = ::Project.imported_from(from).where.not(import_type: nil) # rubocop:disable UsageData/LargeTable
+        if time_period.empty?
+          count(relation)
+        else
+          @project_import_id ||= {}
+          start = time_period[:created_at].first
+          finish = time_period[:created_at].last
+
+          # can be nil values here if no records are in our range and it is possible the same instance
+          # is called with different time periods since it is passed in as a variable
+          unless @project_import_id.key?(start)
+            @project_import_id[start] = ::Project.select(:id).where(Project.arel_table[:created_at].gteq(start)) # rubocop:disable UsageData/LargeTable
+                                                 .order(created_at: :asc).limit(1).first&.id
+          end
+
+          unless @project_import_id.key?(finish)
+            @project_import_id[finish] = ::Project.select(:id).where(Project.arel_table[:created_at].lteq(finish)) # rubocop:disable UsageData/LargeTable
+                                                  .order(created_at: :desc).limit(1).first&.id
+          end
+
+          count(relation, start: @project_import_id[start], finish: @project_import_id[finish])
+        end
+        # rubocop:enable CodeReuse/ActiveRecord
       end
 
       def issue_imports(time_period)