Update projects_imported.total usage ping

  - Current usage ping is timing out and does not
    provide the group::import team correct values
  - Add new index on projects table on import type
    and creator id
  - Update projects_imported.total usage ping to return
    distinct count by creator id, instead of a total count
    of projects

changelogs/unreleased/georgekoltsov-update-project-imported-usage-ping.yml

+---
+title: Update projects_imported.total usage metric
+merge_request: 49568
+author:
+type: fixed

db/migrate/20201210101250_add_index_projects_on_import_type_and_creator_id.rb

+# frozen_string_literal: true
+
+class AddIndexProjectsOnImportTypeAndCreatorId < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :projects, [:creator_id, :import_type, :created_at],
+                         where: 'import_type IS NOT NULL',
+                         name: 'index_projects_on_creator_id_import_type_and_created_at_partial'
+  end
+
+  def down
+    remove_concurrent_index_by_name :projects, 'index_projects_on_creator_id_import_type_and_created_at_partial'
+  end
+end

db/schema_migrations/20201210101250

+734ef1c319549df72bbbfe3acf93ca05f7a6c5547a1efdcaba780195181f5f9a
\ No newline at end of file

db/structure.sql

@@ -22275,6 +22275,8 @@ CREATE INDEX index_projects_on_creator_id_and_created_at_and_id ON projects USIN
 
 CREATE INDEX index_projects_on_creator_id_and_id ON projects USING btree (creator_id, id);
 
+CREATE INDEX index_projects_on_creator_id_import_type_and_created_at_partial ON projects USING btree (creator_id, import_type, created_at) WHERE (import_type IS NOT NULL);
+
 CREATE INDEX index_projects_on_description_trigram ON projects USING gin (description gin_trgm_ops);
 
 CREATE INDEX index_projects_on_id_and_archived_and_pending_delete ON projects USING btree (id) WHERE ((archived = false) AND (pending_delete = false));

lib/gitlab/usage_data.rb

@@ -584,7 +584,7 @@ module Gitlab
             gitlab: distinct_count(::BulkImport.where(time_period, source_type: :gitlab), :user_id)
           },
           projects_imported: {
-            total: count(Project.where(time_period).where.not(import_type: nil)),
+            total: distinct_count(::Project.where(time_period).where.not(import_type: nil), :creator_id),
             gitlab_project: projects_imported_count('gitlab_project', time_period),
             gitlab: projects_imported_count('gitlab', time_period),
             github: projects_imported_count('github', time_period),
@@ -894,7 +894,7 @@ module Gitlab
       end
 
       def projects_imported_count(from, time_period)
-        distinct_count(::Project.imported_from(from).where(time_period), :creator_id) # rubocop: disable CodeReuse/ActiveRecord
+        distinct_count(::Project.imported_from(from).where(time_period).where.not(import_type: nil), :creator_id) # rubocop: disable CodeReuse/ActiveRecord
       end
 
       # rubocop:disable CodeReuse/ActiveRecord

spec/lib/gitlab/usage_data_spec.rb

@@ -224,7 +224,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
             gitlab: 2
           },
           projects_imported: {
-            total: 20,
+            total: 2,
             gitlab_project: 2,
             gitlab: 2,
             github: 2,
@@ -248,7 +248,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
             gitlab: 1
           },
           projects_imported: {
-            total: 10,
+            total: 1,
             gitlab_project: 1,
             gitlab: 1,
             github: 1,