Merge branch 'tc-add-missing-geo-event-indexes' into 'master'

Add indexes to all geo event foreign keys

Closes #7955

See merge request gitlab-org/gitlab-ee!7990

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20181013005024) do
+ActiveRecord::Schema.define(version: 20181017131623) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1123,13 +1123,18 @@ ActiveRecord::Schema.define(version: 20181013005024) do
     t.integer "cache_invalidation_event_id", limit: 8
   end
 
-  add_index "geo_event_log", ["cache_invalidation_event_id"], name: "index_geo_event_log_on_cache_invalidation_event_id", using: :btree
-  add_index "geo_event_log", ["repositories_changed_event_id"], name: "index_geo_event_log_on_repositories_changed_event_id", using: :btree
-  add_index "geo_event_log", ["repository_created_event_id"], name: "index_geo_event_log_on_repository_created_event_id", using: :btree
-  add_index "geo_event_log", ["repository_deleted_event_id"], name: "index_geo_event_log_on_repository_deleted_event_id", using: :btree
-  add_index "geo_event_log", ["repository_renamed_event_id"], name: "index_geo_event_log_on_repository_renamed_event_id", using: :btree
-  add_index "geo_event_log", ["repository_updated_event_id"], name: "index_geo_event_log_on_repository_updated_event_id", using: :btree
-  add_index "geo_event_log", ["reset_checksum_event_id"], name: "index_geo_event_log_on_reset_checksum_event_id", using: :btree
+  add_index "geo_event_log", ["cache_invalidation_event_id"], name: "index_geo_event_log_on_cache_invalidation_event_id", where: "(cache_invalidation_event_id IS NOT NULL)", using: :btree
+  add_index "geo_event_log", ["hashed_storage_attachments_event_id"], name: "index_geo_event_log_on_hashed_storage_attachments_event_id", where: "(hashed_storage_attachments_event_id IS NOT NULL)", using: :btree
+  add_index "geo_event_log", ["hashed_storage_migrated_event_id"], name: "index_geo_event_log_on_hashed_storage_migrated_event_id", where: "(hashed_storage_migrated_event_id IS NOT NULL)", using: :btree
+  add_index "geo_event_log", ["job_artifact_deleted_event_id"], name: "index_geo_event_log_on_job_artifact_deleted_event_id", where: "(job_artifact_deleted_event_id IS NOT NULL)", using: :btree
+  add_index "geo_event_log", ["lfs_object_deleted_event_id"], name: "index_geo_event_log_on_lfs_object_deleted_event_id", where: "(lfs_object_deleted_event_id IS NOT NULL)", using: :btree
+  add_index "geo_event_log", ["repositories_changed_event_id"], name: "index_geo_event_log_on_repositories_changed_event_id", where: "(repositories_changed_event_id IS NOT NULL)", using: :btree
+  add_index "geo_event_log", ["repository_created_event_id"], name: "index_geo_event_log_on_repository_created_event_id", where: "(repository_created_event_id IS NOT NULL)", using: :btree
+  add_index "geo_event_log", ["repository_deleted_event_id"], name: "index_geo_event_log_on_repository_deleted_event_id", where: "(repository_deleted_event_id IS NOT NULL)", using: :btree
+  add_index "geo_event_log", ["repository_renamed_event_id"], name: "index_geo_event_log_on_repository_renamed_event_id", where: "(repository_renamed_event_id IS NOT NULL)", using: :btree
+  add_index "geo_event_log", ["repository_updated_event_id"], name: "index_geo_event_log_on_repository_updated_event_id", where: "(repository_updated_event_id IS NOT NULL)", using: :btree
+  add_index "geo_event_log", ["reset_checksum_event_id"], name: "index_geo_event_log_on_reset_checksum_event_id", where: "(reset_checksum_event_id IS NOT NULL)", using: :btree
+  add_index "geo_event_log", ["upload_deleted_event_id"], name: "index_geo_event_log_on_upload_deleted_event_id", where: "(upload_deleted_event_id IS NOT NULL)", using: :btree
 
   create_table "geo_hashed_storage_attachments_events", id: :bigserial, force: :cascade do |t|
     t.integer "project_id", null: false
@@ -3220,6 +3225,7 @@ ActiveRecord::Schema.define(version: 20181013005024) do
   add_foreign_key "gcp_clusters", "services", on_delete: :nullify
   add_foreign_key "gcp_clusters", "users", on_delete: :nullify
   add_foreign_key "geo_event_log", "geo_cache_invalidation_events", column: "cache_invalidation_event_id", name: "fk_42c3b54bed", on_delete: :cascade
+  add_foreign_key "geo_event_log", "geo_hashed_storage_attachments_events", column: "hashed_storage_attachments_event_id", name: "fk_304067fc30", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_hashed_storage_migrated_events", column: "hashed_storage_migrated_event_id", name: "fk_27548c6db3", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_job_artifact_deleted_events", column: "job_artifact_deleted_event_id", name: "fk_176d3fbb5d", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_lfs_object_deleted_events", column: "lfs_object_deleted_event_id", name: "fk_d5af95fcd9", on_delete: :cascade
@@ -3227,7 +3233,7 @@ ActiveRecord::Schema.define(version: 20181013005024) do
   add_foreign_key "geo_event_log", "geo_repository_created_events", column: "repository_created_event_id", name: "fk_9b9afb1916", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_repository_deleted_events", column: "repository_deleted_event_id", name: "fk_c4b1c1f66e", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_repository_renamed_events", column: "repository_renamed_event_id", name: "fk_86c84214ec", on_delete: :cascade
-  add_foreign_key "geo_event_log", "geo_repository_updated_events", column: "repository_updated_event_id", on_delete: :cascade
+  add_foreign_key "geo_event_log", "geo_repository_updated_events", column: "repository_updated_event_id", name: "fk_78a6492f68", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_reset_checksum_events", column: "reset_checksum_event_id", name: "fk_cff7185ad2", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_upload_deleted_events", column: "upload_deleted_event_id", name: "fk_c1f241c70d", on_delete: :cascade
   add_foreign_key "geo_hashed_storage_attachments_events", "projects", on_delete: :cascade

ee/changelogs/unreleased/tc-add-missing-geo-event-indexes.yml

+---
+title: Add indexes to all geo event foreign keys
+merge_request: 7990
+author:
+type: performance

ee/db/migrate/20181017131623_add_missing_geo_even_log_indexes.rb

+# frozen_string_literal: true
+
+class AddMissingGeoEvenLogIndexes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEXED_COLUMNS = %i[cache_invalidation_event_id
+                       repositories_changed_event_id
+                       repository_created_event_id
+                       repository_deleted_event_id
+                       repository_renamed_event_id
+                       repository_updated_event_id
+                       reset_checksum_event_id]
+
+  NON_INDEXED_COLUMNS = %i[hashed_storage_migrated_event_id
+                           lfs_object_deleted_event_id
+                           hashed_storage_attachments_event_id
+                           job_artifact_deleted_event_id
+                           upload_deleted_event_id]
+
+  MISSING_FK_COLUMNS = %i[hashed_storage_attachments_event_id]
+
+  disable_ddl_transaction!
+
+  def up
+    # Replace existing indexes with partial indexes
+    INDEXED_COLUMNS.each do |col|
+      without_foreign_key(:geo_event_log, col) do
+        remove_concurrent_index(:geo_event_log, col)
+        add_concurrent_index_not_null(:geo_event_log, col)
+      end
+    end
+
+    # Create partial indexes on non-indexed columns
+    NON_INDEXED_COLUMNS.each do |col|
+      add_concurrent_index_not_null(:geo_event_log, col)
+    end
+
+    # Add missing foreign keys
+    MISSING_FK_COLUMNS.each do |col|
+      add_concurrent_foreign_key(:geo_event_log,
+                                 foreign_table_name(col),
+                                 column: col,
+                                 on_delete: :cascade)
+    end
+  end
+
+  def down
+    # Remove missing foreign keys again
+    MISSING_FK_COLUMNS.each do |col|
+      remove_foreign_key(:geo_event_log, column: col)
+    end
+
+    # Drop indexes that didn't exist before
+    NON_INDEXED_COLUMNS.each do |col|
+      without_foreign_key(:geo_event_log, col) do
+        remove_concurrent_index(:geo_event_log, col)
+      end
+    end
+
+    # Recreate full indexes
+    INDEXED_COLUMNS.each do |col|
+      without_foreign_key(:geo_event_log, col) do
+        remove_concurrent_index(:geo_event_log, col)
+        add_concurrent_index(:geo_event_log, col)
+      end
+    end
+  end
+
+  private
+
+  def add_concurrent_index_not_null(table, col)
+    add_concurrent_index(table, col, where: "#{col} IS NOT NULL")
+  end
+
+  def without_foreign_key(table, col)
+    return yield unless drop_foreign_key?(table, col)
+
+    begin
+      remove_foreign_key(table, column: col)
+      yield
+    ensure
+      add_concurrent_foreign_key(table, foreign_table_name(col), column: col, on_delete: :cascade)
+    end
+  end
+
+  def drop_foreign_key?(table, col)
+    Gitlab::Database.mysql? && foreign_key_exists?(table, column: col)
+  end
+
+  def foreign_table_name(col)
+    ('geo_' + col.to_s.sub(/_id$/, '')).pluralize
+  end
+end