Migrate all merge request user mentions to db table

Scan the entire merge requests table in batches of 100K and
then select only records that can contain mentions from each
batch. This way the number of fetched records is constantly
100K and we can ensure query does not timeout due to a overly
large batch of records being queried.

changelogs/unreleased/migrate-all-merge-request-user-mentions.yml

+---
+title: Store user mentions from merge request title or description in the DB
+merge_request: 34378
+author:
+type: changed

db/post_migrate/20200601120434_migrate_all_merge_request_user_mentions_to_db.rb

+# frozen_string_literal: true
+
+class MigrateAllMergeRequestUserMentionsToDb < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  DELAY = 2.minutes.to_i
+  BATCH_SIZE = 100_000
+  MIGRATION = 'UserMentions::CreateResourceUserMention'
+
+  JOIN = "LEFT JOIN merge_request_user_mentions on merge_requests.id = merge_request_user_mentions.merge_request_id"
+  QUERY_CONDITIONS = "(description LIKE '%@%' OR title LIKE '%@%') AND merge_request_user_mentions.merge_request_id IS NULL"
+
+  disable_ddl_transaction!
+
+  class MergeRequest < ActiveRecord::Base
+    include EachBatch
+  end
+
+  def up
+    delay = DELAY
+
+    MergeRequest.each_batch(of: BATCH_SIZE) do |batch, _|
+      range = batch.pluck('MIN(merge_requests.id)', 'MAX(merge_requests.id)').first
+      records_count = MergeRequest.joins(JOIN).where(QUERY_CONDITIONS).where(id: range.first..range.last).count
+
+      if records_count > 0
+        migrate_in(delay, MIGRATION, ['MergeRequest', JOIN, QUERY_CONDITIONS, false, *range])
+        delay += [DELAY, (records_count / 500 + 1).minutes.to_i].max
+      end
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/structure.sql

@@ -23860,6 +23860,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200528123703
 20200528125905
 20200528171933
+20200601120434
 20200601210148
 20200602013900
 20200602013901

ee/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb

@@ -68,6 +68,14 @@ RSpec.describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMent
 
         it_behaves_like 'resource mentions migration', MigrateEpicMentionsToDb, Epic
 
+        context 'when FF disabled' do
+          before do
+            stub_feature_flags(migrate_user_mentions: false)
+          end
+
+          it_behaves_like 'resource migration not run', MigrateEpicMentionsToDb, Epic
+        end
+
         context 'mentions in epic notes' do
           let!(:note1) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: description_mentions) }
           let!(:note2) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: 'sample note') }
@@ -78,6 +86,14 @@ RSpec.describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMent
           let!(:note5) { notes.create!(noteable_id: non_existing_record_id, noteable_type: 'Epic', author_id: author.id, note: description_mentions, project_id: project.id) }
 
           it_behaves_like 'resource notes mentions migration', MigrateEpicNotesMentionsToDb, Epic
+
+          context 'when FF disabled' do
+            before do
+              stub_feature_flags(migrate_user_mentions: false)
+            end
+
+            it_behaves_like 'resource notes migration not run', MigrateEpicNotesMentionsToDb, Epic
+          end
         end
       end
     end
@@ -102,6 +118,14 @@ RSpec.describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMent
       let(:resource) { design }
 
       it_behaves_like 'resource notes mentions migration', MigrateDesignNotesMentionsToDb, DesignManagement::Design
+
+      context 'when FF disabled' do
+        before do
+          stub_feature_flags(migrate_user_mentions: false)
+        end
+
+        it_behaves_like 'resource notes migration not run', MigrateDesignNotesMentionsToDb, DesignManagement::Design
+      end
     end
   end
 

lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb

@@ -12,26 +12,22 @@ module Gitlab
         ISOLATION_MODULE = 'Gitlab::BackgroundMigration::UserMentions::Models'
 
         def perform(resource_model, join, conditions, with_notes, start_id, end_id)
+          return unless Feature.enabled?(:migrate_user_mentions, default_enabled: true)
+
           resource_model = "#{ISOLATION_MODULE}::#{resource_model}".constantize if resource_model.is_a?(String)
           model = with_notes ? Gitlab::BackgroundMigration::UserMentions::Models::Note : resource_model
           resource_user_mention_model = resource_model.user_mention_model
 
           records = model.joins(join).where(conditions).where(id: start_id..end_id)
 
-          records.in_groups_of(BULK_INSERT_SIZE, false).each do |records|
+          records.each_batch(of: BULK_INSERT_SIZE) do |records|
             mentions = []
             records.each do |record|
               mention_record = record.build_mention_values(resource_user_mention_model.resource_foreign_key)
               mentions << mention_record unless mention_record.blank?
             end
 
-            Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert
-              resource_user_mention_model.table_name,
-              mentions,
-              return_ids: true,
-              disable_quote: resource_model.no_quote_columns,
-              on_conflict: :do_nothing
-            )
+            resource_user_mention_model.insert_all(mentions) unless mentions.empty?
           end
         end
       end

lib/gitlab/background_migration/user_mentions/models/commit.rb

@@ -6,6 +6,7 @@ module Gitlab
     module UserMentions
       module Models
         class Commit
+          include EachBatch
           include Concerns::IsolatedMentionable
           include Concerns::MentionableMigrationMethods
 

lib/gitlab/background_migration/user_mentions/models/design_management/design.rb

@@ -7,6 +7,7 @@ module Gitlab
       module Models
         module DesignManagement
           class Design < ActiveRecord::Base
+            include EachBatch
             include Concerns::MentionableMigrationMethods
 
             def self.user_mention_model

lib/gitlab/background_migration/user_mentions/models/epic.rb

@@ -6,6 +6,7 @@ module Gitlab
     module UserMentions
       module Models
         class Epic < ActiveRecord::Base
+          include EachBatch
           include Concerns::IsolatedMentionable
           include Concerns::MentionableMigrationMethods
           include CacheMarkdownField

lib/gitlab/background_migration/user_mentions/models/merge_request.rb

@@ -6,6 +6,7 @@ module Gitlab
     module UserMentions
       module Models
         class MergeRequest < ActiveRecord::Base
+          include EachBatch
           include Concerns::IsolatedMentionable
           include CacheMarkdownField
           include Concerns::MentionableMigrationMethods

lib/gitlab/background_migration/user_mentions/models/note.rb

@@ -6,6 +6,7 @@ module Gitlab
     module UserMentions
       module Models
         class Note < ActiveRecord::Base
+          include EachBatch
           include Concerns::IsolatedMentionable
           include CacheMarkdownField
 

spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb

@@ -75,6 +75,14 @@ RSpec.describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMent
       let(:resource) { merge_request }
 
       it_behaves_like 'resource mentions migration', MigrateMergeRequestMentionsToDb, MergeRequest
+
+      context 'when FF disabled' do
+        before do
+          stub_feature_flags(migrate_user_mentions: false)
+        end
+
+        it_behaves_like 'resource migration not run', MigrateMergeRequestMentionsToDb, MergeRequest
+      end
     end
 
     context 'migrate commit mentions' do
@@ -96,6 +104,14 @@ RSpec.describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMent
       let(:resource) { commit }
 
       it_behaves_like 'resource notes mentions migration', MigrateCommitNotesMentionsToDb, Commit
+
+      context 'when FF disabled' do
+        before do
+          stub_feature_flags(migrate_user_mentions: false)
+        end
+
+        it_behaves_like 'resource notes migration not run', MigrateCommitNotesMentionsToDb, Commit
+      end
     end
   end
 

spec/migrations/migrate_all_merge_request_user_mentions_to_db_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20200601120434_migrate_all_merge_request_user_mentions_to_db')
+
+RSpec.describe MigrateAllMergeRequestUserMentionsToDb, :migration do
+  let(:users) { table(:users) }
+  let(:projects) { table(:projects) }
+  let(:namespaces) { table(:namespaces) }
+  let(:merge_requests) { table(:merge_requests) }
+  let(:merge_request_user_mentions) { table(:merge_request_user_mentions) }
+
+  let(:user) { users.create!(name: 'root', email: 'root@example.com', username: 'root', projects_limit: 0) }
+  let(:group) { namespaces.create!(name: 'group1', path: 'group1', owner_id: user.id, type: 'Group') }
+  let(:project) { projects.create!(name: 'gitlab1', path: 'gitlab1', namespace_id: group.id, visibility_level: 0) }
+
+  let(:opened_state) { 1 }
+  let(:closed_state) { 2 }
+  let(:merged_state) { 3 }
+
+  # migrateable resources
+  let(:common_args) { { source_branch: 'master', source_project_id: project.id, target_project_id: project.id, author_id: user.id, description: 'mr description with @root mention' } }
+  let!(:resource1) { merge_requests.create!(common_args.merge(title: "title 1", state_id: opened_state, target_branch: 'feature1')) }
+  let!(:resource2) { merge_requests.create!(common_args.merge(title: "title 2", state_id: closed_state, target_branch: 'feature2')) }
+  let!(:resource3) { merge_requests.create!(common_args.merge(title: "title 3", state_id: merged_state, target_branch: 'feature3')) }
+
+  # non-migrateable resources
+  # this merge request is already migrated, as it has a record in the merge_request_user_mentions table
+  let!(:resource4) { merge_requests.create!(common_args.merge(title: "title 3", state_id: opened_state, target_branch: 'feature4')) }
+  let!(:user_mention) { merge_request_user_mentions.create!(merge_request_id: resource4.id, mentioned_users_ids: [1]) }
+
+  let!(:resource5) { merge_requests.create!(common_args.merge(title: "title 3", description: 'description with no mention', state_id: opened_state, target_branch: 'feature5')) }
+
+  it_behaves_like 'schedules resource mentions migration', MergeRequest, false
+end

spec/support/shared_examples/lib/gitlab/background_migration/mentions_migration_shared_examples.rb

@@ -82,3 +82,25 @@ RSpec.shared_examples 'schedules resource mentions migration' do |resource_class
     end
   end
 end
+
+RSpec.shared_examples 'resource migration not run' do |migration_class, resource_class|
+  it 'does not migrate mentions' do
+    join = migration_class::JOIN
+    conditions = migration_class::QUERY_CONDITIONS
+
+    expect do
+      subject.perform(resource_class.name, join, conditions, false, resource_class.minimum(:id), resource_class.maximum(:id))
+    end.to change { user_mentions.count }.by(0)
+  end
+end
+
+RSpec.shared_examples 'resource notes migration not run' do |migration_class, resource_class|
+  it 'does not migrate mentions' do
+    join = migration_class::JOIN
+    conditions = migration_class::QUERY_CONDITIONS
+
+    expect do
+      subject.perform(resource_class.name, join, conditions, true, Note.minimum(:id), Note.maximum(:id))
+    end.to change { user_mentions.count }.by(0)
+  end
+end