Merge branch '21801-remigrate-epic-user-mentions' into 'master'

Migrate user mentions from epic and epic notes

Closes #21801

See merge request gitlab-org/gitlab!25259

db/post_migrate/20191115115043_migrate_epic_mentions_to_db.rb

 # frozen_string_literal: true
 
 class MigrateEpicMentionsToDb < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
   DOWNTIME = false
 
   disable_ddl_transaction!
@@ -26,7 +28,7 @@ class MigrateEpicMentionsToDb < ActiveRecord::Migration[5.2]
       .where(QUERY_CONDITIONS)
       .each_batch(of: BATCH_SIZE) do |batch, index|
       range = batch.pluck(Arel.sql('MIN(epics.id)'), Arel.sql('MAX(epics.id)')).first
-      BackgroundMigrationWorker.perform_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, false, *range])
+      migrate_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, false, *range])
     end
   end
 

db/post_migrate/20200214173000_cleanup_empty_epic_user_mentions.rb

+# frozen_string_literal: true
+
+class CleanupEmptyEpicUserMentions < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+  BATCH_SIZE = 10000
+
+  class EpicUserMention < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'epic_user_mentions'
+  end
+
+  def up
+    return unless Gitlab.ee?
+
+    # cleanup epic user mentions with no actual mentions,
+    # re https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24586#note_285982468
+    EpicUserMention
+      .where(mentioned_users_ids: nil)
+      .where(mentioned_groups_ids: nil)
+      .where(mentioned_projects_ids: nil)
+      .each_batch(of: BATCH_SIZE) do |batch|
+      batch.delete_all
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/post_migrate/20200214174519_remigrate_epic_mentions_to_db.rb

+# frozen_string_literal: true
+
+class RemigrateEpicMentionsToDb < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  DELAY = 2.minutes.to_i
+  BATCH_SIZE = 10000
+  MIGRATION = 'UserMentions::CreateResourceUserMention'
+
+  JOIN = "LEFT JOIN epic_user_mentions on epics.id = epic_user_mentions.epic_id"
+  QUERY_CONDITIONS = "(description like '%@%' OR title like '%@%') AND epic_user_mentions.epic_id is null"
+
+  class Epic < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'epics'
+  end
+
+  def up
+    return unless Gitlab.ee?
+
+    Epic
+      .joins(JOIN)
+      .where(QUERY_CONDITIONS)
+      .each_batch(of: BATCH_SIZE) do |batch, index|
+      range = batch.pluck(Arel.sql('MIN(epics.id)'), Arel.sql('MAX(epics.id)')).first
+      migrate_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, false, *range])
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/post_migrate/20200214174607_remigrate_epic_notes_mentions_to_db.rb

+# frozen_string_literal: true
+
+class RemigrateEpicNotesMentionsToDb < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  DELAY = 2.minutes.to_i
+  BATCH_SIZE = 10000
+  MIGRATION = 'UserMentions::CreateResourceUserMention'
+
+  INDEX_NAME = 'epic_mentions_temp_index'
+  INDEX_CONDITION = "note LIKE '%@%'::text AND notes.noteable_type = 'Epic'"
+  QUERY_CONDITIONS = "#{INDEX_CONDITION} AND epic_user_mentions.epic_id IS NULL"
+  JOIN = 'INNER JOIN epics ON epics.id = notes.noteable_id LEFT JOIN epic_user_mentions ON notes.id = epic_user_mentions.note_id'
+
+  class Note < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'notes'
+  end
+
+  def up
+    return unless Gitlab.ee?
+
+    # create temporary index for notes with mentions, may take well over 1h
+    add_concurrent_index(:notes, :id, where: INDEX_CONDITION, name: INDEX_NAME)
+
+    Note
+      .joins(JOIN)
+      .where(QUERY_CONDITIONS)
+      .each_batch(of: BATCH_SIZE) do |batch, index|
+      range = batch.pluck(Arel.sql('MIN(notes.id)'), Arel.sql('MAX(notes.id)')).first
+      migrate_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, true, *range])
+    end
+  end
+
+  def down
+    # no-op
+    # temporary index is to be dropped in a different migration in an upcoming release:
+    # https://gitlab.com/gitlab-org/gitlab/issues/196842
+  end
+end

ee/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb

@@ -3,11 +3,14 @@
 require 'spec_helper'
 require './db/post_migrate/20191115115043_migrate_epic_mentions_to_db'
 require './db/post_migrate/20191115115522_migrate_epic_notes_mentions_to_db'
+require './db/post_migrate/20200214174519_remigrate_epic_mentions_to_db'
+require './db/post_migrate/20200214174607_remigrate_epic_notes_mentions_to_db'
 require './db/post_migrate/20200124110831_migrate_design_notes_mentions_to_db'
 
 describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMention do
   include MigrationsHelpers
 
+  context 'when migrating data' do
     let(:users) { table(:users) }
     let(:namespaces) { table(:namespaces) }
     let(:projects) { table(:projects) }
@@ -64,10 +67,7 @@ describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMention do
         let(:resource) { epic }
 
         it_behaves_like 'resource mentions migration', MigrateEpicMentionsToDb, Epic
-
-      it 'has correct no_quote_columns' do
-        expect(Gitlab::BackgroundMigration::UserMentions::Models::Epic.no_quote_columns).to match([:note_id, :epic_id])
-      end
+        it_behaves_like 'resource mentions migration', RemigrateEpicMentionsToDb, Epic
 
         context 'mentions in epic notes' do
           let!(:note1) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: description_mentions) }
@@ -79,6 +79,7 @@ describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMention do
           let!(:note5) { notes.create!(noteable_id: epics.maximum(:id) + 10, noteable_type: 'Epic', author_id: author.id, note: description_mentions, project_id: project.id) }
 
           it_behaves_like 'resource notes mentions migration', MigrateEpicNotesMentionsToDb, Epic
+          it_behaves_like 'resource notes mentions migration', RemigrateEpicNotesMentionsToDb, Epic
         end
       end
     end
@@ -104,4 +105,15 @@ describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMention do
 
       it_behaves_like 'resource notes mentions migration', MigrateDesignNotesMentionsToDb, DesignManagement::Design
     end
+  end
+
+  context 'checks no_quote_columns' do
+    it 'checks that epic has correct no_quote_columns' do
+      expect(Gitlab::BackgroundMigration::UserMentions::Models::Epic.no_quote_columns).to match([:note_id, :epic_id])
+    end
+
+    it 'checks that Design has correct no_quote_columns' do
+      expect(Gitlab::BackgroundMigration::UserMentions::Models::DesignManagement::Design.no_quote_columns).to match([:note_id, :design_id])
+    end
+  end
 end

ee/spec/migrations/cleanup_empty_epic_user_mentions_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20200214173000_cleanup_empty_epic_user_mentions')
+
+describe CleanupEmptyEpicUserMentions, :migration, :sidekiq do
+  let(:users) { table(:users) }
+  let(:namespaces) { table(:namespaces) }
+  let(:epics) { table(:epics) }
+  let(:notes) { table(:notes) }
+  let(:epic_user_mentions) { table(:epic_user_mentions) }
+
+  let(:user) { users.create!(name: 'root', email: 'root@example.com', username: 'root', projects_limit: 0) }
+  let(:group) { namespaces.create!(name: 'group1', path: 'group1', owner_id: user.id, type: 'Group') }
+  let(:epic) { epics.create!(iid: 1, title: "title", title_html: 'title', description: 'epic description', group_id: group.id, author_id: user.id) }
+
+  # migrateable resources
+  let!(:resource1) { notes.create!(note: 'note1 for @root to check', noteable_id: epic.id, noteable_type: 'Epic') }
+  let!(:resource2) { notes.create!(note: 'note2 for @root to check', noteable_id: epic.id, noteable_type: 'Epic', system: true) }
+  let!(:resource3) { notes.create!(note: 'note3 for @root to check', noteable_id: epic.id, noteable_type: 'Epic') }
+
+  # non-migrateable resources
+  # this note is already migrated, as it has a record in the epic_user_mentions table
+  let!(:resource4) { notes.create!(note: 'note3 for @root to check', noteable_id: epic.id, noteable_type: 'Epic') }
+  let!(:user_mention) { epic_user_mentions.create!(epic_id: epic.id, note_id: resource4.id, mentioned_users_ids: [1]) }
+
+  # these should get cleanup, by the migration
+  let!(:blank_epic_user_mention1) { epic_user_mentions.create!(epic_id: epic.id, note_id: resource1.id)}
+  let!(:blank_epic_user_mention2) { epic_user_mentions.create!(epic_id: epic.id, note_id: resource2.id)}
+  let!(:blank_epic_user_mention3) { epic_user_mentions.create!(epic_id: epic.id, note_id: resource3.id)}
+
+  it 'cleans blank user mentions' do
+    expect(epic_user_mentions.count).to eq 4
+
+    migrate!
+
+    expect(epic_user_mentions.count).to eq 1
+  end
+end

ee/spec/migrations/migrate_design_notes_mentions_to_db_spec.rb

@@ -27,9 +27,5 @@ describe MigrateDesignNotesMentionsToDb, :migration, :sidekiq do
   # this note points to an innexistent noteable record
   let!(:resource5) { notes.create!(note: 'note3 for @root to check', noteable_id: designs.maximum(:id) + 10, noteable_type: 'DesignManagement::Design') }
 
-  before do
-    stub_const("#{described_class.name}::BATCH_SIZE", 1)
-  end
-
   it_behaves_like 'schedules resource mentions migration', DesignManagement::Design, true
 end

ee/spec/migrations/migrate_epic_mentions_to_db_spec.rb

@@ -2,8 +2,9 @@
 
 require 'spec_helper'
 require Rails.root.join('db', 'post_migrate', '20191115115043_migrate_epic_mentions_to_db')
+require Rails.root.join('db', 'post_migrate', '20200214174519_remigrate_epic_mentions_to_db')
 
-describe MigrateEpicMentionsToDb, :migration do
+describe 'epic mentions migration' do
   let(:users) { table(:users) }
   let(:namespaces) { table(:namespaces) }
   let(:epics) { table(:epics) }
@@ -21,10 +22,14 @@ describe MigrateEpicMentionsToDb, :migration do
   # this epic is already migrated, as it has a record in the epic_user_mentions table
   let!(:resource4) { epics.create!(iid: 4, title: "title3", title_html: 'title3', description: 'epic description with @root mention', group_id: group.id, author_id: user.id) }
   let!(:user_mention) { epic_user_mentions.create!(epic_id: resource4.id, mentioned_users_ids: [1]) }
+  # this epic has no mentions so should be filtered out
+  let!(:resource5) { epics.create!(iid: 5, title: "title5", title_html: 'title5', description: 'epic description with no mention', group_id: group.id, author_id: user.id) }
 
-  before do
-    stub_const("#{described_class.name}::BATCH_SIZE", 1)
+  describe MigrateEpicMentionsToDb, :migration do
+    it_behaves_like 'schedules resource mentions migration', Epic, false
   end
 
+  describe RemigrateEpicMentionsToDb, :migration do
     it_behaves_like 'schedules resource mentions migration', Epic, false
+  end
 end

ee/spec/migrations/migrate_epic_notes_mentions_to_db_spec.rb

@@ -2,8 +2,9 @@
 
 require 'spec_helper'
 require Rails.root.join('db', 'post_migrate', '20191115115522_migrate_epic_notes_mentions_to_db')
+require Rails.root.join('db', 'post_migrate', '20200214174607_remigrate_epic_notes_mentions_to_db')
 
-describe MigrateEpicNotesMentionsToDb, :migration do
+describe 'epic notes mentions migration' do
   let(:users) { table(:users) }
   let(:namespaces) { table(:namespaces) }
   let(:epics) { table(:epics) }
@@ -26,9 +27,11 @@ describe MigrateEpicNotesMentionsToDb, :migration do
   # this note points to an innexistent noteable record
   let!(:resource5) { notes.create!(note: 'note3 for @root to check', noteable_id: epics.maximum(:id) + 10, noteable_type: 'Epic') }
 
-  before do
-    stub_const("#{described_class.name}::BATCH_SIZE", 1)
+  describe MigrateEpicNotesMentionsToDb, :migration do
+    it_behaves_like 'schedules resource mentions migration', Epic, true
   end
 
+  describe RemigrateEpicNotesMentionsToDb, :migration do
     it_behaves_like 'schedules resource mentions migration', Epic, true
+  end
 end

lib/gitlab/background_migration/user_mentions/models/epic.rb

@@ -18,7 +18,6 @@ module Gitlab
           self.table_name = 'epics'
 
           belongs_to :author, class_name: "User"
-          belongs_to :project
           belongs_to :group
 
           def self.user_mention_model

spec/support/shared_examples/lib/gitlab/background_migration/mentions_migration_shared_examples.rb

@@ -65,9 +65,16 @@ shared_examples 'resource notes mentions migration' do |migration_class, resourc
 end
 
 shared_examples 'schedules resource mentions migration' do |resource_class, is_for_notes|
+  before do
+    stub_const("#{described_class.name}::BATCH_SIZE", 1)
+  end
+
   it 'schedules background migrations' do
     Sidekiq::Testing.fake! do
       Timecop.freeze do
+        resource_count = is_for_notes ? Note.count : resource_class.count
+        expect(resource_count).to eq 5
+
         migrate!
 
         migration = described_class::MIGRATION