Commit c34a4b6b authored by Toon Claes's avatar Toon Claes

Merge branch '21801-remigrate-epic-user-mentions' into 'master'

Migrate user mentions from epic and epic notes

Closes #21801

See merge request gitlab-org/gitlab!25259
parents 54cd52b6 7660a1c8
# frozen_string_literal: true # frozen_string_literal: true
class MigrateEpicMentionsToDb < ActiveRecord::Migration[5.2] class MigrateEpicMentionsToDb < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false DOWNTIME = false
disable_ddl_transaction! disable_ddl_transaction!
...@@ -26,7 +28,7 @@ class MigrateEpicMentionsToDb < ActiveRecord::Migration[5.2] ...@@ -26,7 +28,7 @@ class MigrateEpicMentionsToDb < ActiveRecord::Migration[5.2]
.where(QUERY_CONDITIONS) .where(QUERY_CONDITIONS)
.each_batch(of: BATCH_SIZE) do |batch, index| .each_batch(of: BATCH_SIZE) do |batch, index|
range = batch.pluck(Arel.sql('MIN(epics.id)'), Arel.sql('MAX(epics.id)')).first range = batch.pluck(Arel.sql('MIN(epics.id)'), Arel.sql('MAX(epics.id)')).first
BackgroundMigrationWorker.perform_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, false, *range]) migrate_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, false, *range])
end end
end end
......
# frozen_string_literal: true
class CleanupEmptyEpicUserMentions < ActiveRecord::Migration[5.2]
DOWNTIME = false
BATCH_SIZE = 10000
class EpicUserMention < ActiveRecord::Base
include EachBatch
self.table_name = 'epic_user_mentions'
end
def up
return unless Gitlab.ee?
# cleanup epic user mentions with no actual mentions,
# re https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24586#note_285982468
EpicUserMention
.where(mentioned_users_ids: nil)
.where(mentioned_groups_ids: nil)
.where(mentioned_projects_ids: nil)
.each_batch(of: BATCH_SIZE) do |batch|
batch.delete_all
end
end
def down
# no-op
end
end
# frozen_string_literal: true
class RemigrateEpicMentionsToDb < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
DELAY = 2.minutes.to_i
BATCH_SIZE = 10000
MIGRATION = 'UserMentions::CreateResourceUserMention'
JOIN = "LEFT JOIN epic_user_mentions on epics.id = epic_user_mentions.epic_id"
QUERY_CONDITIONS = "(description like '%@%' OR title like '%@%') AND epic_user_mentions.epic_id is null"
class Epic < ActiveRecord::Base
include EachBatch
self.table_name = 'epics'
end
def up
return unless Gitlab.ee?
Epic
.joins(JOIN)
.where(QUERY_CONDITIONS)
.each_batch(of: BATCH_SIZE) do |batch, index|
range = batch.pluck(Arel.sql('MIN(epics.id)'), Arel.sql('MAX(epics.id)')).first
migrate_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, false, *range])
end
end
def down
# no-op
end
end
# frozen_string_literal: true
class RemigrateEpicNotesMentionsToDb < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
DELAY = 2.minutes.to_i
BATCH_SIZE = 10000
MIGRATION = 'UserMentions::CreateResourceUserMention'
INDEX_NAME = 'epic_mentions_temp_index'
INDEX_CONDITION = "note LIKE '%@%'::text AND notes.noteable_type = 'Epic'"
QUERY_CONDITIONS = "#{INDEX_CONDITION} AND epic_user_mentions.epic_id IS NULL"
JOIN = 'INNER JOIN epics ON epics.id = notes.noteable_id LEFT JOIN epic_user_mentions ON notes.id = epic_user_mentions.note_id'
class Note < ActiveRecord::Base
include EachBatch
self.table_name = 'notes'
end
def up
return unless Gitlab.ee?
# create temporary index for notes with mentions, may take well over 1h
add_concurrent_index(:notes, :id, where: INDEX_CONDITION, name: INDEX_NAME)
Note
.joins(JOIN)
.where(QUERY_CONDITIONS)
.each_batch(of: BATCH_SIZE) do |batch, index|
range = batch.pluck(Arel.sql('MIN(notes.id)'), Arel.sql('MAX(notes.id)')).first
migrate_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, true, *range])
end
end
def down
# no-op
# temporary index is to be dropped in a different migration in an upcoming release:
# https://gitlab.com/gitlab-org/gitlab/issues/196842
end
end
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20200214173000_cleanup_empty_epic_user_mentions')
describe CleanupEmptyEpicUserMentions, :migration, :sidekiq do
let(:users) { table(:users) }
let(:namespaces) { table(:namespaces) }
let(:epics) { table(:epics) }
let(:notes) { table(:notes) }
let(:epic_user_mentions) { table(:epic_user_mentions) }
let(:user) { users.create!(name: 'root', email: 'root@example.com', username: 'root', projects_limit: 0) }
let(:group) { namespaces.create!(name: 'group1', path: 'group1', owner_id: user.id, type: 'Group') }
let(:epic) { epics.create!(iid: 1, title: "title", title_html: 'title', description: 'epic description', group_id: group.id, author_id: user.id) }
# migrateable resources
let!(:resource1) { notes.create!(note: 'note1 for @root to check', noteable_id: epic.id, noteable_type: 'Epic') }
let!(:resource2) { notes.create!(note: 'note2 for @root to check', noteable_id: epic.id, noteable_type: 'Epic', system: true) }
let!(:resource3) { notes.create!(note: 'note3 for @root to check', noteable_id: epic.id, noteable_type: 'Epic') }
# non-migrateable resources
# this note is already migrated, as it has a record in the epic_user_mentions table
let!(:resource4) { notes.create!(note: 'note3 for @root to check', noteable_id: epic.id, noteable_type: 'Epic') }
let!(:user_mention) { epic_user_mentions.create!(epic_id: epic.id, note_id: resource4.id, mentioned_users_ids: [1]) }
# these should get cleanup, by the migration
let!(:blank_epic_user_mention1) { epic_user_mentions.create!(epic_id: epic.id, note_id: resource1.id)}
let!(:blank_epic_user_mention2) { epic_user_mentions.create!(epic_id: epic.id, note_id: resource2.id)}
let!(:blank_epic_user_mention3) { epic_user_mentions.create!(epic_id: epic.id, note_id: resource3.id)}
it 'cleans blank user mentions' do
expect(epic_user_mentions.count).to eq 4
migrate!
expect(epic_user_mentions.count).to eq 1
end
end
...@@ -27,9 +27,5 @@ describe MigrateDesignNotesMentionsToDb, :migration, :sidekiq do ...@@ -27,9 +27,5 @@ describe MigrateDesignNotesMentionsToDb, :migration, :sidekiq do
# this note points to an innexistent noteable record # this note points to an innexistent noteable record
let!(:resource5) { notes.create!(note: 'note3 for @root to check', noteable_id: designs.maximum(:id) + 10, noteable_type: 'DesignManagement::Design') } let!(:resource5) { notes.create!(note: 'note3 for @root to check', noteable_id: designs.maximum(:id) + 10, noteable_type: 'DesignManagement::Design') }
before do
stub_const("#{described_class.name}::BATCH_SIZE", 1)
end
it_behaves_like 'schedules resource mentions migration', DesignManagement::Design, true it_behaves_like 'schedules resource mentions migration', DesignManagement::Design, true
end end
...@@ -2,8 +2,9 @@ ...@@ -2,8 +2,9 @@
require 'spec_helper' require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20191115115043_migrate_epic_mentions_to_db') require Rails.root.join('db', 'post_migrate', '20191115115043_migrate_epic_mentions_to_db')
require Rails.root.join('db', 'post_migrate', '20200214174519_remigrate_epic_mentions_to_db')
describe MigrateEpicMentionsToDb, :migration do describe 'epic mentions migration' do
let(:users) { table(:users) } let(:users) { table(:users) }
let(:namespaces) { table(:namespaces) } let(:namespaces) { table(:namespaces) }
let(:epics) { table(:epics) } let(:epics) { table(:epics) }
...@@ -21,10 +22,14 @@ describe MigrateEpicMentionsToDb, :migration do ...@@ -21,10 +22,14 @@ describe MigrateEpicMentionsToDb, :migration do
# this epic is already migrated, as it has a record in the epic_user_mentions table # this epic is already migrated, as it has a record in the epic_user_mentions table
let!(:resource4) { epics.create!(iid: 4, title: "title3", title_html: 'title3', description: 'epic description with @root mention', group_id: group.id, author_id: user.id) } let!(:resource4) { epics.create!(iid: 4, title: "title3", title_html: 'title3', description: 'epic description with @root mention', group_id: group.id, author_id: user.id) }
let!(:user_mention) { epic_user_mentions.create!(epic_id: resource4.id, mentioned_users_ids: [1]) } let!(:user_mention) { epic_user_mentions.create!(epic_id: resource4.id, mentioned_users_ids: [1]) }
# this epic has no mentions so should be filtered out
let!(:resource5) { epics.create!(iid: 5, title: "title5", title_html: 'title5', description: 'epic description with no mention', group_id: group.id, author_id: user.id) }
before do describe MigrateEpicMentionsToDb, :migration do
stub_const("#{described_class.name}::BATCH_SIZE", 1) it_behaves_like 'schedules resource mentions migration', Epic, false
end end
it_behaves_like 'schedules resource mentions migration', Epic, false describe RemigrateEpicMentionsToDb, :migration do
it_behaves_like 'schedules resource mentions migration', Epic, false
end
end end
...@@ -2,8 +2,9 @@ ...@@ -2,8 +2,9 @@
require 'spec_helper' require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20191115115522_migrate_epic_notes_mentions_to_db') require Rails.root.join('db', 'post_migrate', '20191115115522_migrate_epic_notes_mentions_to_db')
require Rails.root.join('db', 'post_migrate', '20200214174607_remigrate_epic_notes_mentions_to_db')
describe MigrateEpicNotesMentionsToDb, :migration do describe 'epic notes mentions migration' do
let(:users) { table(:users) } let(:users) { table(:users) }
let(:namespaces) { table(:namespaces) } let(:namespaces) { table(:namespaces) }
let(:epics) { table(:epics) } let(:epics) { table(:epics) }
...@@ -26,9 +27,11 @@ describe MigrateEpicNotesMentionsToDb, :migration do ...@@ -26,9 +27,11 @@ describe MigrateEpicNotesMentionsToDb, :migration do
# this note points to an innexistent noteable record # this note points to an innexistent noteable record
let!(:resource5) { notes.create!(note: 'note3 for @root to check', noteable_id: epics.maximum(:id) + 10, noteable_type: 'Epic') } let!(:resource5) { notes.create!(note: 'note3 for @root to check', noteable_id: epics.maximum(:id) + 10, noteable_type: 'Epic') }
before do describe MigrateEpicNotesMentionsToDb, :migration do
stub_const("#{described_class.name}::BATCH_SIZE", 1) it_behaves_like 'schedules resource mentions migration', Epic, true
end end
it_behaves_like 'schedules resource mentions migration', Epic, true describe RemigrateEpicNotesMentionsToDb, :migration do
it_behaves_like 'schedules resource mentions migration', Epic, true
end
end end
...@@ -18,7 +18,6 @@ module Gitlab ...@@ -18,7 +18,6 @@ module Gitlab
self.table_name = 'epics' self.table_name = 'epics'
belongs_to :author, class_name: "User" belongs_to :author, class_name: "User"
belongs_to :project
belongs_to :group belongs_to :group
def self.user_mention_model def self.user_mention_model
......
...@@ -65,9 +65,16 @@ shared_examples 'resource notes mentions migration' do |migration_class, resourc ...@@ -65,9 +65,16 @@ shared_examples 'resource notes mentions migration' do |migration_class, resourc
end end
shared_examples 'schedules resource mentions migration' do |resource_class, is_for_notes| shared_examples 'schedules resource mentions migration' do |resource_class, is_for_notes|
before do
stub_const("#{described_class.name}::BATCH_SIZE", 1)
end
it 'schedules background migrations' do it 'schedules background migrations' do
Sidekiq::Testing.fake! do Sidekiq::Testing.fake! do
Timecop.freeze do Timecop.freeze do
resource_count = is_for_notes ? Note.count : resource_class.count
expect(resource_count).to eq 5
migrate! migrate!
migration = described_class::MIGRATION migration = described_class::MIGRATION
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment