Commit c34a4b6b authored by Toon Claes's avatar Toon Claes

Merge branch '21801-remigrate-epic-user-mentions' into 'master'

Migrate user mentions from epic and epic notes

Closes #21801

See merge request gitlab-org/gitlab!25259
parents 54cd52b6 7660a1c8
# frozen_string_literal: true
class MigrateEpicMentionsToDb < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
......@@ -26,7 +28,7 @@ class MigrateEpicMentionsToDb < ActiveRecord::Migration[5.2]
.where(QUERY_CONDITIONS)
.each_batch(of: BATCH_SIZE) do |batch, index|
range = batch.pluck(Arel.sql('MIN(epics.id)'), Arel.sql('MAX(epics.id)')).first
BackgroundMigrationWorker.perform_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, false, *range])
migrate_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, false, *range])
end
end
......
# frozen_string_literal: true
class CleanupEmptyEpicUserMentions < ActiveRecord::Migration[5.2]
DOWNTIME = false
BATCH_SIZE = 10000
class EpicUserMention < ActiveRecord::Base
include EachBatch
self.table_name = 'epic_user_mentions'
end
def up
return unless Gitlab.ee?
# cleanup epic user mentions with no actual mentions,
# re https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24586#note_285982468
EpicUserMention
.where(mentioned_users_ids: nil)
.where(mentioned_groups_ids: nil)
.where(mentioned_projects_ids: nil)
.each_batch(of: BATCH_SIZE) do |batch|
batch.delete_all
end
end
def down
# no-op
end
end
# frozen_string_literal: true
class RemigrateEpicMentionsToDb < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
DELAY = 2.minutes.to_i
BATCH_SIZE = 10000
MIGRATION = 'UserMentions::CreateResourceUserMention'
JOIN = "LEFT JOIN epic_user_mentions on epics.id = epic_user_mentions.epic_id"
QUERY_CONDITIONS = "(description like '%@%' OR title like '%@%') AND epic_user_mentions.epic_id is null"
class Epic < ActiveRecord::Base
include EachBatch
self.table_name = 'epics'
end
def up
return unless Gitlab.ee?
Epic
.joins(JOIN)
.where(QUERY_CONDITIONS)
.each_batch(of: BATCH_SIZE) do |batch, index|
range = batch.pluck(Arel.sql('MIN(epics.id)'), Arel.sql('MAX(epics.id)')).first
migrate_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, false, *range])
end
end
def down
# no-op
end
end
# frozen_string_literal: true
class RemigrateEpicNotesMentionsToDb < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
DELAY = 2.minutes.to_i
BATCH_SIZE = 10000
MIGRATION = 'UserMentions::CreateResourceUserMention'
INDEX_NAME = 'epic_mentions_temp_index'
INDEX_CONDITION = "note LIKE '%@%'::text AND notes.noteable_type = 'Epic'"
QUERY_CONDITIONS = "#{INDEX_CONDITION} AND epic_user_mentions.epic_id IS NULL"
JOIN = 'INNER JOIN epics ON epics.id = notes.noteable_id LEFT JOIN epic_user_mentions ON notes.id = epic_user_mentions.note_id'
class Note < ActiveRecord::Base
include EachBatch
self.table_name = 'notes'
end
def up
return unless Gitlab.ee?
# create temporary index for notes with mentions, may take well over 1h
add_concurrent_index(:notes, :id, where: INDEX_CONDITION, name: INDEX_NAME)
Note
.joins(JOIN)
.where(QUERY_CONDITIONS)
.each_batch(of: BATCH_SIZE) do |batch, index|
range = batch.pluck(Arel.sql('MIN(notes.id)'), Arel.sql('MAX(notes.id)')).first
migrate_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, true, *range])
end
end
def down
# no-op
# temporary index is to be dropped in a different migration in an upcoming release:
# https://gitlab.com/gitlab-org/gitlab/issues/196842
end
end
......@@ -3,105 +3,117 @@
require 'spec_helper'
require './db/post_migrate/20191115115043_migrate_epic_mentions_to_db'
require './db/post_migrate/20191115115522_migrate_epic_notes_mentions_to_db'
require './db/post_migrate/20200214174519_remigrate_epic_mentions_to_db'
require './db/post_migrate/20200214174607_remigrate_epic_notes_mentions_to_db'
require './db/post_migrate/20200124110831_migrate_design_notes_mentions_to_db'
describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMention do
include MigrationsHelpers
let(:users) { table(:users) }
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:notes) { table(:notes) }
let(:author) { users.create!(email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active') }
let(:member) { users.create!(email: 'member@example.com', notification_email: 'member@example.com', name: 'member', username: 'member', projects_limit: 10, state: 'active') }
let(:admin) { users.create!(email: 'administrator@example.com', notification_email: 'administrator@example.com', name: 'administrator', username: 'administrator', admin: 1, projects_limit: 10, state: 'active') }
let(:john_doe) { users.create!(email: 'john_doe@example.com', notification_email: 'john_doe@example.com', name: 'john_doe', username: 'john_doe', projects_limit: 10, state: 'active') }
let(:skipped) { users.create!(email: 'skipped@example.com', notification_email: 'skipped@example.com', name: 'skipped', username: 'skipped', projects_limit: 10, state: 'active') }
let(:mentioned_users) { [author, member, admin, john_doe, skipped] }
let(:mentioned_users_refs) { mentioned_users.map { |u| "@#{u.username}" }.join(' ') }
let(:group) { namespaces.create!(name: 'test1', path: 'test1', runners_token: 'my-token1', project_creation_level: 1, visibility_level: 20, type: 'Group') }
let(:inaccessible_group) { namespaces.create!(name: 'test2', path: 'test2', runners_token: 'my-token2', project_creation_level: 1, visibility_level: 0, type: 'Group') }
let(:project) { projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1', namespace_id: group.id, visibility_level: 0) }
let(:mentioned_groups) { [group, inaccessible_group] }
let(:group_mentions) { [group, inaccessible_group].map { |gr| "@#{gr.path}" }.join(' ') }
let(:description_mentions) { "description with mentions #{mentioned_users_refs} and #{group_mentions}" }
before do
# build personal namespaces and routes for users
mentioned_users.each { |u| u.becomes(User).save! }
# build namespaces and routes for groups
mentioned_groups.each do |gr|
gr.name += '-org'
gr.path += '-org'
gr.becomes(Namespace).save!
end
end
context 'when migrating data' do
let(:users) { table(:users) }
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:notes) { table(:notes) }
describe 'epic mentions' do
let(:epics) { table(:epics) }
let(:epic_user_mentions) { table(:epic_user_mentions) }
let(:author) { users.create!(email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active') }
let(:member) { users.create!(email: 'member@example.com', notification_email: 'member@example.com', name: 'member', username: 'member', projects_limit: 10, state: 'active') }
let(:admin) { users.create!(email: 'administrator@example.com', notification_email: 'administrator@example.com', name: 'administrator', username: 'administrator', admin: 1, projects_limit: 10, state: 'active') }
let(:john_doe) { users.create!(email: 'john_doe@example.com', notification_email: 'john_doe@example.com', name: 'john_doe', username: 'john_doe', projects_limit: 10, state: 'active') }
let(:skipped) { users.create!(email: 'skipped@example.com', notification_email: 'skipped@example.com', name: 'skipped', username: 'skipped', projects_limit: 10, state: 'active') }
context 'mentions in epic description' do
let!(:epic) do
epics.create!(iid: 1, group_id: group.id, author_id: author.id, title: "epic title @#{author.username}",
title_html: "epic title @#{author.username}", description: description_mentions)
end
let!(:epic2) do
epics.create!(iid: 2, group_id: group.id, author_id: author.id, title: "epic title}",
title_html: "epic title", description: 'simple description')
end
let!(:epic3) do
epics.create!(iid: 2, group_id: group.id, author_id: author.id, title: "epic title}",
title_html: "epic title", description: 'description with an email@example.com and some other @ char here.')
end
let(:mentioned_users) { [author, member, admin, john_doe, skipped] }
let(:mentioned_users_refs) { mentioned_users.map { |u| "@#{u.username}" }.join(' ') }
let(:user_mentions) { epic_user_mentions }
let(:resource) { epic }
let(:group) { namespaces.create!(name: 'test1', path: 'test1', runners_token: 'my-token1', project_creation_level: 1, visibility_level: 20, type: 'Group') }
let(:inaccessible_group) { namespaces.create!(name: 'test2', path: 'test2', runners_token: 'my-token2', project_creation_level: 1, visibility_level: 0, type: 'Group') }
let(:project) { projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1', namespace_id: group.id, visibility_level: 0) }
it_behaves_like 'resource mentions migration', MigrateEpicMentionsToDb, Epic
let(:mentioned_groups) { [group, inaccessible_group] }
let(:group_mentions) { [group, inaccessible_group].map { |gr| "@#{gr.path}" }.join(' ') }
let(:description_mentions) { "description with mentions #{mentioned_users_refs} and #{group_mentions}" }
it 'has correct no_quote_columns' do
expect(Gitlab::BackgroundMigration::UserMentions::Models::Epic.no_quote_columns).to match([:note_id, :epic_id])
end
before do
# build personal namespaces and routes for users
mentioned_users.each { |u| u.becomes(User).save! }
context 'mentions in epic notes' do
let!(:note1) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: description_mentions) }
let!(:note2) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: 'sample note') }
let!(:note3) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: description_mentions, system: true) }
# this not does not have actual mentions
let!(:note4) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: 'note3 for an email@somesite.com and some other rando @ ref' ) }
# this not points to an innexistent noteable record in desigs table
let!(:note5) { notes.create!(noteable_id: epics.maximum(:id) + 10, noteable_type: 'Epic', author_id: author.id, note: description_mentions, project_id: project.id) }
# build namespaces and routes for groups
mentioned_groups.each do |gr|
gr.name += '-org'
gr.path += '-org'
gr.becomes(Namespace).save!
end
end
it_behaves_like 'resource notes mentions migration', MigrateEpicNotesMentionsToDb, Epic
describe 'epic mentions' do
let(:epics) { table(:epics) }
let(:epic_user_mentions) { table(:epic_user_mentions) }
context 'mentions in epic description' do
let!(:epic) do
epics.create!(iid: 1, group_id: group.id, author_id: author.id, title: "epic title @#{author.username}",
title_html: "epic title @#{author.username}", description: description_mentions)
end
let!(:epic2) do
epics.create!(iid: 2, group_id: group.id, author_id: author.id, title: "epic title}",
title_html: "epic title", description: 'simple description')
end
let!(:epic3) do
epics.create!(iid: 2, group_id: group.id, author_id: author.id, title: "epic title}",
title_html: "epic title", description: 'description with an email@example.com and some other @ char here.')
end
let(:user_mentions) { epic_user_mentions }
let(:resource) { epic }
it_behaves_like 'resource mentions migration', MigrateEpicMentionsToDb, Epic
it_behaves_like 'resource mentions migration', RemigrateEpicMentionsToDb, Epic
context 'mentions in epic notes' do
let!(:note1) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: description_mentions) }
let!(:note2) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: 'sample note') }
let!(:note3) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: description_mentions, system: true) }
# this not does not have actual mentions
let!(:note4) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: 'note3 for an email@somesite.com and some other rando @ ref' ) }
# this not points to an innexistent noteable record in desigs table
let!(:note5) { notes.create!(noteable_id: epics.maximum(:id) + 10, noteable_type: 'Epic', author_id: author.id, note: description_mentions, project_id: project.id) }
it_behaves_like 'resource notes mentions migration', MigrateEpicNotesMentionsToDb, Epic
it_behaves_like 'resource notes mentions migration', RemigrateEpicNotesMentionsToDb, Epic
end
end
end
end
describe 'design mentions' do
let(:projects) { table(:projects) }
let(:designs) { table(:design_management_designs) }
let(:design_user_mentions) { table(:design_user_mentions) }
describe 'design mentions' do
let(:projects) { table(:projects) }
let(:designs) { table(:design_management_designs) }
let(:design_user_mentions) { table(:design_user_mentions) }
let(:project) { projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1', namespace_id: group.id, visibility_level: 0) }
let!(:design) { designs.create!(filename: 'test.png', project_id: project.id) }
let(:project) { projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1', namespace_id: group.id, visibility_level: 0) }
let!(:design) { designs.create!(filename: 'test.png', project_id: project.id) }
let!(:note1) { notes.create!(noteable_id: design.id, noteable_type: 'DesignManagement::Design', project_id: project.id, author_id: author.id, note: description_mentions) }
let!(:note2) { notes.create!(noteable_id: design.id, noteable_type: 'DesignManagement::Design', project_id: project.id, author_id: author.id, note: 'sample note') }
let!(:note3) { notes.create!(noteable_id: design.id, noteable_type: 'DesignManagement::Design', project_id: project.id, author_id: author.id, note: description_mentions, system: true) }
# this not does not have actual mentions
let!(:note4) { notes.create!(noteable_id: design.id, noteable_type: 'DesignManagement::Design', project_id: project.id, author_id: author.id, note: 'note3 for an email@somesite.com and some other rando @ ref' ) }
# this not points to an innexistent noteable record in desigs table
let!(:note5) { notes.create!(noteable_id: designs.maximum(:id) + 10, noteable_type: 'DesignManagement::Design', project_id: project.id, author_id: author.id, note: description_mentions) }
let!(:note1) { notes.create!(noteable_id: design.id, noteable_type: 'DesignManagement::Design', project_id: project.id, author_id: author.id, note: description_mentions) }
let!(:note2) { notes.create!(noteable_id: design.id, noteable_type: 'DesignManagement::Design', project_id: project.id, author_id: author.id, note: 'sample note') }
let!(:note3) { notes.create!(noteable_id: design.id, noteable_type: 'DesignManagement::Design', project_id: project.id, author_id: author.id, note: description_mentions, system: true) }
# this not does not have actual mentions
let!(:note4) { notes.create!(noteable_id: design.id, noteable_type: 'DesignManagement::Design', project_id: project.id, author_id: author.id, note: 'note3 for an email@somesite.com and some other rando @ ref' ) }
# this not points to an innexistent noteable record in desigs table
let!(:note5) { notes.create!(noteable_id: designs.maximum(:id) + 10, noteable_type: 'DesignManagement::Design', project_id: project.id, author_id: author.id, note: description_mentions) }
let(:user_mentions) { design_user_mentions }
let(:resource) { design }
let(:user_mentions) { design_user_mentions }
let(:resource) { design }
it_behaves_like 'resource notes mentions migration', MigrateDesignNotesMentionsToDb, DesignManagement::Design
it_behaves_like 'resource notes mentions migration', MigrateDesignNotesMentionsToDb, DesignManagement::Design
end
end
context 'checks no_quote_columns' do
it 'checks that epic has correct no_quote_columns' do
expect(Gitlab::BackgroundMigration::UserMentions::Models::Epic.no_quote_columns).to match([:note_id, :epic_id])
end
it 'checks that Design has correct no_quote_columns' do
expect(Gitlab::BackgroundMigration::UserMentions::Models::DesignManagement::Design.no_quote_columns).to match([:note_id, :design_id])
end
end
end
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20200214173000_cleanup_empty_epic_user_mentions')
describe CleanupEmptyEpicUserMentions, :migration, :sidekiq do
let(:users) { table(:users) }
let(:namespaces) { table(:namespaces) }
let(:epics) { table(:epics) }
let(:notes) { table(:notes) }
let(:epic_user_mentions) { table(:epic_user_mentions) }
let(:user) { users.create!(name: 'root', email: 'root@example.com', username: 'root', projects_limit: 0) }
let(:group) { namespaces.create!(name: 'group1', path: 'group1', owner_id: user.id, type: 'Group') }
let(:epic) { epics.create!(iid: 1, title: "title", title_html: 'title', description: 'epic description', group_id: group.id, author_id: user.id) }
# migrateable resources
let!(:resource1) { notes.create!(note: 'note1 for @root to check', noteable_id: epic.id, noteable_type: 'Epic') }
let!(:resource2) { notes.create!(note: 'note2 for @root to check', noteable_id: epic.id, noteable_type: 'Epic', system: true) }
let!(:resource3) { notes.create!(note: 'note3 for @root to check', noteable_id: epic.id, noteable_type: 'Epic') }
# non-migrateable resources
# this note is already migrated, as it has a record in the epic_user_mentions table
let!(:resource4) { notes.create!(note: 'note3 for @root to check', noteable_id: epic.id, noteable_type: 'Epic') }
let!(:user_mention) { epic_user_mentions.create!(epic_id: epic.id, note_id: resource4.id, mentioned_users_ids: [1]) }
# these should get cleanup, by the migration
let!(:blank_epic_user_mention1) { epic_user_mentions.create!(epic_id: epic.id, note_id: resource1.id)}
let!(:blank_epic_user_mention2) { epic_user_mentions.create!(epic_id: epic.id, note_id: resource2.id)}
let!(:blank_epic_user_mention3) { epic_user_mentions.create!(epic_id: epic.id, note_id: resource3.id)}
it 'cleans blank user mentions' do
expect(epic_user_mentions.count).to eq 4
migrate!
expect(epic_user_mentions.count).to eq 1
end
end
......@@ -27,9 +27,5 @@ describe MigrateDesignNotesMentionsToDb, :migration, :sidekiq do
# this note points to an innexistent noteable record
let!(:resource5) { notes.create!(note: 'note3 for @root to check', noteable_id: designs.maximum(:id) + 10, noteable_type: 'DesignManagement::Design') }
before do
stub_const("#{described_class.name}::BATCH_SIZE", 1)
end
it_behaves_like 'schedules resource mentions migration', DesignManagement::Design, true
end
......@@ -2,8 +2,9 @@
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20191115115043_migrate_epic_mentions_to_db')
require Rails.root.join('db', 'post_migrate', '20200214174519_remigrate_epic_mentions_to_db')
describe MigrateEpicMentionsToDb, :migration do
describe 'epic mentions migration' do
let(:users) { table(:users) }
let(:namespaces) { table(:namespaces) }
let(:epics) { table(:epics) }
......@@ -21,10 +22,14 @@ describe MigrateEpicMentionsToDb, :migration do
# this epic is already migrated, as it has a record in the epic_user_mentions table
let!(:resource4) { epics.create!(iid: 4, title: "title3", title_html: 'title3', description: 'epic description with @root mention', group_id: group.id, author_id: user.id) }
let!(:user_mention) { epic_user_mentions.create!(epic_id: resource4.id, mentioned_users_ids: [1]) }
# this epic has no mentions so should be filtered out
let!(:resource5) { epics.create!(iid: 5, title: "title5", title_html: 'title5', description: 'epic description with no mention', group_id: group.id, author_id: user.id) }
before do
stub_const("#{described_class.name}::BATCH_SIZE", 1)
describe MigrateEpicMentionsToDb, :migration do
it_behaves_like 'schedules resource mentions migration', Epic, false
end
it_behaves_like 'schedules resource mentions migration', Epic, false
describe RemigrateEpicMentionsToDb, :migration do
it_behaves_like 'schedules resource mentions migration', Epic, false
end
end
......@@ -2,8 +2,9 @@
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20191115115522_migrate_epic_notes_mentions_to_db')
require Rails.root.join('db', 'post_migrate', '20200214174607_remigrate_epic_notes_mentions_to_db')
describe MigrateEpicNotesMentionsToDb, :migration do
describe 'epic notes mentions migration' do
let(:users) { table(:users) }
let(:namespaces) { table(:namespaces) }
let(:epics) { table(:epics) }
......@@ -26,9 +27,11 @@ describe MigrateEpicNotesMentionsToDb, :migration do
# this note points to an innexistent noteable record
let!(:resource5) { notes.create!(note: 'note3 for @root to check', noteable_id: epics.maximum(:id) + 10, noteable_type: 'Epic') }
before do
stub_const("#{described_class.name}::BATCH_SIZE", 1)
describe MigrateEpicNotesMentionsToDb, :migration do
it_behaves_like 'schedules resource mentions migration', Epic, true
end
it_behaves_like 'schedules resource mentions migration', Epic, true
describe RemigrateEpicNotesMentionsToDb, :migration do
it_behaves_like 'schedules resource mentions migration', Epic, true
end
end
......@@ -18,7 +18,6 @@ module Gitlab
self.table_name = 'epics'
belongs_to :author, class_name: "User"
belongs_to :project
belongs_to :group
def self.user_mention_model
......
......@@ -65,9 +65,16 @@ shared_examples 'resource notes mentions migration' do |migration_class, resourc
end
shared_examples 'schedules resource mentions migration' do |resource_class, is_for_notes|
before do
stub_const("#{described_class.name}::BATCH_SIZE", 1)
end
it 'schedules background migrations' do
Sidekiq::Testing.fake! do
Timecop.freeze do
resource_count = is_for_notes ? Note.count : resource_class.count
expect(resource_count).to eq 5
migrate!
migration = described_class::MIGRATION
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment