Commit f88d5132 authored by Andreas Brandl's avatar Andreas Brandl

Simpler migration strategy for MySQL.

MySQL does not return the number of inserted rows (or at least not
expose them through cmd_tuples). Let's do one-shot INSERT instead.
parent f579a087
...@@ -6,31 +6,12 @@ class BuildUserContributedProjectsTable < ActiveRecord::Migration ...@@ -6,31 +6,12 @@ class BuildUserContributedProjectsTable < ActiveRecord::Migration
disable_ddl_transaction! disable_ddl_transaction!
BATCH_SIZE = 100000
SLEEP_TIME = 30
def up def up
with_index(:events, [:author_id, :project_id], name: 'events_user_contributions_temp', where: 'project_id IS NOT NULL') do if Gitlab::Database.postgresql?
iteration = 0 PostgresStrategy.new
records = 0 else
begin MysqlStrategy.new
Rails.logger.info "Building user_contributed_projects table, batch ##{iteration}" end.up
result = execute <<~SQL
INSERT INTO user_contributed_projects (user_id, project_id)
SELECT e.user_id, e.project_id
FROM (SELECT DISTINCT author_id AS user_id, project_id FROM events WHERE project_id IS NOT NULL) AS e
LEFT JOIN user_contributed_projects ucp USING (user_id, project_id)
WHERE ucp.user_id IS NULL
LIMIT #{BATCH_SIZE}
SQL
iteration += 1
records += result.cmd_tuples
Rails.logger.info "Building user_contributed_projects table, batch ##{iteration} complete, created #{records} overall"
Kernel.sleep(SLEEP_TIME) if result.cmd_tuples > 0
end while result.cmd_tuples > 0
end
execute "ANALYZE user_contributed_projects" if Gitlab::Database.postgresql?
end end
def down def down
...@@ -39,10 +20,58 @@ class BuildUserContributedProjectsTable < ActiveRecord::Migration ...@@ -39,10 +20,58 @@ class BuildUserContributedProjectsTable < ActiveRecord::Migration
private private
def with_index(*args) class PostgresStrategy < ActiveRecord::Migration
add_concurrent_index(*args) unless index_exists?(*args) include Gitlab::Database::MigrationHelpers
yield
ensure BATCH_SIZE = 100_000
remove_concurrent_index(*args) if index_exists?(*args) SLEEP_TIME = 30
def up
with_index(:events, [:author_id, :project_id], name: 'events_user_contributions_temp', where: 'project_id IS NOT NULL') do
iteration = 0
records = 0
begin
Rails.logger.info "Building user_contributed_projects table, batch ##{iteration}"
result = execute <<~SQL
INSERT INTO user_contributed_projects (user_id, project_id)
SELECT e.user_id, e.project_id
FROM (SELECT DISTINCT author_id AS user_id, project_id FROM events WHERE project_id IS NOT NULL) AS e
LEFT JOIN user_contributed_projects ucp USING (user_id, project_id)
WHERE ucp.user_id IS NULL
LIMIT #{BATCH_SIZE}
SQL
iteration += 1
records += result.cmd_tuples
Rails.logger.info "Building user_contributed_projects table, batch ##{iteration} complete, created #{records} overall"
Kernel.sleep(SLEEP_TIME) if result.cmd_tuples > 0
end while result.cmd_tuples > 0
end
execute "ANALYZE user_contributed_projects"
end
private
def with_index(*args)
add_concurrent_index(*args) unless index_exists?(*args)
yield
ensure
remove_concurrent_index(*args) if index_exists?(*args)
end
end end
class MysqlStrategy < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
def up
execute <<~SQL
INSERT INTO user_contributed_projects (user_id, project_id)
SELECT e.user_id, e.project_id
FROM (SELECT DISTINCT author_id AS user_id, project_id FROM events WHERE project_id IS NOT NULL) AS e
LEFT JOIN user_contributed_projects ucp USING (user_id, project_id)
WHERE ucp.user_id IS NULL
SQL
end
end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment