Commit d7dbdcf9 authored by Adam Hegyi's avatar Adam Hegyi

Merge branch 'backfill-routes-namespace-id-for-projects' into 'master'

Backfill routes namespace_id for projects

See merge request gitlab-org/gitlab!81121
parents 27ccdd95 8bda0cf8
# frozen_string_literal: true
class AddTmpIndexRoutesIdForProjectNamespaces < Gitlab::Database::Migration[1.0]
INDEX_NAME = 'tmp_index_for_project_namespace_id_migration_on_routes'
disable_ddl_transaction!
def up
# Temporary index to be removed in 15.0
# https://gitlab.com/gitlab-org/gitlab/-/issues/352353
add_concurrent_index :routes, :id, where: "namespace_id IS NULL AND source_type = 'Project'", name: INDEX_NAME
end
def down
remove_concurrent_index_by_name :routes, INDEX_NAME
end
end
# frozen_string_literal: true
class BackfillNamespaceIdForProjectRoutes < Gitlab::Database::Migration[1.0]
disable_ddl_transaction!
MIGRATION = 'BackfillNamespaceIdForProjectRoute'
INTERVAL = 2.minutes
BATCH_SIZE = 1_000
MAX_BATCH_SIZE = 10_000
SUB_BATCH_SIZE = 200
def up
queue_batched_background_migration(
MIGRATION,
:routes,
:id,
job_interval: INTERVAL,
batch_size: BATCH_SIZE,
max_batch_size: MAX_BATCH_SIZE,
sub_batch_size: SUB_BATCH_SIZE
)
end
def down
Gitlab::Database::BackgroundMigration::BatchedMigration
.for_configuration(MIGRATION, :routes, :id, [])
.delete_all
end
end
1aefb5950063a060de1ec20b0808a5488b238b36d86120c34ac5a128c212984e
\ No newline at end of file
1681c19d1f41a05c3dfeded70d128989afb4a81a2e04aacc8879c2c1ab766733
\ No newline at end of file
...@@ -29667,6 +29667,8 @@ CREATE INDEX tmp_index_for_namespace_id_migration_on_routes ON routes USING btre ...@@ -29667,6 +29667,8 @@ CREATE INDEX tmp_index_for_namespace_id_migration_on_routes ON routes USING btre
CREATE INDEX tmp_index_for_null_project_namespace_id ON projects USING btree (id) WHERE (project_namespace_id IS NULL); CREATE INDEX tmp_index_for_null_project_namespace_id ON projects USING btree (id) WHERE (project_namespace_id IS NULL);
CREATE INDEX tmp_index_for_project_namespace_id_migration_on_routes ON routes USING btree (id) WHERE ((namespace_id IS NULL) AND ((source_type)::text = 'Project'::text));
CREATE INDEX tmp_index_issues_on_issue_type_and_id ON issues USING btree (issue_type, id); CREATE INDEX tmp_index_issues_on_issue_type_and_id ON issues USING btree (issue_type, id);
CREATE INDEX tmp_index_members_on_state ON members USING btree (state) WHERE (state = 2); CREATE INDEX tmp_index_members_on_state ON members USING btree (state) WHERE (state = 2);
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# Backfills the `routes.namespace_id` column, by setting it to project.project_namespace_id
class BackfillNamespaceIdForProjectRoute
include Gitlab::Database::DynamicModelHelpers
def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms)
parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id)
parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
cleanup_gin_index('routes')
batch_metrics.time_operation(:update_all) do
ActiveRecord::Base.connection.execute <<~SQL
WITH route_and_ns(route_id, project_namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
#{sub_batch.to_sql}
)
UPDATE routes
SET namespace_id = route_and_ns.project_namespace_id
FROM route_and_ns
WHERE id = route_and_ns.route_id
SQL
end
pause_ms = [0, pause_ms].max
sleep(pause_ms * 0.001)
end
end
def batch_metrics
@batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new
end
private
def cleanup_gin_index(table_name)
sql = "select indexname::text from pg_indexes where tablename = '#{table_name}' and indexdef ilike '%gin%'"
index_names = ActiveRecord::Base.connection.select_values(sql)
index_names.each do |index_name|
ActiveRecord::Base.connection.execute("select gin_clean_pending_list('#{index_name}')")
end
end
def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id)
define_batchable_model(source_table, connection: ActiveRecord::Base.connection)
.joins('INNER JOIN projects ON routes.source_id = projects.id')
.where(source_key_column => start_id..stop_id)
.where(namespace_id: nil)
.where(source_type: 'Project')
.where.not(projects: { project_namespace_id: nil })
.select("routes.id, projects.project_namespace_id")
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceIdForProjectRoute do
let(:migration) { described_class.new }
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:routes) { table(:routes) }
let(:namespace1) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') }
let(:namespace2) { namespaces.create!(name: 'batchtest2', type: 'Group', parent_id: namespace1.id, path: 'space2') }
let(:namespace3) { namespaces.create!(name: 'batchtest3', type: 'Group', parent_id: namespace2.id, path: 'space3') }
let(:proj_namespace1) { namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: namespace1.id) }
let(:proj_namespace2) { namespaces.create!(name: 'proj2', path: 'proj2', type: 'Project', parent_id: namespace2.id) }
let(:proj_namespace3) { namespaces.create!(name: 'proj3', path: 'proj3', type: 'Project', parent_id: namespace3.id) }
let(:proj_namespace4) { namespaces.create!(name: 'proj4', path: 'proj4', type: 'Project', parent_id: namespace3.id) }
# rubocop:disable Layout/LineLength
let(:proj1) { projects.create!(name: 'proj1', path: 'proj1', namespace_id: namespace1.id, project_namespace_id: proj_namespace1.id) }
let(:proj2) { projects.create!(name: 'proj2', path: 'proj2', namespace_id: namespace2.id, project_namespace_id: proj_namespace2.id) }
let(:proj3) { projects.create!(name: 'proj3', path: 'proj3', namespace_id: namespace3.id, project_namespace_id: proj_namespace3.id) }
let(:proj4) { projects.create!(name: 'proj4', path: 'proj4', namespace_id: namespace3.id, project_namespace_id: proj_namespace4.id) }
# rubocop:enable Layout/LineLength
let!(:namespace_route1) { routes.create!(path: 'space1', source_id: namespace1.id, source_type: 'Namespace') }
let!(:namespace_route2) { routes.create!(path: 'space1/space2', source_id: namespace2.id, source_type: 'Namespace') }
let!(:namespace_route3) { routes.create!(path: 'space1/space3', source_id: namespace3.id, source_type: 'Namespace') }
let!(:proj_route1) { routes.create!(path: 'space1/proj1', source_id: proj1.id, source_type: 'Project') }
let!(:proj_route2) { routes.create!(path: 'space1/space2/proj2', source_id: proj2.id, source_type: 'Project') }
let!(:proj_route3) { routes.create!(path: 'space1/space3/proj3', source_id: proj3.id, source_type: 'Project') }
let!(:proj_route4) { routes.create!(path: 'space1/space3/proj4', source_id: proj4.id, source_type: 'Project') }
subject(:perform_migration) { migration.perform(proj_route1.id, proj_route4.id, :routes, :id, 2, 0) }
it 'backfills namespace_id for the selected records', :aggregate_failures do
perform_migration
expected_namespaces = [proj_namespace1.id, proj_namespace2.id, proj_namespace3.id, proj_namespace4.id]
expected_projects = [proj_route1.id, proj_route2.id, proj_route3.id, proj_route4.id]
expect(routes.where.not(namespace_id: nil).pluck(:id)).to match_array(expected_projects)
expect(routes.where.not(namespace_id: nil).pluck(:namespace_id)).to match_array(expected_namespaces)
end
it 'tracks timings of queries' do
expect(migration.batch_metrics.timings).to be_empty
expect { perform_migration }.to change { migration.batch_metrics.timings }
end
end
# frozen_string_literal: true
require 'spec_helper'
require_migration!
RSpec.describe BackfillNamespaceIdForProjectRoutes, :migration do
let(:migration) { described_class::MIGRATION }
describe '#up' do
it 'schedules background jobs for each batch of group members' do
migrate!
expect(migration).to have_scheduled_batched_migration(
table_name: :routes,
column_name: :id,
interval: described_class::INTERVAL
)
end
end
describe '#down' do
it 'deletes all batched migration records' do
migrate!
schema_migrate_down!
expect(migration).not_to have_scheduled_batched_migration
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment