Commit a356b0ef authored by Alex Pooley's avatar Alex Pooley

Retry traversal_ids backfill jobs

Provide a generic method to retry tracked jobs created by #queue_background_migration_jobs_by_range_at_intervals
Retry BackfillNamespaceTraversalIdsRoots jobs
Retry BackfillNamespaceTraversalIdsChildren jobs

Changelog: performance
parent c013d656
# frozen_string_literal: true
class RetryBackfillTraversalIds < ActiveRecord::Migration[6.1]
include Gitlab::Database::MigrationHelpers
ROOTS_MIGRATION = 'BackfillNamespaceTraversalIdsRoots'
CHILDREN_MIGRATION = 'BackfillNamespaceTraversalIdsChildren'
DOWNTIME = false
DELAY_INTERVAL = 2.minutes
disable_ddl_transaction!
def up
duration = requeue_background_migration_jobs_by_range_at_intervals(ROOTS_MIGRATION, DELAY_INTERVAL)
requeue_background_migration_jobs_by_range_at_intervals(CHILDREN_MIGRATION, DELAY_INTERVAL, initial_delay: duration)
end
def down
# no-op
end
end
ec44b7f134de2ea6537c6fe3109fa9d7e32785233f3d1b8e9ea118474d21526a
\ No newline at end of file
......@@ -131,6 +131,42 @@ module Gitlab
final_delay
end
# Requeue pending jobs previously queued with #queue_background_migration_jobs_by_range_at_intervals
#
# This method is useful to schedule jobs that had previously failed.
#
# job_class_name - The background migration job class as a string
# delay_interval - The duration between each job's scheduled time
# batch_size - The maximum number of jobs to fetch to memory from the database.
def requeue_background_migration_jobs_by_range_at_intervals(job_class_name, delay_interval, batch_size: BATCH_SIZE, initial_delay: 0)
# To not overload the worker too much we enforce a minimum interval both
# when scheduling and performing jobs.
delay_interval = [delay_interval, BackgroundMigrationWorker.minimum_interval].max
final_delay = 0
job_counter = 0
jobs = Gitlab::Database::BackgroundMigrationJob.pending.where(class_name: job_class_name)
jobs.each_batch(of: batch_size) do |job_batch|
job_batch.each do |job|
final_delay = initial_delay + delay_interval * job_counter
migrate_in(final_delay, job_class_name, job.arguments)
job_counter += 1
end
end
duration = initial_delay + delay_interval * job_counter
say <<~SAY
Scheduled #{job_counter} #{job_class_name} jobs with an interval of #{delay_interval} seconds.
The migration is expected to take at least #{duration} seconds. Expect all jobs to have completed after #{Time.zone.now + duration}."
SAY
duration
end
# Creates a batched background migration for the given table. A batched migration runs one job
# at a time, computing the bounds of the next batch based on the current migration settings and the previous
# batch bounds. Each job's execution status is tracked in the database as the migration runs. The given job
......
......@@ -242,6 +242,98 @@ RSpec.describe Gitlab::Database::Migrations::BackgroundMigrationHelpers do
end
end
describe '#requeue_background_migration_jobs_by_range_at_intervals' do
let!(:job_class_name) { 'TestJob' }
let!(:pending_job_1) { create(:background_migration_job, class_name: job_class_name, status: :pending, arguments: [1, 2]) }
let!(:pending_job_2) { create(:background_migration_job, class_name: job_class_name, status: :pending, arguments: [3, 4]) }
let!(:successful_job_1) { create(:background_migration_job, class_name: job_class_name, status: :succeeded, arguments: [5, 6]) }
let!(:successful_job_2) { create(:background_migration_job, class_name: job_class_name, status: :succeeded, arguments: [7, 8]) }
around do |example|
freeze_time do
Sidekiq::Testing.fake! do
example.run
end
end
end
subject { model.requeue_background_migration_jobs_by_range_at_intervals(job_class_name, 10.minutes) }
it 'returns the expected duration' do
expect(subject).to eq(20.minutes)
end
context 'when nothing is queued' do
subject { model.requeue_background_migration_jobs_by_range_at_intervals('FakeJob', 10.minutes) }
it 'returns expected duration of zero when nothing gets queued' do
expect(subject).to eq(0)
end
end
it 'queues pending jobs' do
subject
expect(BackgroundMigrationWorker.jobs[0]['args']).to eq([job_class_name, [1, 2]])
expect(BackgroundMigrationWorker.jobs[0]['at']).to be_nil
expect(BackgroundMigrationWorker.jobs[1]['args']).to eq([job_class_name, [3, 4]])
expect(BackgroundMigrationWorker.jobs[1]['at']).to eq(10.minutes.from_now.to_f)
end
context 'with batch_size option' do
subject { model.requeue_background_migration_jobs_by_range_at_intervals(job_class_name, 10.minutes, batch_size: 1) }
it 'returns the expected duration' do
expect(subject).to eq(20.minutes)
end
it 'queues pending jobs' do
subject
expect(BackgroundMigrationWorker.jobs[0]['args']).to eq([job_class_name, [1, 2]])
expect(BackgroundMigrationWorker.jobs[0]['at']).to be_nil
expect(BackgroundMigrationWorker.jobs[1]['args']).to eq([job_class_name, [3, 4]])
expect(BackgroundMigrationWorker.jobs[1]['at']).to eq(10.minutes.from_now.to_f)
end
it 'retrieve jobs in batches' do
jobs = double('jobs')
expect(Gitlab::Database::BackgroundMigrationJob).to receive(:pending) { jobs }
allow(jobs).to receive(:where).with(class_name: job_class_name) { jobs }
expect(jobs).to receive(:each_batch).with(of: 1)
subject
end
end
context 'with initial_delay option' do
let_it_be(:initial_delay) { 3.minutes }
subject { model.requeue_background_migration_jobs_by_range_at_intervals(job_class_name, 10.minutes, initial_delay: initial_delay) }
it 'returns the expected duration' do
expect(subject).to eq(23.minutes)
end
it 'queues pending jobs' do
subject
expect(BackgroundMigrationWorker.jobs[0]['args']).to eq([job_class_name, [1, 2]])
expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(3.minutes.from_now.to_f)
expect(BackgroundMigrationWorker.jobs[1]['args']).to eq([job_class_name, [3, 4]])
expect(BackgroundMigrationWorker.jobs[1]['at']).to eq(13.minutes.from_now.to_f)
end
context 'when nothing is queued' do
subject { model.requeue_background_migration_jobs_by_range_at_intervals('FakeJob', 10.minutes) }
it 'returns expected duration of zero when nothing gets queued' do
expect(subject).to eq(0)
end
end
end
end
describe '#perform_background_migration_inline?' do
it 'returns true in a test environment' do
stub_rails_env('test')
......
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20210604070207_retry_backfill_traversal_ids.rb')
RSpec.describe RetryBackfillTraversalIds, :migration do
include ReloadHelpers
let_it_be(:namespaces_table) { table(:namespaces) }
context 'when BackfillNamespaceTraversalIdsRoots jobs are pending' do
before do
table(:background_migration_jobs).create!(
class_name: 'BackfillNamespaceTraversalIdsRoots',
arguments: [1, 4, 100],
status: Gitlab::Database::BackgroundMigrationJob.statuses['pending']
)
table(:background_migration_jobs).create!(
class_name: 'BackfillNamespaceTraversalIdsRoots',
arguments: [5, 9, 100],
status: Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']
)
end
it 'queues pending jobs' do
migrate!
expect(BackgroundMigrationWorker.jobs.length).to eq(1)
expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['BackfillNamespaceTraversalIdsRoots', [1, 4, 100]])
expect(BackgroundMigrationWorker.jobs[0]['at']).to be_nil
end
end
context 'when BackfillNamespaceTraversalIdsChildren jobs are pending' do
before do
table(:background_migration_jobs).create!(
class_name: 'BackfillNamespaceTraversalIdsChildren',
arguments: [1, 4, 100],
status: Gitlab::Database::BackgroundMigrationJob.statuses['pending']
)
table(:background_migration_jobs).create!(
class_name: 'BackfillNamespaceTraversalIdsRoots',
arguments: [5, 9, 100],
status: Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']
)
end
it 'queues pending jobs' do
migrate!
expect(BackgroundMigrationWorker.jobs.length).to eq(1)
expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['BackfillNamespaceTraversalIdsChildren', [1, 4, 100]])
expect(BackgroundMigrationWorker.jobs[0]['at']).to be_nil
end
end
context 'when BackfillNamespaceTraversalIdsRoots and BackfillNamespaceTraversalIdsChildren jobs are pending' do
before do
table(:background_migration_jobs).create!(
class_name: 'BackfillNamespaceTraversalIdsRoots',
arguments: [1, 4, 100],
status: Gitlab::Database::BackgroundMigrationJob.statuses['pending']
)
table(:background_migration_jobs).create!(
class_name: 'BackfillNamespaceTraversalIdsChildren',
arguments: [5, 9, 100],
status: Gitlab::Database::BackgroundMigrationJob.statuses['pending']
)
table(:background_migration_jobs).create!(
class_name: 'BackfillNamespaceTraversalIdsRoots',
arguments: [11, 14, 100],
status: Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']
)
table(:background_migration_jobs).create!(
class_name: 'BackfillNamespaceTraversalIdsChildren',
arguments: [15, 19, 100],
status: Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']
)
end
it 'queues pending jobs' do
freeze_time do
migrate!
expect(BackgroundMigrationWorker.jobs.length).to eq(2)
expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['BackfillNamespaceTraversalIdsRoots', [1, 4, 100]])
expect(BackgroundMigrationWorker.jobs[0]['at']).to be_nil
expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['BackfillNamespaceTraversalIdsChildren', [5, 9, 100]])
expect(BackgroundMigrationWorker.jobs[1]['at']).to eq(RetryBackfillTraversalIds::DELAY_INTERVAL.from_now.to_f)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment