Retry traversal_ids backfill jobs

Provide a generic method to retry tracked jobs created by #queue_background_migration_jobs_by_range_at_intervals
Retry BackfillNamespaceTraversalIdsRoots jobs
Retry BackfillNamespaceTraversalIdsChildren jobs

Changelog: performance

db/post_migrate/20210604070207_retry_backfill_traversal_ids.rb

+# frozen_string_literal: true
+
+class RetryBackfillTraversalIds < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  ROOTS_MIGRATION = 'BackfillNamespaceTraversalIdsRoots'
+  CHILDREN_MIGRATION = 'BackfillNamespaceTraversalIdsChildren'
+  DOWNTIME = false
+  DELAY_INTERVAL = 2.minutes
+
+  disable_ddl_transaction!
+
+  def up
+    duration = requeue_background_migration_jobs_by_range_at_intervals(ROOTS_MIGRATION, DELAY_INTERVAL)
+    requeue_background_migration_jobs_by_range_at_intervals(CHILDREN_MIGRATION, DELAY_INTERVAL, initial_delay: duration)
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema_migrations/20210604070207

+ec44b7f134de2ea6537c6fe3109fa9d7e32785233f3d1b8e9ea118474d21526a
\ No newline at end of file

lib/gitlab/database/migrations/background_migration_helpers.rb

@@ -131,6 +131,42 @@ module Gitlab
           final_delay
         end
 
+        # Requeue pending jobs previously queued with #queue_background_migration_jobs_by_range_at_intervals
+        #
+        # This method is useful to schedule jobs that had previously failed.
+        #
+        # job_class_name - The background migration job class as a string
+        # delay_interval - The duration between each job's scheduled time
+        # batch_size - The maximum number of jobs to fetch to memory from the database.
+        def requeue_background_migration_jobs_by_range_at_intervals(job_class_name, delay_interval, batch_size: BATCH_SIZE, initial_delay: 0)
+          # To not overload the worker too much we enforce a minimum interval both
+          # when scheduling and performing jobs.
+          delay_interval = [delay_interval, BackgroundMigrationWorker.minimum_interval].max
+
+          final_delay = 0
+          job_counter = 0
+
+          jobs = Gitlab::Database::BackgroundMigrationJob.pending.where(class_name: job_class_name)
+          jobs.each_batch(of: batch_size) do |job_batch|
+            job_batch.each do |job|
+              final_delay = initial_delay + delay_interval * job_counter
+
+              migrate_in(final_delay, job_class_name, job.arguments)
+
+              job_counter += 1
+            end
+          end
+
+          duration = initial_delay + delay_interval * job_counter
+          say <<~SAY
+            Scheduled #{job_counter} #{job_class_name} jobs with an interval of #{delay_interval} seconds.
+
+            The migration is expected to take at least #{duration} seconds. Expect all jobs to have completed after #{Time.zone.now + duration}."
+          SAY
+
+          duration
+        end
+
         # Creates a batched background migration for the given table. A batched migration runs one job
         # at a time, computing the bounds of the next batch based on the current migration settings and the previous
         # batch bounds. Each job's execution status is tracked in the database as the migration runs. The given job

spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb

@@ -242,6 +242,98 @@ RSpec.describe Gitlab::Database::Migrations::BackgroundMigrationHelpers do
     end
   end
 
+  describe '#requeue_background_migration_jobs_by_range_at_intervals' do
+    let!(:job_class_name) { 'TestJob' }
+    let!(:pending_job_1) { create(:background_migration_job, class_name: job_class_name, status: :pending, arguments: [1, 2]) }
+    let!(:pending_job_2) { create(:background_migration_job, class_name: job_class_name, status: :pending, arguments: [3, 4]) }
+    let!(:successful_job_1) { create(:background_migration_job, class_name: job_class_name, status: :succeeded, arguments: [5, 6]) }
+    let!(:successful_job_2) { create(:background_migration_job, class_name: job_class_name, status: :succeeded, arguments: [7, 8]) }
+
+    around do |example|
+      freeze_time do
+        Sidekiq::Testing.fake! do
+          example.run
+        end
+      end
+    end
+
+    subject { model.requeue_background_migration_jobs_by_range_at_intervals(job_class_name, 10.minutes) }
+
+    it 'returns the expected duration' do
+      expect(subject).to eq(20.minutes)
+    end
+
+    context 'when nothing is queued' do
+      subject { model.requeue_background_migration_jobs_by_range_at_intervals('FakeJob', 10.minutes) }
+
+      it 'returns expected duration of zero when nothing gets queued' do
+        expect(subject).to eq(0)
+      end
+    end
+
+    it 'queues pending jobs' do
+      subject
+
+      expect(BackgroundMigrationWorker.jobs[0]['args']).to eq([job_class_name, [1, 2]])
+      expect(BackgroundMigrationWorker.jobs[0]['at']).to be_nil
+      expect(BackgroundMigrationWorker.jobs[1]['args']).to eq([job_class_name, [3, 4]])
+      expect(BackgroundMigrationWorker.jobs[1]['at']).to eq(10.minutes.from_now.to_f)
+    end
+
+    context 'with batch_size option' do
+      subject { model.requeue_background_migration_jobs_by_range_at_intervals(job_class_name, 10.minutes, batch_size: 1) }
+
+      it 'returns the expected duration' do
+        expect(subject).to eq(20.minutes)
+      end
+
+      it 'queues pending jobs' do
+        subject
+
+        expect(BackgroundMigrationWorker.jobs[0]['args']).to eq([job_class_name, [1, 2]])
+        expect(BackgroundMigrationWorker.jobs[0]['at']).to be_nil
+        expect(BackgroundMigrationWorker.jobs[1]['args']).to eq([job_class_name, [3, 4]])
+        expect(BackgroundMigrationWorker.jobs[1]['at']).to eq(10.minutes.from_now.to_f)
+      end
+
+      it 'retrieve jobs in batches' do
+        jobs = double('jobs')
+        expect(Gitlab::Database::BackgroundMigrationJob).to receive(:pending) { jobs }
+        allow(jobs).to receive(:where).with(class_name: job_class_name) { jobs }
+        expect(jobs).to receive(:each_batch).with(of: 1)
+
+        subject
+      end
+    end
+
+    context 'with initial_delay option' do
+      let_it_be(:initial_delay) { 3.minutes }
+
+      subject { model.requeue_background_migration_jobs_by_range_at_intervals(job_class_name, 10.minutes, initial_delay: initial_delay) }
+
+      it 'returns the expected duration' do
+        expect(subject).to eq(23.minutes)
+      end
+
+      it 'queues pending jobs' do
+        subject
+
+        expect(BackgroundMigrationWorker.jobs[0]['args']).to eq([job_class_name, [1, 2]])
+        expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(3.minutes.from_now.to_f)
+        expect(BackgroundMigrationWorker.jobs[1]['args']).to eq([job_class_name, [3, 4]])
+        expect(BackgroundMigrationWorker.jobs[1]['at']).to eq(13.minutes.from_now.to_f)
+      end
+
+      context 'when nothing is queued' do
+        subject { model.requeue_background_migration_jobs_by_range_at_intervals('FakeJob', 10.minutes) }
+
+        it 'returns expected duration of zero when nothing gets queued' do
+          expect(subject).to eq(0)
+        end
+      end
+    end
+  end
+
   describe '#perform_background_migration_inline?' do
     it 'returns true in a test environment' do
       stub_rails_env('test')

spec/migrations/retry_backfill_traversal_ids_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20210604070207_retry_backfill_traversal_ids.rb')
+
+RSpec.describe RetryBackfillTraversalIds, :migration do
+  include ReloadHelpers
+
+  let_it_be(:namespaces_table) { table(:namespaces) }
+
+  context 'when BackfillNamespaceTraversalIdsRoots jobs are pending' do
+    before do
+      table(:background_migration_jobs).create!(
+        class_name: 'BackfillNamespaceTraversalIdsRoots',
+        arguments: [1, 4, 100],
+        status: Gitlab::Database::BackgroundMigrationJob.statuses['pending']
+      )
+      table(:background_migration_jobs).create!(
+        class_name: 'BackfillNamespaceTraversalIdsRoots',
+        arguments: [5, 9, 100],
+        status: Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']
+      )
+    end
+
+    it 'queues pending jobs' do
+      migrate!
+
+      expect(BackgroundMigrationWorker.jobs.length).to eq(1)
+      expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['BackfillNamespaceTraversalIdsRoots', [1, 4, 100]])
+      expect(BackgroundMigrationWorker.jobs[0]['at']).to be_nil
+    end
+  end
+
+  context 'when BackfillNamespaceTraversalIdsChildren jobs are pending' do
+    before do
+      table(:background_migration_jobs).create!(
+        class_name: 'BackfillNamespaceTraversalIdsChildren',
+        arguments: [1, 4, 100],
+        status: Gitlab::Database::BackgroundMigrationJob.statuses['pending']
+      )
+      table(:background_migration_jobs).create!(
+        class_name: 'BackfillNamespaceTraversalIdsRoots',
+        arguments: [5, 9, 100],
+        status: Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']
+      )
+    end
+
+    it 'queues pending jobs' do
+      migrate!
+
+      expect(BackgroundMigrationWorker.jobs.length).to eq(1)
+      expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['BackfillNamespaceTraversalIdsChildren', [1, 4, 100]])
+      expect(BackgroundMigrationWorker.jobs[0]['at']).to be_nil
+    end
+  end
+
+  context 'when BackfillNamespaceTraversalIdsRoots and BackfillNamespaceTraversalIdsChildren jobs are pending' do
+    before do
+      table(:background_migration_jobs).create!(
+        class_name: 'BackfillNamespaceTraversalIdsRoots',
+        arguments: [1, 4, 100],
+        status: Gitlab::Database::BackgroundMigrationJob.statuses['pending']
+      )
+      table(:background_migration_jobs).create!(
+        class_name: 'BackfillNamespaceTraversalIdsChildren',
+        arguments: [5, 9, 100],
+        status: Gitlab::Database::BackgroundMigrationJob.statuses['pending']
+      )
+      table(:background_migration_jobs).create!(
+        class_name: 'BackfillNamespaceTraversalIdsRoots',
+        arguments: [11, 14, 100],
+        status: Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']
+      )
+      table(:background_migration_jobs).create!(
+        class_name: 'BackfillNamespaceTraversalIdsChildren',
+        arguments: [15, 19, 100],
+        status: Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']
+      )
+    end
+
+    it 'queues pending jobs' do
+      freeze_time do
+        migrate!
+
+        expect(BackgroundMigrationWorker.jobs.length).to eq(2)
+        expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['BackfillNamespaceTraversalIdsRoots', [1, 4, 100]])
+        expect(BackgroundMigrationWorker.jobs[0]['at']).to be_nil
+        expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['BackfillNamespaceTraversalIdsChildren', [5, 9, 100]])
+        expect(BackgroundMigrationWorker.jobs[1]['at']).to eq(RetryBackfillTraversalIds::DELAY_INTERVAL.from_now.to_f)
+      end
+    end
+  end
+end