Add nullify job for orphan runner_id columns of ci_builds

When a runner gets deleted, jobs that were run with it still have
runner_id. With this commit, old data will be nullified.

There will be 3 steps;
1. Add a NOT VALID foreign key constraint to the column to ensure
GitLab doesn't create inconsistent records.
2. Add a data migration, to fix or clean up existing records.
3. Validate the whole table by making the foreign key VALID.

This is the 2nd step.

Changelog: other

db/post_migrate/20220223112304_schedule_nullify_orphan_runner_id_on_ci_builds.rb

+# frozen_string_literal: true
+
+class ScheduleNullifyOrphanRunnerIdOnCiBuilds < Gitlab::Database::Migration[1.0]
+  MIGRATION = 'NullifyOrphanRunnerIdOnCiBuilds'
+  INTERVAL = 2.minutes
+  BATCH_SIZE = 100_000
+  MAX_BATCH_SIZE = 25_000 # 100k * 25k = 2.5B ci_builds
+  SUB_BATCH_SIZE = 1_000
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :ci_builds,
+      :id,
+      job_interval: INTERVAL,
+      batch_size: BATCH_SIZE,
+      max_batch_size: MAX_BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    Gitlab::Database::BackgroundMigration::BatchedMigration
+      .for_configuration(MIGRATION, :ci_builds, :id, [])
+      .delete_all
+  end
+end

db/schema_migrations/20220223112304

+57dc23bb2a9faddefe20c1e30a8879ebb1f6f32f17e3cc381acc1d06ad3b598a
\ No newline at end of file

lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # A job to nullify orphan runner_id on ci_builds table
+    class NullifyOrphanRunnerIdOnCiBuilds
+      include Gitlab::Database::DynamicModelHelpers
+
+      def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms)
+        pause_ms = 0 if pause_ms < 0
+
+        batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id)
+        batch_relation.each_batch(column: batch_column, of: sub_batch_size, order_hint: :type) do |sub_batch|
+          batch_metrics.time_operation(:update_all) do
+            sub_batch.update_all(runner_id: nil)
+          end
+
+          sleep(pause_ms * 0.001)
+        end
+      end
+
+      def batch_metrics
+        @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new
+      end
+
+      private
+
+      def connection
+        ActiveRecord::Base.connection
+      end
+
+      def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id)
+        define_batchable_model(source_table, connection: connection)
+          .joins('LEFT OUTER JOIN ci_runners ON ci_runners.id = ci_builds.runner_id')
+          .where('ci_builds.runner_id IS NOT NULL AND ci_runners.id IS NULL')
+          .where(source_key_column => start_id..stop_id)
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::NullifyOrphanRunnerIdOnCiBuilds, :migration, schema: 20220223112304 do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:ci_runners) { table(:ci_runners) }
+  let(:ci_pipelines) { table(:ci_pipelines) }
+  let(:ci_builds) { table(:ci_builds) }
+
+  subject { described_class.new }
+
+  let(:helpers) do
+    ActiveRecord::Migration.new.extend(Gitlab::Database::MigrationHelpers)
+  end
+
+  before do
+    helpers.remove_foreign_key_if_exists(:ci_builds, column: :runner_id)
+  end
+
+  after do
+    helpers.add_concurrent_foreign_key(:ci_builds, :ci_runners, column: :runner_id, on_delete: :nullify, validate: false)
+  end
+
+  describe '#perform' do
+    let(:namespace) { namespaces.create!(name: 'test', path: 'test', type: 'Group') }
+    let(:project) { projects.create!(namespace_id: namespace.id, name: 'test') }
+    let(:pipeline) { ci_pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a', status: 'success') }
+
+    it 'nullifies runner_id for orphan ci_builds in range' do
+      ci_runners.create!(id: 2, runner_type: 'project_type')
+
+      ci_builds.create!(id: 5, type: 'Ci::Build', commit_id: pipeline.id, runner_id: 2)
+      ci_builds.create!(id: 7, type: 'Ci::Build', commit_id: pipeline.id, runner_id: 4)
+      ci_builds.create!(id: 8, type: 'Ci::Build', commit_id: pipeline.id, runner_id: 5)
+      ci_builds.create!(id: 9, type: 'Ci::Build', commit_id: pipeline.id, runner_id: 6)
+
+      subject.perform(4, 8, :ci_builds, :id, 10, 0)
+
+      expect(ci_builds.all).to contain_exactly(
+        an_object_having_attributes(id: 5, runner_id: 2),
+        an_object_having_attributes(id: 7, runner_id: nil),
+        an_object_having_attributes(id: 8, runner_id: nil),
+        an_object_having_attributes(id: 9, runner_id: 6)
+      )
+    end
+  end
+end