Merge branch '333414-fix-bg-migration' into 'master'

Fix and reschedule background migration for backfilling data from `ci_builds` to `security_scans`

See merge request gitlab-org/gitlab!68305

db/post_migrate/20210811214811_schedule_copy_ci_builds_columns_to_security_scans.rb

@@ -3,31 +3,11 @@
 class ScheduleCopyCiBuildsColumnsToSecurityScans < ActiveRecord::Migration[6.1]
   include Gitlab::Database::MigrationHelpers
 
-  INTERVAL = 2.minutes.to_i
-  BATCH_SIZE = 5_000
-  MIGRATION = 'CopyCiBuildsColumnsToSecurityScans'
-
-  disable_ddl_transaction!
-
-  class SecurityScan < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'security_scans'
-  end
-
   def up
-    SecurityScan.reset_column_information
-
-    queue_background_migration_jobs_by_range_at_intervals(
-      SecurityScan,
-      MIGRATION,
-      INTERVAL,
-      batch_size: BATCH_SIZE,
-      track_jobs: true
-    )
+    # no-op as we found an issue with bg migration, we fixed it and rescheduling it again.
   end
 
   def down
-    # noop
+    # no-op
   end
 end

db/post_migrate/20210816183304_schedule_copy_ci_builds_columns_to_security_scans2.rb

+# frozen_string_literal: true
+
+class ScheduleCopyCiBuildsColumnsToSecurityScans2 < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  INTERVAL = 2.minutes.to_i
+  BATCH_SIZE = 5_000
+  MIGRATION = 'CopyCiBuildsColumnsToSecurityScans'
+
+  disable_ddl_transaction!
+
+  class SecurityScan < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'security_scans'
+  end
+
+  def up
+    SecurityScan.reset_column_information
+
+    delete_job_tracking(MIGRATION, status: %w[pending succeeded])
+
+    queue_background_migration_jobs_by_range_at_intervals(
+      SecurityScan,
+      MIGRATION,
+      INTERVAL,
+      batch_size: BATCH_SIZE,
+      track_jobs: true
+    )
+  end
+
+  def down
+    # noop
+  end
+end

db/schema_migrations/20210816183304

+08fed4e3269629304c5036361c237f4c9b860c26212b155869773296a79df01a
\ No newline at end of file

lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb

@@ -30,15 +30,15 @@ module Gitlab
       rescue StandardError => error
         Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
       end
-    end
 
-    private
+      private
 
-    def mark_job_as_succeeded(*arguments)
-      Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
-        'CopyCiBuildsColumnsToSecurityScans',
-        arguments
-      )
+      def mark_job_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
+          'CopyCiBuildsColumnsToSecurityScans',
+          arguments
+        )
+      end
     end
   end
 end

spec/migrations/schedule_copy_ci_builds_columns_to_security_scans_spec.rb → spec/migrations/schedule_copy_ci_builds_columns_to_security_scans2_spec.rb

@@ -3,12 +3,13 @@
 require 'spec_helper'
 require_migration!
 
-RSpec.describe ScheduleCopyCiBuildsColumnsToSecurityScans do
+RSpec.describe ScheduleCopyCiBuildsColumnsToSecurityScans2 do
   let_it_be(:namespaces) { table(:namespaces) }
   let_it_be(:projects) { table(:projects) }
   let_it_be(:ci_pipelines) { table(:ci_pipelines) }
   let_it_be(:ci_builds) { table(:ci_builds) }
   let_it_be(:security_scans) { table(:security_scans) }
+  let_it_be(:background_migration_jobs) { table(:background_migration_jobs) }
 
   let!(:namespace) { namespaces.create!(name: 'namespace', path: 'namespace') }
   let!(:project) { projects.create!(namespace_id: namespace.id) }
@@ -22,6 +23,11 @@ RSpec.describe ScheduleCopyCiBuildsColumnsToSecurityScans do
   let!(:scan2) { security_scans.create!(build_id: build2.id, scan_type: 1) }
   let!(:scan3) { security_scans.create!(build_id: build3.id, scan_type: 1) }
 
+  let!(:job_class_name) { described_class::MIGRATION }
+  let!(:tracked_pending_job) { background_migration_jobs.create!(class_name: job_class_name, status: 0, arguments: [1]) }
+  let!(:tracked_successful_job) { background_migration_jobs.create!(class_name: job_class_name, status: 1, arguments: [2]) }
+  let(:jobs) { Gitlab::Database::BackgroundMigrationJob.where(id: [tracked_pending_job.id, tracked_successful_job.id] ).for_migration_class(job_class_name) }
+
   before do
     stub_const("#{described_class}::BATCH_SIZE", 2)
     allow_next_instance_of(Gitlab::BackgroundMigration::CopyCiBuildsColumnsToSecurityScans) do |instance|
@@ -34,8 +40,11 @@ RSpec.describe ScheduleCopyCiBuildsColumnsToSecurityScans do
   end
 
   it 'schedules background migrations', :aggregate_failures do
+    expect(jobs).not_to be_empty
+
     migrate!
 
+    expect(jobs).to be_empty
     expect(BackgroundMigrationWorker.jobs.size).to eq(2)
     expect(described_class::MIGRATION).to be_scheduled_delayed_migration(2.minutes, scan1.id, scan2.id)
     expect(described_class::MIGRATION).to be_scheduled_delayed_migration(4.minutes, scan3.id, scan3.id)