Merge branch '18792-reschedule-background-migration1' into 'master'

Reschedule background migration to copy container_registry_enabled

See merge request gitlab-org/gitlab!59513

changelogs/unreleased/18792-reschedule-background-migration1.yml

+---
+title: Reschedule background migration to copy projects.container_registry_enabled
+  to project_features.container_registry_access_level
+merge_request: 59513
+author:
+type: fixed

db/post_migrate/20210401131948_move_container_registry_enabled_to_project_features2.rb

@@ -16,9 +16,11 @@ class MoveContainerRegistryEnabledToProjectFeatures2 < ActiveRecord::Migration[6
   end
 
   def up
-    delete_queued_jobs('MoveContainerRegistryEnabledToProjectFeature')
+    # Superceded by db/post_migrate/20210415155043_move_container_registry_enabled_to_project_features3.rb.
 
-    queue_background_migration_jobs_by_range_at_intervals(Project, MIGRATION, 2.minutes, batch_size: BATCH_SIZE, track_jobs: true)
+    # delete_queued_jobs('MoveContainerRegistryEnabledToProjectFeature')
+
+    # queue_background_migration_jobs_by_range_at_intervals(Project, MIGRATION, 2.minutes, batch_size: BATCH_SIZE, track_jobs: true)
   end
 
   def down

db/post_migrate/20210415155043_move_container_registry_enabled_to_project_features3.rb

+# frozen_string_literal: true
+
+class MoveContainerRegistryEnabledToProjectFeatures3 < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+  include Gitlab::Database::DynamicModelHelpers
+
+  BATCH_SIZE = 21_000
+  MIGRATION = 'MoveContainerRegistryEnabledToProjectFeature'
+
+  disable_ddl_transaction!
+
+  def up
+    # Delete any existing jobs from the queue
+    delete_queued_jobs(MIGRATION)
+
+    # Delete existing rows in background_migration_jobs table
+    bg_migration_job_class = define_model('background_migration_jobs')
+    bg_migration_job_class.where(class_name: MIGRATION).delete_all
+
+    batchable_project_class = define_batchable_model('projects')
+    queue_background_migration_jobs_by_range_at_intervals(batchable_project_class, MIGRATION, 2.minutes, batch_size: BATCH_SIZE, track_jobs: true)
+  end
+
+  def down
+    # no-op
+  end
+
+  private
+
+  def define_model(table_name)
+    Class.new(ActiveRecord::Base) do
+      self.table_name = table_name
+      self.inheritance_column = :_type_disabled
+    end
+  end
+end

db/schema_migrations/20210415155043

+af66cb13eca39cc6b9970e6764d08b5b353f212d844267afb5583e888c7d2c94
\ No newline at end of file

lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb

@@ -8,18 +8,15 @@ module Gitlab
     class MoveContainerRegistryEnabledToProjectFeature
       MAX_BATCH_SIZE = 300
 
-      module Migratable
-        # Migration model namespace isolated from application code.
-        class ProjectFeature < ActiveRecord::Base
-          ENABLED = 20
-          DISABLED = 0
-        end
-      end
+      ENABLED = 20
+      DISABLED = 0
 
       def perform(from_id, to_id)
         (from_id..to_id).each_slice(MAX_BATCH_SIZE) do |batch|
           process_batch(batch.first, batch.last)
         end
+
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('MoveContainerRegistryEnabledToProjectFeature', [from_id, to_id])
       end
 
       private
@@ -37,9 +34,9 @@ module Gitlab
         <<~SQL
         UPDATE project_features
         SET container_registry_access_level = (CASE p.container_registry_enabled
-                                              WHEN true THEN #{ProjectFeature::ENABLED}
-                                              WHEN false THEN #{ProjectFeature::DISABLED}
-                                              ELSE #{ProjectFeature::DISABLED}
+                                              WHEN true THEN #{ENABLED}
+                                              WHEN false THEN #{DISABLED}
+                                              ELSE #{DISABLED}
                                               END)
         FROM projects p
         WHERE project_id = p.id AND

spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb

@@ -31,6 +31,15 @@ RSpec.describe Gitlab::BackgroundMigration::MoveContainerRegistryEnabledToProjec
     end
 
     it 'copies values to project_features' do
+      table(:background_migration_jobs).create!(
+        class_name: 'MoveContainerRegistryEnabledToProjectFeature',
+        arguments: [project1.id, project4.id]
+      )
+      table(:background_migration_jobs).create!(
+        class_name: 'MoveContainerRegistryEnabledToProjectFeature',
+        arguments: [-1, -3]
+      )
+
       expect(project1.container_registry_enabled).to eq(true)
       expect(project2.container_registry_enabled).to eq(false)
       expect(project3.container_registry_enabled).to eq(nil)
@@ -57,6 +66,9 @@ RSpec.describe Gitlab::BackgroundMigration::MoveContainerRegistryEnabledToProjec
       expect(project_feature1.reload.container_registry_access_level).to eq(enabled)
       expect(project_feature2.reload.container_registry_access_level).to eq(disabled)
       expect(project_feature3.reload.container_registry_access_level).to eq(disabled)
+
+      expect(table(:background_migration_jobs).first.status).to eq(1) # succeeded
+      expect(table(:background_migration_jobs).second.status).to eq(0) # pending
     end
 
     context 'when no projects exist in range' do

spec/migrations/move_container_registry_enabled_to_project_features2_spec.rb → spec/migrations/move_container_registry_enabled_to_project_features3_spec.rb

 # frozen_string_literal: true
 
 require 'spec_helper'
-require Rails.root.join('db', 'post_migrate', '20210401131948_move_container_registry_enabled_to_project_features2.rb')
+require Rails.root.join('db', 'post_migrate', '20210415155043_move_container_registry_enabled_to_project_features3.rb')
 
-RSpec.describe MoveContainerRegistryEnabledToProjectFeatures2, :migration do
+RSpec.describe MoveContainerRegistryEnabledToProjectFeatures3, :migration do
   let(:namespace) { table(:namespaces).create!(name: 'gitlab', path: 'gitlab-org') }
 
+  let!(:background_jobs) do
+    table(:background_migration_jobs).create!(class_name: described_class::MIGRATION, arguments: [-1, -2])
+    table(:background_migration_jobs).create!(class_name: described_class::MIGRATION, arguments: [-3, -4])
+  end
+
   let!(:projects) do
     [
       table(:projects).create!(namespace_id: namespace.id, name: 'project 1'),
@@ -28,11 +33,18 @@ RSpec.describe MoveContainerRegistryEnabledToProjectFeatures2, :migration do
   end
 
   it 'schedules jobs for ranges of projects' do
+    # old entries in background_migration_jobs should be deleted.
+    expect(table(:background_migration_jobs).count).to eq(2)
+    expect(table(:background_migration_jobs).first.arguments).to eq([-1, -2])
+    expect(table(:background_migration_jobs).second.arguments).to eq([-3, -4])
+
     migrate!
 
     # Since track_jobs is true, each job should have an entry in the background_migration_jobs
     # table.
     expect(table(:background_migration_jobs).count).to eq(2)
+    expect(table(:background_migration_jobs).first.arguments).to eq([projects[0].id, projects[2].id])
+    expect(table(:background_migration_jobs).second.arguments).to eq([projects[3].id, projects[3].id])
 
     expect(described_class::MIGRATION)
       .to be_scheduled_delayed_migration(2.minutes, projects[0].id, projects[2].id)