Merge branch '337100_backfill_project_namespaces' into 'master'

Backfill all project namespaces

See merge request gitlab-org/gitlab!77371

db/post_migrate/20220301093434_backfill_all_project_namespaces.rb

+# frozen_string_literal: true
+
+class BackfillAllProjectNamespaces < Gitlab::Database::Migration[1.0]
+  MIGRATION = 'ProjectNamespaces::BackfillProjectNamespaces'
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1_000
+  MAX_BATCH_SIZE = 5_000
+  SUB_BATCH_SIZE = 10
+
+  disable_ddl_transaction!
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :projects,
+      :id,
+      nil,
+      'up',
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      max_batch_size: MAX_BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    Gitlab::Database::BackgroundMigration::BatchedMigration
+      .for_configuration(MIGRATION, :projects, :id, [nil, 'up']).delete_all
+  end
+end

db/schema_migrations/20220301093434

+ffdd031395c025ea63ea1adcd63636822e62388a8860c93235f3748918fc30ca
\ No newline at end of file

lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb

@@ -34,8 +34,11 @@ module Gitlab
 
         def backfill_project_namespaces(namespace_id)
           project_ids.each_slice(sub_batch_size) do |project_ids|
-            ActiveRecord::Base.connection.execute("select gin_clean_pending_list('index_namespaces_on_name_trigram')")
-            ActiveRecord::Base.connection.execute("select gin_clean_pending_list('index_namespaces_on_path_trigram')")
+            # cleanup gin indexes on namespaces table
+            cleanup_gin_index('namespaces')
+
+            # cleanup gin indexes on projects table
+            cleanup_gin_index('projects')
 
             # We need to lock these project records for the period when we create project namespaces
             # and link them to projects so that if a project is modified in the time between creating
@@ -53,6 +56,14 @@ module Gitlab
           end
         end
 
+        def cleanup_gin_index(table_name)
+          index_names = ActiveRecord::Base.connection.select_values("select indexname::text from pg_indexes where tablename = '#{table_name}' and indexdef ilike '%gin%'")
+
+          index_names.each do |index_name|
+            ActiveRecord::Base.connection.execute("select gin_clean_pending_list('#{index_name}')")
+          end
+        end
+
         def cleanup_backfilled_project_namespaces(namespace_id)
           project_ids.each_slice(sub_batch_size) do |project_ids|
             # IMPORTANT: first nullify project_namespace_id in projects table to avoid removing projects when records

spec/migrations/backfill_all_project_namespaces_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe BackfillAllProjectNamespaces, :migration do
+  let_it_be(:migration) { described_class::MIGRATION }
+
+  let(:projects) { table(:projects) }
+  let(:namespaces) { table(:namespaces) }
+  let(:user_namespace) { namespaces.create!(name: 'user1', path: 'user1', visibility_level: 20, type: 'User') }
+  let(:parent_group1) { namespaces.create!(name: 'parent_group1', path: 'parent_group1', visibility_level: 20, type: 'Group') }
+  let!(:parent_group1_project) { projects.create!(name: 'parent_group1_project', path: 'parent_group1_project', namespace_id: parent_group1.id, visibility_level: 20) }
+  let!(:user_namespace_project) { projects.create!(name: 'user1_project', path: 'user1_project', namespace_id: user_namespace.id, visibility_level: 20) }
+
+  describe '#up' do
+    it 'schedules background jobs for each batch of namespaces' do
+      migrate!
+
+      expect(migration).to have_scheduled_batched_migration(
+        table_name: :projects,
+        column_name: :id,
+        job_arguments: [nil, 'up'],
+        interval: described_class::DELAY_INTERVAL
+      )
+    end
+  end
+
+  describe '#down' do
+    it 'deletes all batched migration records' do
+      migrate!
+      schema_migrate_down!
+
+      expect(migration).not_to have_scheduled_batched_migration
+    end
+  end
+end