Merge branch 'backfill-ci-namespace-project-mirrors' into 'master'

Add backfill migrations for ci namespace/project mirrors

See merge request gitlab-org/gitlab!76327

db/post_migrate/20211208122200_schedule_backfill_ci_namespace_mirrors.rb

+# frozen_string_literal: true
+
+class ScheduleBackfillCiNamespaceMirrors < Gitlab::Database::Migration[1.0]
+  MIGRATION = 'BackfillCiNamespaceMirrors'
+  BATCH_SIZE = 10_000
+  DELAY_INTERVAL = 2.minutes
+
+  disable_ddl_transaction!
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(
+      Gitlab::BackgroundMigration::BackfillCiNamespaceMirrors::Namespace.base_query,
+      MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      track_jobs: true
+    )
+  end
+
+  def down
+    # no-op
+  end
+end

db/post_migrate/20211208122201_schedule_backfill_ci_project_mirrors.rb

+# frozen_string_literal: true
+
+class ScheduleBackfillCiProjectMirrors < Gitlab::Database::Migration[1.0]
+  MIGRATION = 'BackfillCiProjectMirrors'
+  BATCH_SIZE = 10_000
+  DELAY_INTERVAL = 2.minutes
+
+  disable_ddl_transaction!
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(
+      Gitlab::BackgroundMigration::BackfillCiProjectMirrors::Project.base_query,
+      MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      track_jobs: true
+    )
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema_migrations/20211208122200

+8dec4379f34773e979cf6b33ba608185827c37b1a95c74d6472b7562c16d6110
\ No newline at end of file

db/schema_migrations/20211208122201

+f58ccb07fa67ede2a50fa5ff6bddbe4bb89a622e84786fc4bfb404c11b9dbab4
\ No newline at end of file

lib/gitlab/background_migration/backfill_ci_namespace_mirrors.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # A job to create ci_namespace_mirrors entries in batches
+    class BackfillCiNamespaceMirrors
+      class Namespace < ActiveRecord::Base # rubocop:disable Style/Documentation
+        include ::EachBatch
+
+        self.table_name = 'namespaces'
+        self.inheritance_column = nil
+
+        scope :base_query, -> do
+          select(:id, :parent_id)
+        end
+      end
+
+      PAUSE_SECONDS = 0.1
+      SUB_BATCH_SIZE = 500
+
+      def perform(start_id, end_id)
+        batch_query = Namespace.base_query.where(id: start_id..end_id)
+        batch_query.each_batch(of: SUB_BATCH_SIZE) do |sub_batch|
+          first, last = sub_batch.pluck(Arel.sql('MIN(id), MAX(id)')).first
+          ranged_query = Namespace.unscoped.base_query.where(id: first..last)
+
+          update_sql = <<~SQL
+            INSERT INTO ci_namespace_mirrors (namespace_id, traversal_ids)
+            #{insert_values(ranged_query)}
+            ON CONFLICT (namespace_id) DO NOTHING
+          SQL
+          # We do nothing on conflict because we consider they were already filled.
+
+          Namespace.connection.execute(update_sql)
+
+          sleep PAUSE_SECONDS
+        end
+
+        mark_job_as_succeeded(start_id, end_id)
+      end
+
+      private
+
+      def insert_values(batch)
+        calculated_traversal_ids(
+          batch.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433')
+        )
+      end
+
+      # Copied from lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb
+      def calculated_traversal_ids(batch)
+        <<~SQL
+          WITH RECURSIVE cte(source_id, namespace_id, parent_id, height) AS (
+            (
+              SELECT batch.id, batch.id, batch.parent_id, 1
+              FROM (#{batch.to_sql}) AS batch
+            )
+            UNION ALL
+            (
+              SELECT cte.source_id, n.id, n.parent_id, cte.height+1
+              FROM namespaces n, cte
+              WHERE n.id = cte.parent_id
+            )
+          )
+          SELECT flat_hierarchy.source_id as namespace_id,
+                 array_agg(flat_hierarchy.namespace_id ORDER BY flat_hierarchy.height DESC) as traversal_ids
+          FROM (SELECT * FROM cte FOR UPDATE) flat_hierarchy
+          GROUP BY flat_hierarchy.source_id
+        SQL
+      end
+
+      def mark_job_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('BackfillCiNamespaceMirrors', arguments)
+      end
+    end
+  end
+end

lib/gitlab/background_migration/backfill_ci_project_mirrors.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # A job to create ci_project_mirrors entries in batches
+    class BackfillCiProjectMirrors
+      class Project < ActiveRecord::Base # rubocop:disable Style/Documentation
+        include ::EachBatch
+
+        self.table_name = 'projects'
+
+        scope :base_query, -> do
+          select(:id, :namespace_id)
+        end
+      end
+
+      PAUSE_SECONDS = 0.1
+      SUB_BATCH_SIZE = 500
+
+      def perform(start_id, end_id)
+        batch_query = Project.base_query.where(id: start_id..end_id)
+        batch_query.each_batch(of: SUB_BATCH_SIZE) do |sub_batch|
+          first, last = sub_batch.pluck(Arel.sql('MIN(id), MAX(id)')).first
+          ranged_query = Project.unscoped.base_query.where(id: first..last)
+
+          update_sql = <<~SQL
+            INSERT INTO ci_project_mirrors (project_id, namespace_id)
+            #{insert_values(ranged_query)}
+            ON CONFLICT (project_id) DO NOTHING
+          SQL
+          # We do nothing on conflict because we consider they were already filled.
+
+          Project.connection.execute(update_sql)
+
+          sleep PAUSE_SECONDS
+        end
+
+        mark_job_as_succeeded(start_id, end_id)
+      end
+
+      private
+
+      def insert_values(batch)
+        batch.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433').to_sql
+      end
+
+      def mark_job_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('BackfillCiProjectMirrors', arguments)
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/backfill_ci_namespace_mirrors_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillCiNamespaceMirrors, :migration, schema: 20211208122200 do
+  let(:namespaces) { table(:namespaces) }
+  let(:ci_namespace_mirrors) { table(:ci_namespace_mirrors) }
+
+  subject { described_class.new }
+
+  describe '#perform' do
+    it 'creates hierarchies for all namespaces in range' do
+      namespaces.create!(id: 5, name: 'test1', path: 'test1')
+      namespaces.create!(id: 7, name: 'test2', path: 'test2')
+      namespaces.create!(id: 8, name: 'test3', path: 'test3')
+
+      subject.perform(5, 7)
+
+      expect(ci_namespace_mirrors.all).to contain_exactly(
+        an_object_having_attributes(namespace_id: 5, traversal_ids: [5]),
+        an_object_having_attributes(namespace_id: 7, traversal_ids: [7])
+      )
+    end
+
+    it 'handles existing hierarchies gracefully' do
+      namespaces.create!(id: 5, name: 'test1', path: 'test1')
+      test2 = namespaces.create!(id: 7, name: 'test2', path: 'test2')
+      namespaces.create!(id: 8, name: 'test3', path: 'test3', parent_id: 7)
+      namespaces.create!(id: 9, name: 'test4', path: 'test4')
+
+      # Simulate a situation where a user has had a chance to move a group to another parent
+      # before the background migration has had a chance to run
+      test2.update!(parent_id: 5)
+      ci_namespace_mirrors.create!(namespace_id: test2.id, traversal_ids: [5, 7])
+
+      subject.perform(5, 8)
+
+      expect(ci_namespace_mirrors.all).to contain_exactly(
+        an_object_having_attributes(namespace_id: 5, traversal_ids: [5]),
+        an_object_having_attributes(namespace_id: 7, traversal_ids: [5, 7]),
+        an_object_having_attributes(namespace_id: 8, traversal_ids: [5, 7, 8])
+      )
+    end
+  end
+end

spec/lib/gitlab/background_migration/backfill_ci_project_mirrors_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillCiProjectMirrors, :migration, schema: 20211208122201 do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:ci_project_mirrors) { table(:ci_project_mirrors) }
+
+  subject { described_class.new }
+
+  describe '#perform' do
+    it 'creates ci_project_mirrors for all projects in range' do
+      namespaces.create!(id: 10, name: 'namespace1', path: 'namespace1')
+      projects.create!(id: 5, namespace_id: 10, name: 'test1', path: 'test1')
+      projects.create!(id: 7, namespace_id: 10, name: 'test2', path: 'test2')
+      projects.create!(id: 8, namespace_id: 10, name: 'test3', path: 'test3')
+
+      subject.perform(5, 7)
+
+      expect(ci_project_mirrors.all).to contain_exactly(
+        an_object_having_attributes(project_id: 5, namespace_id: 10),
+        an_object_having_attributes(project_id: 7, namespace_id: 10)
+      )
+    end
+
+    it 'handles existing ci_project_mirrors gracefully' do
+      namespaces.create!(id: 10, name: 'namespace1', path: 'namespace1')
+      namespaces.create!(id: 11, name: 'namespace2', path: 'namespace2', parent_id: 10)
+      projects.create!(id: 5, namespace_id: 10, name: 'test1', path: 'test1')
+      projects.create!(id: 7, namespace_id: 11, name: 'test2', path: 'test2')
+      projects.create!(id: 8, namespace_id: 11, name: 'test3', path: 'test3')
+
+      # Simulate a situation where a user has had a chance to move a project to another namespace
+      # before the background migration has had a chance to run
+      ci_project_mirrors.create!(project_id: 7, namespace_id: 10)
+
+      subject.perform(5, 7)
+
+      expect(ci_project_mirrors.all).to contain_exactly(
+        an_object_having_attributes(project_id: 5, namespace_id: 10),
+        an_object_having_attributes(project_id: 7, namespace_id: 10)
+      )
+    end
+  end
+end