Merge branch 'fj-backfill-imported-snippet-repositories' into 'master'

Backfill failed imported snippet repositories

See merge request gitlab-org/gitlab!34052

changelogs/unreleased/fj-backfill-imported-snippet-repositories.yml

+---
+title: Backfill failed imported snippet repositories
+merge_request: 34052
+author:
+type: other

db/post_migrate/20200608072931_backfill_imported_snippet_repositories.rb

+# frozen_string_literal: true
+
+class BackfillImportedSnippetRepositories < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  DELAY_INTERVAL = 2.minutes.to_i
+  BATCH_SIZE = 200
+  MIGRATION = 'BackfillSnippetRepositories'
+
+  disable_ddl_transaction!
+
+  class Snippet < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'snippets'
+    self.inheritance_column = :_type_disabled
+  end
+
+  class SnippetRepository < ActiveRecord::Base
+    self.table_name = 'snippet_repositories'
+  end
+
+  def up
+    index = 1
+
+    Snippet.select(:id).where.not(id: SnippetRepository.select(:snippet_id)).each_batch(of: BATCH_SIZE, column: 'id') do |batch|
+      split_in_consecutive_batches(batch).each do |ids_batch|
+        migrate_in(index * DELAY_INTERVAL, MIGRATION, [ids_batch.first, ids_batch.last])
+
+        index += 1
+      end
+    end
+  end
+
+  def down
+    # no-op
+  end
+
+  private
+
+  def split_in_consecutive_batches(relation)
+    ids = relation.pluck(:id)
+
+    (ids.first..ids.last).to_a.split {|i| !ids.include?(i) }.select(&:present?)
+  end
+end

db/structure.sql

@@ -13886,6 +13886,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200604145731
 20200604174544
 20200604174558
+20200608072931
 20200608075553
 20200609002841
 \.

spec/migrations/backfill_imported_snippet_repositories_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20200608072931_backfill_imported_snippet_repositories.rb')
+
+describe BackfillImportedSnippetRepositories do
+  let(:users) { table(:users) }
+  let(:snippets) { table(:snippets) }
+  let(:user) { users.create(id: 1, email: 'user@example.com', projects_limit: 10, username: 'test', name: 'Test', state: 'active') }
+
+  def create_snippet(id)
+    params = {
+      id: id,
+      type: 'PersonalSnippet',
+      author_id: user.id,
+      file_name: 'foo',
+      content: 'bar'
+    }
+
+    snippets.create!(params)
+  end
+
+  it 'correctly schedules background migrations' do
+    create_snippet(1)
+    create_snippet(2)
+    create_snippet(3)
+    create_snippet(5)
+    create_snippet(7)
+    create_snippet(8)
+    create_snippet(10)
+
+    Sidekiq::Testing.fake! do
+      Timecop.freeze do
+        migrate!
+
+        expect(described_class::MIGRATION)
+          .to be_scheduled_delayed_migration(2.minutes, 1, 3)
+
+        expect(described_class::MIGRATION)
+          .to be_scheduled_delayed_migration(4.minutes, 5, 5)
+
+        expect(described_class::MIGRATION)
+          .to be_scheduled_delayed_migration(6.minutes, 7, 8)
+
+        expect(described_class::MIGRATION)
+          .to be_scheduled_delayed_migration(8.minutes, 10, 10)
+
+        expect(BackgroundMigrationWorker.jobs.size).to eq(4)
+      end
+    end
+  end
+end