Merge branch 'add-bg-migration-to-cleanup-orphaned-deployments' into 'master'

Add BG migration to delete orphaned deployments

See merge request gitlab-org/gitlab!64588

db/post_migrate/20210622141148_schedule_delete_orphaned_deployments.rb

+# frozen_string_literal: true
+
+class ScheduleDeleteOrphanedDeployments < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  MIGRATION = 'DeleteOrphanedDeployments'
+  BATCH_SIZE = 100_000
+  DELAY_INTERVAL = 2.minutes
+
+  disable_ddl_transaction!
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(
+      define_batchable_model('deployments'),
+      MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      track_jobs: true
+    )
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema_migrations/20210622141148

+432954295d6f3a2a45f3deef42b547ffe42501beaea4f376e1be51cf148de671
\ No newline at end of file

lib/gitlab/background_migration/delete_orphaned_deployments.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Background migration for deleting orphaned deployments.
+    class DeleteOrphanedDeployments
+      include Database::MigrationHelpers
+
+      def perform(start_id, end_id)
+        orphaned_deployments
+          .where(id: start_id..end_id)
+          .delete_all
+
+        mark_job_as_succeeded(start_id, end_id)
+      end
+
+      def orphaned_deployments
+        define_batchable_model('deployments')
+          .where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)')
+      end
+
+      private
+
+      def mark_job_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
+          self.class.name.demodulize,
+          arguments
+        )
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedDeployments, :migration, schema: 20210617161348 do
+  let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let!(:project) { table(:projects).create!(namespace_id: namespace.id) }
+  let!(:environment) { table(:environments).create!(name: 'production', slug: 'production', project_id: project.id) }
+  let(:background_migration_jobs) { table(:background_migration_jobs) }
+
+  before do
+    create_deployment!(environment.id, project.id)
+    create_deployment!(non_existing_record_id, project.id)
+  end
+
+  it 'deletes only orphaned deployments' do
+    expect(valid_deployments.pluck(:id)).not_to be_empty
+    expect(orphaned_deployments.pluck(:id)).not_to be_empty
+
+    subject.perform(table(:deployments).minimum(:id), table(:deployments).maximum(:id))
+
+    expect(valid_deployments.pluck(:id)).not_to be_empty
+    expect(orphaned_deployments.pluck(:id)).to be_empty
+  end
+
+  it 'marks jobs as done' do
+    first_job = background_migration_jobs.create!(
+      class_name: 'DeleteOrphanedDeployments',
+      arguments: [table(:deployments).minimum(:id), table(:deployments).minimum(:id)]
+    )
+
+    second_job = background_migration_jobs.create!(
+      class_name: 'DeleteOrphanedDeployments',
+      arguments: [table(:deployments).maximum(:id), table(:deployments).maximum(:id)]
+    )
+
+    subject.perform(table(:deployments).minimum(:id), table(:deployments).minimum(:id))
+
+    expect(first_job.reload.status).to eq(Gitlab::Database::BackgroundMigrationJob.statuses[:succeeded])
+    expect(second_job.reload.status).to eq(Gitlab::Database::BackgroundMigrationJob.statuses[:pending])
+  end
+
+  private
+
+  def valid_deployments
+    table(:deployments).where('EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)')
+  end
+
+  def orphaned_deployments
+    table(:deployments).where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)')
+  end
+
+  def create_deployment!(environment_id, project_id)
+    table(:deployments).create!(
+      environment_id: environment_id,
+      project_id: project_id,
+      ref: 'master',
+      tag: false,
+      sha: 'x',
+      status: 1,
+      iid: table(:deployments).count + 1)
+  end
+end

spec/migrations/schedule_delete_orphaned_deployments_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require_migration!
+
+RSpec.describe ScheduleDeleteOrphanedDeployments, :sidekiq, schema: 20210617161348 do
+  let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let!(:project) { table(:projects).create!(namespace_id: namespace.id) }
+  let!(:environment) { table(:environments).create!(name: 'production', slug: 'production', project_id: project.id) }
+  let(:background_migration_jobs) { table(:background_migration_jobs) }
+
+  before do
+    create_deployment!(environment.id, project.id)
+    create_deployment!(environment.id, project.id)
+    create_deployment!(environment.id, project.id)
+    create_deployment!(non_existing_record_id, project.id)
+    create_deployment!(non_existing_record_id, project.id)
+    create_deployment!(non_existing_record_id, project.id)
+    create_deployment!(non_existing_record_id, project.id)
+
+    stub_const("#{described_class}::BATCH_SIZE", 1)
+  end
+
+  it 'schedules DeleteOrphanedDeployments background jobs' do
+    Sidekiq::Testing.fake! do
+      freeze_time do
+        migrate!
+
+        expect(BackgroundMigrationWorker.jobs.size).to eq(7)
+        table(:deployments).find_each do |deployment|
+          expect(described_class::MIGRATION).to be_scheduled_migration(deployment.id, deployment.id)
+        end
+      end
+    end
+  end
+
+  def create_deployment!(environment_id, project_id)
+    table(:deployments).create!(
+      environment_id: environment_id,
+      project_id: project_id,
+      ref: 'master',
+      tag: false,
+      sha: 'x',
+      status: 1,
+      iid: table(:deployments).count + 1)
+  end
+end