Clean stale background migration jobs

In https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24135, we
reverted the problematic
`ActivatePrometheusServicesForSharedClusterApplications` background
migration, but the migration only dropped jobs that were in the
`background_migration` Sidekiq queue. There may be more jobs in the
Sidekiq scheduled or retry sets, so we need to clear those too.

Closes https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/9160

changelogs/unreleased/sh-cleaup-prom-background-migrations.yml

+---
+title: Clean stale background migration jobs
+merge_request: 25707
+author:
+type: fixed

db/migrate/20200221144534_drop_activate_prometheus_services_background_jobs.rb

+# frozen_string_literal: true
+
+class DropActivatePrometheusServicesBackgroundJobs < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+  DROPPED_JOB_CLASS = 'ActivatePrometheusServicesForSharedClusterApplications'.freeze
+  QUEUE = 'background_migration'.freeze
+
+  def up
+    sidekiq_queues.each do |queue|
+      queue.each do |job|
+        klass, project_id, *should_be_empty = job.args
+        next unless klass == DROPPED_JOB_CLASS && project_id.is_a?(Integer) && should_be_empty.empty?
+
+        job.delete
+      end
+    end
+  end
+
+  def down
+    # no-op
+  end
+
+  def sidekiq_queues
+    [Sidekiq::ScheduledSet.new, Sidekiq::RetrySet.new, Sidekiq::Queue.new(QUEUE)]
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_02_21_105436) do
+ActiveRecord::Schema.define(version: 2020_02_21_144534) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"

spec/migrations/drop_activate_prometheus_services_background_jobs_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20200221144534_drop_activate_prometheus_services_background_jobs.rb')
+
+describe DropActivatePrometheusServicesBackgroundJobs, :sidekiq, :redis, :migration, schema: 2020_02_21_144534 do
+  subject(:migration) { described_class.new }
+
+  describe '#up' do
+    let(:retry_set) { Sidekiq::RetrySet.new }
+    let(:scheduled_set) { Sidekiq::ScheduledSet.new }
+
+    context 'there are only affected jobs on the queue' do
+      let(:payload) { { 'class' => ::BackgroundMigrationWorker, 'args' => [described_class::DROPPED_JOB_CLASS, 1] } }
+      let(:queue_payload) { payload.merge('queue' => described_class::QUEUE) }
+
+      it 'removes enqueued ActivatePrometheusServicesForSharedClusterApplications background jobs' do
+        Sidekiq::Testing.disable! do # https://github.com/mperham/sidekiq/wiki/testing#api Sidekiq's API does not have a testing mode
+          retry_set.schedule(1.hour.from_now, payload)
+          scheduled_set.schedule(1.hour.from_now, payload)
+          Sidekiq::Client.push(queue_payload)
+
+          expect { migration.up }.to change { Sidekiq::Queue.new(described_class::QUEUE).size }.from(1).to(0)
+          expect(retry_set.size).to eq(0)
+          expect(scheduled_set.size).to eq(0)
+        end
+      end
+    end
+
+    context "there aren't any affected jobs on the queue" do
+      let(:payload) { { 'class' => ::BackgroundMigrationWorker, 'args' => ['SomeOtherClass', 1] } }
+      let(:queue_payload) { payload.merge('queue' => described_class::QUEUE) }
+
+      it 'skips other enqueued jobs' do
+        Sidekiq::Testing.disable! do
+          retry_set.schedule(1.hour.from_now, payload)
+          scheduled_set.schedule(1.hour.from_now, payload)
+          Sidekiq::Client.push(queue_payload)
+
+          expect { migration.up }.not_to change { Sidekiq::Queue.new(described_class::QUEUE).size }
+          expect(retry_set.size).to eq(1)
+          expect(scheduled_set.size).to eq(1)
+        end
+      end
+    end
+
+    context "there are multiple types of jobs on the queue" do
+      let(:payload) { { 'class' => ::BackgroundMigrationWorker, 'args' => [described_class::DROPPED_JOB_CLASS, 1] } }
+      let(:queue_payload) { payload.merge('queue' => described_class::QUEUE) }
+
+      it 'skips other enqueued jobs' do
+        Sidekiq::Testing.disable! do
+          queue = Sidekiq::Queue.new(described_class::QUEUE)
+          # these jobs will be deleted
+          retry_set.schedule(1.hour.from_now, payload)
+          scheduled_set.schedule(1.hour.from_now, payload)
+          Sidekiq::Client.push(queue_payload)
+          # this jobs will be skipped
+          skipped_jobs_args = [['SomeOtherClass', 1], [described_class::DROPPED_JOB_CLASS, 'wrong id type'], [described_class::DROPPED_JOB_CLASS, 1, 'some wired argument']]
+          skipped_jobs_args.each do |args|
+            retry_set.schedule(1.hour.from_now, { 'class' => ::BackgroundMigrationWorker, 'args' => args })
+            scheduled_set.schedule(1.hour.from_now, { 'class' => ::BackgroundMigrationWorker, 'args' => args })
+            Sidekiq::Client.push('queue' => described_class::QUEUE, 'class' => ::BackgroundMigrationWorker, 'args' => args)
+          end
+
+          migration.up
+
+          expect(retry_set.size).to be 3
+          expect(scheduled_set.size).to be 3
+          expect(queue.size).to be 3
+          expect(queue.map(&:args)).to match_array skipped_jobs_args
+          expect(retry_set.map(&:args)).to match_array skipped_jobs_args
+          expect(scheduled_set.map(&:args)).to match_array skipped_jobs_args
+        end
+      end
+    end
+
+    context "other queues" do
+      it 'does not modify them' do
+        Sidekiq::Testing.disable! do
+          Sidekiq::Client.push('queue' => 'other', 'class' => ::BackgroundMigrationWorker, 'args' => ['SomeOtherClass', 1])
+          Sidekiq::Client.push('queue' => 'other', 'class' => ::BackgroundMigrationWorker, 'args' => [described_class::DROPPED_JOB_CLASS, 1])
+
+          expect { migration.up }.not_to change { Sidekiq::Queue.new('other').size }
+        end
+      end
+    end
+  end
+end