Automatically enable multiple MR assignees feature flag

This schedules a background migration that checks whether
`multiple_merge_request_assignees` feature flag needs to
be enabled. The rule for it is that
every PopulateMergeRequestAssigneesTable background job
cannot be still running, or being retried or dead.

If there are any enqueued jobs in the state above, the
"checker" (MergeRequestAssigneesMigrationProgressCheck) will
be rescheduled for 3 hours ahead. If there are only dead jobs
in the queue (i.e. no scheduled/enqueued or retrying jobs), we
raise an error and stop the rescheduling.

ee/changelogs/unreleased/osw-schedule-multiple-mr-assignees-ff-auto-enabling.yml

+---
+title: Automatically enable multiple MR assignees feature flag
+merge_request: 10558
+author:
+type: other

ee/db/post_migrate/20190402224749_schedule_merge_request_assignees_migration_progress_check.rb

+# frozen_string_literal: true
+
+class ScheduleMergeRequestAssigneesMigrationProgressCheck < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  MIGRATION = 'MergeRequestAssigneesMigrationProgressCheck'.freeze
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    BackgroundMigrationWorker.perform_async(MIGRATION)
+  end
+
+  def down
+  end
+end

ee/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    class MergeRequestAssigneesMigrationProgressCheck
+      include Gitlab::Utils::StrongMemoize
+
+      RESCHEDULE_DELAY = 3.hours
+      WORKER = 'PopulateMergeRequestAssigneesTable'.freeze
+      DeadJobsError = Class.new(StandardError)
+
+      def perform
+        raise DeadJobsError, "Only dead background jobs in the queue for #{WORKER}" if !ongoing? && dead_jobs?
+
+        if ongoing?
+          BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, self.class.name)
+        else
+          Feature.enable(:multiple_merge_request_assignees)
+        end
+      end
+
+      private
+
+      def dead_jobs?
+        strong_memoize(:dead_jobs) do
+          migration_klass.dead_jobs?(WORKER)
+        end
+      end
+
+      def ongoing?
+        strong_memoize(:ongoing) do
+          migration_klass.exists?(WORKER) || migration_klass.retrying_jobs?(WORKER)
+        end
+      end
+
+      def migration_klass
+        Gitlab::BackgroundMigration
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::MergeRequestAssigneesMigrationProgressCheck do
+  context 'rescheduling' do
+    context 'when there are ongoing and no dead jobs' do
+      it 'reschedules check' do
+        allow(Gitlab::BackgroundMigration).to receive(:exists?)
+                                                .with('PopulateMergeRequestAssigneesTable')
+                                                .and_return(true)
+
+        allow(Gitlab::BackgroundMigration).to receive(:dead_jobs?)
+                                                .with('PopulateMergeRequestAssigneesTable')
+                                                .and_return(false)
+
+        expect(BackgroundMigrationWorker).to receive(:perform_in).with(described_class::RESCHEDULE_DELAY, described_class.name)
+
+        described_class.new.perform
+      end
+    end
+
+    context 'when there are ongoing and dead jobs' do
+      it 'reschedules check' do
+        allow(Gitlab::BackgroundMigration).to receive(:exists?)
+                                                .with('PopulateMergeRequestAssigneesTable')
+                                                .and_return(true)
+
+        allow(Gitlab::BackgroundMigration).to receive(:dead_jobs?)
+                                                .with('PopulateMergeRequestAssigneesTable')
+                                                .and_return(true)
+
+        expect(BackgroundMigrationWorker).to receive(:perform_in).with(described_class::RESCHEDULE_DELAY, described_class.name)
+
+        described_class.new.perform
+      end
+    end
+
+    context 'when there retrying jobs and no scheduled' do
+      it 'reschedules check' do
+        allow(Gitlab::BackgroundMigration).to receive(:exists?)
+                                                .with('PopulateMergeRequestAssigneesTable')
+                                                .and_return(false)
+
+        allow(Gitlab::BackgroundMigration).to receive(:retrying_jobs?)
+                                                .with('PopulateMergeRequestAssigneesTable')
+                                                .and_return(true)
+
+        expect(BackgroundMigrationWorker).to receive(:perform_in).with(described_class::RESCHEDULE_DELAY, described_class.name)
+
+        described_class.new.perform
+      end
+    end
+  end
+
+  context 'when there are no scheduled, or retrying or dead' do
+    it 'enables feature' do
+      allow(Gitlab::BackgroundMigration).to receive(:exists?)
+                                              .with('PopulateMergeRequestAssigneesTable')
+                                              .and_return(false)
+
+      allow(Gitlab::BackgroundMigration).to receive(:retrying_jobs?)
+                                              .with('PopulateMergeRequestAssigneesTable')
+                                              .and_return(false)
+
+      allow(Gitlab::BackgroundMigration).to receive(:dead_jobs?)
+                                              .with('PopulateMergeRequestAssigneesTable')
+                                              .and_return(false)
+
+      expect(Feature).to receive(:enable).with(:multiple_merge_request_assignees)
+
+      described_class.new.perform
+    end
+  end
+
+  context 'when there are only dead jobs' do
+    it 'raises DeadJobsError error' do
+      allow(Gitlab::BackgroundMigration).to receive(:exists?)
+                                              .with('PopulateMergeRequestAssigneesTable')
+                                              .and_return(false)
+
+      allow(Gitlab::BackgroundMigration).to receive(:retrying_jobs?)
+                                              .with('PopulateMergeRequestAssigneesTable')
+                                              .and_return(false)
+
+      allow(Gitlab::BackgroundMigration).to receive(:dead_jobs?)
+                                              .with('PopulateMergeRequestAssigneesTable')
+                                              .and_return(true)
+
+      expect { described_class.new.perform }
+        .to raise_error(described_class::DeadJobsError,
+                        "Only dead background jobs in the queue for #{described_class::WORKER}")
+    end
+  end
+end

ee/spec/migrations/schedule_merge_request_assignees_migration_progress_check_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('ee', 'db', 'post_migrate', '20190402224749_schedule_merge_request_assignees_migration_progress_check.rb')
+
+describe ScheduleMergeRequestAssigneesMigrationProgressCheck do
+  describe '#up' do
+    it 'schedules MergeRequestAssigneesMigrationProgressCheck background job' do
+      expect(BackgroundMigrationWorker).to receive(:perform_async)
+                                             .with(described_class::MIGRATION)
+                                             .and_call_original
+
+      subject.up
+    end
+  end
+end

lib/gitlab/background_migration.rb

@@ -58,11 +58,31 @@ module Gitlab
       migration_class_for(class_name).new.perform(*arguments)
     end
 
-    def self.exists?(migration_class)
+    def self.exists?(migration_class, additional_queues = [])
       enqueued = Sidekiq::Queue.new(self.queue)
       scheduled = Sidekiq::ScheduledSet.new
 
-      [enqueued, scheduled].each do |queue|
+      enqueued_job?([enqueued, scheduled], migration_class)
+    end
+
+    def self.dead_jobs?(migration_class)
+      dead_set = Sidekiq::DeadSet.new
+
+      enqueued_job?([dead_set], migration_class)
+    end
+
+    def self.retrying_jobs?(migration_class)
+      retry_set = Sidekiq::RetrySet.new
+
+      enqueued_job?([retry_set], migration_class)
+    end
+
+    def self.migration_class_for(class_name)
+      const_get(class_name)
+    end
+
+    def self.enqueued_job?(queues, migration_class)
+      queues.each do |queue|
         queue.each do |job|
           return true if job.queue == self.queue && job.args.first == migration_class
         end
@@ -70,9 +90,5 @@ module Gitlab
 
       false
     end
-
-    def self.migration_class_for(class_name)
-      const_get(class_name)
-    end
   end
 end

spec/lib/gitlab/background_migration_spec.rb

@@ -195,4 +195,44 @@ describe Gitlab::BackgroundMigration do
       end
     end
   end
+
+  describe '.dead_jobs?' do
+    let(:queue) do
+      [double(args: ['Foo', [10, 20]], queue: described_class.queue)]
+    end
+
+    context 'when there are dead jobs present' do
+      before do
+        allow(Sidekiq::DeadSet).to receive(:new).and_return(queue)
+      end
+
+      it 'returns true if specific job exists' do
+        expect(described_class.dead_jobs?('Foo')).to eq(true)
+      end
+
+      it 'returns false if specific job does not exist' do
+        expect(described_class.dead_jobs?('Bar')).to eq(false)
+      end
+    end
+  end
+
+  describe '.retrying_jobs?' do
+    let(:queue) do
+      [double(args: ['Foo', [10, 20]], queue: described_class.queue)]
+    end
+
+    context 'when there are dead jobs present' do
+      before do
+        allow(Sidekiq::RetrySet).to receive(:new).and_return(queue)
+      end
+
+      it 'returns true if specific job exists' do
+        expect(described_class.retrying_jobs?('Foo')).to eq(true)
+      end
+
+      it 'returns false if specific job does not exist' do
+        expect(described_class.retrying_jobs?('Bar')).to eq(false)
+      end
+    end
+  end
 end