Populate blocking issues count

Backfill blocking_issues_count for issues table

changelogs/unreleased/issue_237977.yml

+---
+title: Populate issues blocking_issues_count
+merge_request: 42277
+author:
+type: other

db/migrate/20200912193210_add_scheduling_issues_temp_indexes.rb

+# frozen_string_literal: true
+
+class AddSchedulingIssuesTempIndexes < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :issue_links, [:source_id], where: 'link_type = 1', name: 'tmp_idx_blocking_type_links'
+    add_concurrent_index :issue_links, [:target_id], where: 'link_type = 2', name: 'tmp_idx_blocked_by_type_links'
+    add_concurrent_index :issues, :id, where: '(state_id = 1 AND blocking_issues_count = 0)', name: 'tmp_idx_index_issues_with_outdate_blocking_count'
+  end
+
+  def down
+    remove_concurrent_index_by_name(:issue_links, 'tmp_idx_blocking_type_links')
+    remove_concurrent_index_by_name(:issue_links, 'tmp_idx_blocked_by_type_links')
+    remove_concurrent_index_by_name(:issues, 'tmp_idx_index_issues_with_outdate_blocking_count')
+  end
+end

db/post_migrate/20200914185610_schedule_sync_blocking_issues_count.rb

+# frozen_string_literal: true
+
+class ScheduleSyncBlockingIssuesCount < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  BATCH_SIZE = 50
+  DELAY_INTERVAL = 120.seconds.to_i
+  MIGRATION = 'SyncBlockingIssuesCount'.freeze
+
+  disable_ddl_transaction!
+
+  class Issue < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'issues'
+  end
+
+  def up
+    return unless Gitlab.ee?
+
+    blocking_issues_ids = <<-SQL
+      SELECT issue_links.source_id AS blocking_issue_id
+      FROM issue_links
+      INNER JOIN issues ON issue_links.source_id = issues.id
+      WHERE issue_links.link_type = 1
+      AND issues.state_id = 1
+      AND issues.blocking_issues_count = 0
+      UNION
+      SELECT issue_links.target_id AS blocking_issue_id
+      FROM issue_links
+      INNER JOIN issues ON issue_links.target_id = issues.id
+      WHERE issue_links.link_type = 2
+      AND issues.state_id = 1
+      AND issues.blocking_issues_count = 0
+    SQL
+
+    relation =
+      Issue.where("id IN(#{blocking_issues_ids})") # rubocop:disable GitlabSecurity/SqlInjection
+
+    queue_background_migration_jobs_by_range_at_intervals(
+      relation,
+      MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE
+    )
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema_migrations/20200912193210

+bcc84e89e4e9772d3209aa8df1368c188b1c6334114bcf339870cae74e724d01
\ No newline at end of file

db/schema_migrations/20200914185610

+50cdb8d42baa0890b39ca7d07a4c157aea7d52a39dee48ee59e51417442eaaf4
\ No newline at end of file

db/structure.sql

@@ -21627,6 +21627,12 @@ CREATE INDEX terraform_states_verification_failure_partial ON terraform_states U
 
 CREATE INDEX tmp_build_stage_position_index ON ci_builds USING btree (stage_id, stage_idx) WHERE (stage_idx IS NOT NULL);
 
+CREATE INDEX tmp_idx_blocked_by_type_links ON issue_links USING btree (target_id) WHERE (link_type = 2);
+
+CREATE INDEX tmp_idx_blocking_type_links ON issue_links USING btree (source_id) WHERE (link_type = 1);
+
+CREATE INDEX tmp_idx_index_issues_with_outdate_blocking_count ON issues USING btree (id) WHERE ((state_id = 1) AND (blocking_issues_count = 0));
+
 CREATE INDEX tmp_index_for_email_unconfirmation_migration ON emails USING btree (id) WHERE (confirmed_at IS NOT NULL);
 
 CREATE UNIQUE INDEX unique_merge_request_metrics_by_merge_request_id ON merge_request_metrics USING btree (merge_request_id);

ee/lib/ee/gitlab/background_migration/sync_blocking_issues_count.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module BackgroundMigration
+      module SyncBlockingIssuesCount
+        extend ::Gitlab::Utils::Override
+
+        override :perform
+        def perform(start_id, end_id)
+          ActiveRecord::Base.connection.execute <<~SQL
+            UPDATE issues
+            SET blocking_issues_count = grouped_counts.count
+            FROM
+              (
+                SELECT blocking_issue_id, SUM(blocked_count) AS count
+                FROM  (
+                          SELECT COUNT(*) AS blocked_count, issue_links.source_id AS blocking_issue_id
+                          FROM issue_links
+                          INNER JOIN issues ON issue_links.source_id = issues.id
+                          WHERE issue_links.link_type = 1
+                          AND issues.state_id = 1
+                          AND issues.blocking_issues_count = 0
+                          AND issue_links.source_id BETWEEN #{start_id} AND #{end_id}
+                          GROUP BY blocking_issue_id HAVING COUNT(*) > 0
+                UNION ALL
+                          SELECT COUNT(*) AS blocked_count, issue_links.target_id AS blocking_issue_id
+                          FROM issue_links
+                          INNER JOIN issues ON issue_links.target_id = issues.id
+                          WHERE issue_links.link_type = 2
+                          AND issues.state_id = 1
+                          AND issues.blocking_issues_count = 0
+                          AND issue_links.target_id BETWEEN #{start_id} AND #{end_id}
+                          GROUP BY blocking_issue_id HAVING COUNT(*) > 0
+                        ) blocking_counts
+            GROUP BY  blocking_issue_id
+               ) AS grouped_counts
+            WHERE issues.blocking_issues_count = 0
+            AND issues.state_id = 1
+            AND issues.id = grouped_counts.blocking_issue_id
+            AND grouped_counts.count > 0
+          SQL
+        end
+      end
+    end
+  end
+end

ee/spec/migrations/schedule_sync_blocking_issues_count_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20200914185610_schedule_sync_blocking_issues_count')
+
+RSpec.describe ScheduleSyncBlockingIssuesCount do
+  let(:issues) { table(:issues) }
+  let(:links) { table(:issue_links) }
+  let(:migration) { described_class::MIGRATION }
+
+  # Blocking issues
+  let!(:issue_1) { issues.create!(description: 'blocking 1', state_id: 1) }
+  let!(:issue_2) { issues.create!(description: 'blocking 2', state_id: 1) }
+  let!(:issue_3) { issues.create!(description: 'blocking 3', state_id: 2) }
+  let!(:issue_4) { issues.create!(description: 'blocking 4', state_id: 1 ) }
+
+  # Blocked issues
+  let!(:issue_5) { issues.create!(description: 'blocked 1', state_id: 1) }
+  let!(:issue_6) { issues.create!(description: 'blocked 2', state_id: 1) }
+  let!(:issue_7) { issues.create!(description: 'blocked 3', state_id: 1) }
+  let!(:issue_8) { issues.create!(description: 'blocked 4', state_id: 1) }
+  let!(:issue_9) { issues.create!(description: 'blocked 5', state_id: 1) }
+  let!(:issue_10) { issues.create!(description: 'blocked 6', state_id: 1) }
+
+  before do
+    stub_const("#{described_class.name}::BATCH_SIZE", 2)
+
+    # Issue links
+    # -----------
+    # issue_1 blocks two issues with link_type BLOCKS
+    # issue_2 blocks two issues with link_type IS_BLOCKED_BY
+    # issue_3 blocks one issue with link_type IS_BLOCKED_BY but it is closed
+    # issue_4 blocks one issue with link_type BLOCKS
+    links.create!(link_type: 1, source_id: issue_1.id, target_id: issue_5.id)
+    links.create!(link_type: 1, source_id: issue_1.id, target_id: issue_6.id)
+    links.create!(link_type: 2, source_id: issue_7.id, target_id: issue_2.id)
+    links.create!(link_type: 2, source_id: issue_8.id, target_id: issue_2.id)
+    links.create!(link_type: 1, source_id: issue_4.id, target_id: issue_9.id)
+    links.create!(link_type: 2, source_id: issue_10.id, target_id: issue_3.id)
+  end
+
+  context 'scheduling migrations' do
+    before do
+      Sidekiq::Worker.clear_all
+    end
+
+    it 'correctly schedules issuable sync background migration' do
+      Sidekiq::Testing.fake! do
+        freeze_time do
+          migrate!
+
+          expect(migration).to be_scheduled_delayed_migration(120.seconds, issue_1.id, issue_2.id)
+          expect(migration).to be_scheduled_delayed_migration(240.seconds, issue_4.id, issue_4.id)
+          expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+        end
+      end
+    end
+  end
+
+  context 'running background migration' do
+    it 'correctly populates issues blocking_issues_count', :sidekiq_might_not_need_inline do
+      migrate!
+
+      expect(issue_1.reload.blocking_issues_count).to eq(2)
+      expect(issue_2.reload.blocking_issues_count).to eq(2)
+      expect(issue_3.reload.blocking_issues_count).to eq(0)
+      expect(issue_4.reload.blocking_issues_count).to eq(1)
+    end
+  end
+end

lib/gitlab/background_migration/sync_blocking_issues_count.rb

+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class SyncBlockingIssuesCount
+      def perform(start_id, end_id)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::SyncBlockingIssuesCount.prepend_if_ee('EE::Gitlab::BackgroundMigration::SyncBlockingIssuesCount')