Migrate to unique indices in projects runners tokens

Rotate duplicate runner registration tokens in the process
in order to remove duplicates

Changelog: changed

db/post_migrate/20220308115219_schedule_reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb

+# frozen_string_literal: true
+
+class ScheduleResetDuplicateCiRunnersTokenEncryptedValuesOnProjects < Gitlab::Database::Migration[1.0]
+  MIGRATION = 'ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects'
+  TOKEN_COLUMN_NAME = :runners_token_encrypted
+  TEMP_INDEX_NAME = "tmp_index_projects_on_id_and_#{TOKEN_COLUMN_NAME}"
+  BATCH_SIZE = 10_000
+  DELAY_INTERVAL = 2.minutes
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :projects, [:id, TOKEN_COLUMN_NAME], where: "#{TOKEN_COLUMN_NAME} IS NOT NULL", unique: false, name: TEMP_INDEX_NAME
+
+    queue_background_migration_jobs_by_range_at_intervals(
+      Gitlab::BackgroundMigration::ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects::Project.base_query,
+      MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      track_jobs: true
+    )
+  end
+
+  def down
+    remove_concurrent_index_by_name(:projects, name: TEMP_INDEX_NAME)
+  end
+end

db/post_migrate/20220308115502_schedule_reset_duplicate_ci_runners_token_values_on_projects.rb

+# frozen_string_literal: true
+
+class ScheduleResetDuplicateCiRunnersTokenValuesOnProjects < Gitlab::Database::Migration[1.0]
+  MIGRATION = 'ResetDuplicateCiRunnersTokenValuesOnProjects'
+  TOKEN_COLUMN_NAME = :runners_token
+  TEMP_INDEX_NAME = "tmp_index_projects_on_id_and_#{TOKEN_COLUMN_NAME}"
+  BATCH_SIZE = 10_000
+  DELAY_INTERVAL = 2.minutes
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :projects, [:id, TOKEN_COLUMN_NAME], where: "#{TOKEN_COLUMN_NAME} IS NOT NULL", unique: false, name: TEMP_INDEX_NAME
+
+    queue_background_migration_jobs_by_range_at_intervals(
+      Gitlab::BackgroundMigration::ResetDuplicateCiRunnersTokenValuesOnProjects::Project.base_query,
+      MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      track_jobs: true
+    )
+  end
+
+  def down
+    remove_concurrent_index_by_name(:projects, name: TEMP_INDEX_NAME)
+  end
+end

db/schema_migrations/20220308115219

+e18ed9e6b2a98c77190ff2ce33f4d2b1984710b438e851d6a526ec8bb1f33c80
\ No newline at end of file

db/schema_migrations/20220308115502

+0aacf46a4a5b430a718336108f52c1c0bed4283846f36c2ab1de80100dcae0b4
\ No newline at end of file

db/structure.sql

@@ -29551,6 +29551,10 @@ CREATE UNIQUE INDEX tmp_index_on_tmp_project_id_on_namespaces ON namespaces USIN
 
 CREATE INDEX tmp_index_on_vulnerabilities_non_dismissed ON vulnerabilities USING btree (id) WHERE (state <> 2);
 
+CREATE INDEX tmp_index_projects_on_id_and_runners_token ON projects USING btree (id, runners_token) WHERE (runners_token IS NOT NULL);
+
+CREATE INDEX tmp_index_projects_on_id_and_runners_token_encrypted ON projects USING btree (id, runners_token_encrypted) WHERE (runners_token_encrypted IS NOT NULL);
+
 CREATE UNIQUE INDEX uniq_pkgs_deb_grp_architectures_on_distribution_id_and_name ON packages_debian_group_architectures USING btree (distribution_id, name);
 
 CREATE UNIQUE INDEX uniq_pkgs_deb_grp_components_on_distribution_id_and_name ON packages_debian_group_components USING btree (distribution_id, name);

lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # A job to nullify duplicate runners_token_encrypted values in projects table in batches
+    class ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects
+      class Project < ActiveRecord::Base # rubocop:disable Style/Documentation
+        include ::EachBatch
+
+        self.table_name = 'projects'
+
+        scope :base_query, -> do
+          where.not(runners_token_encrypted: nil)
+        end
+      end
+
+      def perform(start_id, end_id)
+        # Reset duplicate runner tokens that would prevent creating an unique index.
+        duplicate_tokens = Project.base_query
+          .where(id: start_id..end_id)
+          .group(:runners_token_encrypted)
+          .having('COUNT(*) > 1')
+          .pluck(:runners_token_encrypted)
+
+        Project.where(runners_token_encrypted: duplicate_tokens).update_all(runners_token_encrypted: nil) if duplicate_tokens.any?
+
+        mark_job_as_succeeded(start_id, end_id)
+      end
+
+      private
+
+      def mark_job_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects', arguments)
+      end
+    end
+  end
+end

lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # A job to nullify duplicate ci_runners_token values in projects table in batches
+    class ResetDuplicateCiRunnersTokenValuesOnProjects
+      class Project < ActiveRecord::Base # rubocop:disable Style/Documentation
+        include ::EachBatch
+
+        self.table_name = 'projects'
+
+        scope :base_query, -> do
+          where.not(runners_token: nil)
+        end
+      end
+
+      def perform(start_id, end_id)
+        # Reset duplicate runner tokens that would prevent creating an unique index.
+        duplicate_tokens = Project.base_query
+          .where(id: start_id..end_id)
+          .group(:runners_token)
+          .having('COUNT(*) > 1')
+          .pluck(:runners_token)
+
+        Project.where(runners_token: duplicate_tokens).update_all(runners_token: nil) if duplicate_tokens.any?
+
+        mark_job_as_succeeded(start_id, end_id)
+      end
+
+      private
+
+      def mark_job_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('ResetDuplicateCiRunnerValuesTokensOnProjects', arguments)
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+
+  let(:perform) { described_class.new.perform(1, 4) }
+
+  before do
+    namespaces.create!(id: 123, name: 'sample', path: 'sample')
+
+    projects.create!(id: 1, namespace_id: 123, runners_token_encrypted: 'duplicate')
+    projects.create!(id: 2, namespace_id: 123, runners_token_encrypted: 'a-runners-token')
+    projects.create!(id: 3, namespace_id: 123, runners_token_encrypted: 'duplicate')
+    projects.create!(id: 4, namespace_id: 123, runners_token_encrypted: nil)
+    projects.create!(id: 5, namespace_id: 123, runners_token_encrypted: 'duplicate-2')
+    projects.create!(id: 6, namespace_id: 123, runners_token_encrypted: 'duplicate-2')
+  end
+
+  describe '#up' do
+    before do
+      stub_const("#{described_class}::SUB_BATCH_SIZE", 2)
+    end
+
+    it 'nullifies duplicate tokens', :aggregate_failures do
+      perform
+
+      expect(projects.count).to eq(6)
+      expect(projects.all.pluck(:id, :runners_token_encrypted).to_h).to eq(
+        { 1 => nil, 2 => 'a-runners-token', 3 => nil, 4 => nil, 5 => 'duplicate-2', 6 => 'duplicate-2' }
+      )
+      expect(projects.pluck(:runners_token_encrypted).uniq).to match_array [nil, 'a-runners-token', 'duplicate-2']
+    end
+  end
+end

spec/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::ResetDuplicateCiRunnersTokenValuesOnProjects do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+
+  let(:perform) { described_class.new.perform(1, 4) }
+
+  before do
+    namespaces.create!(id: 123, name: 'sample', path: 'sample')
+
+    projects.create!(id: 1, namespace_id: 123, runners_token: 'duplicate')
+    projects.create!(id: 2, namespace_id: 123, runners_token: 'a-runners-token')
+    projects.create!(id: 3, namespace_id: 123, runners_token: 'duplicate')
+    projects.create!(id: 4, namespace_id: 123, runners_token: nil)
+    projects.create!(id: 5, namespace_id: 123, runners_token: 'duplicate-2')
+    projects.create!(id: 6, namespace_id: 123, runners_token: 'duplicate-2')
+  end
+
+  describe '#up' do
+    before do
+      stub_const("#{described_class}::SUB_BATCH_SIZE", 2)
+    end
+
+    it 'nullifies duplicate tokens', :aggregate_failures do
+      perform
+
+      expect(projects.count).to eq(6)
+      expect(projects.all.pluck(:id, :runners_token).to_h).to eq(
+        { 1 => nil, 2 => 'a-runners-token', 3 => nil, 4 => nil, 5 => 'duplicate-2', 6 => 'duplicate-2' }
+      )
+      expect(projects.pluck(:runners_token).uniq).to match_array [nil, 'a-runners-token', 'duplicate-2']
+    end
+  end
+end