Merge branch 'feat/ssh-sha256-migration' into 'master'

feat: fingerprint_sha256 for SSH keys background migration

Closes #39455

See merge request gitlab-org/gitlab!21579

changelogs/unreleased/feat-ssh-sha256-migration.yml

+---
+title: add background migration for sha256 fingerprints of ssh keys
+merge_request: 21579
+author: Roger Meier
+type: added

db/post_migrate/20200106071113_update_fingerprint_sha256_within_keys.rb

+# frozen_string_literal: true
+
+class UpdateFingerprintSha256WithinKeys < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  class Key < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'keys'
+    self.inheritance_column = :_type_disabled
+  end
+
+  disable_ddl_transaction!
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(Key, 'MigrateFingerprintSha256WithinKeys', 5.minutes)
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_01_02_170221) do
+ActiveRecord::Schema.define(version: 2020_01_06_071113) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"

lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This class is responsible to update all sha256 fingerprints within the keys table
+    class MigrateFingerprintSha256WithinKeys
+      # Temporary AR table for keys
+      class Key < ActiveRecord::Base
+        include EachBatch
+
+        self.table_name = 'keys'
+        self.inheritance_column = :_type_disabled
+      end
+
+      TEMP_TABLE = 'tmp_fingerprint_sha256_migration'
+
+      def perform(start_id, stop_id)
+        ActiveRecord::Base.transaction do
+          execute(<<~SQL)
+            CREATE TEMPORARY TABLE #{TEMP_TABLE}
+              (id bigint primary key, fingerprint_sha256 bytea not null)
+              ON COMMIT DROP
+          SQL
+
+          fingerprints = []
+          Key.where(id: start_id..stop_id, fingerprint_sha256: nil).find_each do |regular_key|
+            fingerprint = Base64.decode64(generate_ssh_public_key(regular_key.key))
+
+            fingerprints << {
+              id: regular_key.id,
+              fingerprint_sha256: ActiveRecord::Base.connection.escape_bytea(fingerprint)
+            }
+          end
+
+          Gitlab::Database.bulk_insert(TEMP_TABLE, fingerprints)
+
+          execute("ANALYZE #{TEMP_TABLE}")
+
+          execute(<<~SQL)
+            UPDATE keys
+              SET fingerprint_sha256 = t.fingerprint_sha256
+              FROM #{TEMP_TABLE} t
+              WHERE keys.id = t.id
+          SQL
+        end
+      end
+
+      private
+
+      def generate_ssh_public_key(regular_key)
+        Gitlab::SSHPublicKey.new(regular_key).fingerprint("SHA256").gsub("SHA256:", "")
+      end
+
+      def execute(query)
+        ActiveRecord::Base.connection.execute(query)
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::MigrateFingerprintSha256WithinKeys, :migration, schema: 20200106071113 do
+  subject(:fingerprint_migrator) { described_class.new }
+
+  let(:key_table) { table(:keys) }
+
+  before do
+    generate_fingerprints!
+  end
+
+  it 'correctly creates a sha256 fingerprint for a key' do
+    key_1 = Key.find(1017)
+    key_2 = Key.find(1027)
+
+    expect(key_1.fingerprint_md5).to eq('ba:81:59:68:d7:6c:cd:02:02:bf:6a:9b:55:4e:af:d1')
+    expect(key_1.fingerprint_sha256).to eq(nil)
+
+    expect(key_2.fingerprint_md5).to eq('39:e3:64:a6:24:ea:45:a2:8c:55:2a:e9:4d:4f:1f:b4')
+    expect(key_2.fingerprint_sha256).to eq(nil)
+
+    query_count = ActiveRecord::QueryRecorder.new do
+      fingerprint_migrator.perform(1, 10000)
+    end.count
+
+    expect(query_count).to eq(8)
+
+    key_1.reload
+    key_2.reload
+
+    expect(key_1.fingerprint_md5).to eq('ba:81:59:68:d7:6c:cd:02:02:bf:6a:9b:55:4e:af:d1')
+    expect(key_1.fingerprint_sha256).to eq('nUhzNyftwADy8AH3wFY31tAKs7HufskYTte2aXo/lCg')
+
+    expect(key_2.fingerprint_md5).to eq('39:e3:64:a6:24:ea:45:a2:8c:55:2a:e9:4d:4f:1f:b4')
+    expect(key_2.fingerprint_sha256).to eq('zMNbLekgdjtcgDv8VSC0z5lpdACMG3Q4PUoIz5+H2jM')
+  end
+
+  it 'migrates all keys' do
+    expect(Key.where(fingerprint_sha256: nil).count).to eq(Key.all.count)
+
+    fingerprint_migrator.perform(1, 10000)
+
+    expect(Key.where(fingerprint_sha256: nil).count).to eq(0)
+  end
+
+  def generate_fingerprints!
+    values = ""
+    (1000..2000).to_a.each do |record|
+      key = base_key_for(record)
+      fingerprint = fingerprint_for(key)
+
+      values += "(#{record}, #{record}, 'test-#{record}', '#{key}', '#{fingerprint}'),"
+    end
+
+    update_query = <<~SQL
+      INSERT INTO keys ( id, user_id, title, key, fingerprint )
+      VALUES
+      #{values.chomp(",")};
+    SQL
+
+    ActiveRecord::Base.connection.execute(update_query)
+  end
+
+  def base_key_for(record)
+    'ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt0000k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0='
+      .gsub("0000", "%04d" % (record - 1)) # generate arbitrary keys with placeholder 0000 within the key above
+  end
+
+  def fingerprint_for(key)
+    Gitlab::SSHPublicKey.new(key).fingerprint("md5")
+  end
+end

spec/migrations/update_fingerprint_sha256_within_keys_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require Rails.root.join('db', 'post_migrate', '20200106071113_update_fingerprint_sha256_within_keys.rb')
+
+describe UpdateFingerprintSha256WithinKeys, :sidekiq, :migration do
+  let(:key_table) { table(:keys) }
+
+  describe '#up' do
+    it 'the BackgroundMigrationWorker will be triggered and fingerprint_sha256 populated' do
+      key_table.create!(
+        id: 1,
+        user_id: 1,
+        title: 'test',
+        key: 'ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt1016k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=',
+        fingerprint: 'ba:81:59:68:d7:6c:cd:02:02:bf:6a:9b:55:4e:af:d1',
+        fingerprint_sha256: nil
+      )
+
+      expect(Key.first.fingerprint_sha256).to eq(nil)
+
+      described_class.new.up
+
+      expect(BackgroundMigrationWorker.jobs.size).to eq(1)
+      expect(BackgroundMigrationWorker.jobs.first["args"][0]).to eq("MigrateFingerprintSha256WithinKeys")
+      expect(BackgroundMigrationWorker.jobs.first["args"][1]).to eq([1, 1])
+    end
+  end
+end