Commit 4f2af0e9 authored by Andreas Brandl's avatar Andreas Brandl

Merge branch 'feat/ssh-sha256-migration' into 'master'

feat: fingerprint_sha256 for SSH keys background migration

Closes #39455

See merge request gitlab-org/gitlab!21579
parents 297cf2c9 48eaed2c
---
title: add background migration for sha256 fingerprints of ssh keys
merge_request: 21579
author: Roger Meier
type: added
# frozen_string_literal: true
class UpdateFingerprintSha256WithinKeys < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
class Key < ActiveRecord::Base
include EachBatch
self.table_name = 'keys'
self.inheritance_column = :_type_disabled
end
disable_ddl_transaction!
def up
queue_background_migration_jobs_by_range_at_intervals(Key, 'MigrateFingerprintSha256WithinKeys', 5.minutes)
end
def down
# no-op
end
end
......@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2020_01_02_170221) do
ActiveRecord::Schema.define(version: 2020_01_06_071113) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_trgm"
......
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This class is responsible to update all sha256 fingerprints within the keys table
class MigrateFingerprintSha256WithinKeys
# Temporary AR table for keys
class Key < ActiveRecord::Base
include EachBatch
self.table_name = 'keys'
self.inheritance_column = :_type_disabled
end
TEMP_TABLE = 'tmp_fingerprint_sha256_migration'
def perform(start_id, stop_id)
ActiveRecord::Base.transaction do
execute(<<~SQL)
CREATE TEMPORARY TABLE #{TEMP_TABLE}
(id bigint primary key, fingerprint_sha256 bytea not null)
ON COMMIT DROP
SQL
fingerprints = []
Key.where(id: start_id..stop_id, fingerprint_sha256: nil).find_each do |regular_key|
fingerprint = Base64.decode64(generate_ssh_public_key(regular_key.key))
fingerprints << {
id: regular_key.id,
fingerprint_sha256: ActiveRecord::Base.connection.escape_bytea(fingerprint)
}
end
Gitlab::Database.bulk_insert(TEMP_TABLE, fingerprints)
execute("ANALYZE #{TEMP_TABLE}")
execute(<<~SQL)
UPDATE keys
SET fingerprint_sha256 = t.fingerprint_sha256
FROM #{TEMP_TABLE} t
WHERE keys.id = t.id
SQL
end
end
private
def generate_ssh_public_key(regular_key)
Gitlab::SSHPublicKey.new(regular_key).fingerprint("SHA256").gsub("SHA256:", "")
end
def execute(query)
ActiveRecord::Base.connection.execute(query)
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::BackgroundMigration::MigrateFingerprintSha256WithinKeys, :migration, schema: 20200106071113 do
subject(:fingerprint_migrator) { described_class.new }
let(:key_table) { table(:keys) }
before do
generate_fingerprints!
end
it 'correctly creates a sha256 fingerprint for a key' do
key_1 = Key.find(1017)
key_2 = Key.find(1027)
expect(key_1.fingerprint_md5).to eq('ba:81:59:68:d7:6c:cd:02:02:bf:6a:9b:55:4e:af:d1')
expect(key_1.fingerprint_sha256).to eq(nil)
expect(key_2.fingerprint_md5).to eq('39:e3:64:a6:24:ea:45:a2:8c:55:2a:e9:4d:4f:1f:b4')
expect(key_2.fingerprint_sha256).to eq(nil)
query_count = ActiveRecord::QueryRecorder.new do
fingerprint_migrator.perform(1, 10000)
end.count
expect(query_count).to eq(8)
key_1.reload
key_2.reload
expect(key_1.fingerprint_md5).to eq('ba:81:59:68:d7:6c:cd:02:02:bf:6a:9b:55:4e:af:d1')
expect(key_1.fingerprint_sha256).to eq('nUhzNyftwADy8AH3wFY31tAKs7HufskYTte2aXo/lCg')
expect(key_2.fingerprint_md5).to eq('39:e3:64:a6:24:ea:45:a2:8c:55:2a:e9:4d:4f:1f:b4')
expect(key_2.fingerprint_sha256).to eq('zMNbLekgdjtcgDv8VSC0z5lpdACMG3Q4PUoIz5+H2jM')
end
it 'migrates all keys' do
expect(Key.where(fingerprint_sha256: nil).count).to eq(Key.all.count)
fingerprint_migrator.perform(1, 10000)
expect(Key.where(fingerprint_sha256: nil).count).to eq(0)
end
def generate_fingerprints!
values = ""
(1000..2000).to_a.each do |record|
key = base_key_for(record)
fingerprint = fingerprint_for(key)
values += "(#{record}, #{record}, 'test-#{record}', '#{key}', '#{fingerprint}'),"
end
update_query = <<~SQL
INSERT INTO keys ( id, user_id, title, key, fingerprint )
VALUES
#{values.chomp(",")};
SQL
ActiveRecord::Base.connection.execute(update_query)
end
def base_key_for(record)
'ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt0000k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0='
.gsub("0000", "%04d" % (record - 1)) # generate arbitrary keys with placeholder 0000 within the key above
end
def fingerprint_for(key)
Gitlab::SSHPublicKey.new(key).fingerprint("md5")
end
end
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20200106071113_update_fingerprint_sha256_within_keys.rb')
describe UpdateFingerprintSha256WithinKeys, :sidekiq, :migration do
let(:key_table) { table(:keys) }
describe '#up' do
it 'the BackgroundMigrationWorker will be triggered and fingerprint_sha256 populated' do
key_table.create!(
id: 1,
user_id: 1,
title: 'test',
key: 'ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt1016k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=',
fingerprint: 'ba:81:59:68:d7:6c:cd:02:02:bf:6a:9b:55:4e:af:d1',
fingerprint_sha256: nil
)
expect(Key.first.fingerprint_sha256).to eq(nil)
described_class.new.up
expect(BackgroundMigrationWorker.jobs.size).to eq(1)
expect(BackgroundMigrationWorker.jobs.first["args"][0]).to eq("MigrateFingerprintSha256WithinKeys")
expect(BackgroundMigrationWorker.jobs.first["args"][1]).to eq([1, 1])
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment