Allow pull mirrors to use SSH public key authentication (backend)

Gemfile

@@ -407,6 +407,7 @@ gem 'net-ntp'
 
 # SSH host key support
 gem 'net-ssh', '~> 4.1.0'
+gem 'sshkey', '~> 1.9.0'
 
 # Required for ED25519 SSH host key support
 group :ed25519 do

Gemfile.lock

@@ -862,6 +862,7 @@ GEM
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     sqlite3 (1.3.13)
+    sshkey (1.9.0)
     stackprof (0.2.10)
     state_machines (0.4.0)
     state_machines-activemodel (0.4.0)
@@ -1151,6 +1152,7 @@ DEPENDENCIES
   spring-commands-rspec (~> 1.0.4)
   spring-commands-spinach (~> 1.1.0)
   sprockets (~> 3.7.0)
+  sshkey (~> 1.9.0)
   stackprof (~> 0.2.10)
   state_machines-activerecord (~> 0.4.0)
   sys-filesystem (~> 1.1.6)

app/controllers/projects/mirrors_controller.rb

@@ -44,7 +44,17 @@ class Projects::MirrorsController < Projects::ApplicationController
       flash[:alert] = @project.errors.full_messages.join(', ').html_safe
     end
 
-    redirect_to_repository_settings(@project)
+    respond_to do |format|
+      format.html { redirect_to_repository_settings(@project) }
+
+      format.json do
+        if @project.errors.present?
+          render json: @project.errors, status: :unprocessable_entity
+        else
+          render json: ProjectMirrorSerializer.new.represent(@project)
+        end
+      end
+    end
   end
 
   def update_now
@@ -66,13 +76,35 @@ class Projects::MirrorsController < Projects::ApplicationController
   end
 
   def mirror_params
-    params.require(:project).permit(:mirror, :import_url, :mirror_user_id,
-                                    :mirror_trigger_builds, remote_mirrors_attributes: [:url, :id, :enabled])
+    params.require(:project)
+      .permit(
+        :mirror,
+        :import_url,
+        :username_only_import_url,
+        :mirror_user_id,
+        :mirror_trigger_builds,
+        import_data_attributes: [:id, :auth_method, :password, :ssh_known_hosts, :regenerate_ssh_private_key],
+        remote_mirrors_attributes: [:url, :id, :enabled]
+      )
   end
 
   def safe_mirror_params
-    return mirror_params if valid_mirror_user?(mirror_params)
+    params = mirror_params
+
+    params[:mirror_user_id] = current_user.id unless valid_mirror_user?(params)
+
+    import_data = params[:import_data_attributes]
+    if import_data.present?
+      # Prevent Rails from destroying the existing import data
+      import_data[:id] ||= project.import_data&.id
+
+      # If the known hosts data is being set, store details about who and when
+      if import_data[:ssh_known_hosts].present?
+        import_data[:ssh_known_hosts_verified_at] = Time.now
+        import_data[:ssh_known_hosts_verified_by_id] = current_user.id
+      end
+    end
 
-    mirror_params.merge(mirror_user_id: current_user.id)
+    params
   end
 end

app/models/project.rb

@@ -167,7 +167,7 @@ class Project < ActiveRecord::Base
   has_many :todos
   has_many :notification_settings, as: :source, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent
 
-  has_one :import_data, class_name: 'ProjectImportData'
+  has_one :import_data, class_name: 'ProjectImportData', inverse_of: :project, autosave: true
   has_one :project_feature
   has_one :statistics, class_name: 'ProjectStatistics'
 
@@ -196,6 +196,7 @@ class Project < ActiveRecord::Base
 
   accepts_nested_attributes_for :variables, allow_destroy: true
   accepts_nested_attributes_for :project_feature
+  accepts_nested_attributes_for :import_data
 
   delegate :name, to: :owner, allow_nil: true, prefix: true
   delegate :count, to: :forks, prefix: true
@@ -587,8 +588,6 @@ class Project < ActiveRecord::Base
       project_import_data.credentials ||= {}
       project_import_data.credentials = project_import_data.credentials.merge(credentials)
     end
-
-    project_import_data.save
   end
 
   def import?

app/models/project_import_data.rb

 require 'carrierwave/orm/activerecord'
 
 class ProjectImportData < ActiveRecord::Base
-  belongs_to :project
+  prepend ::EE::ProjectImportData
+
+  belongs_to :project, inverse_of: :import_data
   attr_encrypted :credentials,
                  key: Gitlab::Application.secrets.db_key_base,
                  marshal: true,

app/models/repository.rb

@@ -966,7 +966,7 @@ class Repository
 
   def fetch_upstream(url)
     add_remote(Repository::MIRROR_REMOTE, url)
-    fetch_remote(Repository::MIRROR_REMOTE)
+    fetch_remote(Repository::MIRROR_REMOTE, ssh_auth: project&.import_data)
   end
 
   def fetch_geo_mirror(url)
@@ -1088,8 +1088,8 @@ class Repository
     false
   end
 
-  def fetch_remote(remote, forced: false, no_tags: false)
-    gitlab_shell.fetch_remote(repository_storage_path, disk_path, remote, forced: forced, no_tags: no_tags)
+  def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false)
+    gitlab_shell.fetch_remote(repository_storage_path, disk_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
   end
 
   def fetch_ref(source_path, source_ref, target_ref)

app/views/shared/_import_form.html.haml

@@ -16,6 +16,9 @@
           To migrate an SVN repository, check out #{link_to "this document", help_page_path('workflow/importing/migrating_from_svn')}.
         %li
           The Git LFS objects will <strong>not</strong> be imported.
+        %li
+          Once imported, repositories can be mirrored over SSH. Read more
+          = link_to 'here', help_page_path('/workflow/repository_mirroring.md', anchor: 'ssh-authentication')
 
 .form-group
   .col-sm-offset-2.col-sm-10

changelogs/unreleased-ee/98-pull-mirror-ssh-keys.yml

+---
+title: Implement SSH public-key support for repository mirroring
+merge_request: 2423
+author:

config/application.rb

@@ -44,6 +44,7 @@ module Gitlab
       #{config.root}/ee/app/models
       #{config.root}/ee/app/models/concerns
       #{config.root}/ee/app/policies
+      #{config.root}/ee/app/serializers
       #{config.root}/ee/app/services
       #{config.root}/ee/app/workers
     ])

ee/app/models/ee/project.rb

@@ -381,6 +381,27 @@ module EE
       end
     end
 
+    def username_only_import_url
+      bare_url = read_attribute(:import_url)
+      return bare_url unless ::Gitlab::UrlSanitizer.valid?(bare_url)
+      ::Gitlab::UrlSanitizer.new(bare_url, credentials: { user: import_data&.user }).full_url
+    end
+
+    def username_only_import_url=(value)
+      unless ::Gitlab::UrlSanitizer.valid?(value)
+        self.import_url = value
+        return
+      end
+
+      url = ::Gitlab::UrlSanitizer.new(value)
+      creds = url.credentials.slice(:user) if url.credentials[:user].present?
+
+      write_attribute(:import_url, url.sanitized_url)
+      create_or_update_import_data(credentials: creds)
+
+      username_only_import_url
+    end
+
     def mark_remote_mirrors_for_removal
       remote_mirrors.each(&:mark_for_delete_if_blank_url)
     end

ee/app/models/ee/project_import_data.rb

+module EE
+  module ProjectImportData
+    SSH_PRIVATE_KEY_OPTS = {
+      type: 'RSA',
+      bits: 4096
+    }.freeze
+
+    extend ActiveSupport::Concern
+
+    included do
+      validates :auth_method, inclusion: { in: %w[password ssh_public_key] }, allow_blank: true
+
+      # We should generate a key even if there's no SSH URL present
+      before_validation :generate_ssh_private_key!, if: ->(data) do
+        regenerate_ssh_private_key || ( auth_method == 'ssh_public_key' && ssh_private_key.blank? )
+      end
+    end
+
+    attr_accessor :regenerate_ssh_private_key
+
+    def ssh_key_auth?
+      ssh_import? && auth_method == 'ssh_public_key'
+    end
+
+    def ssh_import?
+      project&.import_url&.start_with?('ssh://')
+    end
+
+    %i[auth_method user password ssh_private_key ssh_known_hosts ssh_known_hosts_verified_at ssh_known_hosts_verified_by_id].each do |name|
+      define_method(name) do
+        credentials[name] if credentials.present?
+      end
+
+      define_method("#{name}=") do |value|
+        self.credentials ||= {}
+        self.credentials[name] = value
+      end
+    end
+
+    def ssh_known_hosts_verified_by
+      @ssh_known_hosts_verified_by ||= ::User.find_by(id: ssh_known_hosts_verified_by_id)
+    end
+
+    def ssh_known_hosts_fingerprints
+      ::SshHostKey.fingerprint_host_keys(ssh_known_hosts)
+    end
+
+    def auth_method
+      auth_method = credentials.fetch(:auth_method, nil) if credentials.present?
+
+      auth_method.presence || 'password'
+    end
+
+    def ssh_public_key
+      return nil if ssh_private_key.blank?
+
+      comment = "git@#{::Gitlab.config.gitlab.host}"
+      ::SSHKey.new(ssh_private_key, comment: comment).ssh_public_key
+    end
+
+    def generate_ssh_private_key!
+      self.ssh_private_key = ::SSHKey.generate(SSH_PRIVATE_KEY_OPTS).private_key
+    end
+  end
+end

ee/app/models/ssh_host_key.rb

@@ -54,8 +54,9 @@ class SshHostKey
 
   def as_json(*)
     {
-      known_hosts: known_hosts,
-      fingerprints: fingerprints
+      changes_project_import_data: changes_project_import_data?,
+      fingerprints: fingerprints,
+      known_hosts: known_hosts
     }
   end
 
@@ -67,6 +68,17 @@ class SshHostKey
     @fingerprints ||= self.class.fingerprint_host_keys(known_hosts)
   end
 
+  # Returns true if the known_hosts data differs from that currently set for
+  # `project.import_data.ssh_known_hosts`. Ordering is ignored.
+  #
+  # Ordering is ignored
+  def changes_project_import_data?
+    our_known_hosts = known_hosts
+    project_known_hosts = project.import_data&.ssh_known_hosts
+
+    cleanup(our_known_hosts.to_s) != cleanup(project_known_hosts.to_s)
+  end
+
   def error
     with_reactive_cache { |data| data[:error] }
   end

ee/app/serializers/project_mirror_entity.rb

+class ProjectMirrorEntity < Grape::Entity
+  expose :id
+  expose :mirror
+  expose :import_url
+  expose :username_only_import_url
+  expose :mirror_user_id
+  expose :mirror_trigger_builds
+
+  expose :import_data_attributes do |project|
+    import_data = project.import_data
+    next nil unless import_data.present?
+
+    data = import_data.as_json(
+      only: :id,
+      methods: %i[
+        auth_method
+        ssh_known_hosts
+        ssh_known_hosts_verified_at
+        ssh_known_hosts_verified_by_id
+        ssh_public_key
+      ]
+    )
+
+    data[:ssh_known_hosts_fingerprints] = import_data.ssh_known_hosts_fingerprints.as_json
+
+    data
+  end
+
+  expose :remote_mirrors_attributes do |project|
+    next [] unless project.remote_mirrors.present?
+
+    project.remote_mirrors.map do |remote|
+      remote.as_json(only: %i[id url enabled])
+    end
+  end
+end

ee/app/serializers/project_mirror_serializer.rb

+class ProjectMirrorSerializer < BaseSerializer
+  entity ProjectMirrorEntity
+end

lib/gitlab/shell.rb

@@ -126,6 +126,7 @@ module Gitlab
     #
     # name - project path with namespace
     # remote - remote name
+    # ssh_auth - SSH known_hosts data and a private key to use for public-key authentication
     # forced - should we use --force flag?
     # no_tags - should we use --no-tags flag?
     #
@@ -133,12 +134,24 @@ module Gitlab
     #   fetch_remote("gitlab/gitlab-ci", "upstream")
     #
     # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
-    def fetch_remote(storage, name, remote, forced: false, no_tags: false)
+    def fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false)
       args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"]
       args << '--force' if forced
       args << '--no-tags' if no_tags
 
-      gitlab_shell_fast_execute_raise_error(args)
+      vars = {}
+
+      if ssh_auth&.ssh_import?
+        if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
+          vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key
+        end
+
+        if ssh_auth.ssh_known_hosts.present?
+          vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts
+        end
+      end
+
+      gitlab_shell_fast_execute_raise_error(args, vars)
     end
 
     # Move repository
@@ -448,15 +461,15 @@ module Gitlab
       false
     end
 
-    def gitlab_shell_fast_execute_raise_error(cmd)
-      output, status = gitlab_shell_fast_execute_helper(cmd)
+    def gitlab_shell_fast_execute_raise_error(cmd, vars = {})
+      output, status = gitlab_shell_fast_execute_helper(cmd, vars)
 
       raise Error, output unless status.zero?
       true
     end
 
-    def gitlab_shell_fast_execute_helper(cmd)
-      vars = ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS)
+    def gitlab_shell_fast_execute_helper(cmd, vars = {})
+      vars.merge!(ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS))
 
       # Don't pass along the entire parent environment to prevent gitlab-shell
       # from wasting I/O by searching through GEM_PATH

spec/controllers/projects/mirrors_controller_spec.rb

@@ -128,9 +128,84 @@ describe Projects::MirrorsController do
     end
   end
 
+  describe '#update' do
+    let(:project) { create(:project, :repository, :mirror, :remote_mirror) }
+
+    before do
+      sign_in(project.owner)
+    end
+
+    around(:each) do |example|
+      Sidekiq::Testing.fake! { example.run }
+    end
+
+    context 'JSON' do
+      it 'processes a successful update' do
+        do_put(project, { import_url: 'https://updated.example.com' }, format: :json)
+
+        expect(response).to have_http_status(200)
+        expect(json_response['import_url']).to eq('https://updated.example.com')
+      end
+
+      it 'processes an unsuccessful update' do
+        do_put(project, { import_url: 'ftp://invalid.invalid' }, format: :json)
+
+        expect(response).to have_http_status(422)
+        expect(json_response['import_url'].first).to match /valid URL/
+      end
+
+      it "preserves the import_data object when the ID isn't in the request" do
+        import_data_id = project.import_data.id
+
+        do_put(project, { import_data_attributes: { password: 'update' } }, format: :json)
+
+        expect(response).to have_http_status(200)
+        expect(project.import_data(true).id).to eq(import_data_id)
+      end
+
+      it 'sets ssh_known_hosts_verified_at and verified_by when the update sets known hosts' do
+        do_put(project, { import_data_attributes: { ssh_known_hosts: 'update' } }, format: :json)
+
+        expect(response).to have_http_status(200)
+
+        import_data = project.import_data(true)
+        expect(import_data.ssh_known_hosts_verified_at).to be_within(1.minute).of(Time.now)
+        expect(import_data.ssh_known_hosts_verified_by).to eq(project.owner)
+      end
+
+      it 'unsets ssh_known_hosts_verified_at and verified_by when the update unsets known hosts' do
+        project.import_data.update!(ssh_known_hosts: 'foo')
+
+        do_put(project, { import_data_attributes: { ssh_known_hosts: '' } }, format: :json)
+
+        expect(response).to have_http_status(200)
+
+        import_data = project.import_data(true)
+        expect(import_data.ssh_known_hosts_verified_at).to be_nil
+        expect(import_data.ssh_known_hosts_verified_by).to be_nil
+      end
+    end
+
+    context 'HTML' do
+      it 'processes a successful update' do
+        do_put(project, import_url: 'https://updated.example.com')
+
+        expect(response).to redirect_to(project_settings_repository_path(project))
+        expect(flash[:notice]).to match(/successfully updated/)
+      end
+
+      it 'processes an unsuccessful update' do
+        do_put(project, import_url: 'ftp://invalid.invalid')
+
+        expect(response).to redirect_to(project_settings_repository_path(project))
+        expect(flash[:alert]).to match(/valid URL/)
+      end
+    end
+  end
+
   describe '#ssh_host_keys', use_clean_rails_memory_store_caching: true do
     let(:project) { create(:project) }
-    let(:cache) { SshHostKey.new(project_id: project.id, url: "ssh://example.com:22") }
+    let(:cache) { SshHostKey.new(project: project, url: "ssh://example.com:22") }
 
     before do
       sign_in(project.owner)
@@ -176,7 +251,7 @@ describe Projects::MirrorsController do
         do_get(project)
 
         expect(response).to have_http_status(200)
-        expect(json_response).to eq('known_hosts' => ssh_key, 'fingerprints' => [ssh_fp.stringify_keys])
+        expect(json_response).to eq('known_hosts' => ssh_key, 'fingerprints' => [ssh_fp.stringify_keys], 'changes_project_import_data' => true)
       end
     end
 
@@ -185,8 +260,8 @@ describe Projects::MirrorsController do
     end
   end
 
-  def do_put(project, options)
-    attrs = { namespace_id: project.namespace.to_param, project_id: project.to_param }
+  def do_put(project, options, extra_attrs = {})
+    attrs = extra_attrs.merge(namespace_id: project.namespace.to_param, project_id: project.to_param)
     attrs[:project] = options
 
     put :update, attrs

spec/ee/spec/features/projects/settings/ee/repository_mirrors_settings_spec.rb

@@ -40,13 +40,6 @@ describe 'Project settings > [EE] repository' do
       visit project_settings_repository_path(project)
     end
 
-    it 'shows pull mirror settings' do
-      expect(page).to have_selector('#project_mirror')
-      expect(page).to have_selector('#project_import_url')
-      expect(page).to have_selector('#project_mirror_user_id', visible: false)
-      expect(page).to have_selector('#project_mirror_trigger_builds')
-    end
-
     it 'shows push mirror settings' do
       expect(page).to have_selector('#project_remote_mirrors_attributes_0_enabled')
       expect(page).to have_selector('#project_remote_mirrors_attributes_0_url')

spec/ee/spec/models/ee/project_import_data_spec.rb

+require 'spec_helper'
+
+describe ProjectImportData do
+  let(:import_url) { 'ssh://example.com' }
+  let(:import_data_attrs) { { auth_method: 'ssh_public_key' } }
+  let(:project) { build(:project, :mirror, import_url: import_url, import_data_attributes: import_data_attrs) }
+
+  subject(:import_data) { project.import_data }
+
+  describe 'validations' do
+    it { is_expected.to validate_inclusion_of(:auth_method).in_array([nil, '', 'password', 'ssh_public_key']) }
+  end
+
+  describe '#ssh_key_auth?' do
+    subject { import_data.ssh_key_auth? }
+
+    [
+      { import_url: 'ssh://example.com',  auth_method: 'ssh_public_key', expected: true  },
+      { import_url: 'ssh://example.com',  auth_method: 'password',       expected: false },
+      { import_url: 'http://example.com', auth_method: 'ssh_public_key', expected: false },
+      { import_url: 'http://example.com', auth_method: 'password',       expected: false }
+    ].each do |spec|
+      context spec.inspect do
+        let(:import_url) { spec[:import_url] }
+        let(:import_data_attrs) { { auth_method: spec[:auth_method] } }
+
+        it { is_expected.to spec[:expected] ? be_truthy : be_falsy }
+      end
+    end
+  end
+
+  describe '#ssh_known_hosts_verified_by' do
+    let(:user) { project.owner }
+
+    subject { import_data.ssh_known_hosts_verified_by }
+
+    it 'is a user when ssh_known_hosts_verified_by_id is a valid id' do
+      import_data.ssh_known_hosts_verified_by_id = user.id
+
+      is_expected.to eq(user)
+    end
+
+    it 'is nil when ssh_known_hosts_verified_by_id is an invalid id' do
+      import_data.ssh_known_hosts_verified_by_id = -1
+
+      is_expected.to be_nil
+    end
+
+    it 'is nil when ssh_known_hosts_verified_by_id is nil' do
+      is_expected.to be_nil
+    end
+  end
+
+  describe 'auth_method' do
+    [nil, ''].each do |value|
+      it "returns 'password' when #{value.inspect}" do
+        import_data.auth_method = value
+
+        expect(import_data.auth_method).to eq('password')
+      end
+    end
+  end
+
+  describe '#ssh_import?' do
+    subject { import_data.ssh_import? }
+
+    [
+      { import_url: nil,                   expected: false },
+      { import_url: 'ssh://example.com',   expected: true  },
+      { import_url: 'git://example.com',   expected: false },
+      { import_url: 'http://example.com',  expected: false },
+      { import_url: 'https://example.com', expected: false }
+    ].each do |spec|
+      context spec.inspect do
+        let(:import_url) { spec[:import_url] }
+
+        it { is_expected.to spec[:expected] ? be_truthy : be_falsy }
+      end
+    end
+  end
+
+  describe '#ssh_known_hosts_fingerprints' do
+    subject { import_data.ssh_known_hosts_fingerprints }
+
+    it 'defers to SshHostKey#fingerprint_host_keys' do
+      import_data.ssh_known_hosts = 'known_hosts'
+
+      expect(SshHostKey).to receive(:fingerprint_host_keys).with('known_hosts').and_return(:result)
+
+      is_expected.to eq(:result)
+    end
+  end
+
+  describe '#ssh_public_key' do
+    subject { import_data.ssh_public_key }
+
+    context 'no SSH key' do
+      it { is_expected.to be_nil }
+    end
+
+    context 'with SSH key' do
+      before do
+        # The key should be generated regardless of the URL, as long as the
+        # auth method is correct
+        project.import_url = nil
+
+        # Triggers the `before_validation` callback
+        import_data.valid?
+      end
+
+      it 'returns the public counterpart of the SSH private key' do
+        comment = "git@#{::Gitlab.config.gitlab.host}"
+        expected = SSHKey.new(import_data.ssh_private_key, comment: comment)
+
+        is_expected.to eq(expected.ssh_public_key)
+      end
+    end
+  end
+
+  describe '#regenerate_ssh_private_key' do
+    %w[password ssh_public_key].each do |auth_method|
+      context "auth_method is #{auth_method}" do
+        let(:import_data_attrs) { { auth_method: auth_method } }
+
+        it 'regenerates the SSH private key' do
+          initial = import_data.ssh_private_key
+
+          import_data.regenerate_ssh_private_key = true
+          import_data.valid?
+
+          expect(import_data.ssh_private_key).not_to eq(initial)
+        end
+      end
+    end
+  end
+end

spec/ee/spec/models/ee/project_spec.rb

@@ -814,4 +814,48 @@ describe Project do
       end
     end
   end
+
+  describe '#username_only_import_url' do
+    def build_project(username: 'user', password: 'password')
+      build(:project, import_url: 'http://example.com').tap do |project|
+        project.build_import_data(credentials: { user: username, password: password })
+      end
+    end
+
+    it 'shows the bare url when no username is present' do
+      project = build_project(username: nil)
+
+      expect(project.username_only_import_url).to eq('http://example.com')
+    end
+
+    it 'shows the URL with username when present' do
+      project = build_project(password: nil)
+
+      expect(project.username_only_import_url).to eq('http://user@example.com')
+    end
+
+    it 'excludes the pasword when present' do
+      project = build_project
+
+      expect(project.username_only_import_url).to eq('http://user@example.com')
+    end
+  end
+
+  describe '#username_only_import_url=' do
+    it 'sets the import url and username' do
+      project = build(:project, import_url: 'http://user@example.com')
+
+      expect(project.import_url).to eq('http://user@example.com')
+      expect(project.import_data.user).to eq('user')
+    end
+
+    it 'does not unset the password' do
+      project = build(:project, import_url: 'http://olduser:pass@old.example.com')
+      project.username_only_import_url = 'http://user@example.com'
+
+      expect(project.username_only_import_url).to eq('http://user@example.com')
+      expect(project.import_url).to eq('http://user:pass@example.com')
+      expect(project.import_data.password).to eq('pass')
+    end
+  end
 end

spec/ee/spec/models/ssh_host_key_spec.rb

@@ -9,7 +9,7 @@ describe SshHostKey do
   ]
 
   # Purposefully ordered so that `sort` will make changes
-  known_hosts = <<-EOF.strip_heredoc
+  known_hosts = <<~EOF
     example.com #{keys[0]} git@localhost
     @revoked other.example.com #{keys[1]} git@localhost
   EOF
@@ -46,6 +46,62 @@ describe SshHostKey do
     end
   end
 
+  describe '#fingerprints', use_clean_rails_memory_store_caching: true do
+    it 'returns an array of indexed fingerprints when the cache is filled' do
+      key1 = SSHKeygen.generate
+      key2 = SSHKeygen.generate
+
+      known_hosts = "example.com #{key1} git@localhost\n\n\n@revoked other.example.com #{key2} git@localhost\n"
+      stub_reactive_cache(ssh_host_key, known_hosts: known_hosts)
+
+      expect(ssh_host_key.fingerprints.as_json).to eq(
+        [
+          { bits: 2048, fingerprint: Gitlab::KeyFingerprint.new(key1).fingerprint, type: 'RSA', index: 0 },
+          { bits: 2048, fingerprint: Gitlab::KeyFingerprint.new(key2).fingerprint, type: 'RSA', index: 3 }
+        ]
+      )
+    end
+
+    it 'returns an empty array when the cache is empty' do
+      expect(ssh_host_key.fingerprints).to eq([])
+    end
+  end
+
+  describe '#changes_project_import_data?' do
+    subject { ssh_host_key.changes_project_import_data? }
+
+    reversed = known_hosts.lines.reverse.join
+    extra = known_hosts + "foo\nbar\n"
+
+    [
+      { a: known_hosts, b: extra,       result: true  },
+      { a: known_hosts, b: "foo\n",     result: true  },
+      { a: known_hosts, b: '',          result: true  },
+      { a: known_hosts, b: nil,         result: true  },
+      { a: known_hosts, b: known_hosts, result: false },
+      { a: reversed,    b: known_hosts, result: false },
+      { a: extra,       b: "foo\n",     result: true  },
+      { a: '',          b: '',          result: false },
+      { a: nil,         b: nil,         result: false },
+      { a: '',          b: nil,         result: false }
+    ].each_with_index do |spec, index|
+      it "is #{spec[:result]} for test case #{index}" do
+        expect(ssh_host_key).to receive(:known_hosts).and_return(spec[:a])
+        project.import_data.ssh_known_hosts = spec[:b]
+
+        is_expected.to eq(spec[:result])
+      end
+
+      # Comparisons should be symmetrical, so test the reverse too
+      it "is #{spec[:result]} for test case #{index} (reversed)" do
+        expect(ssh_host_key).to receive(:known_hosts).and_return(spec[:b])
+        project.import_data.ssh_known_hosts = spec[:a]
+
+        is_expected.to eq(spec[:result])
+      end
+    end
+  end
+
   describe '#calculate_reactive_cache' do
     subject(:cache) { ssh_host_key.calculate_reactive_cache }
 

spec/ee/spec/serializers/project_mirror_entity_spec.rb

+require 'spec_helper'
+
+describe ProjectMirrorEntity do
+  subject(:entity) { described_class.new(project).as_json.deep_symbolize_keys }
+
+  describe 'pull mirror' do
+    let(:project) { create(:project, :mirror) }
+    let(:import_data) { project.import_data }
+
+    context 'password authentication' do
+      before do
+        import_data.update!(auth_method: 'password', password: 'fake password')
+      end
+
+      it 'represents the pull mirror' do
+        is_expected.to eq(
+          id: project.id,
+          mirror: true,
+          import_url: project.import_url,
+          username_only_import_url: project.username_only_import_url,
+          mirror_user_id: project.mirror_user_id,
+          mirror_trigger_builds: project.mirror_trigger_builds,
+          import_data_attributes: {
+            id: import_data.id,
+            auth_method: 'password',
+            ssh_known_hosts: nil,
+            ssh_known_hosts_fingerprints: [],
+            ssh_known_hosts_verified_at: nil,
+            ssh_known_hosts_verified_by_id: nil,
+            ssh_public_key: nil
+          },
+          remote_mirrors_attributes: []
+        )
+      end
+    end
+
+    context 'SSH public-key authentication' do
+      before do
+        project.import_url = "ssh://example.com"
+        import_data.update!(auth_method: 'ssh_public_key', ssh_known_hosts: "example.com #{SSHKeygen.generate}")
+      end
+
+      it 'represents the pull mirror' do
+        is_expected.to eq(
+          id: project.id,
+          mirror: true,
+          import_url: project.import_url,
+          username_only_import_url: project.username_only_import_url,
+          mirror_user_id: project.mirror_user_id,
+          mirror_trigger_builds: project.mirror_trigger_builds,
+          import_data_attributes: {
+            id: import_data.id,
+            auth_method: 'ssh_public_key',
+            ssh_known_hosts: import_data.ssh_known_hosts,
+            ssh_known_hosts_fingerprints: import_data.ssh_known_hosts_fingerprints.as_json,
+            ssh_known_hosts_verified_at: import_data.ssh_known_hosts_verified_at,
+            ssh_known_hosts_verified_by_id: import_data.ssh_known_hosts_verified_by_id,
+            ssh_public_key: import_data.ssh_public_key
+          },
+          remote_mirrors_attributes: []
+        )
+      end
+    end
+  end
+
+  describe 'push mirror' do
+    let(:project) { create(:project, :repository, :remote_mirror) }
+    let(:remote_mirror) { project.remote_mirrors.first }
+
+    it 'represents the push mirror' do
+      is_expected.to eq(
+        id: project.id,
+        mirror: false,
+        import_url: nil,
+        username_only_import_url: nil,
+        mirror_user_id: nil,
+        mirror_trigger_builds: false,
+        import_data_attributes: nil,
+        remote_mirrors_attributes: [
+          {
+            id: remote_mirror.id,
+            url: remote_mirror.url,
+            enabled: true
+          }
+        ]
+      )
+    end
+  end
+end

spec/ee/spec/serializers/project_mirror_serializer_spec.rb

+require 'spec_helper'
+
+describe ProjectMirrorSerializer do
+  it 'represents ProjectMirror entities' do
+    expect(described_class.entity_class).to eq(ProjectMirrorEntity)
+  end
+end

spec/lib/gitlab/bitbucket_import/importer_spec.rb

@@ -54,7 +54,7 @@ describe Gitlab::BitbucketImport::Importer do
     create(
       :project,
       import_source: project_identifier,
-      import_data: ProjectImportData.new(credentials: data)
+      import_data_attributes: { credentials: data }
     )
   end
 

spec/lib/gitlab/shell_spec.rb

@@ -509,20 +509,94 @@ describe Gitlab::Shell do
     end
 
     describe '#fetch_remote' do
+      def fetch_remote(ssh_auth = nil)
+        gitlab_shell.fetch_remote('current/storage', 'project/path', 'new/storage', ssh_auth: ssh_auth)
+      end
+
+      def expect_popen(vars = {})
+        popen_args = [
+          projects_path,
+          'fetch-remote',
+          'current/storage',
+          'project/path.git',
+          'new/storage',
+          Gitlab.config.gitlab_shell.git_timeout.to_s
+        ]
+
+        expect(Gitlab::Popen).to receive(:popen).with(popen_args, nil, popen_vars.merge(vars))
+      end
+
+      def build_ssh_auth(opts = {})
+        defaults = {
+          ssh_import?: true,
+          ssh_key_auth?: false,
+          ssh_known_hosts: nil,
+          ssh_private_key: nil
+        }
+
+        double(:ssh_auth, defaults.merge(opts))
+      end
+
       it 'returns true when the command succeeds' do
-        expect(Gitlab::Popen).to receive(:popen)
-          .with([projects_path, 'fetch-remote', 'current/storage', 'project/path.git', 'new/storage', '800'],
-                nil, popen_vars).and_return([nil, 0])
+        expect_popen.and_return([nil, 0])
 
-        expect(gitlab_shell.fetch_remote('current/storage', 'project/path', 'new/storage')).to be true
+        expect(fetch_remote).to be_truthy
       end
 
       it 'raises an exception when the command fails' do
-        expect(Gitlab::Popen).to receive(:popen)
-        .with([projects_path, 'fetch-remote', 'current/storage', 'project/path.git', 'new/storage', '800'],
-              nil, popen_vars).and_return(["error", 1])
+        expect_popen.and_return(["error", 1])
+
+        expect { fetch_remote }.to raise_error(Gitlab::Shell::Error, "error")
+      end
+
+      context 'SSH auth' do
+        it 'passes the SSH key if specified' do
+          expect_popen('GITLAB_SHELL_SSH_KEY' => 'foo').and_return([nil, 0])
+
+          ssh_auth = build_ssh_auth(ssh_key_auth?: true, ssh_private_key: 'foo')
+
+          expect(fetch_remote(ssh_auth)).to be_truthy
+        end
+
+        it 'does not pass an empty SSH key' do
+          expect_popen.and_return([nil, 0])
 
-        expect { gitlab_shell.fetch_remote('current/storage', 'project/path', 'new/storage') }.to raise_error(Gitlab::Shell::Error, "error")
+          ssh_auth = build_ssh_auth(ssh_key_auth: true, ssh_private_key: '')
+
+          expect(fetch_remote(ssh_auth)).to be_truthy
+        end
+
+        it 'does not pass the key unless SSH key auth is to be used' do
+          expect_popen.and_return([nil, 0])
+
+          ssh_auth = build_ssh_auth(ssh_key_auth: false, ssh_private_key: 'foo')
+
+          expect(fetch_remote(ssh_auth)).to be_truthy
+        end
+
+        it 'passes the known_hosts data if specified' do
+          expect_popen('GITLAB_SHELL_KNOWN_HOSTS' => 'foo').and_return([nil, 0])
+
+          ssh_auth = build_ssh_auth(ssh_known_hosts: 'foo')
+
+          expect(fetch_remote(ssh_auth)).to be_truthy
+        end
+
+        it 'does not pass empty known_hosts data' do
+          expect_popen.and_return([nil, 0])
+
+          ssh_auth = build_ssh_auth(ssh_known_hosts: '')
+
+          expect(fetch_remote(ssh_auth)).to be_truthy
+        end
+
+        it 'does not pass known_hosts data unless SSH is to be used' do
+          expect_popen(popen_vars).and_return([nil, 0])
+
+          ssh_auth = build_ssh_auth(ssh_import?: false, ssh_known_hosts: 'foo')
+
+          expect(fetch_remote(ssh_auth)).to be_truthy
+        end
       end
     end