Merge remote-tracking branch 'origin/master' into ce-to-ee-2017-11-22

Signed-off-by: Rémy Coutable <remy@rymai.me>

app/models/repository.rb

@@ -28,7 +28,6 @@ class Repository
   CreateTreeError = Class.new(StandardError)
 
   MIRROR_REMOTE = "upstream".freeze
-  MIRROR_GEO = "geo".freeze
 
   # Methods that cache data from the Git repository.
   #
@@ -935,12 +934,6 @@ class Repository
     fetch_remote(Repository::MIRROR_REMOTE, ssh_auth: project&.import_data)
   end
 
-  def fetch_geo_mirror(url)
-    add_remote(Repository::MIRROR_GEO, url)
-    set_remote_as_mirror(Repository::MIRROR_GEO)
-    fetch_remote(Repository::MIRROR_GEO, forced: true)
-  end
-
   def upstream_branches
     @upstream_branches ||= remote_branches(Repository::MIRROR_REMOTE)
   end
@@ -1022,6 +1015,19 @@ class Repository
     run_git(args).first.lines.map(&:strip)
   end
 
+  def fetch_as_mirror(url, forced: false, refmap: :all_refs, remote_name: nil)
+    unless remote_name
+      remote_name = "tmp-#{SecureRandom.hex}"
+      tmp_remote_name = true
+    end
+
+    add_remote(remote_name, url)
+    set_remote_as_mirror(remote_name, refmap: refmap)
+    fetch_remote(remote_name, forced: forced)
+  ensure
+    remove_remote(remote_name) if tmp_remote_name
+  end
+
   def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false)
     gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
   end
@@ -1125,6 +1131,10 @@ class Repository
     raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref)
   end
 
+  def repository_storage_path
+    @project.repository_storage_path
+  end
+
   private
 
   # TODO Generice finder, later split this on finders by Ref or Oid
@@ -1190,10 +1200,6 @@ class Repository
     raw_repository.run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip
   end
 
-  def repository_storage_path
-    @project.repository_storage_path
-  end
-
   def initialize_raw_repository
     Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, is_wiki))
   end

app/services/geo/base_sync_service.rb

@@ -114,13 +114,13 @@ module Geo
       authorization = ::Gitlab::Geo::BaseRequest.new.authorization
       header = { "http.#{url}.extraHeader" => "Authorization: #{authorization}" }
 
-      repository.with_config(header) { repository.fetch_geo_mirror(url) }
+      repository.with_config(header) { repository.fetch_as_mirror(url, forced: true) }
     end
 
     def fetch_ssh_geo_mirror(repository)
       url = build_repository_url(primary_ssh_path_prefix, repository)
 
-      repository.fetch_geo_mirror(url)
+      repository.fetch_as_mirror(url, forced: true)
     end
 
     def registry

app/services/projects/import_service.rb

@@ -51,10 +51,13 @@ module Projects
 
     def import_repository
       begin
-        if project.gitea_import?
-          fetch_repository
+        refmap = importer_class.try(:refmap) if has_importer?
+
+        if refmap
+          project.ensure_repository
+          project.repository.fetch_as_mirror(project.import_url, refmap: refmap)
         else
-          clone_repository
+          gitlab_shell.import_repository(project.repository_storage_path, project.disk_path, project.import_url)
         end
       rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e
         # Expire cache to prevent scenarios such as:
@@ -66,17 +69,6 @@ module Projects
       end
     end
 
-    def clone_repository
-      gitlab_shell.import_repository(project.repository_storage_path, project.disk_path, project.import_url)
-    end
-
-    def fetch_repository
-      project.ensure_repository
-      project.repository.add_remote(project.import_type, project.import_url)
-      project.repository.set_remote_as_mirror(project.import_type)
-      project.repository.fetch_remote(project.import_type, forced: true)
-    end
-
     def import_data
       return unless has_importer?
 

doc/administration/high_availability/database.md

@@ -539,6 +539,301 @@ gitlab-rake gitlab:db:configure
 At this point, your GitLab instance should be up and running. Verify you are
 able to login, and create issues and merge requests.  If you have troubles check the [Troubleshooting section](#troubleshooting).
 
+### Example configuration
+
+Here we'll show you some fully expanded example configurations.
+
+#### Example recommended setup
+
+This example uses 3 consul servers, 3 postgresql servers, and 1 application node.
+
+We start with all servers on the same 10.6.0.0/16 private network range, they
+can connect to each freely other on those addresses.
+
+Here is a list and description of each machine and the assigned IP:
+
+* `10.6.0.11`: Consul 1
+* `10.6.0.12`: Consul 2
+* `10.6.0.13`: Consul 3
+* `10.6.0.21`: PostgreSQL master
+* `10.6.0.22`: PostgreSQL secondary
+* `10.6.0.23`: PostgreSQL secondary
+* `10.6.0.31`: GitLab application
+
+All passwords are set to `toomanysecrets`, please do not use this password or derived hashes.
+
+The external_url for GitLab is `http://gitlab.example.com`
+
+Please note that after the initial configuration, if a failover occurs, the PostgresSQL master will change to one of the available secondaries until it is failed back.
+
+##### Example recommended setup for Consul servers
+
+On each server edit `/etc/gitlab/gitlab.rb`:
+
+```ruby
+# Disable all components except Consul
+bootstrap['enable'] = false
+gitlab_rails['auto_migrate'] = false
+gitaly['enable'] = false
+gitlab_workhorse['enable'] = false
+mailroom['enable'] = false
+nginx['enable'] = false
+postgresql['enable'] = false
+redis['enable'] = false
+sidekiq['enable'] = false
+prometheus['enable'] = false
+unicorn['enable'] = false
+
+consul['enable'] = true
+consul['configuration'] = {
+  server: true,
+  retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13)
+}
+```
+
+[Reconfigure Omnibus GitLab][reconfigure Gitlab] for the changes to take effect.
+
+##### Example recommended setup for PostgreSQL servers
+
+On each server edit `/etc/gitlab/gitlab.rb`:
+
+```ruby
+# Disable all components except PostgreSQL and Repmgr and Consul
+bootstrap['enable'] = false
+gitaly['enable'] = false
+mailroom['enable'] = false
+nginx['enable'] = false
+unicorn['enable'] = false
+sidekiq['enable'] = false
+redis['enable'] = false
+gitlab_workhorse['enable'] = false
+prometheus_monitoring['enable'] = false
+
+repmgr['enable'] = true
+postgresql['enable'] = true
+consul['enable'] = true
+
+# PostgreSQL configuration
+postgresql['listen_address'] = '0.0.0.0'
+postgresql['hot_standby'] = 'on'
+postgresql['wal_level'] = 'replica'
+postgresql['shared_preload_libraries'] = 'repmgr_funcs'
+
+# Disable automatic database migrations
+gitlab_rails['auto_migrate'] = false
+
+# Configure the consul agent
+consul['services'] = %w(postgresql)
+
+postgresql['pgbouncer_user_password'] = '771a8625958a529132abe6f1a4acb19c'
+postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f'
+postgresql['max_wal_senders'] = 4
+
+postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/16)
+repmgr['trust_auth_cidr_addresses'] = %w(10.6.0.0/16)
+
+consul['configuration'] = {
+  retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13)
+}
+```
+
+[Reconfigure Omnibus GitLab][reconfigure Gitlab] for the changes to take effect.
+
+##### Example recommended setup for application server
+
+On the server edit `/etc/gitlab/gitlab.rb`:
+
+```ruby
+external_url 'http://gitlab.example.com'
+
+gitlab_rails['db_host'] = '127.0.0.1'
+gitlab_rails['db_port'] = 6432
+gitlab_rails['db_password'] = 'toomanysecrets'
+gitlab_rails['auto_migrate'] = false
+
+postgresql['enable'] = false
+pgbouncer['enable'] = true
+consul['enable'] = true
+
+# Configure Pgbouncer
+pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul)
+
+# Configure Consul agent
+consul['watchers'] = %w(postgresql)
+
+pgbouncer['users'] = {
+  'gitlab-consul': {
+    password: '5e0e3263571e3704ad655076301d6ebe'
+  },
+  'pgbouncer': {
+    password: '771a8625958a529132abe6f1a4acb19c'
+  }
+}
+
+consul['configuration'] = {
+  retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13)
+}
+```
+
+[Reconfigure Omnibus GitLab][reconfigure Gitlab] for the changes to take effect.
+
+##### Example recommended setup manual steps
+
+After deploying the configuration follow these steps:
+
+1. On `10.6.0.21`, our primary database
+
+    Enable the `pg_trgm` extension
+
+    ```sh
+    gitlab-psql -d gitlabhq_production
+    ```
+
+    ```
+    CREATE EXTENSION pg_trgm;
+    ```
+
+1. On `10.6.0.22`, our first standby database
+
+    Make this node a standby of the primary
+
+    ```sh
+    gitlab-ctl repmgr standby setup 10.6.0.21
+    ```
+
+1. On `10.6.0.23`, our second standby database
+
+    Make this node a standby of the primary
+
+    ```sh
+    gitlab-ctl repmgr standby setup 10.6.0.21
+    ```
+
+1. On `10.6.0.31`, our application server
+
+    Set gitlab-consul's pgbouncer password to `toomanysecrets`
+
+    ```sh
+    gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul
+    ```
+
+    Run database migrations
+
+    ```sh
+    gitlab-rake gitlab:db:configure
+    ```
+
+#### Example minimal setup
+
+This example uses 3 postgresql servers, and 1 application node.
+
+It differs from the [recommended setup](#example_recommended_setup) by moving the consul servers into the same servers we use for PostgreSQL.
+The trade-off is between reducing server counts, against the increased operational complexity of needing to deal with postgres [failover](#failover_procedure) and [restore](#restore_procedure) procedures in addition to [consul outage recovery](consul.md#outage_recovery) on the same set of machines.
+
+In this example we start with all servers on the same 10.6.0.0/16 private network range, they can connect to each freely other on those addresses.
+
+Here is a list and description of each machine and the assigned IP:
+
+* `10.6.0.21`: PostgreSQL master
+* `10.6.0.22`: PostgreSQL secondary
+* `10.6.0.23`: PostgreSQL secondary
+* `10.6.0.31`: GitLab application
+
+All passwords are set to `toomanysecrets`, please do not use this password or derived hashes.
+
+The external_url for GitLab is `http://gitlab.example.com`
+
+Please note that after the initial configuration, if a failover occurs, the PostgresSQL master will change to one of the available secondaries until it is failed back.
+
+##### Example minimal configuration for database servers
+
+On each server edit `/etc/gitlab/gitlab.rb`:
+
+```ruby
+# Disable all components except PostgreSQL, Repmgr, and Consul
+bootstrap['enable'] = false
+gitaly['enable'] = false
+mailroom['enable'] = false
+nginx['enable'] = false
+unicorn['enable'] = false
+sidekiq['enable'] = false
+redis['enable'] = false
+gitlab_workhorse['enable'] = false
+prometheus_monitoring['enable'] = false
+
+repmgr['enable'] = true
+postgresql['enable'] = true
+consul['enable'] = true
+
+# PostgreSQL configuration
+postgresql['listen_address'] = '0.0.0.0'
+postgresql['hot_standby'] = 'on'
+postgresql['wal_level'] = 'replica'
+postgresql['shared_preload_libraries'] = 'repmgr_funcs'
+
+# Disable automatic database migrations
+gitlab_rails['auto_migrate'] = false
+
+# Configure the consul agent
+consul['services'] = %w(postgresql)
+
+postgresql['pgbouncer_user_password'] = '771a8625958a529132abe6f1a4acb19c'
+postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f'
+postgresql['max_wal_senders'] = 4
+
+postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/16)
+repmgr['trust_auth_cidr_addresses'] = %w(10.6.0.0/16)
+
+consul['configuration'] = {
+  server: true,
+  retry_join: %w(10.6.0.21 10.6.0.22 10.6.0.23)
+}
+```
+
+[Reconfigure Omnibus GitLab][reconfigure Gitlab] for the changes to take effect.
+
+##### Example minimal configuration for application server
+
+On the server edit `/etc/gitlab/gitlab.rb`:
+
+```ruby
+external_url 'http://gitlab.example.com'
+
+gitlab_rails['db_host'] = '127.0.0.1'
+gitlab_rails['db_port'] = 6432
+gitlab_rails['db_password'] = 'toomanysecrets'
+gitlab_rails['auto_migrate'] = false
+
+postgresql['enable'] = false
+pgbouncer['enable'] = true
+consul['enable'] = true
+
+# Configure Pgbouncer
+pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul)
+
+# Configure Consul agent
+consul['watchers'] = %w(postgresql)
+
+pgbouncer['users'] = {
+  'gitlab-consul': {
+    password: '5e0e3263571e3704ad655076301d6ebe'
+  },
+  'pgbouncer': {
+    password: '771a8625958a529132abe6f1a4acb19c'
+  }
+}
+
+consul['configuration'] = {
+  retry_join: %w(10.6.0.21 10.6.0.22 10.6.0.23)
+}
+```
+
+[Reconfigure Omnibus GitLab][reconfigure Gitlab] for the changes to take effect.
+
+##### Example minimal setup manual steps
+
+The manual steps for this configuration are the same as for the [example recommended setup](#example_recommended_setup_manual_steps).
+
 ### Failover procedure
 
 By default, if the master database fails, `repmgrd` should promote one of the

ee/app/models/concerns/ee/repository_mirroring.rb

-module EE
-  module RepositoryMirroring
-    def storage_path
-      @project.repository_storage_path
-    end
-
-    def push_remote_branches(remote, branches)
-      gitlab_shell.push_remote_branches(storage_path, disk_path, remote, branches)
-    end
-
-    def delete_remote_branches(remote, branches)
-      gitlab_shell.delete_remote_branches(storage_path, disk_path, remote, branches)
-    end
-  end
-end

ee/app/models/ee/repository.rb

@@ -4,7 +4,6 @@ module EE
   # This module is intended to encapsulate EE-specific model logic
   # and be prepended in the `Repository` model
   module Repository
-    prepend RepositoryMirroring
     extend ActiveSupport::Concern
 
     # Transiently sets a configuration variable
@@ -31,5 +30,13 @@ module EE
 
       refs.map { |sha| commit(sha.strip) }
     end
+
+    def push_remote_branches(remote, branches)
+      gitlab_shell.push_remote_branches(repository_storage_path, disk_path, remote, branches)
+    end
+
+    def delete_remote_branches(remote, branches)
+      gitlab_shell.delete_remote_branches(repository_storage_path, disk_path, remote, branches)
+    end
   end
 end

ee/app/models/remote_mirror.rb

@@ -165,7 +165,6 @@ class RemoteMirror < ActiveRecord::Base
   def refresh_remote
     return unless project
 
-    project.repository.remove_remote(ref_name)
     project.repository.add_remote(ref_name, url)
   end
 

ee/app/services/projects/update_repository_storage_service.rb

@@ -35,7 +35,7 @@ module Projects
                                                    repository.relative_path,
                                                    repository.gl_repository)
 
-      new_repository.fetch_mirror(repository.path)
+      new_repository.fetch_as_mirror_without_shell(repository.path)
     end
 
     def mark_old_paths_for_archive

lib/gitlab/git/repository.rb

@@ -1150,10 +1150,12 @@ module Gitlab
         @has_visible_content = has_local_branches?
       end
 
-      def fetch(remote = 'origin')
-        args = %W(#{Gitlab.config.git.bin_path} fetch #{remote})
-
-        popen(args, @path).last.zero?
+      # Like all public `Gitlab::Git::Repository` methods, this method is part
+      # of `Repository`'s interface through `method_missing`.
+      # `Repository` has its own `fetch_remote` which uses `gitlab-shell` and
+      # takes some extra attributes, so we qualify this method name to prevent confusion.
+      def fetch_remote_without_shell(remote = 'origin')
+        run_git(['fetch', remote]).last.zero?
       end
 
       def blob_at(sha, path)

lib/gitlab/git/repository_mirroring.rb

 module Gitlab
   module Git
     module RepositoryMirroring
-      IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze
-      IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze
-      MIRROR_REMOTE = 'mirror'.freeze
+      REFMAPS = {
+        # With `:all_refs`, the repository is equivalent to the result of `git clone --mirror`
+        all_refs: '+refs/*:refs/*',
+        heads: '+refs/heads/*:refs/heads/*',
+        tags: '+refs/tags/*:refs/tags/*'
+      }.freeze
 
       RemoteError = Class.new(StandardError)
 
-      def set_remote_as_mirror(remote_name)
-        # This is used to define repository as equivalent as "git clone --mirror"
-        rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*'
-        rugged.config["remote.#{remote_name}.mirror"] = true
-        rugged.config["remote.#{remote_name}.prune"] = true
-      end
-
-      def set_import_remote_as_mirror(remote_name)
-        # Add first fetch with Rugged so it does not create its own.
-        rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS
-
-        add_remote_fetch_config(remote_name, IMPORT_TAG_REFS)
+      def set_remote_as_mirror(remote_name, refmap: :all_refs)
+        set_remote_refmap(remote_name, refmap)
 
         rugged.config["remote.#{remote_name}.mirror"] = true
         rugged.config["remote.#{remote_name}.prune"] = true
       end
 
-      def add_remote_fetch_config(remote_name, refspec)
-        run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
+      def set_remote_refmap(remote_name, refmap)
+        Array(refmap).each_with_index do |refspec, i|
+          refspec = REFMAPS[refspec] || refspec
+
+          # We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it.
+          # To make sure we start from scratch, we set the first using rugged, and use `git` for any others
+          if i == 0
+            rugged.config["remote.#{remote_name}.fetch"] = refspec
+          else
+            run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
+          end
+        end
       end
 
-      def fetch_mirror(url)
-        add_remote(MIRROR_REMOTE, url)
-        set_remote_as_mirror(MIRROR_REMOTE)
-        fetch(MIRROR_REMOTE)
-        remove_remote(MIRROR_REMOTE)
+      # Like all_refs public `Gitlab::Git::Repository` methods, this method is part
+      # of `Repository`'s interface through `method_missing`.
+      # `Repository` has its own `fetch_as_mirror` which uses `gitlab-shell` and
+      # takes some extra attributes, so we qualify this method name to prevent confusion.
+      def fetch_as_mirror_without_shell(url)
+        remote_name = "tmp-#{SecureRandom.hex}"
+        add_remote(remote_name, url)
+        set_remote_as_mirror(remote_name)
+        fetch_remote_without_shell(remote_name)
+      ensure
+        remove_remote(remote_name) if remote_name
       end
 
       def remote_tags(remote)

lib/gitlab/github_import.rb

 module Gitlab
   module GithubImport
+    def self.refmap
+      [:heads, :tags, '+refs/pull/*/head:refs/merge-requests/*/head']
+    end
+
     def self.new_client_for(project, token: nil, parallel: true)
       token_to_use = token || project.import_data&.credentials&.fetch(:user)
 

lib/gitlab/github_import/importer/repository_importer.rb

@@ -45,27 +45,14 @@ module Gitlab
         def import_repository
           project.ensure_repository
 
-          configure_repository_remote
-
-          project.repository.fetch_remote('github', forced: true)
+          refmap = Gitlab::GithubImport.refmap
+          project.repository.fetch_as_mirror(project.import_url, refmap: refmap, forced: true, remote_name: 'github')
 
           true
         rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e
           fail_import("Failed to import the repository: #{e.message}")
         end
 
-        def configure_repository_remote
-          return if project.repository.remote_exists?('github')
-
-          project.repository.add_remote('github', project.import_url)
-          project.repository.set_import_remote_as_mirror('github')
-
-          project.repository.add_remote_fetch_config(
-            'github',
-            '+refs/pull/*/head:refs/merge-requests/*/head'
-          )
-        end
-
         def import_wiki_repository
           wiki_path = "#{project.disk_path}.wiki"
           wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git')

lib/gitlab/legacy_github_import/importer.rb

@@ -3,6 +3,10 @@ module Gitlab
     class Importer
       include Gitlab::ShellAdapter
 
+      def self.refmap
+        Gitlab::GithubImport.refmap
+      end
+
       attr_reader :errors, :project, :repo, :repo_url
 
       def initialize(project)

spec/lib/gitlab/git/repository_spec.rb

@@ -588,12 +588,12 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
-  describe '#fetch_mirror' do
+  describe '#fetch_as_mirror_without_shell' do
     let(:new_repository) do
       Gitlab::Git::Repository.new('default', 'my_project.git', '')
     end
 
-    subject { new_repository.fetch_mirror(repository.path) }
+    subject { new_repository.fetch_as_mirror_without_shell(repository.path) }
 
     before do
       Gitlab::Shell.new.add_repository('default', 'my_project')
@@ -1652,15 +1652,15 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
-  describe '#fetch' do
+  describe '#fetch_remote_without_shell' do
     let(:git_path) { Gitlab.config.git.bin_path }
     let(:remote_name) { 'my_remote' }
 
-    subject { repository.fetch(remote_name) }
+    subject { repository.fetch_remote_without_shell(remote_name) }
 
     it 'fetches the remote and returns true if the command was successful' do
       expect(repository).to receive(:popen)
-        .with(%W(#{git_path} fetch #{remote_name}), repository.path)
+        .with(%W(#{git_path} fetch #{remote_name}), repository.path, {})
         .and_return(['', 0])
 
       expect(subject).to be(true)
@@ -1777,6 +1777,7 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
+<<<<<<< HEAD
   describe '#fetch' do
     let(:git_path) { Gitlab.config.git.bin_path }
     let(:remote_name) { 'my_remote' }
@@ -1818,6 +1819,8 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
+=======
+>>>>>>> origin/master
   def create_remote_branch(repository, remote_name, branch_name, source_branch_name)
     source_branch = repository.branches.find { |branch| branch.name == source_branch_name }
     rugged = repository.rugged

spec/lib/gitlab/github_import/importer/repository_importer_spec.rb

@@ -164,12 +164,9 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do
       expect(project)
         .to receive(:ensure_repository)
 
-      expect(importer)
-        .to receive(:configure_repository_remote)
-
       expect(repository)
-        .to receive(:fetch_remote)
-        .with('github', forced: true)
+        .to receive(:fetch_as_mirror)
+        .with(project.import_url, refmap: Gitlab::GithubImport.refmap, forced: true, remote_name: 'github')
 
       expect(importer.import_repository).to eq(true)
     end
@@ -186,40 +183,6 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do
     end
   end
 
-  describe '#configure_repository_remote' do
-    it 'configures the remote details' do
-      expect(repository)
-        .to receive(:remote_exists?)
-        .with('github')
-        .and_return(false)
-
-      expect(repository)
-        .to receive(:add_remote)
-        .with('github', 'foo.git')
-
-      expect(repository)
-        .to receive(:set_import_remote_as_mirror)
-        .with('github')
-
-      expect(repository)
-        .to receive(:add_remote_fetch_config)
-
-      importer.configure_repository_remote
-    end
-
-    it 'does not configure the remote if already configured' do
-      expect(repository)
-        .to receive(:remote_exists?)
-        .with('github')
-        .and_return(true)
-
-      expect(repository)
-        .not_to receive(:add_remote)
-
-      importer.configure_repository_remote
-    end
-  end
-
   describe '#import_wiki_repository' do
     it 'imports the wiki repository' do
       expect(importer.gitlab_shell)

spec/models/repository_spec.rb

@@ -2140,7 +2140,7 @@ describe Repository do
   describe '#push_remote_branches' do
     it 'push branches to the remote repo' do
       expect_any_instance_of(Gitlab::Shell).to receive(:push_remote_branches)
-        .with(repository.storage_path, repository.disk_path, 'remote_name', ['branch'])
+        .with(repository.repository_storage_path, repository.disk_path, 'remote_name', ['branch'])
 
       repository.push_remote_branches('remote_name', ['branch'])
     end
@@ -2149,7 +2149,7 @@ describe Repository do
   describe '#delete_remote_branches' do
     it 'delete branches to the remote repo' do
       expect_any_instance_of(Gitlab::Shell).to receive(:delete_remote_branches)
-        .with(repository.storage_path, repository.disk_path, 'remote_name', ['branch'])
+        .with(repository.repository_storage_path, repository.disk_path, 'remote_name', ['branch'])
 
       repository.delete_remote_branches('remote_name', ['branch'])
     end

spec/services/geo/repository_sync_service_spec.rb

@@ -26,13 +26,13 @@ describe Geo::RepositorySyncService do
         .with(subject.lease_key, anything)
         .and_return(lease)
 
-      allow_any_instance_of(Repository).to receive(:fetch_geo_mirror)
+      allow_any_instance_of(Repository).to receive(:fetch_as_mirror)
         .and_return(true)
     end
 
     it 'fetches project repository with JWT credentials' do
       expect(repository).to receive(:with_config).with("http.#{url_to_repo}.extraHeader" => anything).and_call_original
-      expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
+      expect(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true).once
 
       subject.execute
     end
@@ -53,7 +53,7 @@ describe Geo::RepositorySyncService do
     end
 
     it 'returns the lease when sync fail' do
-      allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Shell::Error }
+      allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true) { raise Gitlab::Shell::Error }
 
       expect(Gitlab::ExclusiveLease).to receive(:cancel).once.with(
         subject.__send__(:lease_key), anything).and_call_original
@@ -64,26 +64,26 @@ describe Geo::RepositorySyncService do
     it 'does not fetch project repository if cannot obtain a lease' do
       allow(lease).to receive(:try_obtain) { false }
 
-      expect(repository).not_to receive(:fetch_geo_mirror)
+      expect(repository).not_to receive(:fetch_as_mirror)
 
       subject.execute
     end
 
     it 'rescues when Gitlab::Shell::Error is raised' do
-      allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Shell::Error }
+      allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true) { raise Gitlab::Shell::Error }
 
       expect { subject.execute }.not_to raise_error
     end
 
     it 'rescues when Gitlab::Git::RepositoryMirroring::RemoteError is raised' do
-      allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo)
+      allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true)
         .and_raise(Gitlab::Git::RepositoryMirroring::RemoteError)
 
       expect { subject.execute }.not_to raise_error
     end
 
     it 'rescues exception and fires after_create hook when Gitlab::Git::Repository::NoRepository is raised' do
-      allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Git::Repository::NoRepository }
+      allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true) { raise Gitlab::Git::Repository::NoRepository }
 
       expect(repository).to receive(:after_create)
 
@@ -137,7 +137,7 @@ describe Geo::RepositorySyncService do
         let(:registry) { Geo::ProjectRegistry.find_by(project_id: project.id) }
 
         before do
-          allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Shell::Error }
+          allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true) { raise Gitlab::Shell::Error }
 
           subject.execute
         end
@@ -180,7 +180,7 @@ describe Geo::RepositorySyncService do
 
         # gitlab-shell always appends .git to the end of the repository, so
         # we're relying on the fact that projects can't contain + in the name
-        deleted_dir = File.join(project.repository.storage_path, project.path) + "+failed-geo-sync.git"
+        deleted_dir = File.join(project.repository_storage_path, project.path) + "+failed-geo-sync.git"
         expect(File.directory?(deleted_dir)).to be false
         expect(File.directory?(project.repository.path)).to be true
       end
@@ -228,7 +228,7 @@ describe Geo::RepositorySyncService do
       end
 
       it 'fetches wiki repository over SSH' do
-        expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
+        expect(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true).once
 
         subject.execute
       end

spec/services/geo/wiki_sync_service_spec.rb

@@ -26,13 +26,13 @@ RSpec.describe Geo::WikiSyncService do
         .with(subject.lease_key, anything)
         .and_return(lease)
 
-      allow_any_instance_of(Repository).to receive(:fetch_geo_mirror)
+      allow_any_instance_of(Repository).to receive(:fetch_as_mirror)
         .and_return(true)
     end
 
     it 'fetches wiki repository with JWT credentials' do
       expect(repository).to receive(:with_config).with("http.#{url_to_repo}.extraHeader" => anything).and_call_original
-      expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
+      expect(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true).once
 
       subject.execute
     end
@@ -47,26 +47,26 @@ RSpec.describe Geo::WikiSyncService do
     it 'does not fetch wiki repository if cannot obtain a lease' do
       allow(lease).to receive(:try_obtain) { false }
 
-      expect(repository).not_to receive(:fetch_geo_mirror)
+      expect(repository).not_to receive(:fetch_as_mirror)
 
       subject.execute
     end
 
     it 'rescues exception when Gitlab::Shell::Error is raised' do
-      allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Shell::Error }
+      allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true) { raise Gitlab::Shell::Error }
 
       expect { subject.execute }.not_to raise_error
     end
 
     it 'rescues exception when Gitlab::Git::RepositoryMirroring::RemoteError is raised' do
-      allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo)
+      allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true)
         .and_raise(Gitlab::Git::RepositoryMirroring::RemoteError)
 
       expect { subject.execute }.not_to raise_error
     end
 
     it 'rescues exception when Gitlab::Git::Repository::NoRepository is raised' do
-      allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Git::Repository::NoRepository }
+      allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true) { raise Gitlab::Git::Repository::NoRepository }
 
       expect { subject.execute }.not_to raise_error
     end
@@ -109,7 +109,7 @@ RSpec.describe Geo::WikiSyncService do
         let(:registry) { Geo::ProjectRegistry.find_by(project_id: project.id) }
 
         before do
-          allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Shell::Error }
+          allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true) { raise Gitlab::Shell::Error }
 
           subject.execute
         end
@@ -134,7 +134,7 @@ RSpec.describe Geo::WikiSyncService do
       end
 
       it 'fetches wiki repository over SSH' do
-        expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
+        expect(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true).once
 
         subject.execute
       end

spec/services/projects/update_repository_storage_service_spec.rb

@@ -31,7 +31,7 @@ describe Projects::UpdateRepositoryStorageService do
 
       context 'when the move succeeds' do
         it 'moves the repository to the new storage and unmarks the repository as read only' do
-          expect_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_mirror)
+          expect_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_as_mirror_without_shell)
             .with(project.repository.raw.path).and_return(true)
           expect(GitlabShellWorker).to receive(:perform_async)
             .with(:mv_repository,
@@ -48,7 +48,7 @@ describe Projects::UpdateRepositoryStorageService do
 
       context 'when the move fails' do
         it 'unmarks the repository as read-only without updating the repository storage' do
-          expect_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_mirror)
+          expect_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_as_mirror_without_shell)
             .with(project.repository.raw.path).and_return(false)
           expect(GitlabShellWorker).not_to receive(:perform_async)
 
@@ -84,7 +84,7 @@ describe Projects::UpdateRepositoryStorageService do
 
       context 'when the move succeeds' do
         it 'moves the repository and its wiki to the new storage and unmarks the repository as read only' do
-          expect(repository_double).to receive(:fetch_mirror)
+          expect(repository_double).to receive(:fetch_as_mirror_without_shell)
             .with(project.repository.raw.path).and_return(true)
           expect(GitlabShellWorker).to receive(:perform_async)
             .with(:mv_repository,
@@ -92,7 +92,7 @@ describe Projects::UpdateRepositoryStorageService do
               project.disk_path,
               "#{project.disk_path}+#{project.id}+moved+#{time.to_i}")
 
-          expect(wiki_repository_double).to receive(:fetch_mirror)
+          expect(wiki_repository_double).to receive(:fetch_as_mirror_without_shell)
             .with(project.wiki.repository.raw.path).and_return(true)
           expect(GitlabShellWorker).to receive(:perform_async)
             .with(:mv_repository,
@@ -109,9 +109,9 @@ describe Projects::UpdateRepositoryStorageService do
 
       context 'when the move of the wiki fails' do
         it 'unmarks the repository as read-only without updating the repository storage' do
-          expect(repository_double).to receive(:fetch_mirror)
+          expect(repository_double).to receive(:fetch_as_mirror_without_shell)
             .with(project.repository.raw.path).and_return(true)
-          expect(wiki_repository_double).to receive(:fetch_mirror)
+          expect(wiki_repository_double).to receive(:fetch_as_mirror_without_shell)
             .with(project.wiki.repository.raw.path).and_return(false)
           expect(GitlabShellWorker).not_to receive(:perform_async)