Use Gitlab::Git operations for repository mirroring

app/models/concerns/repository_mirroring.rb

 module RepositoryMirroring
-  IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze
-  IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze
-
   def storage_path
     @project.repository_storage_path
   end
@@ -13,56 +10,4 @@ module RepositoryMirroring
   def delete_remote_branches(remote, branches)
     gitlab_shell.delete_remote_branches(storage_path, disk_path, remote, branches)
   end
-
-  def set_remote_as_mirror(name)
-    # This is used to define repository as equivalent as "git clone --mirror"
-    raw_repository.rugged.config["remote.#{name}.fetch"] = 'refs/*:refs/*'
-    raw_repository.rugged.config["remote.#{name}.mirror"] = true
-    raw_repository.rugged.config["remote.#{name}.prune"] = true
-  end
-
-  def set_import_remote_as_mirror(remote_name)
-    # Add first fetch with Rugged so it does not create its own.
-    raw_repository.rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS
-
-    add_remote_fetch_config(remote_name, IMPORT_TAG_REFS)
-
-    raw_repository.rugged.config["remote.#{remote_name}.mirror"] = true
-    raw_repository.rugged.config["remote.#{remote_name}.prune"] = true
-  end
-
-  def add_remote_fetch_config(remote_name, refspec)
-    run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
-  end
-
-  def fetch_mirror(remote, url)
-    add_remote(remote, url)
-    set_remote_as_mirror(remote)
-    fetch_remote(remote, forced: true)
-    remove_remote(remote)
-  end
-
-  def remote_tags(remote)
-    gitlab_shell.list_remote_tags(storage_path, disk_path, remote).map do |name, target|
-      target_commit = Gitlab::Git::Commit.find(raw_repository, target)
-      Gitlab::Git::Tag.new(raw_repository, name, target, target_commit)
-    end
-  end
-
-  def remote_branches(remote_name)
-    branches = []
-
-    rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref|
-      name = ref.name.sub(/\Arefs\/remotes\/#{remote_name}\//, '')
-
-      begin
-        target_commit = Gitlab::Git::Commit.find(raw_repository, ref.target)
-        branches << Gitlab::Git::Branch.new(raw_repository, name, ref.target, target_commit)
-      rescue Rugged::ReferenceError
-        # Omit invalid branch
-      end
-    end
-
-    branches
-  end
 end

app/services/geo/repository_sync_service.rb

@@ -21,7 +21,9 @@ module Geo
         log_info("Finished repository sync",
                  update_delay_s: update_delay_in_seconds,
                  download_time_s: download_time_in_seconds)
-      rescue Gitlab::Shell::Error, Geo::EmptyCloneUrlPrefixError => e
+      rescue Gitlab::Shell::Error,
+             Gitlab::Git::RepositoryMirroring::RemoteError,
+             Geo::EmptyCloneUrlPrefixError => e
         log_error('Error syncing repository', e)
       rescue Gitlab::Git::Repository::NoRepository => e
         log_error('Invalid repository', e)

app/services/geo/wiki_sync_service.rb

@@ -21,6 +21,7 @@ module Geo
                  update_delay_s: update_delay_in_seconds,
                  download_time_s: download_time_in_seconds)
       rescue Gitlab::Git::Repository::NoRepository,
+             Gitlab::Git::RepositoryMirroring::RemoteError,
              Gitlab::Shell::Error,
              ProjectWiki::CouldNotCreateWikiError,
              Geo::EmptyCloneUrlPrefixError => e

app/services/projects/update_mirror_service.rb

@@ -19,7 +19,7 @@ module Projects
       update_branches
 
       success
-    rescue Gitlab::Shell::Error, UpdateError => e
+    rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError, UpdateError => e
       error(e.message)
     end
 

app/services/projects/update_repository_storage_service.rb

@@ -7,11 +7,10 @@ module Projects
     end
 
     def execute(new_repository_storage_key)
-      new_storage_path = Gitlab.config.repositories.storages[new_repository_storage_key]['path']
-      result = move_storage(project.disk_path, new_storage_path)
+      result = mirror_repository(new_repository_storage_key)
 
       if project.wiki.repository_exists?
-        result &&= move_storage(project.wiki.disk_path, new_storage_path)
+        result &&= mirror_repository(new_repository_storage_key, wiki: true)
       end
 
       if result
@@ -25,8 +24,18 @@ module Projects
 
     private
 
-    def move_storage(project_path, new_storage_path)
-      gitlab_shell.mv_storage(project.repository_storage_path, project_path, new_storage_path)
+    def mirror_repository(new_storage_key, wiki: false)
+      return false unless wait_for_pushes(wiki)
+
+      repository = (wiki ? project.wiki.repository : project.repository).raw
+
+      # Initialize a git repository on the target path
+      gitlab_shell.add_repository(new_storage_key, repository.relative_path)
+      new_repository = Gitlab::Git::Repository.new(new_storage_key,
+                                                   repository.relative_path,
+                                                   repository.gl_repository)
+
+      new_repository.fetch_mirror(repository.path)
     end
 
     def mark_old_paths_for_archive
@@ -53,5 +62,17 @@ module Projects
     def moved_path(path)
       "#{path}+#{project.id}+moved+#{Time.now.to_i}"
     end
+
+    def wait_for_pushes(wiki)
+      reference_counter = project.reference_counter(wiki: wiki)
+
+      # Try for 30 seconds, polling every 10
+      3.times do
+        return true if reference_counter.value == 0
+        sleep 10
+      end
+
+      false
+    end
   end
 end

spec/services/geo/repository_sync_service_spec.rb

@@ -66,6 +66,13 @@ RSpec.describe Geo::RepositorySyncService do
       expect { subject.execute }.not_to raise_error
     end
 
+    it 'rescues when Gitlab::Git::RepositoryMirroring::RemoteError is raised' do
+      allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo)
+        .and_raise(Gitlab::Git::RepositoryMirroring::RemoteError)
+
+      expect { subject.execute }.not_to raise_error
+    end
+
     it 'rescues exception and fires after_create hook when Gitlab::Git::Repository::NoRepository is raised' do
       allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Git::Repository::NoRepository }
 

spec/services/geo/wiki_sync_service_spec.rb

@@ -58,6 +58,13 @@ RSpec.describe Geo::WikiSyncService do
       expect { subject.execute }.not_to raise_error
     end
 
+    it 'rescues exception when Gitlab::Git::RepositoryMirroring::RemoteError is raised' do
+      allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo)
+        .and_raise(Gitlab::Git::RepositoryMirroring::RemoteError)
+
+      expect { subject.execute }.not_to raise_error
+    end
+
     it 'rescues exception when Gitlab::Git::Repository::NoRepository is raised' do
       allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Git::Repository::NoRepository }
 

spec/services/projects/update_repository_storage_service_spec.rb

@@ -6,7 +6,6 @@ describe Projects::UpdateRepositoryStorageService do
   subject { described_class.new(project) }
 
   describe "#execute" do
-    let(:gitlab_shell) { Gitlab::Shell.new }
     let(:time) { Time.now }
 
     before do
@@ -18,7 +17,6 @@ describe Projects::UpdateRepositoryStorageService do
         'b' => { 'path' => 'tmp/tests/storage_b' }
       }
       stub_storage_settings(storages)
-      allow(subject).to receive(:gitlab_shell).and_return(gitlab_shell)
 
       allow(Time).to receive(:now).and_return(time)
     end
@@ -33,9 +31,8 @@ describe Projects::UpdateRepositoryStorageService do
 
       context 'when the move succeeds' do
         it 'moves the repository to the new storage and unmarks the repository as read only' do
-          expect(gitlab_shell).to receive(:mv_storage)
-            .with('tmp/tests/storage_a', project.disk_path, 'tmp/tests/storage_b')
-            .and_return(true)
+          expect_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_mirror)
+            .with(project.repository.raw.path).and_return(true)
           expect(GitlabShellWorker).to receive(:perform_async)
             .with(:mv_repository,
               'tmp/tests/storage_a',
@@ -51,9 +48,8 @@ describe Projects::UpdateRepositoryStorageService do
 
       context 'when the move fails' do
         it 'unmarks the repository as read-only without updating the repository storage' do
-          expect(gitlab_shell).to receive(:mv_storage)
-            .with('tmp/tests/storage_a', project.disk_path, 'tmp/tests/storage_b')
-            .and_return(false)
+          expect_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_mirror)
+            .with(project.repository.raw.path).and_return(false)
           expect(GitlabShellWorker).not_to receive(:perform_async)
 
           subject.execute('b')
@@ -66,25 +62,38 @@ describe Projects::UpdateRepositoryStorageService do
 
     context 'with wiki', :skip_gitaly_mock do
       let(:project) { create(:project, :repository, repository_storage: 'a', repository_read_only: true, wiki_enabled: true) }
+      let(:repository_double) { double(:repository) }
+      let(:wiki_repository_double) { double(:repository) }
 
       before do
         project.create_wiki
+
+        # Default stub for non-specified params
+        allow(Gitlab::Git::Repository).to receive(:new).and_call_original
+
+        relative_path = project.repository.raw.relative_path
+        allow(Gitlab::Git::Repository).to receive(:new)
+          .with('b', relative_path, "project-#{project.id}")
+          .and_return(repository_double)
+
+        wiki_relative_path = project.wiki.repository.raw.relative_path
+        allow(Gitlab::Git::Repository).to receive(:new)
+          .with('b', wiki_relative_path, "wiki-#{project.id}")
+          .and_return(wiki_repository_double)
       end
 
       context 'when the move succeeds' do
         it 'moves the repository and its wiki to the new storage and unmarks the repository as read only' do
-          expect(gitlab_shell).to receive(:mv_storage)
-            .with('tmp/tests/storage_a', project.disk_path, 'tmp/tests/storage_b')
-            .and_return(true)
+          expect(repository_double).to receive(:fetch_mirror)
+            .with(project.repository.raw.path).and_return(true)
           expect(GitlabShellWorker).to receive(:perform_async)
             .with(:mv_repository,
               'tmp/tests/storage_a',
               project.disk_path,
               "#{project.disk_path}+#{project.id}+moved+#{time.to_i}")
 
-          expect(gitlab_shell).to receive(:mv_storage)
-            .with('tmp/tests/storage_a', project.wiki.disk_path, 'tmp/tests/storage_b')
-            .and_return(true)
+          expect(wiki_repository_double).to receive(:fetch_mirror)
+            .with(project.wiki.repository.raw.path).and_return(true)
           expect(GitlabShellWorker).to receive(:perform_async)
             .with(:mv_repository,
               'tmp/tests/storage_a',
@@ -100,12 +109,10 @@ describe Projects::UpdateRepositoryStorageService do
 
       context 'when the move of the wiki fails' do
         it 'unmarks the repository as read-only without updating the repository storage' do
-          expect(gitlab_shell).to receive(:mv_storage)
-            .with('tmp/tests/storage_a', project.disk_path, 'tmp/tests/storage_b')
-            .and_return(true)
-          expect(gitlab_shell).to receive(:mv_storage)
-            .with('tmp/tests/storage_a', project.wiki.disk_path, 'tmp/tests/storage_b')
-            .and_return(false)
+          expect(repository_double).to receive(:fetch_mirror)
+            .with(project.repository.raw.path).and_return(true)
+          expect(wiki_repository_double).to receive(:fetch_mirror)
+            .with(project.wiki.repository.raw.path).and_return(false)
           expect(GitlabShellWorker).not_to receive(:perform_async)
 
           subject.execute('b')