Encapsulate git operations for mirroring in Gitlab::Git

app/models/project.rb

@@ -1679,6 +1679,10 @@ class Project < ActiveRecord::Base
     Gitlab::GlRepository.gl_repository(self, is_wiki)
   end
 
+  def reference_counter(wiki: false)
+    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: wiki))
+  end
+
   private
 
   def storage
@@ -1697,11 +1701,11 @@ class Project < ActiveRecord::Base
   end
 
   def repo_reference_count
-    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: false)).value
+    reference_counter.value
   end
 
   def wiki_reference_count
-    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: true)).value
+    reference_counter(wiki: true).value
   end
 
   def check_repository_absence!

app/models/repository.rb

@@ -1021,19 +1021,6 @@ class Repository
     run_git(args).first.lines.map(&:strip)
   end
 
-  def add_remote(name, url)
-    raw_repository.remote_add(name, url)
-  rescue Rugged::ConfigError
-    raw_repository.remote_update(name, url: url)
-  end
-
-  def remove_remote(name)
-    raw_repository.remote_delete(name)
-    true
-  rescue Rugged::ConfigError
-    false
-  end
-
   def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false)
     gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
   end

app/services/projects/import_service.rb

@@ -44,7 +44,7 @@ module Projects
         else
           clone_repository
         end
-      rescue Gitlab::Shell::Error => e
+      rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e
         # Expire cache to prevent scenarios such as:
         # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true
         # 2. Retried import, repo is broken or not imported but +exists?+ still returns true

lib/github/import.rb

@@ -60,7 +60,9 @@ module Github
         project.repository.set_import_remote_as_mirror('github')
         project.repository.add_remote_fetch_config('github', '+refs/pull/*/head:refs/merge-requests/*/head')
         fetch_remote(forced: true)
-      rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e
+      rescue Gitlab::Git::Repository::NoRepository,
+             Gitlab::Git::RepositoryMirroring::RemoteError,
+             Gitlab::Shell::Error => e
         error(:project, repo_url, e.message)
         raise Github::RepositoryFetchError
       end

lib/gitlab/git/repository.rb

@@ -6,6 +6,7 @@ require "rubygems/package"
 module Gitlab
   module Git
     class Repository
+      include Gitlab::Git::RepositoryMirroring
       include Gitlab::Git::Popen
 
       ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[
@@ -898,16 +899,25 @@ module Gitlab
         end
       end
 
-      # Delete the specified remote from this repository.
-      def remote_delete(remote_name)
-        rugged.remotes.delete(remote_name)
-        nil
+      def add_remote(remote_name, url)
+        rugged.remotes.create(remote_name, url)
+      rescue Rugged::ConfigError
+        remote_update(remote_name, url: url)
       end
 
-      # Add a new remote to this repository.
-      def remote_add(remote_name, url)
-        rugged.remotes.create(remote_name, url)
-        nil
+      def remove_remote(remote_name)
+        # When a remote is deleted all its remote refs are deleted too, but in
+        # the case of mirrors we map its refs (that would usualy go under
+        # [remote_name]/) to the top level namespace. We clean the mapping so
+        # those don't get deleted.
+        if rugged.config["remote.#{remote_name}.mirror"]
+          rugged.config.delete("remote.#{remote_name}.fetch")
+        end
+
+        rugged.remotes.delete(remote_name)
+        true
+      rescue Rugged::ConfigError
+        false
       end
 
       # Update the specified remote using the values in the +options+ hash

lib/gitlab/git/repository_mirroring.rb

+module Gitlab
+  module Git
+    module RepositoryMirroring
+      IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze
+      IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze
+      MIRROR_REMOTE = 'mirror'.freeze
+
+      RemoteError = Class.new(StandardError)
+
+      def set_remote_as_mirror(remote_name)
+        # This is used to define repository as equivalent as "git clone --mirror"
+        rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*'
+        rugged.config["remote.#{remote_name}.mirror"] = true
+        rugged.config["remote.#{remote_name}.prune"] = true
+      end
+
+      def set_import_remote_as_mirror(remote_name)
+        # Add first fetch with Rugged so it does not create its own.
+        rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS
+
+        add_remote_fetch_config(remote_name, IMPORT_TAG_REFS)
+
+        rugged.config["remote.#{remote_name}.mirror"] = true
+        rugged.config["remote.#{remote_name}.prune"] = true
+      end
+
+      def add_remote_fetch_config(remote_name, refspec)
+        run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
+      end
+
+      def fetch_mirror(url)
+        add_remote(MIRROR_REMOTE, url)
+        set_remote_as_mirror(MIRROR_REMOTE)
+        fetch(MIRROR_REMOTE)
+        remove_remote(MIRROR_REMOTE)
+      end
+
+      def remote_tags(remote)
+        # Each line has this format: "dc872e9fa6963f8f03da6c8f6f264d0845d6b092\trefs/tags/v1.10.0\n"
+        # We want to convert it to: [{ 'v1.10.0' => 'dc872e9fa6963f8f03da6c8f6f264d0845d6b092' }, ...]
+        list_remote_tags(remote).map do |line|
+          target, path = line.strip.split("\t")
+
+          # When the remote repo does not have tags.
+          if target.nil? || path.nil?
+            Rails.logger.info "Empty or invalid list of tags for remote: #{remote}. Output: #{output}"
+            return []
+          end
+
+          name = path.split('/', 3).last
+          # We're only interested in tag references
+          # See: http://stackoverflow.com/questions/15472107/when-listing-git-ls-remote-why-theres-after-the-tag-name
+          next if name =~ /\^\{\}\Z/
+
+          target_commit = Gitlab::Git::Commit.find(self, target)
+          Gitlab::Git::Tag.new(self, name, target, target_commit)
+        end.compact
+      end
+
+      def remote_branches(remote_name)
+        branches = []
+
+        rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref|
+          name = ref.name.sub(/\Arefs\/remotes\/#{remote_name}\//, '')
+
+          begin
+            target_commit = Gitlab::Git::Commit.find(self, ref.target)
+            branches << Gitlab::Git::Branch.new(self, name, ref.target, target_commit)
+          rescue Rugged::ReferenceError
+            # Omit invalid branch
+          end
+        end
+
+        branches
+      end
+
+      private
+
+      def list_remote_tags(remote)
+        tag_list, exit_code, error = nil
+        cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{full_path} ls-remote --tags #{remote})
+
+        Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr|
+          tag_list  = stdout.read
+          error     = stderr.read
+          exit_code = wait_thr.value.exitstatus
+        end
+
+        raise RemoteError, error unless exit_code.zero?
+
+        tag_list.split('\n')
+      end
+    end
+  end
+end

lib/gitlab/git/wiki.rb

@@ -10,6 +10,8 @@ module Gitlab
       end
       PageBlob = Struct.new(:name)
 
+      attr_reader :repository
+
       def self.default_ref
         'master'
       end

lib/gitlab/shell.rb

@@ -108,34 +108,6 @@ module Gitlab
       gitlab_shell_fast_execute_raise_error(cmd)
     end
 
-    def list_remote_tags(storage, name, remote)
-      output, status = Popen.popen([gitlab_shell_projects_path, 'list-remote-tags', storage, "#{name}.git", remote])
-      tags_with_targets = []
-
-      raise Error, output unless status.zero?
-
-      # Each line has this format: "dc872e9fa6963f8f03da6c8f6f264d0845d6b092\trefs/tags/v1.10.0\n"
-      # We want to convert it to: [{ 'v1.10.0' => 'dc872e9fa6963f8f03da6c8f6f264d0845d6b092' }, ...]
-      output.lines.each do |line|
-        target, path = line.strip!.split("\t")
-
-        # When the remote repo does not have tags.
-        if target.nil? || path.nil?
-          Rails.logger.info "Empty or invalid list of tags for remote: #{remote}. Output: #{output}"
-          break
-        end
-
-        name = path.split('/', 3).last
-        # We're only interested in tag references
-        # See: http://stackoverflow.com/questions/15472107/when-listing-git-ls-remote-why-theres-after-the-tag-name
-        next if name =~ /\^\{\}\Z/
-
-        tags_with_targets.concat([name, target])
-      end
-
-      Hash[*tags_with_targets]
-    end
-
     # Fetch remote for repository
     #
     # repository - an instance of Git::Repository

spec/lib/gitlab/git/repository_spec.rb

@@ -559,10 +559,10 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
-  describe "#remote_delete" do
+  describe "#remove_remote" do
     before(:all) do
       @repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '')
-      @repo.remote_delete("expendable")
+      @repo.remove_remote("expendable")
     end
 
     it "should remove the remote" do
@@ -575,14 +575,16 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
-  describe "#remote_add" do
+  describe "#remote_update" do
     before(:all) do
       @repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '')
-      @repo.remote_add("new_remote", SeedHelper::GITLAB_GIT_TEST_REPO_URL)
+      @repo.remote_update("expendable", url: TEST_NORMAL_REPO_PATH)
     end
 
     it "should add the remote" do
-      expect(@repo.rugged.remotes.each_name.to_a).to include("new_remote")
+      expect(@repo.rugged.remotes["expendable"].url).to(
+        eq(TEST_NORMAL_REPO_PATH)
+      )
     end
 
     after(:all) do
@@ -591,21 +593,58 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
-  describe "#remote_update" do
-    before(:all) do
-      @repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '')
-      @repo.remote_update("expendable", url: TEST_NORMAL_REPO_PATH)
+  describe '#fetch_mirror' do
+    let(:new_repository) do
+      Gitlab::Git::Repository.new('default', 'my_project.git', '')
     end
 
-    it "should add the remote" do
-      expect(@repo.rugged.remotes["expendable"].url).to(
-        eq(TEST_NORMAL_REPO_PATH)
-      )
+    subject { new_repository.fetch_mirror(repository.path) }
+
+    before do
+      Gitlab::Shell.new.add_repository('default', 'my_project')
     end
 
-    after(:all) do
-      FileUtils.rm_rf(TEST_MUTABLE_REPO_PATH)
-      ensure_seeds
+    after do
+      Gitlab::Shell.new.remove_repository(TestEnv.repos_path, 'my_project')
+    end
+
+    it 'fetches a url as a mirror remote' do
+      subject
+
+      expect(refs(new_repository.path)).to eq(refs(repository.path))
+    end
+
+    context 'with keep-around refs' do
+      let(:sha) { SeedRepo::Commit::ID }
+      let(:keep_around_ref) { "refs/keep-around/#{sha}" }
+      let(:tmp_ref) { "refs/tmp/#{SecureRandom.hex}" }
+
+      before do
+        repository.rugged.references.create(keep_around_ref, sha, force: true)
+        repository.rugged.references.create(tmp_ref, sha, force: true)
+      end
+
+      it 'includes the temporary and keep-around refs' do
+        subject
+
+        expect(refs(new_repository.path)).to include(keep_around_ref)
+        expect(refs(new_repository.path)).to include(tmp_ref)
+      end
+    end
+  end
+
+  describe '#remote_tags' do
+    let(:target_commit_id) { SeedRepo::Commit::ID }
+
+    subject { repository.remote_tags('upstream') }
+
+    it 'gets the remote tags' do
+      expect(repository).to receive(:list_remote_tags).with('upstream')
+        .and_return(["#{target_commit_id}\trefs/tags/v0.0.1\n"])
+
+      expect(subject.first).to be_an_instance_of(Gitlab::Git::Tag)
+      expect(subject.first.name).to eq('v0.0.1')
+      expect(subject.first.dereferenced_target.id).to eq(target_commit_id)
     end
   end
 
@@ -1757,4 +1796,10 @@ describe Gitlab::Git::Repository, seed_helper: true do
     sha = Rugged::Commit.create(repo, options)
     repo.lookup(sha)
   end
+
+  def refs(dir)
+    IO.popen(%W[git -C #{dir} for-each-ref], &:read).split("\n").map do |line|
+      line.split("\t").last
+    end
+  end
 end

spec/models/repository_spec.rb

@@ -2148,30 +2148,6 @@ describe Repository do
     end
   end
 
-  describe '#remove_remote' do
-    it 'remove a remote reference' do
-      repository.add_remote('upstream', 'http://repo.test')
-
-      expect(repository.remove_remote('upstream')).to eq(true)
-    end
-  end
-
-  describe '#remote_tags' do
-    it 'gets the remote tags' do
-      masterrev = repository.find_branch('master').dereferenced_target.id
-
-      expect_any_instance_of(Gitlab::Shell).to receive(:list_remote_tags)
-        .with(repository.storage_path, repository.disk_path, 'upstream')
-        .and_return({ 'v0.0.1' => masterrev })
-
-      tags = repository.remote_tags('upstream')
-
-      expect(tags.first).to be_an_instance_of(Gitlab::Git::Tag)
-      expect(tags.first.name).to eq('v0.0.1')
-      expect(tags.first.dereferenced_target.id).to eq(masterrev)
-    end
-  end
-
   describe '#local_branches' do
     it 'returns the local branches' do
       masterrev = repository.find_branch('master').dereferenced_target