Fetch GitHub project as a mirror to get all refs at once

app/models/concerns/repository_mirroring.rb

+module RepositoryMirroring
+  def storage_path
+    @project.repository_storage_path
+  end
+
+  def add_remote(name, url)
+    raw_repository.remote_add(name, url)
+  rescue Rugged::ConfigError
+    raw_repository.remote_update(name, url: url)
+  end
+
+  def remove_remote(name)
+    raw_repository.remote_delete(name)
+    true
+  rescue Rugged::ConfigError
+    false
+  end
+
+  def set_remote_as_mirror(name)
+    config = raw_repository.rugged.config
+
+    # This is used to define repository as equivalent as "git clone --mirror"
+    config["remote.#{name}.fetch"] = 'refs/*:refs/*'
+    config["remote.#{name}.mirror"] = true
+    config["remote.#{name}.prune"] = true
+  end
+
+  def fetch_remote(remote, forced: false, no_tags: false)
+    gitlab_shell.fetch_remote(storage_path, path_with_namespace, remote, forced: forced, no_tags: no_tags)
+  end
+end

app/models/project.rb

@@ -551,6 +551,10 @@ class Project < ActiveRecord::Base
     import_type == 'gitea'
   end
 
+  def github_import?
+    import_type == 'github'
+  end
+
   def check_limit
     unless creator.can_create_project? || namespace.kind == 'group'
       projects_limit = creator.projects_limit

app/models/repository.rb

@@ -2,6 +2,7 @@ require 'securerandom'
 
 class Repository
   include Gitlab::ShellAdapter
+  include RepositoryMirroring
 
   attr_accessor :path_with_namespace, :project
 
@@ -1033,6 +1034,13 @@ class Repository
     rugged.references.delete(tmp_ref) if tmp_ref
   end
 
+  def fetch_mirror(remote, url)
+    add_remote(remote, url)
+    set_remote_as_mirror(remote)
+    fetch_remote(remote, forced: true)
+    remove_remote(remote)
+  end
+
   def fetch_ref(source_path, source_ref, target_ref)
     args = %W(#{Gitlab.config.git.bin_path} fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref})
     Gitlab::Popen.popen(args, path_to_repo)

app/services/projects/import_service.rb

@@ -21,9 +21,13 @@ module Projects
         # In this case, we only want to import issues, not a repository.
         create_repository
       elsif !project.repository_exists?
+        if project.github_import? || project.gitea_import?
+          fetch_repository
+        else
           import_repository
         end
       end
+    end
 
     def create_repository
       unless project.create_repository
@@ -45,6 +49,25 @@ module Projects
       end
     end
 
+    def fetch_repository
+      begin
+        raise Error, 'Blocked import URL.' if Gitlab::UrlBlocker.blocked_url?(project.import_url)
+
+        project.create_repository
+        project.repository.add_remote(project.import_type, project.import_url)
+        project.repository.set_remote_as_mirror (project.import_type)
+        project.repository.fetch_remote(project.import_type, forced: true)
+        project.repository.remove_remote(project.import_type)
+      rescue => e
+        # Expire cache to prevent scenarios such as:
+        # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true
+        # 2. Retried import, repo is broken or not imported but +exists?+ still returns true
+        project.repository.before_import if project.repository_exists?
+
+        raise Error, "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
+      end
+    end
+
     def import_data
       return unless has_importer?
 

lib/gitlab/github_import/importer.rb

@@ -157,7 +157,7 @@ module Gitlab
       end
 
       def restore_source_branch(pull_request)
-        project.repository.fetch_ref(repo_url, "pull/#{pull_request.number}/head", pull_request.source_branch_name)
+        project.repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha)
       end
 
       def restore_target_branch(pull_request)

lib/gitlab/shell.rb

@@ -88,6 +88,25 @@ module Gitlab
       true
     end
 
+    # Fetch remote for repository
+    #
+    # name - project path with namespace
+    # remote - remote name
+    # forced - should we use --force flag?
+    #
+    # Ex.
+    #   fetch_remote("gitlab/gitlab-ci", "upstream")
+    #
+    def fetch_remote(storage, name, remote, forced: false, no_tags: false)
+      args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, '2100']
+      args << '--force' if forced
+      args << '--no-tags' if no_tags
+
+      output, status = Popen.popen(args)
+      raise Error, output unless status.zero?
+      true
+    end
+
     # Move repository
     # storage - project's storage path
     # path - project path with namespace