Commit 0e6beaf5 authored by Douwe Maan's avatar Douwe Maan

Clean up repository fetch and mirror methods

parent 7084c096
...@@ -972,6 +972,19 @@ class Repository ...@@ -972,6 +972,19 @@ class Repository
run_git(args).first.lines.map(&:strip) run_git(args).first.lines.map(&:strip)
end end
def fetch_as_mirror(url, forced: false, fetch_refs: :all, remote_name: nil)
unless remote_name
remote_name = "tmp-#{SecureRandom.hex}"
tmp_remote_name = true
end
add_remote(remote_name, url)
set_remote_as_mirror(remote_name, fetch_refs: fetch_refs)
fetch_remote(remote_name, forced: forced)
ensure
remove_remote(remote_name) if tmp_remote_name
end
def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false) def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false)
gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
end end
...@@ -1069,6 +1082,10 @@ class Repository ...@@ -1069,6 +1082,10 @@ class Repository
raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref) raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref)
end end
def repository_storage_path
@project.repository_storage_path
end
private private
# TODO Generice finder, later split this on finders by Ref or Oid # TODO Generice finder, later split this on finders by Ref or Oid
...@@ -1134,10 +1151,6 @@ class Repository ...@@ -1134,10 +1151,6 @@ class Repository
raw_repository.run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip raw_repository.run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip
end end
def repository_storage_path
@project.repository_storage_path
end
def initialize_raw_repository def initialize_raw_repository
Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, is_wiki)) Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, is_wiki))
end end
......
...@@ -51,10 +51,13 @@ module Projects ...@@ -51,10 +51,13 @@ module Projects
def import_repository def import_repository
begin begin
if project.gitea_import? fetch_refs = importer_class.try(:fetch_refs) if has_importer?
fetch_repository
if fetch_refs
project.ensure_repository
project.repository.fetch_as_mirror(project.import_url, fetch_refs: fetch_refs)
else else
clone_repository gitlab_shell.import_repository(project.repository_storage_path, project.disk_path, project.import_url)
end end
rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e
# Expire cache to prevent scenarios such as: # Expire cache to prevent scenarios such as:
...@@ -66,17 +69,6 @@ module Projects ...@@ -66,17 +69,6 @@ module Projects
end end
end end
def clone_repository
gitlab_shell.import_repository(project.repository_storage_path, project.disk_path, project.import_url)
end
def fetch_repository
project.ensure_repository
project.repository.add_remote(project.import_type, project.import_url)
project.repository.set_remote_as_mirror(project.import_type)
project.repository.fetch_remote(project.import_type, forced: true)
end
def import_data def import_data
return unless has_importer? return unless has_importer?
......
...@@ -1150,10 +1150,12 @@ module Gitlab ...@@ -1150,10 +1150,12 @@ module Gitlab
@has_visible_content = has_local_branches? @has_visible_content = has_local_branches?
end end
def fetch(remote = 'origin') # Like all public `Gitlab::Git::Repository` methods, this method is part
args = %W(#{Gitlab.config.git.bin_path} fetch #{remote}) # of `Repository`'s interface through `method_missing`.
# `Repository` has its own `fetch_remote` which uses `gitlab-shell` and
popen(args, @path).last.zero? # takes some extra attributes, so we qualify this method name to prevent confusion.
def fetch_remote_without_shell(remote = 'origin')
run_git(['fetch', remote]).last.zero?
end end
def blob_at(sha, path) def blob_at(sha, path)
......
module Gitlab module Gitlab
module Git module Git
module RepositoryMirroring module RepositoryMirroring
IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze FETCH_REFS = {
IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze # `:all` is used to define repository as equivalent as "git clone --mirror"
MIRROR_REMOTE = 'mirror'.freeze all: '+refs/*:refs/*',
heads: '+refs/heads/*:refs/heads/*',
tags: '+refs/tags/*:refs/tags/*'
}.freeze
RemoteError = Class.new(StandardError) RemoteError = Class.new(StandardError)
def set_remote_as_mirror(remote_name) def set_remote_as_mirror(remote_name, fetch_refs: :all)
# This is used to define repository as equivalent as "git clone --mirror" Array(fetch_refs).each_with_index do |fetch_ref, i|
rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*' fetch_ref = FETCH_REFS[fetch_ref] || fetch_ref
rugged.config["remote.#{remote_name}.mirror"] = true
rugged.config["remote.#{remote_name}.prune"] = true
end
def set_import_remote_as_mirror(remote_name) # Add first fetch with Rugged so it does not create its own.
# Add first fetch with Rugged so it does not create its own. if i == 0
rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS rugged.config["remote.#{remote_name}.fetch"] = fetch_ref
else
add_remote_fetch_config(remote_name, IMPORT_TAG_REFS) add_remote_fetch_config(remote_name, fetch_ref)
end
end
rugged.config["remote.#{remote_name}.mirror"] = true rugged.config["remote.#{remote_name}.mirror"] = true
rugged.config["remote.#{remote_name}.prune"] = true rugged.config["remote.#{remote_name}.prune"] = true
...@@ -28,11 +30,17 @@ module Gitlab ...@@ -28,11 +30,17 @@ module Gitlab
run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
end end
def fetch_mirror(url) # Like all public `Gitlab::Git::Repository` methods, this method is part
add_remote(MIRROR_REMOTE, url) # of `Repository`'s interface through `method_missing`.
set_remote_as_mirror(MIRROR_REMOTE) # `Repository` has its own `fetch_as_mirror` which uses `gitlab-shell` and
fetch(MIRROR_REMOTE) # takes some extra attributes, so we qualify this method name to prevent confusion.
remove_remote(MIRROR_REMOTE) def fetch_as_mirror_without_shell(url)
remote_name = "tmp-#{SecureRandom.hex}"
add_remote(remote_name, url)
set_remote_as_mirror(remote_name)
fetch_remote_without_shell(remote_name)
ensure
remove_remote(remote_name) if remote_name
end end
def remote_tags(remote) def remote_tags(remote)
......
module Gitlab module Gitlab
module GithubImport module GithubImport
def self.fetch_refs
[:heads, :tags, '+refs/pull/*/head:refs/merge-requests/*/head']
end
def self.new_client_for(project, token: nil, parallel: true) def self.new_client_for(project, token: nil, parallel: true)
token_to_use = token || project.import_data&.credentials&.fetch(:user) token_to_use = token || project.import_data&.credentials&.fetch(:user)
......
...@@ -45,27 +45,14 @@ module Gitlab ...@@ -45,27 +45,14 @@ module Gitlab
def import_repository def import_repository
project.ensure_repository project.ensure_repository
configure_repository_remote fetch_refs = Gitlab::GithubImport.fetch_refs
project.repository.fetch_as_mirror(project.import_url, fetch_refs: fetch_refs, forced: true, remote_name: 'github')
project.repository.fetch_remote('github', forced: true)
true true
rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e
fail_import("Failed to import the repository: #{e.message}") fail_import("Failed to import the repository: #{e.message}")
end end
def configure_repository_remote
return if project.repository.remote_exists?('github')
project.repository.add_remote('github', project.import_url)
project.repository.set_import_remote_as_mirror('github')
project.repository.add_remote_fetch_config(
'github',
'+refs/pull/*/head:refs/merge-requests/*/head'
)
end
def import_wiki_repository def import_wiki_repository
wiki_path = "#{project.disk_path}.wiki" wiki_path = "#{project.disk_path}.wiki"
wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git') wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git')
......
...@@ -3,6 +3,10 @@ module Gitlab ...@@ -3,6 +3,10 @@ module Gitlab
class Importer class Importer
include Gitlab::ShellAdapter include Gitlab::ShellAdapter
def self.fetch_refs
Gitlab::GithubImport.fetch_refs
end
attr_reader :errors, :project, :repo, :repo_url attr_reader :errors, :project, :repo, :repo_url
def initialize(project) def initialize(project)
......
...@@ -588,12 +588,12 @@ describe Gitlab::Git::Repository, seed_helper: true do ...@@ -588,12 +588,12 @@ describe Gitlab::Git::Repository, seed_helper: true do
end end
end end
describe '#fetch_mirror' do describe '#fetch_as_mirror_without_shell' do
let(:new_repository) do let(:new_repository) do
Gitlab::Git::Repository.new('default', 'my_project.git', '') Gitlab::Git::Repository.new('default', 'my_project.git', '')
end end
subject { new_repository.fetch_mirror(repository.path) } subject { new_repository.fetch_as_mirror_without_shell(repository.path) }
before do before do
Gitlab::Shell.new.add_repository('default', 'my_project') Gitlab::Shell.new.add_repository('default', 'my_project')
...@@ -1643,15 +1643,15 @@ describe Gitlab::Git::Repository, seed_helper: true do ...@@ -1643,15 +1643,15 @@ describe Gitlab::Git::Repository, seed_helper: true do
end end
end end
describe '#fetch' do describe '#fetch_remote_without_shell' do
let(:git_path) { Gitlab.config.git.bin_path } let(:git_path) { Gitlab.config.git.bin_path }
let(:remote_name) { 'my_remote' } let(:remote_name) { 'my_remote' }
subject { repository.fetch(remote_name) } subject { repository.fetch_remote_without_shell(remote_name) }
it 'fetches the remote and returns true if the command was successful' do it 'fetches the remote and returns true if the command was successful' do
expect(repository).to receive(:popen) expect(repository).to receive(:popen)
.with(%W(#{git_path} fetch #{remote_name}), repository.path) .with(%W(#{git_path} fetch #{remote_name}), repository.path, {})
.and_return(['', 0]) .and_return(['', 0])
expect(subject).to be(true) expect(subject).to be(true)
...@@ -1768,21 +1768,6 @@ describe Gitlab::Git::Repository, seed_helper: true do ...@@ -1768,21 +1768,6 @@ describe Gitlab::Git::Repository, seed_helper: true do
end end
end end
describe '#fetch' do
let(:git_path) { Gitlab.config.git.bin_path }
let(:remote_name) { 'my_remote' }
subject { repository.fetch(remote_name) }
it 'fetches the remote and returns true if the command was successful' do
expect(repository).to receive(:popen)
.with(%W(#{git_path} fetch #{remote_name}), repository.path)
.and_return(['', 0])
expect(subject).to be(true)
end
end
describe '#delete_all_refs_except' do describe '#delete_all_refs_except' do
let(:repository) do let(:repository) do
Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '')
......
...@@ -164,12 +164,9 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do ...@@ -164,12 +164,9 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do
expect(project) expect(project)
.to receive(:ensure_repository) .to receive(:ensure_repository)
expect(importer)
.to receive(:configure_repository_remote)
expect(repository) expect(repository)
.to receive(:fetch_remote) .to receive(:fetch_as_mirror)
.with('github', forced: true) .with(project.import_url, fetch_refs: Gitlab::GithubImport.fetch_refs, forced: true, remote_name: 'github')
expect(importer.import_repository).to eq(true) expect(importer.import_repository).to eq(true)
end end
...@@ -186,40 +183,6 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do ...@@ -186,40 +183,6 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do
end end
end end
describe '#configure_repository_remote' do
it 'configures the remote details' do
expect(repository)
.to receive(:remote_exists?)
.with('github')
.and_return(false)
expect(repository)
.to receive(:add_remote)
.with('github', 'foo.git')
expect(repository)
.to receive(:set_import_remote_as_mirror)
.with('github')
expect(repository)
.to receive(:add_remote_fetch_config)
importer.configure_repository_remote
end
it 'does not configure the remote if already configured' do
expect(repository)
.to receive(:remote_exists?)
.with('github')
.and_return(true)
expect(repository)
.not_to receive(:add_remote)
importer.configure_repository_remote
end
end
describe '#import_wiki_repository' do describe '#import_wiki_repository' do
it 'imports the wiki repository' do it 'imports the wiki repository' do
expect(importer.gitlab_shell) expect(importer.gitlab_shell)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment