Commit f53d2731 authored by Sean McGivern's avatar Sean McGivern

Merge branch 'dm-cleanup-fetch-and-mirror-methods-ee' into 'master'

Cleanup fetch and mirror methods

See merge request gitlab-org/gitlab-ee!3396
parents 7e49a1ef 313b1e95
......@@ -28,7 +28,6 @@ class Repository
CreateTreeError = Class.new(StandardError)
MIRROR_REMOTE = "upstream".freeze
MIRROR_GEO = "geo".freeze
# Methods that cache data from the Git repository.
#
......@@ -934,12 +933,6 @@ class Repository
fetch_remote(Repository::MIRROR_REMOTE, ssh_auth: project&.import_data)
end
def fetch_geo_mirror(url)
add_remote(Repository::MIRROR_GEO, url)
set_remote_as_mirror(Repository::MIRROR_GEO)
fetch_remote(Repository::MIRROR_GEO, forced: true)
end
def upstream_branches
@upstream_branches ||= remote_branches(Repository::MIRROR_REMOTE)
end
......@@ -1021,6 +1014,19 @@ class Repository
run_git(args).first.lines.map(&:strip)
end
def fetch_as_mirror(url, forced: false, refmap: :all_refs, remote_name: nil)
unless remote_name
remote_name = "tmp-#{SecureRandom.hex}"
tmp_remote_name = true
end
add_remote(remote_name, url)
set_remote_as_mirror(remote_name, refmap: refmap)
fetch_remote(remote_name, forced: forced)
ensure
remove_remote(remote_name) if tmp_remote_name
end
def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false)
gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
end
......@@ -1124,6 +1130,10 @@ class Repository
raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref)
end
def repository_storage_path
@project.repository_storage_path
end
private
# TODO Generice finder, later split this on finders by Ref or Oid
......@@ -1189,10 +1199,6 @@ class Repository
raw_repository.run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip
end
def repository_storage_path
@project.repository_storage_path
end
def initialize_raw_repository
Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, is_wiki))
end
......
......@@ -114,13 +114,13 @@ module Geo
authorization = ::Gitlab::Geo::BaseRequest.new.authorization
header = { "http.#{url}.extraHeader" => "Authorization: #{authorization}" }
repository.with_config(header) { repository.fetch_geo_mirror(url) }
repository.with_config(header) { repository.fetch_as_mirror(url, forced: true) }
end
def fetch_ssh_geo_mirror(repository)
url = build_repository_url(primary_ssh_path_prefix, repository)
repository.fetch_geo_mirror(url)
repository.fetch_as_mirror(url, forced: true)
end
def registry
......
......@@ -51,10 +51,13 @@ module Projects
def import_repository
begin
if project.gitea_import?
fetch_repository
refmap = importer_class.try(:refmap) if has_importer?
if refmap
project.ensure_repository
project.repository.fetch_as_mirror(project.import_url, refmap: refmap)
else
clone_repository
gitlab_shell.import_repository(project.repository_storage_path, project.disk_path, project.import_url)
end
rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e
# Expire cache to prevent scenarios such as:
......@@ -66,17 +69,6 @@ module Projects
end
end
def clone_repository
gitlab_shell.import_repository(project.repository_storage_path, project.disk_path, project.import_url)
end
def fetch_repository
project.ensure_repository
project.repository.add_remote(project.import_type, project.import_url)
project.repository.set_remote_as_mirror(project.import_type)
project.repository.fetch_remote(project.import_type, forced: true)
end
def import_data
return unless has_importer?
......
module EE
module RepositoryMirroring
def storage_path
@project.repository_storage_path
end
def push_remote_branches(remote, branches)
gitlab_shell.push_remote_branches(storage_path, disk_path, remote, branches)
end
def delete_remote_branches(remote, branches)
gitlab_shell.delete_remote_branches(storage_path, disk_path, remote, branches)
end
end
end
......@@ -4,7 +4,6 @@ module EE
# This module is intended to encapsulate EE-specific model logic
# and be prepended in the `Repository` model
module Repository
prepend RepositoryMirroring
extend ActiveSupport::Concern
# Transiently sets a configuration variable
......@@ -31,5 +30,13 @@ module EE
refs.map { |sha| commit(sha.strip) }
end
def push_remote_branches(remote, branches)
gitlab_shell.push_remote_branches(repository_storage_path, disk_path, remote, branches)
end
def delete_remote_branches(remote, branches)
gitlab_shell.delete_remote_branches(repository_storage_path, disk_path, remote, branches)
end
end
end
......@@ -165,7 +165,6 @@ class RemoteMirror < ActiveRecord::Base
def refresh_remote
return unless project
project.repository.remove_remote(ref_name)
project.repository.add_remote(ref_name, url)
end
......
......@@ -35,7 +35,7 @@ module Projects
repository.relative_path,
repository.gl_repository)
new_repository.fetch_mirror(repository.path)
new_repository.fetch_as_mirror_without_shell(repository.path)
end
def mark_old_paths_for_archive
......
......@@ -1145,10 +1145,12 @@ module Gitlab
@has_visible_content = has_local_branches?
end
def fetch(remote = 'origin')
args = %W(#{Gitlab.config.git.bin_path} fetch #{remote})
popen(args, @path).last.zero?
# Like all public `Gitlab::Git::Repository` methods, this method is part
# of `Repository`'s interface through `method_missing`.
# `Repository` has its own `fetch_remote` which uses `gitlab-shell` and
# takes some extra attributes, so we qualify this method name to prevent confusion.
def fetch_remote_without_shell(remote = 'origin')
run_git(['fetch', remote]).last.zero?
end
def blob_at(sha, path)
......
module Gitlab
module Git
module RepositoryMirroring
IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze
IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze
MIRROR_REMOTE = 'mirror'.freeze
REFMAPS = {
# With `:all_refs`, the repository is equivalent to the result of `git clone --mirror`
all_refs: '+refs/*:refs/*',
heads: '+refs/heads/*:refs/heads/*',
tags: '+refs/tags/*:refs/tags/*'
}.freeze
RemoteError = Class.new(StandardError)
def set_remote_as_mirror(remote_name)
# This is used to define repository as equivalent as "git clone --mirror"
rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*'
rugged.config["remote.#{remote_name}.mirror"] = true
rugged.config["remote.#{remote_name}.prune"] = true
end
def set_import_remote_as_mirror(remote_name)
# Add first fetch with Rugged so it does not create its own.
rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS
add_remote_fetch_config(remote_name, IMPORT_TAG_REFS)
def set_remote_as_mirror(remote_name, refmap: :all_refs)
set_remote_refmap(remote_name, refmap)
rugged.config["remote.#{remote_name}.mirror"] = true
rugged.config["remote.#{remote_name}.prune"] = true
end
def add_remote_fetch_config(remote_name, refspec)
def set_remote_refmap(remote_name, refmap)
Array(refmap).each_with_index do |refspec, i|
refspec = REFMAPS[refspec] || refspec
# We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it.
# To make sure we start from scratch, we set the first using rugged, and use `git` for any others
if i == 0
rugged.config["remote.#{remote_name}.fetch"] = refspec
else
run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
end
end
end
def fetch_mirror(url)
add_remote(MIRROR_REMOTE, url)
set_remote_as_mirror(MIRROR_REMOTE)
fetch(MIRROR_REMOTE)
remove_remote(MIRROR_REMOTE)
# Like all_refs public `Gitlab::Git::Repository` methods, this method is part
# of `Repository`'s interface through `method_missing`.
# `Repository` has its own `fetch_as_mirror` which uses `gitlab-shell` and
# takes some extra attributes, so we qualify this method name to prevent confusion.
def fetch_as_mirror_without_shell(url)
remote_name = "tmp-#{SecureRandom.hex}"
add_remote(remote_name, url)
set_remote_as_mirror(remote_name)
fetch_remote_without_shell(remote_name)
ensure
remove_remote(remote_name) if remote_name
end
def remote_tags(remote)
......
module Gitlab
module GithubImport
def self.refmap
[:heads, :tags, '+refs/pull/*/head:refs/merge-requests/*/head']
end
def self.new_client_for(project, token: nil, parallel: true)
token_to_use = token || project.import_data&.credentials&.fetch(:user)
......
......@@ -45,27 +45,14 @@ module Gitlab
def import_repository
project.ensure_repository
configure_repository_remote
project.repository.fetch_remote('github', forced: true)
refmap = Gitlab::GithubImport.refmap
project.repository.fetch_as_mirror(project.import_url, refmap: refmap, forced: true, remote_name: 'github')
true
rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e
fail_import("Failed to import the repository: #{e.message}")
end
def configure_repository_remote
return if project.repository.remote_exists?('github')
project.repository.add_remote('github', project.import_url)
project.repository.set_import_remote_as_mirror('github')
project.repository.add_remote_fetch_config(
'github',
'+refs/pull/*/head:refs/merge-requests/*/head'
)
end
def import_wiki_repository
wiki_path = "#{project.disk_path}.wiki"
wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git')
......
......@@ -3,6 +3,10 @@ module Gitlab
class Importer
include Gitlab::ShellAdapter
def self.refmap
Gitlab::GithubImport.refmap
end
attr_reader :errors, :project, :repo, :repo_url
def initialize(project)
......
......@@ -588,12 +588,12 @@ describe Gitlab::Git::Repository, seed_helper: true do
end
end
describe '#fetch_mirror' do
describe '#fetch_as_mirror_without_shell' do
let(:new_repository) do
Gitlab::Git::Repository.new('default', 'my_project.git', '')
end
subject { new_repository.fetch_mirror(repository.path) }
subject { new_repository.fetch_as_mirror_without_shell(repository.path) }
before do
Gitlab::Shell.new.add_repository('default', 'my_project')
......@@ -1652,15 +1652,15 @@ describe Gitlab::Git::Repository, seed_helper: true do
end
end
describe '#fetch' do
describe '#fetch_remote_without_shell' do
let(:git_path) { Gitlab.config.git.bin_path }
let(:remote_name) { 'my_remote' }
subject { repository.fetch(remote_name) }
subject { repository.fetch_remote_without_shell(remote_name) }
it 'fetches the remote and returns true if the command was successful' do
expect(repository).to receive(:popen)
.with(%W(#{git_path} fetch #{remote_name}), repository.path)
.with(%W(#{git_path} fetch #{remote_name}), repository.path, {})
.and_return(['', 0])
expect(subject).to be(true)
......@@ -1777,21 +1777,6 @@ describe Gitlab::Git::Repository, seed_helper: true do
end
end
describe '#fetch' do
let(:git_path) { Gitlab.config.git.bin_path }
let(:remote_name) { 'my_remote' }
subject { repository.fetch(remote_name) }
it 'fetches the remote and returns true if the command was successful' do
expect(repository).to receive(:popen)
.with(%W(#{git_path} fetch #{remote_name}), repository.path)
.and_return(['', 0])
expect(subject).to be(true)
end
end
def create_remote_branch(repository, remote_name, branch_name, source_branch_name)
source_branch = repository.branches.find { |branch| branch.name == source_branch_name }
rugged = repository.rugged
......
......@@ -164,12 +164,9 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do
expect(project)
.to receive(:ensure_repository)
expect(importer)
.to receive(:configure_repository_remote)
expect(repository)
.to receive(:fetch_remote)
.with('github', forced: true)
.to receive(:fetch_as_mirror)
.with(project.import_url, refmap: Gitlab::GithubImport.refmap, forced: true, remote_name: 'github')
expect(importer.import_repository).to eq(true)
end
......@@ -186,40 +183,6 @@ describe Gitlab::GithubImport::Importer::RepositoryImporter do
end
end
describe '#configure_repository_remote' do
it 'configures the remote details' do
expect(repository)
.to receive(:remote_exists?)
.with('github')
.and_return(false)
expect(repository)
.to receive(:add_remote)
.with('github', 'foo.git')
expect(repository)
.to receive(:set_import_remote_as_mirror)
.with('github')
expect(repository)
.to receive(:add_remote_fetch_config)
importer.configure_repository_remote
end
it 'does not configure the remote if already configured' do
expect(repository)
.to receive(:remote_exists?)
.with('github')
.and_return(true)
expect(repository)
.not_to receive(:add_remote)
importer.configure_repository_remote
end
end
describe '#import_wiki_repository' do
it 'imports the wiki repository' do
expect(importer.gitlab_shell)
......
......@@ -2115,7 +2115,7 @@ describe Repository do
describe '#push_remote_branches' do
it 'push branches to the remote repo' do
expect_any_instance_of(Gitlab::Shell).to receive(:push_remote_branches)
.with(repository.storage_path, repository.disk_path, 'remote_name', ['branch'])
.with(repository.repository_storage_path, repository.disk_path, 'remote_name', ['branch'])
repository.push_remote_branches('remote_name', ['branch'])
end
......@@ -2124,7 +2124,7 @@ describe Repository do
describe '#delete_remote_branches' do
it 'delete branches to the remote repo' do
expect_any_instance_of(Gitlab::Shell).to receive(:delete_remote_branches)
.with(repository.storage_path, repository.disk_path, 'remote_name', ['branch'])
.with(repository.repository_storage_path, repository.disk_path, 'remote_name', ['branch'])
repository.delete_remote_branches('remote_name', ['branch'])
end
......
......@@ -26,13 +26,13 @@ describe Geo::RepositorySyncService do
.with(subject.lease_key, anything)
.and_return(lease)
allow_any_instance_of(Repository).to receive(:fetch_geo_mirror)
allow_any_instance_of(Repository).to receive(:fetch_as_mirror)
.and_return(true)
end
it 'fetches project repository with JWT credentials' do
expect(repository).to receive(:with_config).with("http.#{url_to_repo}.extraHeader" => anything).and_call_original
expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
expect(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true).once
subject.execute
end
......@@ -53,7 +53,7 @@ describe Geo::RepositorySyncService do
end
it 'returns the lease when sync fail' do
allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Shell::Error }
allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true) { raise Gitlab::Shell::Error }
expect(Gitlab::ExclusiveLease).to receive(:cancel).once.with(
subject.__send__(:lease_key), anything).and_call_original
......@@ -64,26 +64,26 @@ describe Geo::RepositorySyncService do
it 'does not fetch project repository if cannot obtain a lease' do
allow(lease).to receive(:try_obtain) { false }
expect(repository).not_to receive(:fetch_geo_mirror)
expect(repository).not_to receive(:fetch_as_mirror)
subject.execute
end
it 'rescues when Gitlab::Shell::Error is raised' do
allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Shell::Error }
allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true) { raise Gitlab::Shell::Error }
expect { subject.execute }.not_to raise_error
end
it 'rescues when Gitlab::Git::RepositoryMirroring::RemoteError is raised' do
allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo)
allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true)
.and_raise(Gitlab::Git::RepositoryMirroring::RemoteError)
expect { subject.execute }.not_to raise_error
end
it 'rescues exception and fires after_create hook when Gitlab::Git::Repository::NoRepository is raised' do
allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Git::Repository::NoRepository }
allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true) { raise Gitlab::Git::Repository::NoRepository }
expect(repository).to receive(:after_create)
......@@ -137,7 +137,7 @@ describe Geo::RepositorySyncService do
let(:registry) { Geo::ProjectRegistry.find_by(project_id: project.id) }
before do
allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Shell::Error }
allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true) { raise Gitlab::Shell::Error }
subject.execute
end
......@@ -180,7 +180,7 @@ describe Geo::RepositorySyncService do
# gitlab-shell always appends .git to the end of the repository, so
# we're relying on the fact that projects can't contain + in the name
deleted_dir = File.join(project.repository.storage_path, project.path) + "+failed-geo-sync.git"
deleted_dir = File.join(project.repository_storage_path, project.path) + "+failed-geo-sync.git"
expect(File.directory?(deleted_dir)).to be false
expect(File.directory?(project.repository.path)).to be true
end
......@@ -228,7 +228,7 @@ describe Geo::RepositorySyncService do
end
it 'fetches wiki repository over SSH' do
expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
expect(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true).once
subject.execute
end
......
......@@ -26,13 +26,13 @@ RSpec.describe Geo::WikiSyncService do
.with(subject.lease_key, anything)
.and_return(lease)
allow_any_instance_of(Repository).to receive(:fetch_geo_mirror)
allow_any_instance_of(Repository).to receive(:fetch_as_mirror)
.and_return(true)
end
it 'fetches wiki repository with JWT credentials' do
expect(repository).to receive(:with_config).with("http.#{url_to_repo}.extraHeader" => anything).and_call_original
expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
expect(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true).once
subject.execute
end
......@@ -47,26 +47,26 @@ RSpec.describe Geo::WikiSyncService do
it 'does not fetch wiki repository if cannot obtain a lease' do
allow(lease).to receive(:try_obtain) { false }
expect(repository).not_to receive(:fetch_geo_mirror)
expect(repository).not_to receive(:fetch_as_mirror)
subject.execute
end
it 'rescues exception when Gitlab::Shell::Error is raised' do
allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Shell::Error }
allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true) { raise Gitlab::Shell::Error }
expect { subject.execute }.not_to raise_error
end
it 'rescues exception when Gitlab::Git::RepositoryMirroring::RemoteError is raised' do
allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo)
allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true)
.and_raise(Gitlab::Git::RepositoryMirroring::RemoteError)
expect { subject.execute }.not_to raise_error
end
it 'rescues exception when Gitlab::Git::Repository::NoRepository is raised' do
allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Git::Repository::NoRepository }
allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true) { raise Gitlab::Git::Repository::NoRepository }
expect { subject.execute }.not_to raise_error
end
......@@ -109,7 +109,7 @@ RSpec.describe Geo::WikiSyncService do
let(:registry) { Geo::ProjectRegistry.find_by(project_id: project.id) }
before do
allow(repository).to receive(:fetch_geo_mirror).with(url_to_repo) { raise Gitlab::Shell::Error }
allow(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true) { raise Gitlab::Shell::Error }
subject.execute
end
......@@ -134,7 +134,7 @@ RSpec.describe Geo::WikiSyncService do
end
it 'fetches wiki repository over SSH' do
expect(repository).to receive(:fetch_geo_mirror).with(url_to_repo).once
expect(repository).to receive(:fetch_as_mirror).with(url_to_repo, forced: true).once
subject.execute
end
......
......@@ -31,7 +31,7 @@ describe Projects::UpdateRepositoryStorageService do
context 'when the move succeeds' do
it 'moves the repository to the new storage and unmarks the repository as read only' do
expect_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_mirror)
expect_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_as_mirror_without_shell)
.with(project.repository.raw.path).and_return(true)
expect(GitlabShellWorker).to receive(:perform_async)
.with(:mv_repository,
......@@ -48,7 +48,7 @@ describe Projects::UpdateRepositoryStorageService do
context 'when the move fails' do
it 'unmarks the repository as read-only without updating the repository storage' do
expect_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_mirror)
expect_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_as_mirror_without_shell)
.with(project.repository.raw.path).and_return(false)
expect(GitlabShellWorker).not_to receive(:perform_async)
......@@ -84,7 +84,7 @@ describe Projects::UpdateRepositoryStorageService do
context 'when the move succeeds' do
it 'moves the repository and its wiki to the new storage and unmarks the repository as read only' do
expect(repository_double).to receive(:fetch_mirror)
expect(repository_double).to receive(:fetch_as_mirror_without_shell)
.with(project.repository.raw.path).and_return(true)
expect(GitlabShellWorker).to receive(:perform_async)
.with(:mv_repository,
......@@ -92,7 +92,7 @@ describe Projects::UpdateRepositoryStorageService do
project.disk_path,
"#{project.disk_path}+#{project.id}+moved+#{time.to_i}")
expect(wiki_repository_double).to receive(:fetch_mirror)
expect(wiki_repository_double).to receive(:fetch_as_mirror_without_shell)
.with(project.wiki.repository.raw.path).and_return(true)
expect(GitlabShellWorker).to receive(:perform_async)
.with(:mv_repository,
......@@ -109,9 +109,9 @@ describe Projects::UpdateRepositoryStorageService do
context 'when the move of the wiki fails' do
it 'unmarks the repository as read-only without updating the repository storage' do
expect(repository_double).to receive(:fetch_mirror)
expect(repository_double).to receive(:fetch_as_mirror_without_shell)
.with(project.repository.raw.path).and_return(true)
expect(wiki_repository_double).to receive(:fetch_mirror)
expect(wiki_repository_double).to receive(:fetch_as_mirror_without_shell)
.with(project.wiki.repository.raw.path).and_return(false)
expect(GitlabShellWorker).not_to receive(:perform_async)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment