Commit 3ab38383 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'gitaly-backup-restore-delete-repositories' into 'master'

Use Gitaly's DeleteAllRepositories RPC during backup restore

Closes gitaly#1194

See merge request gitlab-org/gitlab-ce!19086
parents a45b9fc9 b529d04b
...@@ -412,7 +412,7 @@ group :ed25519 do ...@@ -412,7 +412,7 @@ group :ed25519 do
end end
# Gitaly GRPC client # Gitaly GRPC client
gem 'gitaly-proto', '~> 0.99.0', require: 'gitaly' gem 'gitaly-proto', '~> 0.100.0', require: 'gitaly'
gem 'grpc', '~> 1.11.0' gem 'grpc', '~> 1.11.0'
# Locked until https://github.com/google/protobuf/issues/4210 is closed # Locked until https://github.com/google/protobuf/issues/4210 is closed
......
...@@ -281,7 +281,7 @@ GEM ...@@ -281,7 +281,7 @@ GEM
gettext_i18n_rails (>= 0.7.1) gettext_i18n_rails (>= 0.7.1)
po_to_json (>= 1.0.0) po_to_json (>= 1.0.0)
rails (>= 3.2.0) rails (>= 3.2.0)
gitaly-proto (0.99.0) gitaly-proto (0.100.0)
google-protobuf (~> 3.1) google-protobuf (~> 3.1)
grpc (~> 1.10) grpc (~> 1.10)
github-linguist (5.3.3) github-linguist (5.3.3)
...@@ -1036,7 +1036,7 @@ DEPENDENCIES ...@@ -1036,7 +1036,7 @@ DEPENDENCIES
gettext (~> 3.2.2) gettext (~> 3.2.2)
gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails (~> 1.8.0)
gettext_i18n_rails_js (~> 1.3) gettext_i18n_rails_js (~> 1.3)
gitaly-proto (~> 0.99.0) gitaly-proto (~> 0.100.0)
github-linguist (~> 5.3.3) github-linguist (~> 5.3.3)
gitlab-flowdock-git-hook (~> 1.0.1) gitlab-flowdock-git-hook (~> 1.0.1)
gitlab-gollum-lib (~> 4.2) gitlab-gollum-lib (~> 4.2)
......
...@@ -73,29 +73,40 @@ module Backup ...@@ -73,29 +73,40 @@ module Backup
end end
def prepare_directories def prepare_directories
# TODO: Need to find a way to do this for gitaly
# Gitaly discussion issue: https://gitlab.com/gitlab-org/gitaly/issues/1194
Gitlab.config.repositories.storages.each do |name, repository_storage| Gitlab.config.repositories.storages.each do |name, repository_storage|
path = repository_storage.legacy_disk_path delete_all_repositories(name, repository_storage)
next unless File.exist?(path) end
end
# Move all files in the existing repos directory except . and .. to
# repositories.old.<timestamp> directory def delete_all_repositories(name, repository_storage)
bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) gitaly_migrate(:delete_all_repositories) do |is_enabled|
FileUtils.mkdir_p(bk_repos_path, mode: 0700) if is_enabled
files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
else
begin local_delete_all_repositories(name, repository_storage)
FileUtils.mv(files, bk_repos_path)
rescue Errno::EACCES
access_denied_error(path)
rescue Errno::EBUSY
resource_busy_error(path)
end end
end end
end end
def local_delete_all_repositories(name, repository_storage)
path = repository_storage.legacy_disk_path
return unless File.exist?(path)
# Move all files in the existing repos directory except . and .. to
# repositories.old.<timestamp> directory
bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
FileUtils.mkdir_p(bk_repos_path, mode: 0700)
files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
begin
FileUtils.mv(files, bk_repos_path)
rescue Errno::EACCES
access_denied_error(path)
rescue Errno::EBUSY
resource_busy_error(path)
end
end
def restore_custom_hooks(project) def restore_custom_hooks(project)
# TODO: Need to find a way to do this for gitaly # TODO: Need to find a way to do this for gitaly
# Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195 # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195
...@@ -113,6 +124,7 @@ module Backup ...@@ -113,6 +124,7 @@ module Backup
def restore def restore
prepare_directories prepare_directories
gitlab_shell = Gitlab::Shell.new gitlab_shell = Gitlab::Shell.new
Project.find_each(batch_size: 1000) do |project| Project.find_each(batch_size: 1000) do |project|
progress.print " * #{project.full_path} ... " progress.print " * #{project.full_path} ... "
path_to_project_bundle = path_to_bundle(project) path_to_project_bundle = path_to_bundle(project)
...@@ -121,7 +133,6 @@ module Backup ...@@ -121,7 +133,6 @@ module Backup
restore_repo_success = nil restore_repo_success = nil
if File.exist?(path_to_project_bundle) if File.exist?(path_to_project_bundle)
begin begin
gitlab_shell.remove_repository(project.repository_storage, project.disk_path) if project.repository_exists?
project.repository.create_from_bundle path_to_project_bundle project.repository.create_from_bundle path_to_project_bundle
restore_repo_success = true restore_repo_success = true
rescue => e rescue => e
...@@ -146,7 +157,6 @@ module Backup ...@@ -146,7 +157,6 @@ module Backup
if File.exist?(path_to_wiki_bundle) if File.exist?(path_to_wiki_bundle)
progress.print " * #{wiki.full_path} ... " progress.print " * #{wiki.full_path} ... "
begin begin
gitlab_shell.remove_repository(wiki.repository_storage, wiki.disk_path) if wiki.repository_exists?
wiki.repository.create_from_bundle(path_to_wiki_bundle) wiki.repository.create_from_bundle(path_to_wiki_bundle)
progress.puts "[DONE]".color(:green) progress.puts "[DONE]".color(:green)
rescue => e rescue => e
...@@ -224,5 +234,11 @@ module Backup ...@@ -224,5 +234,11 @@ module Backup
def display_repo_path(project) def display_repo_path(project)
project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
end end
def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
Gitlab::GitalyClient.migrate(method, status: status, &block)
rescue GRPC::NotFound, GRPC::BadStatus => e
raise Error, e
end
end end
end end
module Gitlab
module GitalyClient
class StorageService
def initialize(storage)
@storage = storage
end
# Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation.
def delete_all_repositories
request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage)
GitalyClient.call(@storage, :storage_service, :delete_all_repositories, request)
end
end
end
end
...@@ -71,6 +71,40 @@ describe Backup::Repository do ...@@ -71,6 +71,40 @@ describe Backup::Repository do
end end
end end
describe '#delete_all_repositories', :seed_helper do
shared_examples('delete_all_repositories') do
before do
allow(FileUtils).to receive(:mkdir_p).and_call_original
allow(FileUtils).to receive(:mv).and_call_original
end
after(:all) do
ensure_seeds
end
it 'removes all repositories' do
# Sanity check: there should be something for us to delete
expect(list_repositories).to include(File.join(SEED_STORAGE_PATH, TEST_REPO_PATH))
subject.delete_all_repositories('default', Gitlab.config.repositories.storages['default'])
expect(list_repositories).to be_empty
end
def list_repositories
Dir[SEED_STORAGE_PATH + '/*.git']
end
end
context 'with gitaly' do
it_behaves_like 'delete_all_repositories'
end
context 'without gitaly', :skip_gitaly_mock do
it_behaves_like 'delete_all_repositories'
end
end
describe '#empty_repo?' do describe '#empty_repo?' do
context 'for a wiki' do context 'for a wiki' do
let(:wiki) { create(:project_wiki) } let(:wiki) { create(:project_wiki) }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment