Commit 2ca2d3d6 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'use-backup-custom-hooks-gitaly' into 'master'

Use gitaly for backup rake task

See merge request gitlab-org/gitlab-ce!19742
parents a88a9e22 9d7e7327
0.105.1 0.107.0
...@@ -419,7 +419,7 @@ group :ed25519 do ...@@ -419,7 +419,7 @@ group :ed25519 do
end end
# Gitaly GRPC client # Gitaly GRPC client
gem 'gitaly-proto', '~> 0.101.0', require: 'gitaly' gem 'gitaly-proto', '~> 0.102.0', require: 'gitaly'
gem 'grpc', '~> 1.11.0' gem 'grpc', '~> 1.11.0'
# Locked until https://github.com/google/protobuf/issues/4210 is closed # Locked until https://github.com/google/protobuf/issues/4210 is closed
......
...@@ -283,7 +283,7 @@ GEM ...@@ -283,7 +283,7 @@ GEM
gettext_i18n_rails (>= 0.7.1) gettext_i18n_rails (>= 0.7.1)
po_to_json (>= 1.0.0) po_to_json (>= 1.0.0)
rails (>= 3.2.0) rails (>= 3.2.0)
gitaly-proto (0.101.0) gitaly-proto (0.102.0)
google-protobuf (~> 3.1) google-protobuf (~> 3.1)
grpc (~> 1.10) grpc (~> 1.10)
github-linguist (5.3.3) github-linguist (5.3.3)
...@@ -1039,7 +1039,7 @@ DEPENDENCIES ...@@ -1039,7 +1039,7 @@ DEPENDENCIES
gettext (~> 3.2.2) gettext (~> 3.2.2)
gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails (~> 1.8.0)
gettext_i18n_rails_js (~> 1.3) gettext_i18n_rails_js (~> 1.3)
gitaly-proto (~> 0.101.0) gitaly-proto (~> 0.102.0)
github-linguist (~> 5.3.3) github-linguist (~> 5.3.3)
gitlab-flowdock-git-hook (~> 1.0.1) gitlab-flowdock-git-hook (~> 1.0.1)
gitlab-gollum-lib (~> 4.2) gitlab-gollum-lib (~> 4.2)
......
...@@ -286,7 +286,7 @@ GEM ...@@ -286,7 +286,7 @@ GEM
gettext_i18n_rails (>= 0.7.1) gettext_i18n_rails (>= 0.7.1)
po_to_json (>= 1.0.0) po_to_json (>= 1.0.0)
rails (>= 3.2.0) rails (>= 3.2.0)
gitaly-proto (0.101.0) gitaly-proto (0.102.0)
google-protobuf (~> 3.1) google-protobuf (~> 3.1)
grpc (~> 1.10) grpc (~> 1.10)
github-linguist (5.3.3) github-linguist (5.3.3)
...@@ -1049,7 +1049,7 @@ DEPENDENCIES ...@@ -1049,7 +1049,7 @@ DEPENDENCIES
gettext (~> 3.2.2) gettext (~> 3.2.2)
gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails (~> 1.8.0)
gettext_i18n_rails_js (~> 1.3) gettext_i18n_rails_js (~> 1.3)
gitaly-proto (~> 0.101.0) gitaly-proto (~> 0.102.0)
github-linguist (~> 5.3.3) github-linguist (~> 5.3.3)
gitlab-flowdock-git-hook (~> 1.0.1) gitlab-flowdock-git-hook (~> 1.0.1)
gitlab-gollum-lib (~> 4.2) gitlab-gollum-lib (~> 4.2)
......
---
title: migrate backup rake task to gitaly
merge_request:
author:
type: added
...@@ -4,7 +4,6 @@ require_relative 'helper' ...@@ -4,7 +4,6 @@ require_relative 'helper'
module Backup module Backup
class Repository class Repository
include Backup::Helper include Backup::Helper
# rubocop:disable Metrics/AbcSize
attr_reader :progress attr_reader :progress
...@@ -18,61 +17,26 @@ module Backup ...@@ -18,61 +17,26 @@ module Backup
Project.find_each(batch_size: 1000) do |project| Project.find_each(batch_size: 1000) do |project|
progress.print " * #{display_repo_path(project)} ... " progress.print " * #{display_repo_path(project)} ... "
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
path_to_repo(project)
end
path_to_project_bundle = path_to_bundle(project)
# Create namespace dir or hashed path if missing
if project.hashed_storage?(:repository) if project.hashed_storage?(:repository)
FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path))) FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path)))
else else
FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace
end end
if empty_repo?(project) if !empty_repo?(project)
progress.puts "[SKIPPED]".color(:cyan) backup_project(project)
progress.puts "[DONE]".color(:green)
else else
in_path(path_to_project_repo) do |dir| progress.puts "[SKIPPED]".color(:cyan)
FileUtils.mkdir_p(path_to_tars(project))
cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
output, status = Gitlab::Popen.popen(cmd)
unless status.zero?
progress_warn(project, cmd.join(' '), output)
end
end
cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
output, status = Gitlab::Popen.popen(cmd)
if status.zero?
progress.puts "[DONE]".color(:green)
else
progress_warn(project, cmd.join(' '), output)
end
end end
wiki = ProjectWiki.new(project) wiki = ProjectWiki.new(project)
path_to_wiki_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
path_to_repo(wiki)
end
path_to_wiki_bundle = path_to_bundle(wiki)
if File.exist?(path_to_wiki_repo) if !empty_repo?(wiki)
progress.print " * #{display_repo_path(wiki)} ... " backup_project(wiki)
progress.puts "[DONE] Wiki".color(:green)
if empty_repo?(wiki) else
progress.puts " [SKIPPED]".color(:cyan) progress.puts "[SKIPPED] Wiki".color(:cyan)
else
cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all)
output, status = Gitlab::Popen.popen(cmd)
if status.zero?
progress.puts " [DONE]".color(:green)
else
progress_warn(wiki, cmd.join(' '), output)
end
end
end end
end end
end end
...@@ -83,6 +47,38 @@ module Backup ...@@ -83,6 +47,38 @@ module Backup
end end
end end
def backup_project(project)
gitaly_migrate(:repository_backup) do |is_enabled|
if is_enabled
backup_project_gitaly(project)
else
backup_project_local(project)
end
end
backup_custom_hooks(project)
rescue => e
progress_warn(project, e, 'Failed to backup repo')
end
def backup_project_gitaly(project)
path_to_project_bundle = path_to_bundle(project)
Gitlab::GitalyClient::RepositoryService.new(project.repository)
.create_bundle(path_to_project_bundle)
end
def backup_project_local(project)
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
path_to_repo(project)
end
path_to_project_bundle = path_to_bundle(project)
cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
output, status = Gitlab::Popen.popen(cmd)
progress_warn(project, cmd.join(' '), output) unless status.zero?
end
def delete_all_repositories(name, repository_storage) def delete_all_repositories(name, repository_storage)
gitaly_migrate(:delete_all_repositories) do |is_enabled| gitaly_migrate(:delete_all_repositories) do |is_enabled|
if is_enabled if is_enabled
...@@ -97,8 +93,6 @@ module Backup ...@@ -97,8 +93,6 @@ module Backup
path = repository_storage.legacy_disk_path path = repository_storage.legacy_disk_path
return unless File.exist?(path) return unless File.exist?(path)
# Move all files in the existing repos directory except . and .. to
# repositories.old.<timestamp> directory
bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
FileUtils.mkdir_p(bk_repos_path, mode: 0700) FileUtils.mkdir_p(bk_repos_path, mode: 0700)
files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
...@@ -129,13 +123,47 @@ module Backup ...@@ -129,13 +123,47 @@ module Backup
.restore_custom_hooks(custom_hooks_path) .restore_custom_hooks(custom_hooks_path)
end end
def local_backup_custom_hooks(project)
in_path(path_to_tars(project)) do |dir|
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
path_to_repo(project)
end
break unless File.exist?(File.join(path_to_project_repo, dir))
FileUtils.mkdir_p(path_to_tars(project))
cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir})
output, status = Gitlab::Popen.popen(cmd)
unless status.zero?
progress_warn(project, cmd.join(' '), output)
end
end
end
def gitaly_backup_custom_hooks(project)
FileUtils.mkdir_p(path_to_tars(project))
custom_hooks_path = path_to_tars(project, 'custom_hooks')
Gitlab::GitalyClient::RepositoryService.new(project.repository)
.backup_custom_hooks(custom_hooks_path)
end
def backup_custom_hooks(project)
gitaly_migrate(:backup_custom_hooks) do |is_enabled|
if is_enabled
gitaly_backup_custom_hooks(project)
else
local_backup_custom_hooks(project)
end
end
end
def restore_custom_hooks(project) def restore_custom_hooks(project)
in_path(path_to_tars(project)) do |dir| in_path(path_to_tars(project)) do |dir|
gitaly_migrate(:restore_custom_hooks) do |is_enabled| gitaly_migrate(:restore_custom_hooks) do |is_enabled|
if is_enabled if is_enabled
local_restore_custom_hooks(project, dir)
else
gitaly_restore_custom_hooks(project, dir) gitaly_restore_custom_hooks(project, dir)
else
local_restore_custom_hooks(project, dir)
end end
end end
end end
...@@ -186,7 +214,6 @@ module Backup ...@@ -186,7 +214,6 @@ module Backup
end end
end end
end end
# rubocop:enable Metrics/AbcSize
protected protected
...@@ -224,9 +251,7 @@ module Backup ...@@ -224,9 +251,7 @@ module Backup
def prepare def prepare
FileUtils.rm_rf(backup_repos_path) FileUtils.rm_rf(backup_repos_path)
# Ensure the parent dir of backup_repos_path exists
FileUtils.mkdir_p(Gitlab.config.backup.path) FileUtils.mkdir_p(Gitlab.config.backup.path)
# Fail if somebody raced to create backup_repos_path before us
FileUtils.mkdir(backup_repos_path, mode: 0700) FileUtils.mkdir(backup_repos_path, mode: 0700)
end end
...@@ -242,7 +267,6 @@ module Backup ...@@ -242,7 +267,6 @@ module Backup
end end
def empty_repo?(project_or_wiki) def empty_repo?(project_or_wiki)
# Protect against stale caches
project_or_wiki.repository.expire_emptiness_caches project_or_wiki.repository.expire_emptiness_caches
project_or_wiki.repository.empty? project_or_wiki.repository.empty?
end end
......
...@@ -196,20 +196,21 @@ module Gitlab ...@@ -196,20 +196,21 @@ module Gitlab
end end
def create_bundle(save_path) def create_bundle(save_path)
request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo) gitaly_fetch_stream_to_file(
response = GitalyClient.call( save_path,
@storage,
:repository_service,
:create_bundle, :create_bundle,
request, Gitaly::CreateBundleRequest,
timeout: GitalyClient.default_timeout GitalyClient.default_timeout
) )
end
File.open(save_path, 'wb') do |f| def backup_custom_hooks(save_path)
response.each do |message| gitaly_fetch_stream_to_file(
f.write(message.data) save_path,
end :backup_custom_hooks,
end Gitaly::BackupCustomHooksRequest,
GitalyClient.default_timeout
)
end end
def create_from_bundle(bundle_path) def create_from_bundle(bundle_path)
...@@ -309,6 +310,25 @@ module Gitlab ...@@ -309,6 +310,25 @@ module Gitlab
private private
def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout)
request = request_class.new(repository: @gitaly_repo)
response = GitalyClient.call(
@storage,
:repository_service,
rpc_name,
request,
timeout: timeout
)
File.open(save_path, 'wb') do |f|
response.each do |message|
f.write(message.data)
end
end
# If the file is empty means that we recieved an empty stream, we delete the file
FileUtils.rm(save_path) if File.zero?(save_path)
end
def gitaly_repo_stream_request(file_path, rpc_name, request_class, timeout) def gitaly_repo_stream_request(file_path, rpc_name, request_class, timeout)
request = request_class.new(repository: @gitaly_repo) request = request_class.new(repository: @gitaly_repo)
enum = Enumerator.new do |y| enum = Enumerator.new do |y|
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment