Commit d7afed34 authored by Zeger-Jan van de Weg's avatar Zeger-Jan van de Weg

Remove feature flags from lib/backup

Moved to OPT_OUT in 7d14b725, Now, by
removing the feature gates, this is an mandatory feature.

Related issues:
- https://gitlab.com/gitlab-org/gitaly/issues/526
- https://gitlab.com/gitlab-org/gitaly/issues/1194

Closes https://gitlab.com/gitlab-org/gitaly/issues/749
Closes https://gitlab.com/gitlab-org/gitaly/issues/1212
Closes https://gitlab.com/gitlab-org/gitaly/issues/1195
parent 50352503
require 'yaml' require 'yaml'
require_relative 'helper'
module Backup module Backup
class Repository class Repository
include Backup::Helper
attr_reader :progress attr_reader :progress
def initialize(progress) def initialize(progress)
...@@ -42,131 +39,36 @@ module Backup ...@@ -42,131 +39,36 @@ module Backup
end end
def prepare_directories def prepare_directories
Gitlab.config.repositories.storages.each do |name, repository_storage| Gitlab.config.repositories.storages.each do |name, _repository_storage|
delete_all_repositories(name, repository_storage) Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
end end
end end
def backup_project(project) def backup_project(project)
gitaly_migrate(:repository_backup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
if is_enabled
backup_project_gitaly(project)
else
backup_project_local(project)
end
end
backup_custom_hooks(project)
rescue => e
progress_warn(project, e, 'Failed to backup repo')
end
def backup_project_gitaly(project)
path_to_project_bundle = path_to_bundle(project) path_to_project_bundle = path_to_bundle(project)
Gitlab::GitalyClient::RepositoryService.new(project.repository) Gitlab::GitalyClient::RepositoryService.new(project.repository)
.create_bundle(path_to_project_bundle) .create_bundle(path_to_project_bundle)
end
def backup_project_local(project)
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
path_to_repo(project)
end
path_to_project_bundle = path_to_bundle(project)
cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
output, status = Gitlab::Popen.popen(cmd)
progress_warn(project, cmd.join(' '), output) unless status.zero?
end
def delete_all_repositories(name, repository_storage)
gitaly_migrate(:delete_all_repositories, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
if is_enabled
Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
else
local_delete_all_repositories(name, repository_storage)
end
end
end
def local_delete_all_repositories(name, repository_storage)
path = repository_storage.legacy_disk_path
return unless File.exist?(path)
bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
FileUtils.mkdir_p(bk_repos_path, mode: 0700)
files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
begin
FileUtils.mv(files, bk_repos_path)
rescue Errno::EACCES
access_denied_error(path)
rescue Errno::EBUSY
resource_busy_error(path)
end
end
def local_restore_custom_hooks(project, dir) backup_custom_hooks(project)
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do rescue => e
path_to_repo(project) progress_warn(project, e, 'Failed to backup repo')
end
cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
output, status = Gitlab::Popen.popen(cmd)
unless status.zero?
progress_warn(project, cmd.join(' '), output)
end
end
def gitaly_restore_custom_hooks(project, dir)
custom_hooks_path = path_to_tars(project, dir)
Gitlab::GitalyClient::RepositoryService.new(project.repository)
.restore_custom_hooks(custom_hooks_path)
end end
def local_backup_custom_hooks(project) def backup_custom_hooks(project)
in_path(path_to_tars(project)) do |dir| FileUtils.mkdir_p(project_backup_path(project))
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
path_to_repo(project)
end
break unless File.exist?(File.join(path_to_project_repo, dir))
FileUtils.mkdir_p(path_to_tars(project))
cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir})
output, status = Gitlab::Popen.popen(cmd)
unless status.zero?
progress_warn(project, cmd.join(' '), output)
end
end
end
def gitaly_backup_custom_hooks(project) custom_hooks_path = custom_hooks_tar(project)
FileUtils.mkdir_p(path_to_tars(project))
custom_hooks_path = path_to_tars(project, 'custom_hooks')
Gitlab::GitalyClient::RepositoryService.new(project.repository) Gitlab::GitalyClient::RepositoryService.new(project.repository)
.backup_custom_hooks(custom_hooks_path) .backup_custom_hooks(custom_hooks_path)
end end
def backup_custom_hooks(project)
gitaly_migrate(:backup_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
if is_enabled
gitaly_backup_custom_hooks(project)
else
local_backup_custom_hooks(project)
end
end
end
def restore_custom_hooks(project) def restore_custom_hooks(project)
in_path(path_to_tars(project)) do |dir| return unless Dir.exist?(project_backup_path(project))
gitaly_migrate(:restore_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| return if Dir.glob("#{project_backup_path(project)}/custom_hooks*").none?
if is_enabled
gitaly_restore_custom_hooks(project, dir) custom_hooks_path = custom_hooks_tar(project)
else Gitlab::GitalyClient::RepositoryService.new(project.repository)
local_restore_custom_hooks(project, dir) .restore_custom_hooks(custom_hooks_path)
end
end
end
end end
def restore def restore
...@@ -181,7 +83,8 @@ module Backup ...@@ -181,7 +83,8 @@ module Backup
restore_repo_success = nil restore_repo_success = nil
if File.exist?(path_to_project_bundle) if File.exist?(path_to_project_bundle)
begin begin
project.repository.create_from_bundle path_to_project_bundle project.repository.create_from_bundle(path_to_project_bundle)
restore_custom_hooks(project)
restore_repo_success = true restore_repo_success = true
rescue => e rescue => e
restore_repo_success = false restore_repo_success = false
...@@ -197,8 +100,6 @@ module Backup ...@@ -197,8 +100,6 @@ module Backup
progress.puts "[Failed] restoring #{project.full_path} repository".color(:red) progress.puts "[Failed] restoring #{project.full_path} repository".color(:red)
end end
restore_custom_hooks(project)
wiki = ProjectWiki.new(project) wiki = ProjectWiki.new(project)
path_to_wiki_bundle = path_to_bundle(wiki) path_to_wiki_bundle = path_to_bundle(wiki)
...@@ -219,48 +120,28 @@ module Backup ...@@ -219,48 +120,28 @@ module Backup
protected protected
def path_to_repo(project)
project.repository.path_to_repo
end
def path_to_bundle(project) def path_to_bundle(project)
File.join(backup_repos_path, project.disk_path + '.bundle') File.join(backup_repos_path, project.disk_path + '.bundle')
end end
def path_to_tars(project, dir = nil) def project_backup_path(project)
path = File.join(backup_repos_path, project.disk_path) File.join(backup_repos_path, project.disk_path)
end
if dir def custom_hooks_tar(project)
File.join(path, "#{dir}.tar") File.join(project_backup_path(project), "custom_hooks.tar")
else
path
end
end end
def backup_repos_path def backup_repos_path
File.join(Gitlab.config.backup.path, 'repositories') File.join(Gitlab.config.backup.path, 'repositories')
end end
def in_path(path)
return unless Dir.exist?(path)
dir_entries = Dir.entries(path)
if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar')
yield('custom_hooks')
end
end
def prepare def prepare
FileUtils.rm_rf(backup_repos_path) FileUtils.rm_rf(backup_repos_path)
FileUtils.mkdir_p(Gitlab.config.backup.path) FileUtils.mkdir_p(Gitlab.config.backup.path)
FileUtils.mkdir(backup_repos_path, mode: 0700) FileUtils.mkdir(backup_repos_path, mode: 0700)
end end
def silent
{ err: '/dev/null', out: '/dev/null' }
end
private private
def progress_warn(project, cmd, output) def progress_warn(project, cmd, output)
...@@ -273,18 +154,8 @@ module Backup ...@@ -273,18 +154,8 @@ module Backup
project_or_wiki.repository.empty? project_or_wiki.repository.empty?
end end
def repository_storage_paths_args
Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path }
end
def display_repo_path(project) def display_repo_path(project)
project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
end end
def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
Gitlab::GitalyClient.migrate(method, status: status, &block)
rescue GRPC::NotFound, GRPC::BadStatus => e
raise Error, e
end
end end
end end
...@@ -73,37 +73,27 @@ describe Backup::Repository do ...@@ -73,37 +73,27 @@ describe Backup::Repository do
end end
end end
describe '#delete_all_repositories', :seed_helper do describe '#prepare_directories', :seed_helper do
shared_examples('delete_all_repositories') do before do
before do allow(FileUtils).to receive(:mkdir_p).and_call_original
allow(FileUtils).to receive(:mkdir_p).and_call_original allow(FileUtils).to receive(:mv).and_call_original
allow(FileUtils).to receive(:mv).and_call_original end
end
after(:all) do
ensure_seeds
end
it 'removes all repositories' do
# Sanity check: there should be something for us to delete
expect(list_repositories).to include(File.join(SEED_STORAGE_PATH, TEST_REPO_PATH))
subject.delete_all_repositories('default', Gitlab.config.repositories.storages['default']) after(:all) do
ensure_seeds
end
expect(list_repositories).to be_empty it' removes all repositories' do
end # Sanity check: there should be something for us to delete
expect(list_repositories).to include(File.join(SEED_STORAGE_PATH, TEST_REPO_PATH))
def list_repositories subject.prepare_directories
Dir[File.join(SEED_STORAGE_PATH, '*.git')]
end
end
context 'with gitaly' do expect(list_repositories).to be_empty
it_behaves_like 'delete_all_repositories'
end end
context 'without gitaly', :skip_gitaly_mock do def list_repositories
it_behaves_like 'delete_all_repositories' Dir[File.join(SEED_STORAGE_PATH, '*.git')]
end end
end end
......
require 'spec_helper'
describe Gitlab::GitalyClient::StorageService do
describe '#delete_all_repositories' do
let!(:project) { create(:project, :repository) }
it 'removes all repositories' do
described_class.new(project.repository_storage).delete_all_repositories
expect(project.repository.exists?).to be(false)
end
end
end
...@@ -87,6 +87,27 @@ describe 'gitlab:app namespace rake task' do ...@@ -87,6 +87,27 @@ describe 'gitlab:app namespace rake task' do
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
end end
end end
context 'when the restore directory is not empty' do
before do
# We only need a backup of the repositories for this test
stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
end
it 'removes stale data' do
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
excluded_project = create(:project, :repository, name: 'mepmep')
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
raw_repo = excluded_project.repository.raw
# The restore will not find the repository in the backup, but will create
# an empty one in its place
expect(raw_repo.empty?).to be(true)
end
end
end # backup_restore task end # backup_restore task
describe 'backup' do describe 'backup' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment