Commit 2a4d6ef4 authored by Dylan Griffith's avatar Dylan Griffith

Merge branch 'gitaly-backup' into 'master'

Add a new repository backup strategy using gitaly-backup [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!61436
parents 186eade4 5d245b97
---
name: gitaly_backup
introduced_by_url: https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3554
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/333034
milestone: '14.0'
type: development
group: group::gitaly
default_enabled: false
# frozen_string_literal: true
module Backup
# Backup and restores repositories using gitaly-backup
class GitalyBackup
def initialize(progress)
@progress = progress
end
def start(type)
raise Error, 'already started' if started?
command = case type
when :create
'create'
when :restore
'restore'
else
raise Error, "unknown backup type: #{type}"
end
@read_io, @write_io = IO.pipe
@pid = Process.spawn(bin_path, command, '-path', backup_repos_path, in: @read_io, out: progress)
end
def wait
return unless started?
@write_io.close
Process.wait(@pid)
status = $?
@pid = nil
raise Error, "gitaly-backup exit status #{status.exitstatus}" if status.exitstatus != 0
end
def enqueue(container, repo_type)
raise Error, 'not started' unless started?
repository = repo_type.repository_for(container)
@write_io.puts({
storage_name: repository.storage,
relative_path: repository.relative_path,
gl_project_path: repository.gl_project_path,
always_create: repo_type.project?
}.merge(Gitlab::GitalyClient.connection_data(repository.storage)).to_json)
end
private
attr_reader :progress
def started?
@pid.present?
end
def backup_repos_path
File.absolute_path(File.join(Gitlab.config.backup.path, 'repositories'))
end
def bin_path
File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-backup'))
end
end
end
......@@ -4,7 +4,7 @@ require 'yaml'
module Backup
class Repositories
def initialize(progress, strategy: GitalyRpcBackup.new(progress))
def initialize(progress, strategy:)
@progress = progress
@strategy = strategy
end
......
......@@ -109,7 +109,7 @@ namespace :gitlab do
puts "GITLAB_BACKUP_MAX_CONCURRENCY and GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY must have a value of at least 1".color(:red)
exit 1
else
Backup::Repositories.new(progress).dump(
Backup::Repositories.new(progress, strategy: repository_backup_strategy).dump(
max_concurrency: max_concurrency,
max_storage_concurrency: max_storage_concurrency
)
......@@ -119,7 +119,7 @@ namespace :gitlab do
task restore: :gitlab_environment do
puts_time "Restoring repositories ...".color(:blue)
Backup::Repositories.new(progress).restore
Backup::Repositories.new(progress, strategy: repository_backup_strategy).restore
puts_time "done".color(:green)
end
end
......@@ -294,6 +294,14 @@ namespace :gitlab do
$stdout
end
end
def repository_backup_strategy
if Feature.enabled?(:gitaly_backup)
Backup::GitalyBackup.new(progress)
else
Backup::GitalyRpcBackup.new(progress)
end
end
end
# namespace end: backup
end
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Backup::GitalyBackup do
let(:progress) do
Tempfile.new('progress').tap do |progress|
progress.unlink
end
end
after do
progress.close
end
subject { described_class.new(progress) }
context 'unknown' do
it 'fails to start unknown' do
expect { subject.start(:unknown) }.to raise_error(::Backup::Error, 'unknown backup type: unknown')
end
end
context 'create' do
RSpec.shared_examples 'creates a repository backup' do
it 'creates repository bundles', :aggregate_failures do
# Add data to the wiki, design repositories, and snippets, so they will be included in the dump.
create(:wiki_page, container: project)
create(:design, :with_file, issue: create(:issue, project: project))
project_snippet = create(:project_snippet, :repository, project: project)
personal_snippet = create(:personal_snippet, :repository, author: project.owner)
subject.start(:create)
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.enqueue(project, Gitlab::GlRepository::WIKI)
subject.enqueue(project, Gitlab::GlRepository::DESIGN)
subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET)
subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
subject.wait
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.wiki.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.design.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', personal_snippet.disk_path + '.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project_snippet.disk_path + '.bundle'))
end
it 'raises when the exit code not zero' do
expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false'))
subject.start(:create)
expect { subject.wait }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1')
end
end
context 'hashed storage' do
let_it_be(:project) { create(:project, :repository) }
it_behaves_like 'creates a repository backup'
end
context 'legacy storage' do
let_it_be(:project) { create(:project, :repository, :legacy_storage) }
it_behaves_like 'creates a repository backup'
end
end
context 'restore' do
let_it_be(:project) { create(:project, :repository) }
let_it_be(:personal_snippet) { create(:personal_snippet, author: project.owner) }
let_it_be(:project_snippet) { create(:project_snippet, project: project, author: project.owner) }
def copy_bundle_to_backup_path(bundle_name, destination)
FileUtils.mkdir_p(File.join(Gitlab.config.backup.path, 'repositories', File.dirname(destination)))
FileUtils.cp(Rails.root.join('spec/fixtures/lib/backup', bundle_name), File.join(Gitlab.config.backup.path, 'repositories', destination))
end
it 'restores from repository bundles', :aggregate_failures do
copy_bundle_to_backup_path('project_repo.bundle', project.disk_path + '.bundle')
copy_bundle_to_backup_path('wiki_repo.bundle', project.disk_path + '.wiki.bundle')
copy_bundle_to_backup_path('design_repo.bundle', project.disk_path + '.design.bundle')
copy_bundle_to_backup_path('personal_snippet_repo.bundle', personal_snippet.disk_path + '.bundle')
copy_bundle_to_backup_path('project_snippet_repo.bundle', project_snippet.disk_path + '.bundle')
subject.start(:restore)
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.enqueue(project, Gitlab::GlRepository::WIKI)
subject.enqueue(project, Gitlab::GlRepository::DESIGN)
subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET)
subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
subject.wait
collect_commit_shas = -> (repo) { repo.commits('master', limit: 10).map(&:sha) }
expect(collect_commit_shas.call(project.repository)).to eq(['393a7d860a5a4c3cc736d7eb00604e3472bb95ec'])
expect(collect_commit_shas.call(project.wiki.repository)).to eq(['c74b9948d0088d703ee1fafeddd9ed9add2901ea'])
expect(collect_commit_shas.call(project.design_repository)).to eq(['c3cd4d7bd73a51a0f22045c3a4c871c435dc959d'])
expect(collect_commit_shas.call(personal_snippet.repository)).to eq(['3b3c067a3bc1d1b695b51e2be30c0f8cf698a06e'])
expect(collect_commit_shas.call(project_snippet.repository)).to eq(['6e44ba56a4748be361a841e759c20e421a1651a1'])
end
it 'raises when the exit code not zero' do
expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false'))
subject.start(:restore)
expect { subject.wait }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1')
end
end
end
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment