Commit 151d87e4 authored by Stan Hu's avatar Stan Hu

Merge branch 'da-gitaly-calculate-repository-checksum' into 'master'

Repository checksum calculation is handled by Gitaly when feature is enabled

See merge request gitlab-org/gitlab-ce!18210
parents fe96152d 76cd2d78
---
title: Repository checksum calculation is handled by Gitaly when feature is enabled
merge_request:
author:
type: changed
module Gitlab
module Git
class Checksum
include Gitlab::Git::Popen
EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze
Failure = Class.new(StandardError)
attr_reader :path, :relative_path, :storage, :storage_path
def initialize(storage, relative_path)
@storage = storage
@storage_path = Gitlab.config.repositories.storages[storage].legacy_disk_path
@relative_path = "#{relative_path}.git"
@path = File.join(storage_path, @relative_path)
end
def calculate
unless repository_exists?
failure!(Gitlab::Git::Repository::NoRepository, 'No repository for such path')
end
calculate_checksum_by_shelling_out
end
private
def repository_exists?
raw_repository.exists?
end
def calculate_checksum_by_shelling_out
args = %W(--git-dir=#{path} show-ref --heads --tags)
output, status = run_git(args)
if status&.zero?
refs = output.split("\n")
result = refs.inject(nil) do |checksum, ref|
value = Digest::SHA1.hexdigest(ref).hex
if checksum.nil?
value
else
checksum ^ value
end
end
result.to_s(16)
else
# Empty repositories return with a non-zero status and an empty output.
if output&.empty?
EMPTY_REPOSITORY_CHECKSUM
else
failure!(Gitlab::Git::Checksum::Failure, output)
end
end
end
def failure!(klass, message)
Gitlab::GitLogger.error("'git show-ref --heads --tags' in #{path}: #{message}")
raise klass.new("Could not calculate the checksum for #{path}: #{message}")
end
def circuit_breaker
@circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage)
end
def raw_repository
Gitlab::Git::Repository.new(storage, relative_path, nil)
end
def run_git(args)
circuit_breaker.perform do
popen([Gitlab.config.git.bin_path, *args], path)
end
end
end
end
end
......@@ -23,6 +23,7 @@ module Gitlab
SQUASH_WORKTREE_PREFIX = 'squash'.freeze
GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze
GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout
EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze
NoRepository = Class.new(StandardError)
InvalidBlobName = Class.new(StandardError)
......@@ -31,6 +32,7 @@ module Gitlab
DeleteBranchError = Class.new(StandardError)
CreateTreeError = Class.new(StandardError)
TagExistsError = Class.new(StandardError)
ChecksumError = Class.new(StandardError)
class << self
# Unlike `new`, `create` takes the repository path
......@@ -1502,6 +1504,16 @@ module Gitlab
FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path)
end
def checksum
gitaly_migrate(:calculate_checksum) do |is_enabled|
if is_enabled
gitaly_repository_client.calculate_checksum
else
calculate_checksum_by_shelling_out
end
end
end
private
def local_write_ref(ref_path, ref, old_ref: nil, shell: true)
......@@ -2420,6 +2432,34 @@ module Gitlab
def sha_from_ref(ref)
rev_parse_target(ref).oid
end
def calculate_checksum_by_shelling_out
raise NoRepository unless exists?
args = %W(--git-dir=#{path} show-ref --heads --tags)
output, status = run_git(args)
if status.nil? || !status.zero?
# Empty repositories return with a non-zero status and an empty output.
return EMPTY_REPOSITORY_CHECKSUM if output&.empty?
raise ChecksumError, output
end
refs = output.split("\n")
result = refs.inject(nil) do |checksum, ref|
value = Digest::SHA1.hexdigest(ref).hex
if checksum.nil?
value
else
checksum ^ value
end
end
result.to_s(16)
end
end
end
end
......@@ -262,6 +262,12 @@ module Gitlab
response.license_short_name.presence
end
def calculate_checksum
request = Gitaly::CalculateChecksumRequest.new(repository: @gitaly_repo)
response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request)
response.checksum.presence
end
end
end
end
require 'spec_helper'
describe Gitlab::Git::Checksum, seed_helper: true do
let(:storage) { 'default' }
it 'raises Gitlab::Git::Repository::NoRepository when there is no repo' do
checksum = described_class.new(storage, 'nonexistent-repo')
expect { checksum.calculate }.to raise_error Gitlab::Git::Repository::NoRepository
end
it 'pretends that checksum is 000000... when the repo is empty' do
FileUtils.rm_rf(File.join(SEED_STORAGE_PATH, 'empty-repo.git'))
system(git_env, *%W(#{Gitlab.config.git.bin_path} init --bare empty-repo.git),
chdir: SEED_STORAGE_PATH,
out: '/dev/null',
err: '/dev/null')
checksum = described_class.new(storage, 'empty-repo')
expect(checksum.calculate).to eq '0000000000000000000000000000000000000000'
end
it 'raises Gitlab::Git::Repository::Failure when shelling out to git return non-zero status' do
checksum = described_class.new(storage, 'gitlab-git-test')
allow(checksum).to receive(:popen).and_return(['output', nil])
expect { checksum.calculate }.to raise_error Gitlab::Git::Checksum::Failure
end
it 'calculates the checksum when there is a repo' do
checksum = described_class.new(storage, 'gitlab-git-test')
expect(checksum.calculate).to eq '54f21be4c32c02f6788d72207fa03ad3bce725e4'
end
end
......@@ -2184,6 +2184,55 @@ describe Gitlab::Git::Repository, seed_helper: true do
end
end
describe '#checksum' do
shared_examples 'calculating checksum' do
it 'calculates the checksum for non-empty repo' do
expect(repository.checksum).to eq '54f21be4c32c02f6788d72207fa03ad3bce725e4'
end
it 'returns 0000000000000000000000000000000000000000 for an empty repo' do
FileUtils.rm_rf(File.join(storage_path, 'empty-repo.git'))
system(git_env, *%W(#{Gitlab.config.git.bin_path} init --bare empty-repo.git),
chdir: storage_path,
out: '/dev/null',
err: '/dev/null')
empty_repo = described_class.new('default', 'empty-repo.git', '')
expect(empty_repo.checksum).to eq '0000000000000000000000000000000000000000'
end
it 'raises a no repository exception when there is no repo' do
broken_repo = described_class.new('default', 'a/path.git', '')
expect { broken_repo.checksum }.to raise_error(Gitlab::Git::Repository::NoRepository)
end
end
context 'when calculate_checksum Gitaly feature is enabled' do
it_behaves_like 'calculating checksum'
end
context 'when calculate_checksum Gitaly feature is disabled', :disable_gitaly do
it_behaves_like 'calculating checksum'
describe 'when storage is broken', :broken_storage do
it 'raises a storage exception when storage is not available' do
broken_repo = described_class.new('broken', 'a/path.git', '')
expect { broken_repo.rugged }.to raise_error(Gitlab::Git::Storage::Inaccessible)
end
end
it "raises a Gitlab::Git::Repository::Failure error if the `popen` call to git returns a non-zero exit code" do
allow(repository).to receive(:popen).and_return(['output', nil])
expect { repository.checksum }.to raise_error Gitlab::Git::Repository::ChecksumError
end
end
end
context 'gitlab_projects commands' do
let(:gitlab_projects) { repository.gitlab_projects }
let(:timeout) { Gitlab.config.gitlab_shell.git_timeout }
......
......@@ -134,4 +134,15 @@ describe Gitlab::GitalyClient::RepositoryService do
client.squash_in_progress?(squash_id)
end
end
describe '#calculate_checksum' do
it 'sends a calculate_checksum message' do
expect_any_instance_of(Gitaly::RepositoryService::Stub)
.to receive(:calculate_checksum)
.with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash))
.and_return(double(checksum: 0))
client.calculate_checksum
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment