Commit 6766a0a1 authored by Patrick Bajao's avatar Patrick Bajao

Download a folder from repository

Add `GetArchiveRequest` to git-archive params.

Modifies `Git::Repository#archive_metadata` to append `path`
to `ArchivePrefix` so it'll not hit the cache of repository archive
when it already exists.
parent e028276d
1.30.0 =24704-get-archive-by-path
8.3.3 =24704-git-archive-by-path
...@@ -419,7 +419,7 @@ group :ed25519 do ...@@ -419,7 +419,7 @@ group :ed25519 do
end end
# Gitaly GRPC client # Gitaly GRPC client
gem 'gitaly-proto', '~> 1.13.0', require: 'gitaly' gem 'gitaly-proto', '~> 1.19.0', require: 'gitaly'
gem 'grpc', '~> 1.15.0' gem 'grpc', '~> 1.15.0'
......
...@@ -281,7 +281,7 @@ GEM ...@@ -281,7 +281,7 @@ GEM
gettext_i18n_rails (>= 0.7.1) gettext_i18n_rails (>= 0.7.1)
po_to_json (>= 1.0.0) po_to_json (>= 1.0.0)
rails (>= 3.2.0) rails (>= 3.2.0)
gitaly-proto (1.13.0) gitaly-proto (1.19.0)
grpc (~> 1.0) grpc (~> 1.0)
github-markup (1.7.0) github-markup (1.7.0)
gitlab-default_value_for (3.1.1) gitlab-default_value_for (3.1.1)
...@@ -1017,7 +1017,7 @@ DEPENDENCIES ...@@ -1017,7 +1017,7 @@ DEPENDENCIES
gettext (~> 3.2.2) gettext (~> 3.2.2)
gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails (~> 1.8.0)
gettext_i18n_rails_js (~> 1.3) gettext_i18n_rails_js (~> 1.3)
gitaly-proto (~> 1.13.0) gitaly-proto (~> 1.19.0)
github-markup (~> 1.7.0) github-markup (~> 1.7.0)
gitlab-default_value_for (~> 3.1.1) gitlab-default_value_for (~> 3.1.1)
gitlab-markup (~> 1.7.0) gitlab-markup (~> 1.7.0)
......
...@@ -23,7 +23,7 @@ class Projects::RepositoriesController < Projects::ApplicationController ...@@ -23,7 +23,7 @@ class Projects::RepositoriesController < Projects::ApplicationController
append_sha = false if @filename == shortname append_sha = false if @filename == shortname
end end
send_git_archive @repository, ref: @ref, subdirectory: params[:subdirectory], format: params[:format], append_sha: append_sha send_git_archive @repository, ref: @ref, path: params[:path], format: params[:format], append_sha: append_sha
rescue => ex rescue => ex
logger.error("#{self.class.name}: #{ex}") logger.error("#{self.class.name}: #{ex}")
git_not_found! git_not_found!
......
...@@ -26,14 +26,6 @@ module ProjectsHelper ...@@ -26,14 +26,6 @@ module ProjectsHelper
image_tag(src, width: opts[:size], class: classes, alt: '', "data-src" => avatar) image_tag(src, width: opts[:size], class: classes, alt: '', "data-src" => avatar)
end end
def is_directory
@path.empty? ? false : true
end
def get_directory_path
@path ? "#{@path}/" : ''
end
def author_content_tag(author, opts = {}) def author_content_tag(author, opts = {})
default_opts = { author_class: 'author', tooltip: false, by_username: false } default_opts = { author_class: 'author', tooltip: false, by_username: false }
...@@ -307,6 +299,10 @@ module ProjectsHelper ...@@ -307,6 +299,10 @@ module ProjectsHelper
}.to_json }.to_json
end end
def directory?
@path.present?
end
private private
def get_project_nav_tabs(project, current_user) def get_project_nav_tabs(project, current_user)
......
...@@ -31,7 +31,6 @@ module WorkhorseHelper ...@@ -31,7 +31,6 @@ module WorkhorseHelper
# Archive a Git repository and send it through Workhorse # Archive a Git repository and send it through Workhorse
def send_git_archive(repository, **kwargs) def send_git_archive(repository, **kwargs)
kwargs.delete(:subdirectory) if kwargs[:subdirectory].nil?
headers.store(*Gitlab::Workhorse.send_git_archive(repository, **kwargs)) headers.store(*Gitlab::Workhorse.send_git_archive(repository, **kwargs))
head :ok head :ok
end end
......
...@@ -299,13 +299,14 @@ class Repository ...@@ -299,13 +299,14 @@ class Repository
end end
end end
def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:) def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:, path: nil)
raw_repository.archive_metadata( raw_repository.archive_metadata(
ref, ref,
storage_path, storage_path,
project.path, project.path,
format, format,
append_sha: append_sha append_sha: append_sha,
path: path
) )
end end
......
...@@ -8,14 +8,23 @@ ...@@ -8,14 +8,23 @@
%span.sr-only= _('Select Archive Format') %span.sr-only= _('Select Archive Format')
= sprite_icon("arrow-down") = sprite_icon("arrow-down")
%ul.dropdown-menu.dropdown-menu-right{ role: 'menu' } %ul.dropdown-menu.dropdown-menu-right{ role: 'menu' }
- if is_directory - if directory?
%li.dropdown-header %li.dropdown-header
#{ _('Directory') } #{ _('Directory') }
%li %li
= link_to project_archive_path(project, subdirectory: get_directory_path, id: tree_join(ref, archive_prefix), format: 'zip'), rel: 'nofollow', download: '' do = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), path: @path, format: 'zip'), rel: 'nofollow', download: '' do
%span= _('Download zip') %span= _('Download zip')
%li
= link_to project_archive_path(project, id: tree_join(ref, archive_prefix), path: @path, format: 'tar.gz'), rel: 'nofollow', download: '' do
%span= _('Download tar.gz')
%li
= link_to project_archive_path(project, id: tree_join(ref, archive_prefix), path: @path, format: 'tar.bz2'), rel: 'nofollow', download: '' do
%span= _('Download tar.bz2')
%li
= link_to project_archive_path(project, id: tree_join(ref, archive_prefix), path: @path, format: 'tar'), rel: 'nofollow', download: '' do
%span= _('Download tar')
%li.dropdown-header %li.dropdown-header
#{ _('Repository') } #{ _('Source code') }
%li %li
= link_to project_archive_path(project, id: tree_join(ref, archive_prefix), format: 'zip'), rel: 'nofollow', download: '' do = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), format: 'zip'), rel: 'nofollow', download: '' do
%span= _('Download zip') %span= _('Download zip')
......
---
title: Download a folder from repository
merge_request: 26532
author: kiameisomabes
type: added
...@@ -231,12 +231,12 @@ module Gitlab ...@@ -231,12 +231,12 @@ module Gitlab
end end
end end
def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:) def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:, path: nil)
ref ||= root_ref ref ||= root_ref
commit = Gitlab::Git::Commit.find(self, ref) commit = Gitlab::Git::Commit.find(self, ref)
return {} if commit.nil? return {} if commit.nil?
prefix = archive_prefix(ref, commit.id, project_path, append_sha: append_sha) prefix = archive_prefix(ref, commit.id, project_path, append_sha: append_sha, path: path)
{ {
'ArchivePrefix' => prefix, 'ArchivePrefix' => prefix,
...@@ -248,13 +248,14 @@ module Gitlab ...@@ -248,13 +248,14 @@ module Gitlab
# This is both the filename of the archive (missing the extension) and the # This is both the filename of the archive (missing the extension) and the
# name of the top-level member of the archive under which all files go # name of the top-level member of the archive under which all files go
def archive_prefix(ref, sha, project_path, append_sha:) def archive_prefix(ref, sha, project_path, append_sha:, path:)
append_sha = (ref != sha) if append_sha.nil? append_sha = (ref != sha) if append_sha.nil?
formatted_ref = ref.tr('/', '-') formatted_ref = ref.tr('/', '-')
prefix_segments = [project_path, formatted_ref] prefix_segments = [project_path, formatted_ref]
prefix_segments << sha if append_sha prefix_segments << sha if append_sha
prefix_segments << path.tr('/', '-').gsub(%r{^/|/$}, '') if path
prefix_segments.join('-') prefix_segments.join('-')
end end
......
...@@ -63,13 +63,26 @@ module Gitlab ...@@ -63,13 +63,26 @@ module Gitlab
] ]
end end
def send_git_archive(repository, ref:, format:, append_sha:) def send_git_archive(repository, ref:, format:, append_sha:, path: nil)
format ||= 'tar.gz' format ||= 'tar.gz'
format = format.downcase format = format.downcase
params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha) metadata = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha, path: path)
raise "Repository or ref not found" if params.empty?
params['GitalyServer'] = gitaly_server_hash(repository) raise "Repository or ref not found" if metadata.empty?
params = {
'GitalyServer' => gitaly_server_hash(repository),
'ArchivePath' => metadata['ArchivePath'],
'GetArchiveRequest' => encode_binary(
Gitaly::GetArchiveRequest.new(
repository: repository.gitaly_repository,
commit_id: metadata['CommitId'],
prefix: metadata['ArchivePrefix'],
format: archive_format(format),
path: path.presence || ""
).to_proto
)
}
# If present DisableCache must be a Boolean. Otherwise workhorse ignores it. # If present DisableCache must be a Boolean. Otherwise workhorse ignores it.
params['DisableCache'] = true if git_archive_cache_disabled? params['DisableCache'] = true if git_archive_cache_disabled?
...@@ -220,6 +233,10 @@ module Gitlab ...@@ -220,6 +233,10 @@ module Gitlab
Base64.urlsafe_encode64(JSON.dump(hash)) Base64.urlsafe_encode64(JSON.dump(hash))
end end
def encode_binary(binary)
Base64.urlsafe_encode64(binary)
end
def gitaly_server_hash(repository) def gitaly_server_hash(repository)
{ {
address: Gitlab::GitalyClient.address(repository.project.repository_storage), address: Gitlab::GitalyClient.address(repository.project.repository_storage),
...@@ -238,6 +255,19 @@ module Gitlab ...@@ -238,6 +255,19 @@ module Gitlab
def git_archive_cache_disabled? def git_archive_cache_disabled?
ENV['WORKHORSE_ARCHIVE_CACHE_DISABLED'].present? || Feature.enabled?(:workhorse_archive_cache_disabled) ENV['WORKHORSE_ARCHIVE_CACHE_DISABLED'].present? || Feature.enabled?(:workhorse_archive_cache_disabled)
end end
def archive_format(format)
case format
when "tar.bz2", "tbz", "tbz2", "tb2", "bz2"
Gitaly::GetArchiveRequest::Format::TAR_BZ2
when "tar"
Gitaly::GetArchiveRequest::Format::TAR
when "zip"
Gitaly::GetArchiveRequest::Format::ZIP
else
Gitaly::GetArchiveRequest::Format::TAR_GZ
end
end
end end
end end
end end
...@@ -152,13 +152,14 @@ describe Gitlab::Git::Repository, :seed_helper do ...@@ -152,13 +152,14 @@ describe Gitlab::Git::Repository, :seed_helper do
let(:append_sha) { true } let(:append_sha) { true }
let(:ref) { 'master' } let(:ref) { 'master' }
let(:format) { nil } let(:format) { nil }
let(:path) { nil }
let(:expected_extension) { 'tar.gz' } let(:expected_extension) { 'tar.gz' }
let(:expected_filename) { "#{expected_prefix}.#{expected_extension}" } let(:expected_filename) { "#{expected_prefix}.#{expected_extension}" }
let(:expected_path) { File.join(storage_path, cache_key, expected_filename) } let(:expected_path) { File.join(storage_path, cache_key, expected_filename) }
let(:expected_prefix) { "gitlab-git-test-#{ref}-#{SeedRepo::LastCommit::ID}" } let(:expected_prefix) { "gitlab-git-test-#{ref}-#{SeedRepo::LastCommit::ID}" }
subject(:metadata) { repository.archive_metadata(ref, storage_path, 'gitlab-git-test', format, append_sha: append_sha) } subject(:metadata) { repository.archive_metadata(ref, storage_path, 'gitlab-git-test', format, append_sha: append_sha, path: path) }
it 'sets CommitId to the commit SHA' do it 'sets CommitId to the commit SHA' do
expect(metadata['CommitId']).to eq(SeedRepo::LastCommit::ID) expect(metadata['CommitId']).to eq(SeedRepo::LastCommit::ID)
...@@ -176,6 +177,14 @@ describe Gitlab::Git::Repository, :seed_helper do ...@@ -176,6 +177,14 @@ describe Gitlab::Git::Repository, :seed_helper do
expect(metadata['ArchivePath']).to eq(expected_path) expect(metadata['ArchivePath']).to eq(expected_path)
end end
context 'path is set' do
let(:path) { 'foo/bar' }
it 'appends the path to the prefix' do
expect(metadata['ArchivePrefix']).to eq("#{expected_prefix}-foo-bar")
end
end
context 'append_sha varies archive path and filename' do context 'append_sha varies archive path and filename' do
where(:append_sha, :ref, :expected_prefix) do where(:append_sha, :ref, :expected_prefix) do
sha = SeedRepo::LastCommit::ID sha = SeedRepo::LastCommit::ID
......
...@@ -16,20 +16,12 @@ describe Gitlab::Workhorse do ...@@ -16,20 +16,12 @@ describe Gitlab::Workhorse do
let(:ref) { 'master' } let(:ref) { 'master' }
let(:format) { 'zip' } let(:format) { 'zip' }
let(:storage_path) { Gitlab.config.gitlab.repository_downloads_path } let(:storage_path) { Gitlab.config.gitlab.repository_downloads_path }
let(:base_params) { repository.archive_metadata(ref, storage_path, format, append_sha: nil) } let(:path) { 'some/path' }
let(:gitaly_params) do let(:metadata) { repository.archive_metadata(ref, storage_path, format, append_sha: nil, path: path) }
base_params.merge(
'GitalyServer' => {
'address' => Gitlab::GitalyClient.address(project.repository_storage),
'token' => Gitlab::GitalyClient.token(project.repository_storage)
},
'GitalyRepository' => repository.gitaly_repository.to_h.deep_stringify_keys
)
end
let(:cache_disabled) { false } let(:cache_disabled) { false }
subject do subject do
described_class.send_git_archive(repository, ref: ref, format: format, append_sha: nil) described_class.send_git_archive(repository, ref: ref, format: format, append_sha: nil, path: path)
end end
before do before do
...@@ -41,7 +33,22 @@ describe Gitlab::Workhorse do ...@@ -41,7 +33,22 @@ describe Gitlab::Workhorse do
expect(key).to eq('Gitlab-Workhorse-Send-Data') expect(key).to eq('Gitlab-Workhorse-Send-Data')
expect(command).to eq('git-archive') expect(command).to eq('git-archive')
expect(params).to include(gitaly_params) expect(params).to eq({
'GitalyServer' => {
address: Gitlab::GitalyClient.address(project.repository_storage),
token: Gitlab::GitalyClient.token(project.repository_storage)
},
'ArchivePath' => metadata['ArchivePath'],
'GetArchiveRequest' => Base64.urlsafe_encode64(
Gitaly::GetArchiveRequest.new(
repository: repository.gitaly_repository,
commit_id: metadata['CommitId'],
prefix: metadata['ArchivePrefix'],
format: Gitaly::GetArchiveRequest::Format::ZIP,
path: path
).to_proto
)
}.deep_stringify_keys)
end end
context 'when archive caching is disabled' do context 'when archive caching is disabled' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment