Download a folder from repository

Add `GetArchiveRequest` to git-archive params.

Modifies `Git::Repository#archive_metadata` to append `path`
to `ArchivePrefix` so it'll not hit the cache of repository archive
when it already exists.

GITALY_SERVER_VERSION

-1.30.0
+=24704-get-archive-by-path

GITLAB_WORKHORSE_VERSION

-8.3.3
+=24704-git-archive-by-path

Gemfile

@@ -419,7 +419,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly-proto', '~> 1.13.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 1.19.0', require: 'gitaly'
 
 gem 'grpc', '~> 1.15.0'
 

Gemfile.lock

@@ -281,7 +281,7 @@ GEM
       gettext_i18n_rails (>= 0.7.1)
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
-    gitaly-proto (1.13.0)
+    gitaly-proto (1.19.0)
       grpc (~> 1.0)
     github-markup (1.7.0)
     gitlab-default_value_for (3.1.1)
@@ -1017,7 +1017,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.3)
-  gitaly-proto (~> 1.13.0)
+  gitaly-proto (~> 1.19.0)
   github-markup (~> 1.7.0)
   gitlab-default_value_for (~> 3.1.1)
   gitlab-markup (~> 1.7.0)

app/controllers/projects/repositories_controller.rb

@@ -23,7 +23,7 @@ class Projects::RepositoriesController < Projects::ApplicationController
       append_sha = false if @filename == shortname
     end
 
-    send_git_archive @repository, ref: @ref, subdirectory: params[:subdirectory], format: params[:format], append_sha: append_sha
+    send_git_archive @repository, ref: @ref, path: params[:path], format: params[:format], append_sha: append_sha
   rescue => ex
     logger.error("#{self.class.name}: #{ex}")
     git_not_found!

app/helpers/projects_helper.rb

@@ -27,14 +27,6 @@ module ProjectsHelper
     image_tag(src, width: opts[:size], class: classes, alt: '', "data-src" => avatar)
   end
 
-  def is_directory
-    @path.empty? ? false : true
-  end
-
-  def get_directory_path
-    @path ? "#{@path}/" : ''
-  end
-
   def author_content_tag(author, opts = {})
     default_opts = { author_class: 'author', tooltip: false, by_username: false }
     opts = default_opts.merge(opts)
@@ -307,6 +299,10 @@ module ProjectsHelper
     }.to_json
   end
 
+  def directory?
+    @path.present?
+  end
+
   private
 
   def get_project_nav_tabs(project, current_user)

app/helpers/workhorse_helper.rb

@@ -31,7 +31,6 @@ module WorkhorseHelper
 
   # Archive a Git repository and send it through Workhorse
   def send_git_archive(repository, **kwargs)
-    kwargs.delete(:subdirectory) if kwargs[:subdirectory].nil?
     headers.store(*Gitlab::Workhorse.send_git_archive(repository, **kwargs))
     head :ok
   end

app/models/repository.rb

@@ -299,13 +299,14 @@ class Repository
     end
   end
 
-  def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:)
+  def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:, path: nil)
     raw_repository.archive_metadata(
       ref,
       storage_path,
       project.path,
       format,
-      append_sha: append_sha
+      append_sha: append_sha,
+      path: path
     )
   end
 

app/views/projects/buttons/_download.html.haml

@@ -8,14 +8,23 @@
       %span.sr-only=  _('Select Archive Format')
       = sprite_icon("arrow-down")
     %ul.dropdown-menu.dropdown-menu-right{ role: 'menu' }
-      - if is_directory
+      - if directory?
         %li.dropdown-header
           #{ _('Directory') }
         %li
-          = link_to project_archive_path(project, subdirectory: get_directory_path, id: tree_join(ref, archive_prefix), format: 'zip'), rel: 'nofollow', download: '' do
+          = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), path: @path, format: 'zip'), rel: 'nofollow', download: '' do
             %span=  _('Download zip')
+        %li
+          = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), path: @path, format: 'tar.gz'), rel: 'nofollow', download: '' do
+            %span=  _('Download tar.gz')
+        %li
+          = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), path: @path, format: 'tar.bz2'), rel: 'nofollow', download: '' do
+            %span=  _('Download tar.bz2')
+        %li
+          = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), path: @path, format: 'tar'), rel: 'nofollow', download: '' do
+            %span=  _('Download tar')
       %li.dropdown-header
-        #{ _('Repository') }
+        #{ _('Source code') }
       %li
         = link_to project_archive_path(project, id: tree_join(ref, archive_prefix), format: 'zip'), rel: 'nofollow', download: '' do
           %span=  _('Download zip')

changelogs/unreleased/24704-download-repository-path.yml

+---
+title: Download a folder from repository
+merge_request: 26532
+author: kiameisomabes
+type: added

lib/gitlab/git/repository.rb

@@ -231,12 +231,12 @@ module Gitlab
         end
       end
 
-      def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:)
+      def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:, path: nil)
         ref ||= root_ref
         commit = Gitlab::Git::Commit.find(self, ref)
         return {} if commit.nil?
 
-        prefix = archive_prefix(ref, commit.id, project_path, append_sha: append_sha)
+        prefix = archive_prefix(ref, commit.id, project_path, append_sha: append_sha, path: path)
 
         {
           'ArchivePrefix' => prefix,
@@ -248,13 +248,14 @@ module Gitlab
 
       # This is both the filename of the archive (missing the extension) and the
       # name of the top-level member of the archive under which all files go
-      def archive_prefix(ref, sha, project_path, append_sha:)
+      def archive_prefix(ref, sha, project_path, append_sha:, path:)
         append_sha = (ref != sha) if append_sha.nil?
 
         formatted_ref = ref.tr('/', '-')
 
         prefix_segments = [project_path, formatted_ref]
         prefix_segments << sha if append_sha
+        prefix_segments << path.tr('/', '-').gsub(%r{^/|/$}, '') if path
 
         prefix_segments.join('-')
       end

lib/gitlab/workhorse.rb

@@ -63,13 +63,26 @@ module Gitlab
         ]
       end
 
-      def send_git_archive(repository, ref:, format:, append_sha:)
+      def send_git_archive(repository, ref:, format:, append_sha:, path: nil)
         format ||= 'tar.gz'
         format = format.downcase
-        params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha)
-        raise "Repository or ref not found" if params.empty?
+        metadata = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha, path: path)
 
-        params['GitalyServer'] = gitaly_server_hash(repository)
+        raise "Repository or ref not found" if metadata.empty?
+
+        params = {
+          'GitalyServer' => gitaly_server_hash(repository),
+          'ArchivePath' => metadata['ArchivePath'],
+          'GetArchiveRequest' => encode_binary(
+            Gitaly::GetArchiveRequest.new(
+              repository: repository.gitaly_repository,
+              commit_id: metadata['CommitId'],
+              prefix: metadata['ArchivePrefix'],
+              format: archive_format(format),
+              path: path.presence || ""
+            ).to_proto
+          )
+        }
 
         # If present DisableCache must be a Boolean. Otherwise workhorse ignores it.
         params['DisableCache'] = true if git_archive_cache_disabled?
@@ -220,6 +233,10 @@ module Gitlab
         Base64.urlsafe_encode64(JSON.dump(hash))
       end
 
+      def encode_binary(binary)
+        Base64.urlsafe_encode64(binary)
+      end
+
       def gitaly_server_hash(repository)
         {
           address: Gitlab::GitalyClient.address(repository.project.repository_storage),
@@ -238,6 +255,19 @@ module Gitlab
       def git_archive_cache_disabled?
         ENV['WORKHORSE_ARCHIVE_CACHE_DISABLED'].present? || Feature.enabled?(:workhorse_archive_cache_disabled)
       end
+
+      def archive_format(format)
+        case format
+        when "tar.bz2", "tbz", "tbz2", "tb2", "bz2"
+          Gitaly::GetArchiveRequest::Format::TAR_BZ2
+        when "tar"
+          Gitaly::GetArchiveRequest::Format::TAR
+        when "zip"
+          Gitaly::GetArchiveRequest::Format::ZIP
+        else
+          Gitaly::GetArchiveRequest::Format::TAR_GZ
+        end
+      end
     end
   end
 end

spec/lib/gitlab/git/repository_spec.rb

@@ -152,13 +152,14 @@ describe Gitlab::Git::Repository, :seed_helper do
     let(:append_sha) { true }
     let(:ref) { 'master' }
     let(:format) { nil }
+    let(:path) { nil }
 
     let(:expected_extension) { 'tar.gz' }
     let(:expected_filename) { "#{expected_prefix}.#{expected_extension}" }
     let(:expected_path) { File.join(storage_path, cache_key, expected_filename) }
     let(:expected_prefix) { "gitlab-git-test-#{ref}-#{SeedRepo::LastCommit::ID}" }
 
-    subject(:metadata) { repository.archive_metadata(ref, storage_path, 'gitlab-git-test', format, append_sha: append_sha) }
+    subject(:metadata) { repository.archive_metadata(ref, storage_path, 'gitlab-git-test', format, append_sha: append_sha, path: path) }
 
     it 'sets CommitId to the commit SHA' do
       expect(metadata['CommitId']).to eq(SeedRepo::LastCommit::ID)
@@ -176,6 +177,14 @@ describe Gitlab::Git::Repository, :seed_helper do
       expect(metadata['ArchivePath']).to eq(expected_path)
     end
 
+    context 'path is set' do
+      let(:path) { 'foo/bar' }
+
+      it 'appends the path to the prefix' do
+        expect(metadata['ArchivePrefix']).to eq("#{expected_prefix}-foo-bar")
+      end
+    end
+
     context 'append_sha varies archive path and filename' do
       where(:append_sha, :ref, :expected_prefix) do
         sha = SeedRepo::LastCommit::ID

spec/lib/gitlab/workhorse_spec.rb

@@ -16,20 +16,12 @@ describe Gitlab::Workhorse do
     let(:ref) { 'master' }
     let(:format) { 'zip' }
     let(:storage_path) { Gitlab.config.gitlab.repository_downloads_path }
-    let(:base_params) { repository.archive_metadata(ref, storage_path, format, append_sha: nil) }
-    let(:gitaly_params) do
-      base_params.merge(
-        'GitalyServer' => {
-          'address' => Gitlab::GitalyClient.address(project.repository_storage),
-          'token' => Gitlab::GitalyClient.token(project.repository_storage)
-        },
-        'GitalyRepository' => repository.gitaly_repository.to_h.deep_stringify_keys
-      )
-    end
+    let(:path) { 'some/path' }
+    let(:metadata) { repository.archive_metadata(ref, storage_path, format, append_sha: nil, path: path) }
     let(:cache_disabled) { false }
 
     subject do
-      described_class.send_git_archive(repository, ref: ref, format: format, append_sha: nil)
+      described_class.send_git_archive(repository, ref: ref, format: format, append_sha: nil, path: path)
     end
 
     before do
@@ -41,7 +33,22 @@ describe Gitlab::Workhorse do
 
       expect(key).to eq('Gitlab-Workhorse-Send-Data')
       expect(command).to eq('git-archive')
-      expect(params).to include(gitaly_params)
+      expect(params).to eq({
+        'GitalyServer' => {
+          address: Gitlab::GitalyClient.address(project.repository_storage),
+          token: Gitlab::GitalyClient.token(project.repository_storage)
+        },
+        'ArchivePath' => metadata['ArchivePath'],
+        'GetArchiveRequest' => Base64.urlsafe_encode64(
+          Gitaly::GetArchiveRequest.new(
+            repository: repository.gitaly_repository,
+            commit_id: metadata['CommitId'],
+            prefix: metadata['ArchivePrefix'],
+            format: Gitaly::GetArchiveRequest::Format::ZIP,
+            path: path
+          ).to_proto
+        )
+      }.deep_stringify_keys)
     end
 
     context 'when archive caching is disabled' do