Merge branch '39345-get-raw-archive' into 'master'

Add an API endpoint to download git repository snapshots

Closes #39345

See merge request gitlab-org/gitlab-ce!18173

GITALY_SERVER_VERSION

-0.95.0
+0.96.1

changelogs/unreleased/39345-get-raw-archive.yml

+---
+title: Add an API endpoint to download git repository snapshots
+merge_request: 18173
+author:
+type: added

doc/api/projects.md

@@ -1399,3 +1399,26 @@ Read more in the [Project Badges](project_badges.md) documentation.
 ## Issue and merge request description templates
 
 The non-default [issue and merge request description templates](../user/project/description_templates.md) are managed inside the project's repository. So you can manage them via the API through the [Repositories API](repositories.md) and the [Repository Files API](repository_files.md).
+
+## Download snapshot of a git repository
+
+> Introduced in GitLab 10.7
+
+This endpoint may only be accessed by an administrative user.
+
+Download a snapshot of the project (or wiki, if requested) git repository. This
+snapshot is always in uncompressed [tar](https://en.wikipedia.org/wiki/Tar_(computing))
+format.
+
+If a repository is corrupted to the point where `git clone` does not work, the
+snapshot may allow some of the data to be retrieved.
+
+```
+GET /projects/:id/snapshot
+```
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `id`      | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) |
+| `wiki`    | boolean | no | Whether to download the wiki, rather than project, repository |
+

lib/api/api.rb

@@ -154,6 +154,7 @@ module API
     mount ::API::ProjectHooks
     mount ::API::Projects
     mount ::API::ProjectMilestones
+    mount ::API::ProjectSnapshots
     mount ::API::ProjectSnippets
     mount ::API::ProtectedBranches
     mount ::API::Repositories

lib/api/helpers/project_snapshots_helpers.rb

+module API
+  module Helpers
+    module ProjectSnapshotsHelpers
+      def authorize_read_git_snapshot!
+        authenticated_with_full_private_access!
+      end
+
+      def send_git_snapshot(repository)
+        header(*Gitlab::Workhorse.send_git_snapshot(repository))
+      end
+
+      def snapshot_project
+        user_project
+      end
+
+      def snapshot_repository
+        if to_boolean(params[:wiki])
+          snapshot_project.wiki.repository
+        else
+          snapshot_project.repository
+        end
+      end
+    end
+  end
+end

lib/api/project_snapshots.rb

+module API
+  class ProjectSnapshots < Grape::API
+    helpers ::API::Helpers::ProjectSnapshotsHelpers
+
+    before { authorize_read_git_snapshot! }
+
+    resource :projects do
+      desc 'Download a (possibly inconsistent) snapshot of a repository' do
+        detail 'This feature was introduced in GitLab 10.7'
+      end
+      params do
+        optional :wiki, type: Boolean, desc: 'Set to true to receive the wiki repository'
+      end
+      get ':id/snapshot' do
+        send_git_snapshot(snapshot_repository)
+      end
+    end
+  end
+end

lib/gitlab/git/repository.rb

@@ -1258,6 +1258,10 @@ module Gitlab
         true
       end
 
+      def create_from_snapshot(url, auth)
+        gitaly_repository_client.create_from_snapshot(url, auth)
+      end
+
       def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:)
         gitaly_migrate(:rebase) do |is_enabled|
           if is_enabled

lib/gitlab/gitaly_client/repository_service.rb

@@ -235,6 +235,22 @@ module Gitlab
         )
       end
 
+      def create_from_snapshot(http_url, http_auth)
+        request = Gitaly::CreateRepositoryFromSnapshotRequest.new(
+          repository: @gitaly_repo,
+          http_url: http_url,
+          http_auth: http_auth
+        )
+
+        GitalyClient.call(
+          @storage,
+          :repository_service,
+          :create_repository_from_snapshot,
+          request,
+          timeout: GitalyClient.default_timeout
+        )
+      end
+
       def write_ref(ref_path, ref, old_ref, shell)
         request = Gitaly::WriteRefRequest.new(
           repository: @gitaly_repo,

lib/gitlab/workhorse.rb

@@ -81,6 +81,20 @@ module Gitlab
         ]
       end
 
+      def send_git_snapshot(repository)
+        params = {
+          'GitalyServer' => gitaly_server_hash(repository),
+          'GetSnapshotRequest' => Gitaly::GetSnapshotRequest.new(
+            repository: repository.gitaly_repository
+          ).to_json
+        }
+
+        [
+          SEND_DATA_HEADER,
+          "git-snapshot:#{encode(params)}"
+        ]
+      end
+
       def send_git_diff(repository, diff_refs)
         params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT)
                    {

spec/lib/gitlab/gitaly_client/repository_service_spec.rb

@@ -156,4 +156,15 @@ describe Gitlab::GitalyClient::RepositoryService do
       client.calculate_checksum
     end
   end
+
+  describe '#create_from_snapshot' do
+    it 'sends a create_repository_from_snapshot message' do
+      expect_any_instance_of(Gitaly::RepositoryService::Stub)
+        .to receive(:create_repository_from_snapshot)
+        .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash))
+        .and_return(double)
+
+      client.create_from_snapshot('http://example.com?wiki=1', 'Custom xyz')
+    end
+  end
 end

spec/lib/gitlab/workhorse_spec.rb

@@ -482,4 +482,26 @@ describe Gitlab::Workhorse do
       }.deep_stringify_keys)
     end
   end
+
+  describe '.send_git_snapshot' do
+    let(:url) { 'http://example.com' }
+
+    subject(:request) { described_class.send_git_snapshot(repository) }
+
+    it 'sets the header correctly' do
+      key, command, params = decode_workhorse_header(request)
+
+      expect(key).to eq("Gitlab-Workhorse-Send-Data")
+      expect(command).to eq('git-snapshot')
+      expect(params).to eq(
+        'GitalyServer' => {
+          'address' => Gitlab::GitalyClient.address(project.repository_storage),
+          'token' => Gitlab::GitalyClient.token(project.repository_storage)
+        },
+        'GetSnapshotRequest' => Gitaly::GetSnapshotRequest.new(
+          repository: repository.gitaly_repository
+        ).to_json
+      )
+    end
+  end
 end

spec/requests/api/project_snapshots_spec.rb

+require 'spec_helper'
+
+describe API::ProjectSnapshots do
+  include WorkhorseHelpers
+
+  let(:project) { create(:project) }
+  let(:admin) { create(:admin) }
+
+  describe 'GET /projects/:id/snapshot' do
+    def expect_snapshot_response_for(repository)
+      type, params = workhorse_send_data
+
+      expect(type).to eq('git-snapshot')
+      expect(params).to eq(
+        'GitalyServer' => {
+          'address' => Gitlab::GitalyClient.address(repository.project.repository_storage),
+          'token' => Gitlab::GitalyClient.token(repository.project.repository_storage)
+        },
+        'GetSnapshotRequest' => Gitaly::GetSnapshotRequest.new(
+          repository: repository.gitaly_repository
+        ).to_json
+      )
+    end
+
+    it 'returns authentication error as project owner' do
+      get api("/projects/#{project.id}/snapshot", project.owner)
+
+      expect(response).to have_gitlab_http_status(403)
+    end
+
+    it 'returns authentication error as unauthenticated user' do
+      get api("/projects/#{project.id}/snapshot", nil)
+
+      expect(response).to have_gitlab_http_status(401)
+    end
+
+    it 'requests project repository raw archive as administrator' do
+      get api("/projects/#{project.id}/snapshot", admin), wiki: '0'
+
+      expect(response).to have_gitlab_http_status(200)
+      expect_snapshot_response_for(project.repository)
+    end
+
+    it 'requests wiki repository raw archive as administrator' do
+      get api("/projects/#{project.id}/snapshot", admin), wiki: '1'
+
+      expect(response).to have_gitlab_http_status(200)
+      expect_snapshot_response_for(project.wiki.repository)
+    end
+  end
+end