Commit 807c1a99 authored by Marin Jankovski's avatar Marin Jankovski

Merge branch 'lfs-batch-download' into 'master'

Add support for batch download operation

- Drops Accept for all download requests,
- Allows to do batch download for public projects and non-authorized users
- Returns 501 for legacy API with message to upgrade the client

/cc @marin @jacobvosmaer @yorickpeterse 

See merge request !1842
parents 261698f0 dbc0be1b
...@@ -10,23 +10,9 @@ module Gitlab ...@@ -10,23 +10,9 @@ module Gitlab
@request = request @request = request
end end
# Return a response for a download request
# Can be a response to:
# Request from a user to get the file
# Request from gitlab-workhorse which file to serve to the user
def render_download_hypermedia_response(oid)
render_response_to_download do
if check_download_accept_header?
render_lfs_download_hypermedia(oid)
else
render_not_found
end
end
end
def render_download_object_response(oid) def render_download_object_response(oid)
render_response_to_download do render_response_to_download do
if check_download_sendfile_header? && check_download_accept_header? if check_download_sendfile_header?
render_lfs_sendfile(oid) render_lfs_sendfile(oid)
else else
render_not_found render_not_found
...@@ -34,20 +20,15 @@ module Gitlab ...@@ -34,20 +20,15 @@ module Gitlab
end end
end end
def render_lfs_api_auth def render_batch_operation_response
render_response_to_push do
request_body = JSON.parse(@request.body.read) request_body = JSON.parse(@request.body.read)
return render_not_found if request_body.empty? || request_body['objects'].empty? case request_body["operation"]
when "download"
response = build_response(request_body['objects']) render_batch_download(request_body)
[ when "upload"
200, render_batch_upload(request_body)
{ else
"Content-Type" => "application/json; charset=utf-8", render_not_found
"Cache-Control" => "private",
},
[JSON.dump(response)]
]
end end
end end
...@@ -71,13 +52,24 @@ module Gitlab ...@@ -71,13 +52,24 @@ module Gitlab
end end
end end
def render_unsupported_deprecated_api
[
501,
{ "Content-Type" => "application/json; charset=utf-8" },
[JSON.dump({
'message' => 'Server supports batch API only, please update your Git LFS client to version 0.6.0 and up.',
'documentation_url' => "#{Gitlab.config.gitlab.url}/help",
})]
]
end
private private
def render_not_enabled def render_not_enabled
[ [
501, 501,
{ {
"Content-Type" => "application/vnd.git-lfs+json", "Content-Type" => "application/json; charset=utf-8",
}, },
[JSON.dump({ [JSON.dump({
'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.', 'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.',
...@@ -142,18 +134,35 @@ module Gitlab ...@@ -142,18 +134,35 @@ module Gitlab
end end
end end
def render_lfs_download_hypermedia(oid) def render_batch_upload(body)
return render_not_found unless oid.present? return render_not_found if body.empty? || body['objects'].nil?
lfs_object = object_for_download(oid) render_response_to_push do
if lfs_object response = build_upload_batch_response(body['objects'])
[ [
200, 200,
{ "Content-Type" => "application/vnd.git-lfs+json" }, {
[JSON.dump(download_hypermedia(oid))] "Content-Type" => "application/json; charset=utf-8",
"Cache-Control" => "private",
},
[JSON.dump(response)]
]
end
end
def render_batch_download(body)
return render_not_found if body.empty? || body['objects'].nil?
render_response_to_download do
response = build_download_batch_response(body['objects'])
[
200,
{
"Content-Type" => "application/json; charset=utf-8",
"Cache-Control" => "private",
},
[JSON.dump(response)]
] ]
else
render_not_found
end end
end end
...@@ -199,10 +208,6 @@ module Gitlab ...@@ -199,10 +208,6 @@ module Gitlab
@env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile" @env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile"
end end
def check_download_accept_header?
@env['HTTP_ACCEPT'].to_s == "application/vnd.git-lfs+json; charset=utf-8"
end
def user_can_fetch? def user_can_fetch?
# Check user access against the project they used to initiate the pull # Check user access against the project they used to initiate the pull
@user.can?(:download_code, @origin_project) @user.can?(:download_code, @origin_project)
...@@ -266,42 +271,56 @@ module Gitlab ...@@ -266,42 +271,56 @@ module Gitlab
@project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set @project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set
end end
def build_response(objects) def build_upload_batch_response(objects)
selected_objects = select_existing_objects(objects) selected_objects = select_existing_objects(objects)
upload_hypermedia(objects, selected_objects) upload_hypermedia_links(objects, selected_objects)
end end
def download_hypermedia(oid) def build_download_batch_response(objects)
{ selected_objects = select_existing_objects(objects)
'_links' => {
'download' => download_hypermedia_links(objects, selected_objects)
{ end
'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{oid}",
def download_hypermedia_links(all_objects, existing_objects)
all_objects.each do |object|
if existing_objects.include?(object['oid'])
object['actions'] = {
'download' => {
'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}",
'header' => { 'header' => {
'Accept' => "application/vnd.git-lfs+json; charset=utf-8",
'Authorization' => @env['HTTP_AUTHORIZATION'] 'Authorization' => @env['HTTP_AUTHORIZATION']
}.compact }.compact
} }
} }
else
object['error'] = {
'code' => 404,
'message' => "Object does not exist on the server or you don't have permissions to access it",
} }
end end
def upload_hypermedia(all_objects, existing_objects)
all_objects.each do |object|
object['_links'] = hypermedia_links(object) unless existing_objects.include?(object['oid'])
end end
{ 'objects' => all_objects } { 'objects' => all_objects }
end end
def hypermedia_links(object) def upload_hypermedia_links(all_objects, existing_objects)
{ all_objects.each do |object|
"upload" => { # generate actions only for non-existing objects
next if existing_objects.include?(object['oid'])
object['actions'] = {
'upload' => {
'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}", 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}",
'header' => { 'Authorization' => @env['HTTP_AUTHORIZATION'] } 'header' => {
'Authorization' => @env['HTTP_AUTHORIZATION']
}.compact }.compact
} }
}
end
{ 'objects' => all_objects }
end end
end end
end end
......
...@@ -34,7 +34,7 @@ module Gitlab ...@@ -34,7 +34,7 @@ module Gitlab
case path_match[1] case path_match[1]
when "info/lfs" when "info/lfs"
lfs.render_download_hypermedia_response(oid) lfs.render_unsupported_deprecated_api
when "gitlab-lfs" when "gitlab-lfs"
lfs.render_download_object_response(oid) lfs.render_download_object_response(oid)
else else
...@@ -48,7 +48,9 @@ module Gitlab ...@@ -48,7 +48,9 @@ module Gitlab
# Check for Batch API # Check for Batch API
if post_path[0].ends_with?("/info/lfs/objects/batch") if post_path[0].ends_with?("/info/lfs/objects/batch")
lfs.render_lfs_api_auth lfs.render_batch_operation_response
elsif post_path[0].ends_with?("/info/lfs/objects")
lfs.render_unsupported_deprecated_api
else else
nil nil
end end
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment