Commit 28113478 authored by Kamil Trzcinski's avatar Kamil Trzcinski Committed by Marin Jankovski

Add support for batch download operation

parent 6d139565
...@@ -26,7 +26,7 @@ module Gitlab ...@@ -26,7 +26,7 @@ module Gitlab
def render_download_object_response(oid) def render_download_object_response(oid)
render_response_to_download do render_response_to_download do
if check_download_sendfile_header? && check_download_accept_header? if check_download_sendfile_header?
render_lfs_sendfile(oid) render_lfs_sendfile(oid)
else else
render_not_found render_not_found
...@@ -34,20 +34,15 @@ module Gitlab ...@@ -34,20 +34,15 @@ module Gitlab
end end
end end
def render_lfs_api_auth def render_batch_operation_response
render_response_to_push do request_body = JSON.parse(@request.body.read)
request_body = JSON.parse(@request.body.read) case request_body["operation"]
return render_not_found if request_body.empty? || request_body['objects'].empty? when "download"
render_batch_download(request_body)
response = build_response(request_body['objects']) when "upload"
[ render_batch_upload(request_body)
200, else
{ render_forbidden
"Content-Type" => "application/json; charset=utf-8",
"Cache-Control" => "private",
},
[JSON.dump(response)]
]
end end
end end
...@@ -142,6 +137,38 @@ module Gitlab ...@@ -142,6 +137,38 @@ module Gitlab
end end
end end
def render_batch_upload(body)
return render_not_found if body.empty? || body['objects'].nil?
render_response_to_push do
response = build_upload_batch_response(body['objects'])
[
200,
{
"Content-Type" => "application/json; charset=utf-8",
"Cache-Control" => "private",
},
[JSON.dump(response)]
]
end
end
def render_batch_download(body)
return render_not_found if body.empty? || body['objects'].nil?
render_response_to_download do
response = build_download_batch_response(body['objects'])
[
200,
{
"Content-Type" => "application/json; charset=utf-8",
"Cache-Control" => "private",
},
[JSON.dump(response)]
]
end
end
def render_lfs_download_hypermedia(oid) def render_lfs_download_hypermedia(oid)
return render_not_found unless oid.present? return render_not_found unless oid.present?
...@@ -266,10 +293,16 @@ module Gitlab ...@@ -266,10 +293,16 @@ module Gitlab
@project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set @project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set
end end
def build_response(objects) def build_upload_batch_response(objects)
selected_objects = select_existing_objects(objects) selected_objects = select_existing_objects(objects)
upload_hypermedia(objects, selected_objects) upload_hypermedia_links(objects, selected_objects)
end
def build_download_batch_response(objects)
selected_objects = select_existing_objects(objects)
download_hypermedia_links(objects, selected_objects)
end end
def download_hypermedia(oid) def download_hypermedia(oid)
...@@ -279,7 +312,6 @@ module Gitlab ...@@ -279,7 +312,6 @@ module Gitlab
{ {
'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{oid}", 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{oid}",
'header' => { 'header' => {
'Accept' => "application/vnd.git-lfs+json; charset=utf-8",
'Authorization' => @env['HTTP_AUTHORIZATION'] 'Authorization' => @env['HTTP_AUTHORIZATION']
}.compact }.compact
} }
...@@ -287,21 +319,40 @@ module Gitlab ...@@ -287,21 +319,40 @@ module Gitlab
} }
end end
def upload_hypermedia(all_objects, existing_objects) def download_hypermedia_links(all_objects, existing_objects)
all_objects.each do |object| all_objects.each do |object|
object['_links'] = hypermedia_links(object) unless existing_objects.include?(object['oid']) # generate links only for existing objects
next unless existing_objects.include?(object['oid'])
object['_links'] = {
'download' => {
'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}",
'header' => {
'Authorization' => @env['HTTP_AUTHORIZATION']
}.compact
}
}
end end
{ 'objects' => all_objects } { 'objects' => all_objects }
end end
def hypermedia_links(object) def upload_hypermedia_links(all_objects, existing_objects)
{ all_objects.each do |object|
"upload" => { # generate links only for non-existing objects
'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}", next if existing_objects.include?(object['oid'])
'header' => { 'Authorization' => @env['HTTP_AUTHORIZATION'] }
}.compact object['_links'] = {
} 'upload' => {
'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}",
'header' => {
'Authorization' => @env['HTTP_AUTHORIZATION']
}.compact
}
}
end
{ 'objects' => all_objects }
end end
end end
end end
......
...@@ -48,7 +48,7 @@ module Gitlab ...@@ -48,7 +48,7 @@ module Gitlab
# Check for Batch API # Check for Batch API
if post_path[0].ends_with?("/info/lfs/objects/batch") if post_path[0].ends_with?("/info/lfs/objects/batch")
lfs.render_lfs_api_auth lfs.render_batch_operation_response
else else
nil nil
end end
......
...@@ -66,7 +66,7 @@ describe Gitlab::Lfs::Router do ...@@ -66,7 +66,7 @@ describe Gitlab::Lfs::Router do
json_response = ActiveSupport::JSON.decode(lfs_router_auth.try_call.last.first) json_response = ActiveSupport::JSON.decode(lfs_router_auth.try_call.last.first)
expect(json_response['_links']['download']['href']).to eq("#{Gitlab.config.gitlab.url}/#{project.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}") expect(json_response['_links']['download']['href']).to eq("#{Gitlab.config.gitlab.url}/#{project.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}")
expect(json_response['_links']['download']['header']).to eq("Authorization" => @auth, "Accept" => "application/vnd.git-lfs+json; charset=utf-8") expect(json_response['_links']['download']['header']).to eq("Authorization" => @auth)
end end
end end
...@@ -107,7 +107,7 @@ describe Gitlab::Lfs::Router do ...@@ -107,7 +107,7 @@ describe Gitlab::Lfs::Router do
json_response = ActiveSupport::JSON.decode(lfs_router_public_auth.try_call.last.first) json_response = ActiveSupport::JSON.decode(lfs_router_public_auth.try_call.last.first)
expect(json_response['_links']['download']['href']).to eq("#{Gitlab.config.gitlab.url}/#{public_project.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}") expect(json_response['_links']['download']['href']).to eq("#{Gitlab.config.gitlab.url}/#{public_project.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}")
expect(json_response['_links']['download']['header']).to eq("Accept" => "application/vnd.git-lfs+json; charset=utf-8") expect(json_response['_links']['download']['header']).to eq({})
end end
end end
...@@ -117,7 +117,7 @@ describe Gitlab::Lfs::Router do ...@@ -117,7 +117,7 @@ describe Gitlab::Lfs::Router do
json_response = ActiveSupport::JSON.decode(lfs_router_public_noauth.try_call.last.first) json_response = ActiveSupport::JSON.decode(lfs_router_public_noauth.try_call.last.first)
expect(json_response['_links']['download']['href']).to eq("#{Gitlab.config.gitlab.url}/#{public_project.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}") expect(json_response['_links']['download']['href']).to eq("#{Gitlab.config.gitlab.url}/#{public_project.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}")
expect(json_response['_links']['download']['header']).to eq("Accept" => "application/vnd.git-lfs+json; charset=utf-8") expect(json_response['_links']['download']['header']).to eq({})
end end
end end
end end
...@@ -191,7 +191,7 @@ describe Gitlab::Lfs::Router do ...@@ -191,7 +191,7 @@ describe Gitlab::Lfs::Router do
json_response = ActiveSupport::JSON.decode(lfs_router_forked_auth.try_call.last.first) json_response = ActiveSupport::JSON.decode(lfs_router_forked_auth.try_call.last.first)
expect(json_response['_links']['download']['href']).to eq("#{Gitlab.config.gitlab.url}/#{forked_project.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}") expect(json_response['_links']['download']['href']).to eq("#{Gitlab.config.gitlab.url}/#{forked_project.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}")
expect(json_response['_links']['download']['header']).to eq("Accept" => "application/vnd.git-lfs+json; charset=utf-8") expect(json_response['_links']['download']['header']).to eq({})
end end
end end
...@@ -219,7 +219,7 @@ describe Gitlab::Lfs::Router do ...@@ -219,7 +219,7 @@ describe Gitlab::Lfs::Router do
json_response = ActiveSupport::JSON.decode(lfs_router_forked_auth.try_call.last.first) json_response = ActiveSupport::JSON.decode(lfs_router_forked_auth.try_call.last.first)
expect(json_response['_links']['download']['href']).to eq("#{Gitlab.config.gitlab.url}/#{forked_project.path_with_namespace}.git/gitlab-lfs/objects/91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897") expect(json_response['_links']['download']['href']).to eq("#{Gitlab.config.gitlab.url}/#{forked_project.path_with_namespace}.git/gitlab-lfs/objects/91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897")
expect(json_response['_links']['download']['header']).to eq("Accept" => "application/vnd.git-lfs+json; charset=utf-8", "Authorization" => @auth) expect(json_response['_links']['download']['header']).to eq("Authorization" => @auth)
end end
end end
...@@ -250,7 +250,8 @@ describe Gitlab::Lfs::Router do ...@@ -250,7 +250,8 @@ describe Gitlab::Lfs::Router do
body = { 'objects' => [{ body = { 'objects' => [{
'oid' => sample_oid, 'oid' => sample_oid,
'size' => sample_size 'size' => sample_size
}] }],
'operation' => 'upload'
}.to_json }.to_json
env['rack.input'] = StringIO.new(body) env['rack.input'] = StringIO.new(body)
end end
...@@ -286,7 +287,8 @@ describe Gitlab::Lfs::Router do ...@@ -286,7 +287,8 @@ describe Gitlab::Lfs::Router do
'objects' => [{ 'objects' => [{
'oid' => '91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897', 'oid' => '91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897',
'size' => 1575078 'size' => 1575078
}] }],
'operation' => 'upload'
}.to_json }.to_json
env['rack.input'] = StringIO.new(body) env['rack.input'] = StringIO.new(body)
end end
...@@ -315,7 +317,8 @@ describe Gitlab::Lfs::Router do ...@@ -315,7 +317,8 @@ describe Gitlab::Lfs::Router do
{ 'oid' => sample_oid, { 'oid' => sample_oid,
'size' => sample_size 'size' => sample_size
} }
] ],
'operation' => 'upload'
}.to_json }.to_json
env['rack.input'] = StringIO.new(body) env['rack.input'] = StringIO.new(body)
public_project.lfs_objects << lfs_object public_project.lfs_objects << lfs_object
...@@ -351,6 +354,12 @@ describe Gitlab::Lfs::Router do ...@@ -351,6 +354,12 @@ describe Gitlab::Lfs::Router do
end end
context 'when user is not authenticated' do context 'when user is not authenticated' do
before do
env['rack.input'] = StringIO.new(
{ 'objects' => [], 'operation' => 'upload' }.to_json
)
end
context 'when user has push access' do context 'when user has push access' do
before do before do
project.team << [user, :master] project.team << [user, :master]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment