Commit b736a7b1 authored by Dmitriy Zaporozhets's avatar Dmitriy Zaporozhets

Merge branch 'lfs' into 'master'

Git LFS  support

Part of gitlab-org/gitlab-ce#2955

See merge request !1727
parents 9179fcec 14032d8e
...@@ -43,3 +43,4 @@ rails_best_practices_output.html ...@@ -43,3 +43,4 @@ rails_best_practices_output.html
tmp/ tmp/
vendor/bundle/* vendor/bundle/*
builds/* builds/*
shared/*
...@@ -72,8 +72,7 @@ class ProjectsController < ApplicationController ...@@ -72,8 +72,7 @@ class ProjectsController < ApplicationController
def remove_fork def remove_fork
return access_denied! unless can?(current_user, :remove_fork_project, @project) return access_denied! unless can?(current_user, :remove_fork_project, @project)
if @project.forked? if @project.unlink_fork
@project.forked_project_link.destroy
flash[:notice] = 'The fork relationship has been removed.' flash[:notice] = 'The fork relationship has been removed.'
end end
end end
......
class LfsObject < ActiveRecord::Base
has_many :lfs_objects_projects, dependent: :destroy
has_many :projects, through: :lfs_objects_projects
validates :oid, presence: true, uniqueness: true
mount_uploader :file, LfsObjectUploader
end
class LfsObjectsProject < ActiveRecord::Base
belongs_to :project
belongs_to :lfs_object
validates :lfs_object_id, presence: true
validates :lfs_object_id, uniqueness: { scope: [:project_id], message: "already exists in project" }
validates :project_id, presence: true
end
...@@ -124,6 +124,8 @@ class Project < ActiveRecord::Base ...@@ -124,6 +124,8 @@ class Project < ActiveRecord::Base
has_many :ci_commits, dependent: :destroy, class_name: 'Ci::Commit', foreign_key: :gl_project_id has_many :ci_commits, dependent: :destroy, class_name: 'Ci::Commit', foreign_key: :gl_project_id
has_many :ci_builds, through: :ci_commits, source: :builds, dependent: :destroy, class_name: 'Ci::Build' has_many :ci_builds, through: :ci_commits, source: :builds, dependent: :destroy, class_name: 'Ci::Build'
has_many :releases, dependent: :destroy has_many :releases, dependent: :destroy
has_many :lfs_objects_projects, dependent: :destroy
has_many :lfs_objects, through: :lfs_objects_projects
has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" has_one :import_data, dependent: :destroy, class_name: "ProjectImportData"
has_one :gitlab_ci_project, dependent: :destroy, class_name: "Ci::Project", foreign_key: :gitlab_id has_one :gitlab_ci_project, dependent: :destroy, class_name: "Ci::Project", foreign_key: :gitlab_id
...@@ -798,4 +800,14 @@ class Project < ActiveRecord::Base ...@@ -798,4 +800,14 @@ class Project < ActiveRecord::Base
def enable_ci def enable_ci
self.builds_enabled = true self.builds_enabled = true
end end
def unlink_fork
if forked?
forked_from_project.lfs_objects.find_each do |lfs_object|
lfs_object.projects << self
end
forked_project_link.destroy
end
end
end end
# encoding: utf-8
class LfsObjectUploader < CarrierWave::Uploader::Base
storage :file
def store_dir
"#{Gitlab.config.lfs.storage_path}/#{model.oid[0,2]}/#{model.oid[2,2]}"
end
def cache_dir
"#{Gitlab.config.lfs.storage_path}/tmp/cache"
end
def move_to_cache
true
end
def move_to_store
true
end
def exists?
file.try(:exists?)
end
def filename
model.oid[4..-1]
end
end
...@@ -124,6 +124,12 @@ production: &base ...@@ -124,6 +124,12 @@ production: &base
# The mailbox where incoming mail will end up. Usually "inbox". # The mailbox where incoming mail will end up. Usually "inbox".
mailbox: "inbox" mailbox: "inbox"
## Git LFS
lfs:
enabled: false
# The location where LFS objects are stored (default: shared/lfs-objects).
# storage_path: shared/lfs-objects
## Gravatar ## Gravatar
## For Libravatar see: http://doc.gitlab.com/ce/customization/libravatar.html ## For Libravatar see: http://doc.gitlab.com/ce/customization/libravatar.html
gravatar: gravatar:
...@@ -317,8 +323,6 @@ production: &base ...@@ -317,8 +323,6 @@ production: &base
# path: /mnt/gitlab # Default: shared # path: /mnt/gitlab # Default: shared
# #
# 4. Advanced settings # 4. Advanced settings
# ========================== # ==========================
...@@ -419,6 +423,8 @@ test: ...@@ -419,6 +423,8 @@ test:
<<: *base <<: *base
gravatar: gravatar:
enabled: true enabled: true
lfs:
enabled: false
gitlab: gitlab:
host: localhost host: localhost
port: 80 port: 80
......
...@@ -199,6 +199,13 @@ Settings.incoming_email['ssl'] = false if Settings.incoming_email['ssl']. ...@@ -199,6 +199,13 @@ Settings.incoming_email['ssl'] = false if Settings.incoming_email['ssl'].
Settings.incoming_email['start_tls'] = false if Settings.incoming_email['start_tls'].nil? Settings.incoming_email['start_tls'] = false if Settings.incoming_email['start_tls'].nil?
Settings.incoming_email['mailbox'] = "inbox" if Settings.incoming_email['mailbox'].nil? Settings.incoming_email['mailbox'] = "inbox" if Settings.incoming_email['mailbox'].nil?
#
# Git LFS
#
Settings['lfs'] ||= Settingslogic.new({})
Settings.lfs['enabled'] = false if Settings.lfs['enabled'].nil?
Settings.lfs['storage_path'] = File.expand_path(Settings.lfs['storage_path'] || File.join(Settings.shared['path'], "lfs-objects"), Rails.root)
# #
# Gravatar # Gravatar
# #
......
...@@ -93,7 +93,7 @@ Gitlab::Application.routes.draw do ...@@ -93,7 +93,7 @@ Gitlab::Application.routes.draw do
end end
# Enable Grack support # Enable Grack support
mount Grack::AuthSpawner, at: '/', constraints: lambda { |request| /[-\/\w\.]+\.git\//.match(request.path_info) }, via: [:get, :post] mount Grack::AuthSpawner, at: '/', constraints: lambda { |request| /[-\/\w\.]+\.git\//.match(request.path_info) }, via: [:get, :post, :put]
# Help # Help
get 'help' => 'help#index' get 'help' => 'help#index'
......
class CreateLfsObjects < ActiveRecord::Migration
def change
create_table :lfs_objects do |t|
t.string :oid, null: false, unique: true
t.integer :size, null: false
t.timestamps
end
end
end
class CreateLfsObjectsProjects < ActiveRecord::Migration
def change
create_table :lfs_objects_projects do |t|
t.integer :lfs_object_id, null: false
t.integer :project_id, null: false
t.timestamps
end
add_index :lfs_objects_projects, :project_id
end
end
class AddFileToLfsObjects < ActiveRecord::Migration
def change
add_column :lfs_objects, :file, :string
end
end
class AddIndexForLfsOidAndSize < ActiveRecord::Migration
def change
add_index :lfs_objects, :oid
add_index :lfs_objects, [:oid, :size]
end
end
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20151109100728) do ActiveRecord::Schema.define(version: 20151114113410) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
...@@ -422,6 +422,26 @@ ActiveRecord::Schema.define(version: 20151109100728) do ...@@ -422,6 +422,26 @@ ActiveRecord::Schema.define(version: 20151109100728) do
add_index "labels", ["project_id"], name: "index_labels_on_project_id", using: :btree add_index "labels", ["project_id"], name: "index_labels_on_project_id", using: :btree
create_table "lfs_objects", force: true do |t|
t.string "oid", null: false, unique: true
t.integer "size", null: false
t.datetime "created_at"
t.datetime "updated_at"
t.string "file"
end
add_index "lfs_objects", ["oid", "size"], name: "index_lfs_objects_on_oid_and_size", using: :btree
add_index "lfs_objects", ["oid"], name: "index_lfs_objects_on_oid", using: :btree
create_table "lfs_objects_projects", force: true do |t|
t.integer "lfs_object_id", null: false
t.integer "project_id", null: false
t.datetime "created_at"
t.datetime "updated_at"
end
add_index "lfs_objects_projects", ["project_id"], name: "index_lfs_objects_projects_on_project_id", using: :btree
create_table "members", force: true do |t| create_table "members", force: true do |t|
t.integer "access_level", null: false t.integer "access_level", null: false
t.integer "source_id", null: false t.integer "source_id", null: false
......
...@@ -33,6 +33,9 @@ module Grack ...@@ -33,6 +33,9 @@ module Grack
auth! auth!
lfs_response = Gitlab::Lfs::Router.new(project, @user, @request).try_call
return lfs_response unless lfs_response.nil?
if project && authorized_request? if project && authorized_request?
# Tell gitlab-workhorse the request is OK, and what the GL_ID is # Tell gitlab-workhorse the request is OK, and what the GL_ID is
render_grack_auth_ok render_grack_auth_ok
......
module Gitlab
module Lfs
class Response
def initialize(project, user, request)
@origin_project = project
@project = storage_project(project)
@user = user
@env = request.env
@request = request
end
# Return a response for a download request
# Can be a response to:
# Request from a user to get the file
# Request from gitlab-workhorse which file to serve to the user
def render_download_hypermedia_response(oid)
render_response_to_download do
if check_download_accept_header?
render_lfs_download_hypermedia(oid)
else
render_not_found
end
end
end
def render_download_object_response(oid)
render_response_to_download do
if check_download_sendfile_header? && check_download_accept_header?
render_lfs_sendfile(oid)
else
render_not_found
end
end
end
def render_lfs_api_auth
render_response_to_push do
request_body = JSON.parse(@request.body.read)
return render_not_found if request_body.empty? || request_body['objects'].empty?
response = build_response(request_body['objects'])
[
200,
{
"Content-Type" => "application/json; charset=utf-8",
"Cache-Control" => "private",
},
[JSON.dump(response)]
]
end
end
def render_storage_upload_authorize_response(oid, size)
render_response_to_push do
[
200,
{ "Content-Type" => "application/json; charset=utf-8" },
[JSON.dump({
'StoreLFSPath' => "#{Gitlab.config.lfs.storage_path}/tmp/upload",
'LfsOid' => oid,
'LfsSize' => size
})]
]
end
end
def render_storage_upload_store_response(oid, size, tmp_file_name)
render_response_to_push do
render_lfs_upload_ok(oid, size, tmp_file_name)
end
end
private
def render_not_enabled
[
501,
{
"Content-Type" => "application/vnd.git-lfs+json",
},
[JSON.dump({
'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.',
'documentation_url' => "#{Gitlab.config.gitlab.url}/help",
})]
]
end
def render_unauthorized
[
401,
{
'Content-Type' => 'text/plain'
},
['Unauthorized']
]
end
def render_not_found
[
404,
{
"Content-Type" => "application/vnd.git-lfs+json"
},
[JSON.dump({
'message' => 'Not found.',
'documentation_url' => "#{Gitlab.config.gitlab.url}/help",
})]
]
end
def render_forbidden
[
403,
{
"Content-Type" => "application/vnd.git-lfs+json"
},
[JSON.dump({
'message' => 'Access forbidden. Check your access level.',
'documentation_url' => "#{Gitlab.config.gitlab.url}/help",
})]
]
end
def render_lfs_sendfile(oid)
return render_not_found unless oid.present?
lfs_object = object_for_download(oid)
if lfs_object && lfs_object.file.exists?
[
200,
{
# GitLab-workhorse will forward Content-Type header
"Content-Type" => "application/octet-stream",
"X-Sendfile" => lfs_object.file.path
},
[]
]
else
render_not_found
end
end
def render_lfs_download_hypermedia(oid)
return render_not_found unless oid.present?
lfs_object = object_for_download(oid)
if lfs_object
[
200,
{ "Content-Type" => "application/vnd.git-lfs+json" },
[JSON.dump(download_hypermedia(oid))]
]
else
render_not_found
end
end
def render_lfs_upload_ok(oid, size, tmp_file)
if store_file(oid, size, tmp_file)
[
200,
{
'Content-Type' => 'text/plain',
'Content-Length' => 0
},
[]
]
else
[
422,
{ 'Content-Type' => 'text/plain' },
["Unprocessable entity"]
]
end
end
def render_response_to_download
return render_not_enabled unless Gitlab.config.lfs.enabled
unless @project.public?
return render_unauthorized unless @user
return render_forbidden unless user_can_fetch?
end
yield
end
def render_response_to_push
return render_not_enabled unless Gitlab.config.lfs.enabled
return render_unauthorized unless @user
return render_forbidden unless user_can_push?
yield
end
def check_download_sendfile_header?
@env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile"
end
def check_download_accept_header?
@env['HTTP_ACCEPT'].to_s == "application/vnd.git-lfs+json; charset=utf-8"
end
def user_can_fetch?
# Check user access against the project they used to initiate the pull
@user.can?(:download_code, @origin_project)
end
def user_can_push?
# Check user access against the project they used to initiate the push
@user.can?(:push_code, @origin_project)
end
def storage_project(project)
if project.forked?
project.forked_from_project
else
project
end
end
def store_file(oid, size, tmp_file)
tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file)
object = LfsObject.find_or_create_by(oid: oid, size: size)
if object.file.exists?
success = true
else
success = move_tmp_file_to_storage(object, tmp_file_path)
end
if success
success = link_to_project(object)
end
success
ensure
# Ensure that the tmp file is removed
FileUtils.rm_f(tmp_file_path)
end
def object_for_download(oid)
@project.lfs_objects.find_by(oid: oid)
end
def move_tmp_file_to_storage(object, path)
File.open(path) do |f|
object.file = f
end
object.file.store!
object.save
end
def link_to_project(object)
if object && !object.projects.exists?(@project)
object.projects << @project
object.save
end
end
def select_existing_objects(objects)
objects_oids = objects.map { |o| o['oid'] }
@project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set
end
def build_response(objects)
selected_objects = select_existing_objects(objects)
upload_hypermedia(objects, selected_objects)
end
def download_hypermedia(oid)
{
'_links' => {
'download' =>
{
'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{oid}",
'header' => {
'Accept' => "application/vnd.git-lfs+json; charset=utf-8",
'Authorization' => @env['HTTP_AUTHORIZATION']
}.compact
}
}
}
end
def upload_hypermedia(all_objects, existing_objects)
all_objects.each do |object|
object['_links'] = hypermedia_links(object) unless existing_objects.include?(object['oid'])
end
{ 'objects' => all_objects }
end
def hypermedia_links(object)
{
"upload" => {
'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}",
'header' => { 'Authorization' => @env['HTTP_AUTHORIZATION'] }
}.compact
}
end
end
end
end
module Gitlab
module Lfs
class Router
def initialize(project, user, request)
@project = project
@user = user
@env = request.env
@request = request
end
def try_call
return unless @request && @request.path.present?
case @request.request_method
when 'GET'
get_response
when 'POST'
post_response
when 'PUT'
put_response
else
nil
end
end
private
def get_response
path_match = @request.path.match(/\/(info\/lfs|gitlab-lfs)\/objects\/([0-9a-f]{64})$/)
return nil unless path_match
oid = path_match[2]
return nil unless oid
case path_match[1]
when "info/lfs"
lfs.render_download_hypermedia_response(oid)
when "gitlab-lfs"
lfs.render_download_object_response(oid)
else
nil
end
end
def post_response
post_path = @request.path.match(/\/info\/lfs\/objects(\/batch)?$/)
return nil unless post_path
# Check for Batch API
if post_path[0].ends_with?("/info/lfs/objects/batch")
lfs.render_lfs_api_auth
else
nil
end
end
def put_response
object_match = @request.path.match(/\/gitlab-lfs\/objects\/([0-9a-f]{64})\/([0-9]+)(|\/authorize){1}$/)
return nil if object_match.nil?
oid = object_match[1]
size = object_match[2].try(:to_i)
return nil if oid.nil? || size.nil?
# GitLab-workhorse requests
# 1. Try to authorize the request
# 2. send a request with a header containing the name of the temporary file
if object_match[3] && object_match[3] == '/authorize'
lfs.render_storage_upload_authorize_response(oid, size)
else
tmp_file_name = sanitize_tmp_filename(@request.env['HTTP_X_GITLAB_LFS_TMP'])
return nil unless tmp_file_name
lfs.render_storage_upload_store_response(oid, size, tmp_file_name)
end
end
def lfs
return unless @project
Gitlab::Lfs::Response.new(@project, @user, @request)
end
def sanitize_tmp_filename(name)
if name.present?
name.gsub!(/^.*(\\|\/)/, '')
name = name.match(/[0-9a-f]{73}/)
name[0] if name
else
nil
end
end
end
end
end
...@@ -113,6 +113,13 @@ server { ...@@ -113,6 +113,13 @@ server {
proxy_pass http://gitlab; proxy_pass http://gitlab;
} }
location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects {
client_max_body_size 0;
# 'Error' 418 is a hack to re-use the @gitlab-workhorse block
error_page 418 = @gitlab-workhorse;
return 418;
}
location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ {
# 'Error' 418 is a hack to re-use the @gitlab-workhorse block # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
error_page 418 = @gitlab-workhorse; error_page 418 = @gitlab-workhorse;
......
...@@ -160,6 +160,13 @@ server { ...@@ -160,6 +160,13 @@ server {
proxy_pass http://gitlab; proxy_pass http://gitlab;
} }
location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects {
client_max_body_size 0;
# 'Error' 418 is a hack to re-use the @gitlab-workhorse block
error_page 418 = @gitlab-workhorse;
return 418;
}
location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ {
# 'Error' 418 is a hack to re-use the @gitlab-workhorse block # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
error_page 418 = @gitlab-workhorse; error_page 418 = @gitlab-workhorse;
......
# Read about factories at https://github.com/thoughtbot/factory_girl
FactoryGirl.define do
factory :lfs_object do
oid "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a17f80"
size 499013
end
trait :with_file do
file { fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "`/png") }
end
end
# Read about factories at https://github.com/thoughtbot/factory_girl
FactoryGirl.define do
factory :lfs_objects_project do
lfs_object
project
end
end
require 'spec_helper'
describe Gitlab::Lfs::Router do
let(:project) { create(:project) }
let(:public_project) { create(:project, :public) }
let(:forked_project) { fork_project(public_project, user) }
let(:user) { create(:user) }
let(:user_two) { create(:user) }
let!(:lfs_object) { create(:lfs_object, :with_file) }
let(:request) { Rack::Request.new(env) }
let(:env) do
{
'rack.input' => '',
'REQUEST_METHOD' => 'GET',
}
end
let(:lfs_router_auth) { new_lfs_router(project, user) }
let(:lfs_router_noauth) { new_lfs_router(project, nil) }
let(:lfs_router_public_auth) { new_lfs_router(public_project, user) }
let(:lfs_router_public_noauth) { new_lfs_router(public_project, nil) }
let(:lfs_router_forked_noauth) { new_lfs_router(forked_project, nil) }
let(:lfs_router_forked_auth) { new_lfs_router(forked_project, user_two) }
let(:sample_oid) { "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a17f80" }
let(:sample_size) { 499013 }
describe 'when lfs is disabled' do
before do
allow(Gitlab.config.lfs).to receive(:enabled).and_return(false)
env["PATH_INFO"] = "#{project.repository.path_with_namespace}.git/info/lfs/objects/#{sample_oid}"
end
it 'responds with 501' do
respond_with_disabled = [ 501,
{ "Content-Type"=>"application/vnd.git-lfs+json" },
["{\"message\":\"Git LFS is not enabled on this GitLab server, contact your admin.\",\"documentation_url\":\"#{Gitlab.config.gitlab.url}/help\"}"]
]
expect(lfs_router_auth.try_call).to match_array(respond_with_disabled)
end
end
describe 'when fetching lfs object' do
before do
enable_lfs
env['HTTP_ACCEPT'] = "application/vnd.git-lfs+json; charset=utf-8"
env["PATH_INFO"] = "#{project.repository.path_with_namespace}.git/info/lfs/objects/#{sample_oid}"
end
describe 'when user is authenticated' do
context 'and user has project download access' do
before do
@auth = authorize(user)
env["HTTP_AUTHORIZATION"] = @auth
project.lfs_objects << lfs_object
project.team << [user, :master]
end
it "responds with status 200" do
expect(lfs_router_auth.try_call.first).to eq(200)
end
it "responds with download hypermedia" do
json_response = ActiveSupport::JSON.decode(lfs_router_auth.try_call.last.first)
expect(json_response['_links']['download']['href']).to eq("#{Gitlab.config.gitlab.url}/#{project.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}")
expect(json_response['_links']['download']['header']).to eq("Authorization" => @auth, "Accept" => "application/vnd.git-lfs+json; charset=utf-8")
end
end
context 'and user does not have project access' do
it "responds with status 403" do
expect(lfs_router_auth.try_call.first).to eq(403)
end
end
end
describe 'when user is unauthenticated' do
context 'and user does not have download access' do
it "responds with status 401" do
expect(lfs_router_noauth.try_call.first).to eq(401)
end
end
context 'and user has download access' do
before do
project.team << [user, :master]
end
it "responds with status 401" do
expect(lfs_router_noauth.try_call.first).to eq(401)
end
end
end
describe 'and project is public' do
context 'and project has access to the lfs object' do
before do
public_project.lfs_objects << lfs_object
end
context 'and user is authenticated' do
it "responds with status 200 and sends download hypermedia" do
expect(lfs_router_public_auth.try_call.first).to eq(200)
json_response = ActiveSupport::JSON.decode(lfs_router_public_auth.try_call.last.first)
expect(json_response['_links']['download']['href']).to eq("#{Gitlab.config.gitlab.url}/#{public_project.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}")
expect(json_response['_links']['download']['header']).to eq("Accept" => "application/vnd.git-lfs+json; charset=utf-8")
end
end
context 'and user is unauthenticated' do
it "responds with status 200 and sends download hypermedia" do
expect(lfs_router_public_noauth.try_call.first).to eq(200)
json_response = ActiveSupport::JSON.decode(lfs_router_public_noauth.try_call.last.first)
expect(json_response['_links']['download']['href']).to eq("#{Gitlab.config.gitlab.url}/#{public_project.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}")
expect(json_response['_links']['download']['header']).to eq("Accept" => "application/vnd.git-lfs+json; charset=utf-8")
end
end
end
context 'and project does not have access to the lfs object' do
it "responds with status 404" do
expect(lfs_router_public_auth.try_call.first).to eq(404)
end
end
end
describe 'and request comes from gitlab-workhorse' do
before do
env["PATH_INFO"] = "#{project.repository.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}"
end
context 'without user being authorized' do
it "responds with status 401" do
expect(lfs_router_noauth.try_call.first).to eq(401)
end
end
context 'with required headers' do
before do
env['HTTP_X_SENDFILE_TYPE'] = "X-Sendfile"
end
context 'when user does not have project access' do
it "responds with status 403" do
expect(lfs_router_auth.try_call.first).to eq(403)
end
end
context 'when user has project access' do
before do
project.lfs_objects << lfs_object
project.team << [user, :master]
end
it "responds with status 200" do
expect(lfs_router_auth.try_call.first).to eq(200)
end
it "responds with the file location" do
expect(lfs_router_auth.try_call[1]['Content-Type']).to eq("application/octet-stream")
expect(lfs_router_auth.try_call[1]['X-Sendfile']).to eq(lfs_object.file.path)
end
end
end
context 'without required headers' do
it "responds with status 403" do
expect(lfs_router_auth.try_call.first).to eq(403)
end
end
end
describe 'from a forked public project' do
before do
env['HTTP_ACCEPT'] = "application/vnd.git-lfs+json; charset=utf-8"
env["PATH_INFO"] = "#{forked_project.repository.path_with_namespace}.git/info/lfs/objects/#{sample_oid}"
end
context "when fetching a lfs object" do
context "and user has project download access" do
before do
public_project.lfs_objects << lfs_object
end
it "can download the lfs object" do
expect(lfs_router_forked_auth.try_call.first).to eq(200)
json_response = ActiveSupport::JSON.decode(lfs_router_forked_auth.try_call.last.first)
expect(json_response['_links']['download']['href']).to eq("#{Gitlab.config.gitlab.url}/#{forked_project.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}")
expect(json_response['_links']['download']['header']).to eq("Accept" => "application/vnd.git-lfs+json; charset=utf-8")
end
end
context "and user is not authenticated but project is public" do
before do
public_project.lfs_objects << lfs_object
end
it "can download the lfs object" do
expect(lfs_router_forked_auth.try_call.first).to eq(200)
end
end
context "and user has project download access" do
before do
env["PATH_INFO"] = "#{forked_project.repository.path_with_namespace}.git/info/lfs/objects/91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897"
@auth = ActionController::HttpAuthentication::Basic.encode_credentials(user.username, user.password)
env["HTTP_AUTHORIZATION"] = @auth
lfs_object_two = create(:lfs_object, :with_file, oid: "91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897", size: 1575078)
public_project.lfs_objects << lfs_object_two
end
it "can get a lfs object that is not in the forked project" do
expect(lfs_router_forked_auth.try_call.first).to eq(200)
json_response = ActiveSupport::JSON.decode(lfs_router_forked_auth.try_call.last.first)
expect(json_response['_links']['download']['href']).to eq("#{Gitlab.config.gitlab.url}/#{forked_project.path_with_namespace}.git/gitlab-lfs/objects/91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897")
expect(json_response['_links']['download']['header']).to eq("Accept" => "application/vnd.git-lfs+json; charset=utf-8", "Authorization" => @auth)
end
end
context "and user has project download access" do
before do
env["PATH_INFO"] = "#{forked_project.repository.path_with_namespace}.git/info/lfs/objects/267c8b1d876743971e3a9978405818ff5ca731c4c870b06507619cd9b1847b6b"
lfs_object_three = create(:lfs_object, :with_file, oid: "267c8b1d876743971e3a9978405818ff5ca731c4c870b06507619cd9b1847b6b", size: 127192524)
project.lfs_objects << lfs_object_three
end
it "cannot get a lfs object that is not in the project" do
expect(lfs_router_forked_auth.try_call.first).to eq(404)
end
end
end
end
end
describe 'when initiating pushing of the lfs object' do
before do
enable_lfs
env['REQUEST_METHOD'] = 'POST'
env["PATH_INFO"] = "#{project.repository.path_with_namespace}.git/info/lfs/objects/batch"
end
describe 'when user is authenticated' do
before do
body = { 'objects' => [{
'oid' => sample_oid,
'size' => sample_size
}]
}.to_json
env['rack.input'] = StringIO.new(body)
end
describe 'when user has project push access' do
before do
@auth = authorize(user)
env["HTTP_AUTHORIZATION"] = @auth
project.team << [user, :master]
end
context 'when pushing an lfs object that already exists' do
before do
public_project.lfs_objects << lfs_object
end
it "responds with status 200 and links the object to the project" do
response_body = lfs_router_auth.try_call.last
response = ActiveSupport::JSON.decode(response_body.first)
expect(response['objects']).to be_kind_of(Array)
expect(response['objects'].first['oid']).to eq(sample_oid)
expect(response['objects'].first['size']).to eq(sample_size)
expect(lfs_object.projects.pluck(:id)).to_not include(project.id)
expect(lfs_object.projects.pluck(:id)).to include(public_project.id)
expect(response['objects'].first).to have_key('_links')
end
end
context 'when pushing a lfs object that does not exist' do
before do
body = {
'objects' => [{
'oid' => '91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897',
'size' => 1575078
}]
}.to_json
env['rack.input'] = StringIO.new(body)
end
it "responds with status 200 and upload hypermedia link" do
response = lfs_router_auth.try_call
expect(response.first).to eq(200)
response_body = ActiveSupport::JSON.decode(response.last.first)
expect(response_body['objects']).to be_kind_of(Array)
expect(response_body['objects'].first['oid']).to eq("91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897")
expect(response_body['objects'].first['size']).to eq(1575078)
expect(lfs_object.projects.pluck(:id)).not_to include(project.id)
expect(response_body['objects'].first['_links']['upload']['href']).to eq("#{Gitlab.config.gitlab.url}/#{project.path_with_namespace}.git/gitlab-lfs/objects/91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897/1575078")
expect(response_body['objects'].first['_links']['upload']['header']).to eq("Authorization" => @auth)
end
end
context 'when pushing one new and one existing lfs object' do
before do
body = {
'objects' => [
{ 'oid' => '91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897',
'size' => 1575078
},
{ 'oid' => sample_oid,
'size' => sample_size
}
]
}.to_json
env['rack.input'] = StringIO.new(body)
public_project.lfs_objects << lfs_object
end
it "responds with status 200 with upload hypermedia link for the new object" do
response = lfs_router_auth.try_call
expect(response.first).to eq(200)
response_body = ActiveSupport::JSON.decode(response.last.first)
expect(response_body['objects']).to be_kind_of(Array)
expect(response_body['objects'].first['oid']).to eq("91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897")
expect(response_body['objects'].first['size']).to eq(1575078)
expect(response_body['objects'].first['_links']['upload']['href']).to eq("#{Gitlab.config.gitlab.url}/#{project.path_with_namespace}.git/gitlab-lfs/objects/91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897/1575078")
expect(response_body['objects'].first['_links']['upload']['header']).to eq("Authorization" => @auth)
expect(response_body['objects'].last['oid']).to eq(sample_oid)
expect(response_body['objects'].last['size']).to eq(sample_size)
expect(lfs_object.projects.pluck(:id)).to_not include(project.id)
expect(lfs_object.projects.pluck(:id)).to include(public_project.id)
expect(response_body['objects'].last).to have_key('_links')
end
end
end
context 'when user does not have push access' do
it 'responds with 403' do
expect(lfs_router_auth.try_call.first).to eq(403)
end
end
end
context 'when user is not authenticated' do
context 'when user has push access' do
before do
project.team << [user, :master]
end
it "responds with status 401" do
expect(lfs_router_public_noauth.try_call.first).to eq(401)
end
end
context 'when user does not have push access' do
it "responds with status 401" do
expect(lfs_router_public_noauth.try_call.first).to eq(401)
end
end
end
end
describe 'when pushing a lfs object' do
before do
enable_lfs
env['REQUEST_METHOD'] = 'PUT'
end
describe 'to one project' do
describe 'when user has push access to the project' do
before do
project.team << [user, :master]
end
describe 'when user is authenticated' do
context 'and request is sent by gitlab-workhorse to authorize the request' do
before do
header_for_upload_authorize(project)
end
it 'responds with status 200, location of lfs store and object details' do
json_response = ActiveSupport::JSON.decode(lfs_router_auth.try_call.last.first)
expect(lfs_router_auth.try_call.first).to eq(200)
expect(json_response['StoreLFSPath']).to eq("#{Gitlab.config.shared.path}/lfs-objects/tmp/upload")
expect(json_response['LfsOid']).to eq(sample_oid)
expect(json_response['LfsSize']).to eq(sample_size)
end
end
context 'and request is sent by gitlab-workhorse to finalize the upload' do
before do
headers_for_upload_finalize(project)
end
it 'responds with status 200 and lfs object is linked to the project' do
expect(lfs_router_auth.try_call.first).to eq(200)
expect(lfs_object.projects.pluck(:id)).to include(project.id)
end
end
end
describe 'when user is unauthenticated' do
let(:lfs_router_noauth) { new_lfs_router(project, nil) }
context 'and request is sent by gitlab-workhorse to authorize the request' do
before do
header_for_upload_authorize(project)
end
it 'responds with status 401' do
expect(lfs_router_noauth.try_call.first).to eq(401)
end
end
context 'and request is sent by gitlab-workhorse to finalize the upload' do
before do
headers_for_upload_finalize(project)
end
it 'responds with status 401' do
expect(lfs_router_noauth.try_call.first).to eq(401)
end
end
context 'and request is sent with a malformed headers' do
before do
env["PATH_INFO"] = "#{project.repository.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}/#{sample_size}"
env["HTTP_X_GITLAB_LFS_TMP"] = "cat /etc/passwd"
end
it 'does not recognize it as a valid lfs command' do
expect(lfs_router_noauth.try_call).to eq(nil)
end
end
end
end
describe 'and user does not have push access' do
describe 'when user is authenticated' do
context 'and request is sent by gitlab-workhorse to authorize the request' do
before do
header_for_upload_authorize(project)
end
it 'responds with 403' do
expect(lfs_router_auth.try_call.first).to eq(403)
end
end
context 'and request is sent by gitlab-workhorse to finalize the upload' do
before do
headers_for_upload_finalize(project)
end
it 'responds with 403' do
expect(lfs_router_auth.try_call.first).to eq(403)
end
end
end
describe 'when user is unauthenticated' do
let(:lfs_router_noauth) { new_lfs_router(project, nil) }
context 'and request is sent by gitlab-workhorse to authorize the request' do
before do
header_for_upload_authorize(project)
end
it 'responds with 401' do
expect(lfs_router_noauth.try_call.first).to eq(401)
end
end
context 'and request is sent by gitlab-workhorse to finalize the upload' do
before do
headers_for_upload_finalize(project)
end
it 'responds with 401' do
expect(lfs_router_noauth.try_call.first).to eq(401)
end
end
end
end
end
describe "to a forked project" do
let(:forked_project) { fork_project(public_project, user) }
describe 'when user has push access to the project' do
before do
forked_project.team << [user_two, :master]
end
describe 'when user is authenticated' do
context 'and request is sent by gitlab-workhorse to authorize the request' do
before do
header_for_upload_authorize(forked_project)
end
it 'responds with status 200, location of lfs store and object details' do
json_response = ActiveSupport::JSON.decode(lfs_router_forked_auth.try_call.last.first)
expect(lfs_router_forked_auth.try_call.first).to eq(200)
expect(json_response['StoreLFSPath']).to eq("#{Gitlab.config.shared.path}/lfs-objects/tmp/upload")
expect(json_response['LfsOid']).to eq(sample_oid)
expect(json_response['LfsSize']).to eq(sample_size)
end
end
context 'and request is sent by gitlab-workhorse to finalize the upload' do
before do
headers_for_upload_finalize(forked_project)
end
it 'responds with status 200 and lfs object is linked to the source project' do
expect(lfs_router_forked_auth.try_call.first).to eq(200)
expect(lfs_object.projects.pluck(:id)).to include(public_project.id)
end
end
end
describe 'when user is unauthenticated' do
context 'and request is sent by gitlab-workhorse to authorize the request' do
before do
header_for_upload_authorize(forked_project)
end
it 'responds with status 401' do
expect(lfs_router_forked_noauth.try_call.first).to eq(401)
end
end
context 'and request is sent by gitlab-workhorse to finalize the upload' do
before do
headers_for_upload_finalize(forked_project)
end
it 'responds with status 401' do
expect(lfs_router_forked_noauth.try_call.first).to eq(401)
end
end
end
end
describe 'and user does not have push access' do
describe 'when user is authenticated' do
context 'and request is sent by gitlab-workhorse to authorize the request' do
before do
header_for_upload_authorize(forked_project)
end
it 'responds with 403' do
expect(lfs_router_forked_auth.try_call.first).to eq(403)
end
end
context 'and request is sent by gitlab-workhorse to finalize the upload' do
before do
headers_for_upload_finalize(forked_project)
end
it 'responds with 403' do
expect(lfs_router_forked_auth.try_call.first).to eq(403)
end
end
end
describe 'when user is unauthenticated' do
context 'and request is sent by gitlab-workhorse to authorize the request' do
before do
header_for_upload_authorize(forked_project)
end
it 'responds with 401' do
expect(lfs_router_forked_noauth.try_call.first).to eq(401)
end
end
context 'and request is sent by gitlab-workhorse to finalize the upload' do
before do
headers_for_upload_finalize(forked_project)
end
it 'responds with 401' do
expect(lfs_router_forked_noauth.try_call.first).to eq(401)
end
end
end
end
describe 'and second project not related to fork or a source project' do
let(:second_project) { create(:project) }
let(:lfs_router_second_project) { new_lfs_router(second_project, user) }
before do
public_project.lfs_objects << lfs_object
headers_for_upload_finalize(second_project)
end
context 'when pushing the same lfs object to the second project' do
before do
second_project.team << [user, :master]
end
it 'responds with 200 and links the lfs object to the project' do
expect(lfs_router_second_project.try_call.first).to eq(200)
expect(lfs_object.projects.pluck(:id)).to include(second_project.id, public_project.id)
end
end
end
end
end
def enable_lfs
allow(Gitlab.config.lfs).to receive(:enabled).and_return(true)
end
def authorize(user)
ActionController::HttpAuthentication::Basic.encode_credentials(user.username, user.password)
end
def new_lfs_router(project, user)
Gitlab::Lfs::Router.new(project, user, request)
end
def header_for_upload_authorize(project)
env["PATH_INFO"] = "#{project.repository.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}/#{sample_size}/authorize"
end
def headers_for_upload_finalize(project)
env["PATH_INFO"] = "#{project.repository.path_with_namespace}.git/gitlab-lfs/objects/#{sample_oid}/#{sample_size}"
env["HTTP_X_GITLAB_LFS_TMP"] = "#{sample_oid}6e561c9d4"
end
def fork_project(project, user, object = nil)
allow(RepositoryForkWorker).to receive(:perform_async).and_return(true)
Projects::ForkService.new(project, user, {}).execute
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment