Commit a841d79d authored by Patricio Cano's avatar Patricio Cano

Enforce repository size limit across all projects and groups, includes LFS objects in that limit.

Limit can be set globally, and overridden per group, and/or project.
parent 7f8d2416
......@@ -127,6 +127,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
:usage_ping_enabled,
:repository_storage,
:enabled_git_access_protocol,
:repository_size_limit,
restricted_visibility_levels: [],
import_sources: [],
disabled_oauth_sign_in_sources: []
......
......@@ -66,6 +66,7 @@ class Admin::GroupsController < Admin::ApplicationController
:lfs_enabled,
:name,
:path,
:repository_size_limit,
:request_access_enabled,
:visibility_level
)
......
......@@ -131,12 +131,13 @@ class GroupsController < Groups::ApplicationController
:avatar,
:description,
:lfs_enabled,
:membership_lock,
:name,
:path,
:public,
:repository_size_limit,
:request_access_enabled,
:share_with_group_lock,
:membership_lock,
:visibility_level
)
end
......
......@@ -68,7 +68,9 @@ class Projects::GitHttpController < Projects::GitHttpClientController
def render_denied
if user && user.can?(:read_project, project)
render plain: 'Access denied', status: :forbidden
message = project.above_size_limit? ? access_check.message : 'Access denied'
render plain: message, status: :forbidden
else
# Do not leak information about project existence
render_not_found
......
......@@ -320,6 +320,12 @@ class Projects::MergeRequestsController < Projects::ApplicationController
return access_denied! unless @merge_request.can_be_merged_by?(current_user)
return render_404 unless @merge_request.approved?
# user is not able to merge if project is above size limit
if @merge_request.target_project.above_size_limit?
@status = :size_limit_reached
return
end
# Disable the CI check if merge_when_build_succeeds is enabled since we have
# to wait until CI completes to know
unless @merge_request.mergeable?(skip_ci_check: merge_when_build_succeeds_active?)
......
......@@ -16,7 +16,11 @@ module LfsHelper
return if upload_request? && lfs_upload_access?
if project.public? || (user && user.can?(:read_project, project))
render_lfs_forbidden
if project.above_size_limit? || objects_exceeded_limit?
render_size_error
else
render_lfs_forbidden
end
else
render_lfs_not_found
end
......@@ -30,10 +34,27 @@ module LfsHelper
def lfs_upload_access?
return false unless project.lfs_enabled?
return false if project.above_size_limit? || objects_exceed_repo_limit?
user && user.can?(:push_code, project)
end
def objects_exceed_repo_limit?
return false unless project.size_limit_enabled?
objects_size = 0
objects.each do |object|
objects_size += object[:size]
end
@limit_exceeded = true if (project.aggregated_repository_size + objects_size.to_mb) > project.repo_size_limit
end
def objects_exceeded_limit?
@limit_exceeded ||= false
end
def render_lfs_forbidden
render(
json: {
......@@ -56,6 +77,17 @@ module LfsHelper
)
end
def render_size_error
render(
json: {
message: 'This repository has exceeded its storage limit. Please contact your GitLab admin.',
documentation_url: "#{Gitlab.config.gitlab.url}/help",
},
content_type: "application/vnd.git-lfs+json",
status: 406
)
end
def storage_project
@storage_project ||= begin
result = project
......
......@@ -63,6 +63,10 @@ class ApplicationSetting < ActiveRecord::Base
presence: true,
numericality: { only_integer: true, greater_than: 0 }
validates :repository_size_limit,
presence: true,
numericality: { only_integer: true, greater_than_or_equal_to: 0 }
validates :container_registry_token_expire_delay,
presence: true,
numericality: { only_integer: true, greater_than: 0 }
......
......@@ -33,6 +33,9 @@ class Group < Namespace
validates :avatar, file_size: { maximum: 200.kilobytes.to_i }
validates :repository_size_limit,
numericality: { only_integer: true, greater_than_or_equal_to: 0, allow_nil: true }
mount_uploader :avatar, AvatarUploader
after_create :post_create_hook
......@@ -199,6 +202,12 @@ class Group < Namespace
system_hook_service.execute_hooks_for(self, :destroy)
end
def repo_size_limit
return current_application_settings.repository_size_limit if repository_size_limit.nil?
repository_size_limit
end
def system_hook_service
SystemHooksService.new
end
......
......@@ -289,6 +289,10 @@ class MergeRequest < ActiveRecord::Base
end
end
def is_valid?
!(target_project.above_size_limit? || target_project.blank? || source_branch.blank?)
end
def branch_merge_base_sha
branch_merge_base_commit.try(:sha)
end
......
......@@ -147,6 +147,10 @@ class Namespace < ActiveRecord::Base
Gitlab.config.lfs.enabled
end
def repo_size_limit
current_application_settings.repository_size_limit
end
private
def repository_storage_paths
......
......@@ -182,6 +182,9 @@ class Project < ActiveRecord::Base
presence: true,
inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
validates :repository_size_limit,
numericality: { only_integer: true, greater_than_or_equal_to: 0, allow_nil: true }
with_options if: :mirror? do |project|
project.validates :import_url, presence: true
project.validates :mirror_user, presence: true
......@@ -1537,6 +1540,30 @@ class Project < ActiveRecord::Base
Gitlab::Redis.with { |redis| redis.del(pushes_since_gc_redis_key) }
end
def aggregated_repository_size
repository_size + lfs_objects.sum(:size).to_i.to_mb
end
def above_size_limit?
return false if repo_size_limit == 0
aggregated_repository_size > repo_size_limit
end
def size_to_remove
aggregated_repository_size - repo_size_limit
end
def repo_size_limit
return namespace.repo_size_limit if repository_size_limit.nil?
repository_size_limit
end
def size_limit_enabled?
repo_size_limit != 0
end
private
def pushes_since_gc_redis_key
......
......@@ -34,6 +34,10 @@ module Files
error(ex.message)
end
def size_limit_error_message
"Your changes could not be committed, because this repository has exceeded its size limit of #{project.repo_size_limit}MB by #{project.size_to_remove}MB"
end
private
def different_branch?
......
......@@ -9,6 +9,10 @@ module Files
def validate
super
if project.above_size_limit?
raise_error(size_limit_error_message)
end
unless @file_path =~ Gitlab::Regex.file_path_regex
raise_error(
'Your changes could not be committed, because the file path ' +
......
......@@ -9,6 +9,10 @@ module Files
def validate
super
if project.above_size_limit?
raise_error(size_limit_error_message)
end
if @file_path =~ Gitlab::Regex.directory_traversal_regex
raise_error(
'Your changes could not be committed, because the file name ' +
......
......@@ -16,6 +16,10 @@ module Files
def validate
super
if project.above_size_limit?
raise_error(size_limit_error_message)
end
if file_has_changed?
raise FileChangedError.new("You are attempting to update a file that has changed since you started editing it.")
end
......
......@@ -35,6 +35,10 @@ module MergeRequests
end
end
def render_size_limit_message(project)
"The target's repository size (#{project.aggregated_repository_size}MB) exceeds the limit of #{project.repo_size_limit}MB by #{project.size_to_remove}MB"
end
private
def filter_params
......
......@@ -13,15 +13,23 @@ module MergeRequests
merge_request.target_project ||= (project.forked_from_project || project)
merge_request.target_branch ||= merge_request.target_project.default_branch
if merge_request.target_project.above_size_limit?
message = render_size_limit_message(merge_request.target_project)
merge_request.errors.add(:base, message)
end
if merge_request.target_branch.blank? || merge_request.source_branch.blank?
message =
if params[:source_branch] || params[:target_branch]
"You must select source and target branch"
end
return build_failed(merge_request, message)
merge_request.errors.add(:base, message) unless message.nil?
end
return build_failed(merge_request) unless merge_request.is_valid?
compare = CompareService.new.execute(
merge_request.source_project,
merge_request.source_branch,
......@@ -97,8 +105,7 @@ module MergeRequests
merge_request
end
def build_failed(merge_request, message)
merge_request.errors.add(:base, message) unless message.nil?
def build_failed(merge_request)
merge_request.compare_commits = []
merge_request.can_be_created = false
merge_request
......
......@@ -98,6 +98,11 @@
= f.label :max_attachment_size, 'Maximum attachment size (MB)', class: 'control-label col-sm-2'
.col-sm-10
= f.number_field :max_attachment_size, class: 'form-control'
.form-group
= f.label :repository_size_limit, 'Per repository size limit (MB)', class: 'control-label col-sm-2'
.col-sm-10
= f.number_field :repository_size_limit, class: 'form-control', min: 0
%span.help-block#repository_size_limit_help_block Includes LFS objects. It can be overridden per group, or per project. 0 for unlimited
.form-group
= f.label :session_expire_delay, 'Session duration (minutes)', class: 'control-label col-sm-2'
.col-sm-10
......
......@@ -2,6 +2,8 @@
= form_errors(@group)
= render 'shared/group_form', f: f
= render 'groups/repository_size_limit_setting', f: f
.form-group.group-description-holder
= f.label :avatar, "Group avatar", class: 'control-label'
.col-sm-10
......
- if current_user.admin?
.form-group
= f.label :repository_size_limit, class: 'control-label' do
Repository size limit (MB)
.col-sm-10
= f.number_field :repository_size_limit, class: 'form-control', min: 0
%span.help-block#repository_size_limit_help_block
Repositories within this group will be restricted to this maximum size (includes LFS objects). Can be overridden inside each project. 0 for unlimited.
......@@ -6,6 +6,8 @@
= form_errors(@group)
= render 'shared/group_form', f: f
= render 'repository_size_limit_setting', f: f
.form-group
.col-sm-offset-2.col-sm-10
= image_tag group_icon(@group), alt: '', class: 'avatar group-avatar s160'
......
......@@ -34,6 +34,14 @@
= visibility_level_label(@project.visibility_level)
.light= visibility_level_description(@project.visibility_level, @project)
- if current_user.admin?
.form-group
= f.label :repository_size_limit, class: 'label-light' do
Repository size limit (MB)
= f.number_field :repository_size_limit, class: 'form-control', min: 0
%span.help-block#repository_size_limit_help_block
This project's repository will be restricted to this maximum size (includes LFS objects). 0 for unlimited.
.form-group
= render 'shared/allow_request_access', form: f
......
......@@ -11,6 +11,9 @@
- when :sha_mismatch
:plain
$('.mr-widget-body').html("#{escape_javascript(render('projects/merge_requests/widget/open/sha_mismatch'))}");
- when :size_limit_reached
:plain
$('.mr-widget-body').html("#{escape_javascript(render('projects/merge_requests/widget/open/size_limit_reached'))}");
- else
:plain
$('.mr-widget-body').html("#{escape_javascript(render('projects/merge_requests/widget/open/reload'))}");
%h4.size-limit-reached
= icon("exclamation-triangle")
This repository has reached its size limit.
%p
Please contact your GitLab administrator for more information.
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddRepositorySizeLimitToApplicationSettings < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :application_settings, :repository_size_limit, :integer, default: 0
end
end
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddRepositorySizeLimitToProjects < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :projects, :repository_size_limit, :integer
end
end
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddRepositorySizeLimitToNamespaces < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :namespaces, :repository_size_limit, :integer
end
end
......@@ -100,6 +100,7 @@ ActiveRecord::Schema.define(version: 20160915201649) do
t.boolean "usage_ping_enabled", default: true, null: false
t.boolean "koding_enabled"
t.string "koding_url"
t.integer "repository_size_limit", default: 0
end
create_table "approvals", force: :cascade do |t|
......@@ -736,6 +737,7 @@ ActiveRecord::Schema.define(version: 20160915201649) do
t.datetime "ldap_sync_last_successful_update_at"
t.datetime "ldap_sync_last_sync_at"
t.boolean "lfs_enabled"
t.integer "repository_size_limit"
end
add_index "namespaces", ["created_at"], name: "index_namespaces_on_created_at", using: :btree
......@@ -956,6 +958,7 @@ ActiveRecord::Schema.define(version: 20160915201649) do
t.boolean "has_external_wiki"
t.boolean "repository_read_only"
t.boolean "lfs_enabled"
t.integer "repository_size_limit"
end
add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
......
......@@ -70,6 +70,8 @@ module Gitlab
return build_status_object(true) if git_annex_branch_sync?(changes)
if user
return build_status_object(false, above_size_limit_message) if project.above_size_limit?
user_push_access_check(changes)
elsif deploy_key
build_status_object(false, "Deploy keys are not allowed to push code.")
......@@ -102,6 +104,8 @@ module Gitlab
changes_list = Gitlab::ChangesList.new(changes)
push_size_in_bytes = 0
# Iterate over all changes to find if user allowed all of them to be applied
changes_list.each do |change|
status = change_access_check(change)
......@@ -109,11 +113,44 @@ module Gitlab
# If user does not have access to make at least one change - cancel all push
return status
end
if project.size_limit_enabled?
push_size_in_bytes += delta_size_check(change, project.repository)
end
end
if project.size_limit_enabled? && changes_above_limit(push_size_in_bytes.to_mb)
return build_status_object(false, will_go_over_limit_message)
end
build_status_object(true)
end
def delta_size_check(change, repo)
oldrev, newrev = change.values_at(:oldrev, :newrev)
size_of_deltas = 0
begin
tree_a = repo.lookup(oldrev)
tree_b = repo.lookup(newrev)
diff = tree_a.diff(tree_b)
diff.each_delta do |d|
new_file_size = d.deleted? ? 0 : Gitlab::Git::Blob.raw(repo, d.new_file[:oid]).size
size_of_deltas += new_file_size
end
size_of_deltas
rescue Rugged::OdbError, Rugged::ReferenceError, Rugged::InvalidError
size_of_deltas
end
end
def changes_above_limit(size_mb)
size_mb > project.repo_size_limit || size_mb + project.aggregated_repository_size > project.repo_size_limit
end
def change_access_check(change)
Checks::ChangeAccess.new(change, user_access: user_access, project: project).exec
end
......@@ -122,6 +159,21 @@ module Gitlab
Gitlab::ProtocolAccess.allowed?(protocol)
end
def above_size_limit_message
[
"This repository's size (#{project.aggregated_repository_size}MB) exceeds the limit of #{project.repo_size_limit}MB",
"GitLab: by #{project.size_to_remove}MB and as a result you are unable to push to it.",
"GitLab: Please contact your Gitlab administrator for more information.",
].join("\n") + "\n"
end
def will_go_over_limit_message
[
"Your push to this repository would cause it to exceed the limit of #{project.repo_size_limit}MB.",
"GitLab: As a result it has been rejected. Please contact your Gitlab administrator for more information.",
].join("\n") + "\n"
end
def matching_merge_request?(newrev, branch_name)
Checks::MatchingMergeRequest.new(newrev, branch_name, project).match?
end
......
......@@ -355,6 +355,18 @@ describe Projects::MergeRequestsController do
end
end
context 'when the repository is above size limit' do
before do
allow_any_instance_of(Project).to receive(:above_size_limit?).and_return(true)
post :merge, base_params.merge(sha: merge_request.diff_head_sha)
end
it 'returns :size_limit_reached' do
expect(assigns(:status)).to eq(:size_limit_reached)
end
end
context 'when the merge request is not mergeable' do
before do
merge_request.update_attributes(title: "WIP: #{merge_request.title}")
......
......@@ -581,6 +581,24 @@ describe Gitlab::GitAccess, lib: true do
expect(access.push_access_check('cfe32cf61b73a0d5e9f13e774abde7ff789b1660 913c66a37b4a45b9769037c55c2d238bd0942d2e refs/heads/master')).to be_allowed
end
end
describe 'repository size restrictions' do
before do
project.update_attribute(:repository_size_limit, 50)
end
it 'returns false when blob is too big' do
allow_any_instance_of(Gitlab::Git::Blob).to receive(:size).and_return(100.megabytes.to_i)
expect(access.push_access_check('cfe32cf61b73a0d5e9f13e774abde7ff789b1660 913c66a37b4a45b9769037c55c2d238bd0942d2e refs/heads/master')).not_to be_allowed
end
it 'returns true when blob is just right' do
allow_any_instance_of(Gitlab::Git::Blob).to receive(:size).and_return(2.megabytes.to_i)
expect(access.push_access_check('cfe32cf61b73a0d5e9f13e774abde7ff789b1660 913c66a37b4a45b9769037c55c2d238bd0942d2e refs/heads/master')).to be_allowed
end
end
end
end
......
......@@ -85,4 +85,22 @@ describe Group, models: true do
expect { group.mark_ldap_sync_as_failed('Error') }.not_to raise_error
end
end
describe '#repo_size_limit' do
let(:group) { build(:group) }
before do
allow_any_instance_of(ApplicationSetting).to receive(:repository_size_limit).and_return(50)
end
it 'returns the value set globally' do
expect(group.repo_size_limit).to eq(50)
end
it 'returns the value set locally' do
group.update_attribute(:repository_size_limit, 75)
expect(group.repo_size_limit).to eq(75)
end
end
end
......@@ -479,6 +479,67 @@ describe Project, models: true do
end
end
describe 'repository size restrictions' do
let(:project) { build(:empty_project) }
before do
allow_any_instance_of(ApplicationSetting).to receive(:repository_size_limit).and_return(50)
end
describe '#repo_size_limit' do
it 'returns the limit set in the application settings' do
expect(project.repo_size_limit).to eq(50)
end
it 'returns the value set in the group' do
group = create(:group, repository_size_limit: 100)
project.update_attribute(:namespace_id, group.id)
expect(project.repo_size_limit).to eq(100)
end
it 'returns the value set locally' do
project.update_attribute(:repository_size_limit, 75)
expect(project.repo_size_limit).to eq(75)
end
end
describe '#size_limit_enabled?' do
it 'returns false when disabled' do
project.update_attribute(:repository_size_limit, 0)
expect(project.size_limit_enabled?).to be_falsey
end
it 'returns true when a limit is set' do
project.update_attribute(:repository_size_limit, 75)
expect(project.size_limit_enabled?).to be_truthy
end
end
describe '#above_size_limit?' do
it 'returns true when above the limit' do
allow(project).to receive(:aggregated_repository_size).and_return(100)
expect(project.above_size_limit?).to be_truthy
end
it 'returns false when not over the limit' do
expect(project.above_size_limit?).to be_falsey
end
end
describe '#size_to_remove' do
it 'returns the correct value' do
allow(project).to receive(:aggregated_repository_size).and_return(100)
expect(project.size_to_remove).to eq(50)
end
end
end
describe '#default_issues_tracker?' do
let(:project) { create(:project) }
let(:ext_project) { create(:redmine_project) }
......
......@@ -222,6 +222,22 @@ describe 'Git HTTP requests', lib: true do
end
end
context "when repository is above size limit" do
let(:env) { { user: user.username, password: user.password } }
before do
project.team << [user, :master]
end
it 'responds with status 403' do
allow_any_instance_of(Project).to receive(:above_size_limit?).and_return(true)
upload(path, env) do |response|
expect(response).to have_http_status(403)
end
end
end
context "when username and password are provided" do
let(:env) { { user: user.username, password: 'nope' } }
......
......@@ -520,7 +520,7 @@ describe 'Git LFS API and storage' do
{ 'operation' => 'upload',
'objects' => [
{ 'oid' => '91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897',
'size' => 1575078
'size' => 157507855
}]
}
end
......@@ -532,10 +532,30 @@ describe 'Git LFS API and storage' do
it 'responds with upload hypermedia link' do
expect(json_response['objects']).to be_kind_of(Array)
expect(json_response['objects'].first['oid']).to eq("91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897")
expect(json_response['objects'].first['size']).to eq(1575078)
expect(json_response['objects'].first['actions']['upload']['href']).to eq("#{Gitlab.config.gitlab.url}/#{project.path_with_namespace}.git/gitlab-lfs/objects/91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897/1575078")
expect(json_response['objects'].first['size']).to eq(157507855)
expect(json_response['objects'].first['actions']['upload']['href']).to eq("#{Gitlab.config.gitlab.url}/#{project.path_with_namespace}.git/gitlab-lfs/objects/91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897/157507855")
expect(json_response['objects'].first['actions']['upload']['header']).to eq('Authorization' => authorization)
end
context 'and project is above the limit' do
let(:update_lfs_permissions) do
allow_any_instance_of(Project).to receive(:above_size_limit?).and_return(true)
end
it 'responds with status 406' do
expect(response).to have_http_status(406)
end
end
context 'and project will go over the limit' do
let(:update_lfs_permissions) do
allow_any_instance_of(Project).to receive_messages(repo_size_limit: 145, size_limit_enabled?: true)
end
it 'responds with status 406' do
expect(response).to have_http_status(406)
end
end
end
context 'when pushing one new and one existing lfs object' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment