Commit 1a68a0a6 authored by Yorick Peterse's avatar Yorick Peterse

Merge branch 'shards' into 'master'

Implement multiple repository mount points

See merge request !4578
parents 2efee5f6 1735fdd4
...@@ -3,6 +3,8 @@ Please view this file on the master branch, on stable branches it's out of date. ...@@ -3,6 +3,8 @@ Please view this file on the master branch, on stable branches it's out of date.
v 8.10.0 (unreleased) v 8.10.0 (unreleased)
- Fix commit builds API, return all builds for all pipelines for given commit. !4849 - Fix commit builds API, return all builds for all pipelines for given commit. !4849
- Replace Haml with Hamlit to make view rendering faster. !3666 - Replace Haml with Hamlit to make view rendering faster. !3666
- Refactor repository paths handling to allow multiple git mount points
- Add Application Setting to configure default Repository Path for new projects
- Wrap code blocks on Activies and Todos page. !4783 (winniehell) - Wrap code blocks on Activies and Todos page. !4783 (winniehell)
- Align flash messages with left side of page content !4959 (winniehell) - Align flash messages with left side of page content !4959 (winniehell)
- Display last commit of deleted branch in push events !4699 (winniehell) - Display last commit of deleted branch in push events !4699 (winniehell)
......
...@@ -109,6 +109,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController ...@@ -109,6 +109,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
:metrics_packet_size, :metrics_packet_size,
:send_user_confirmation_email, :send_user_confirmation_email,
:container_registry_token_expire_delay, :container_registry_token_expire_delay,
:repository_storage,
restricted_visibility_levels: [], restricted_visibility_levels: [],
import_sources: [], import_sources: [],
disabled_oauth_sign_in_sources: [] disabled_oauth_sign_in_sources: []
......
...@@ -78,4 +78,12 @@ module ApplicationSettingsHelper ...@@ -78,4 +78,12 @@ module ApplicationSettingsHelper
end end
end end
end end
def repository_storage_options_for_select
options = Gitlab.config.repositories.storages.map do |name, path|
["#{name} - #{path}", name]
end
options_for_select(options, @application_setting.repository_storage)
end
end end
...@@ -327,9 +327,9 @@ module ProjectsHelper ...@@ -327,9 +327,9 @@ module ProjectsHelper
end end
end end
def sanitize_repo_path(message) def sanitize_repo_path(project, message)
return '' unless message.present? return '' unless message.present?
message.strip.gsub(Gitlab.config.gitlab_shell.repos_path.chomp('/'), "[REPOS PATH]") message.strip.gsub(project.repository_storage_path.chomp('/'), "[REPOS PATH]")
end end
end end
...@@ -55,6 +55,10 @@ class ApplicationSetting < ActiveRecord::Base ...@@ -55,6 +55,10 @@ class ApplicationSetting < ActiveRecord::Base
presence: true, presence: true,
numericality: { only_integer: true, greater_than: 0 } numericality: { only_integer: true, greater_than: 0 }
validates :repository_storage,
presence: true,
inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
validates_each :restricted_visibility_levels do |record, attr, value| validates_each :restricted_visibility_levels do |record, attr, value|
unless value.nil? unless value.nil?
value.each do |level| value.each do |level|
...@@ -134,6 +138,7 @@ class ApplicationSetting < ActiveRecord::Base ...@@ -134,6 +138,7 @@ class ApplicationSetting < ActiveRecord::Base
disabled_oauth_sign_in_sources: [], disabled_oauth_sign_in_sources: [],
send_user_confirmation_email: false, send_user_confirmation_email: false,
container_registry_token_expire_delay: 5, container_registry_token_expire_delay: 5,
repository_storage: 'default',
) )
end end
......
...@@ -21,8 +21,10 @@ class Namespace < ActiveRecord::Base ...@@ -21,8 +21,10 @@ class Namespace < ActiveRecord::Base
delegate :name, to: :owner, allow_nil: true, prefix: true delegate :name, to: :owner, allow_nil: true, prefix: true
after_create :ensure_dir_exist
after_update :move_dir, if: :path_changed? after_update :move_dir, if: :path_changed?
# Save the storage paths before the projects are destroyed to use them on after destroy
before_destroy(prepend: true) { @old_repository_storage_paths = repository_storage_paths }
after_destroy :rm_dir after_destroy :rm_dir
scope :root, -> { where('type IS NULL') } scope :root, -> { where('type IS NULL') }
...@@ -87,51 +89,35 @@ class Namespace < ActiveRecord::Base ...@@ -87,51 +89,35 @@ class Namespace < ActiveRecord::Base
owner_name owner_name
end end
def ensure_dir_exist
gitlab_shell.add_namespace(path)
end
def rm_dir
# Move namespace directory into trash.
# We will remove it later async
new_path = "#{path}+#{id}+deleted"
if gitlab_shell.mv_namespace(path, new_path)
message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
Gitlab::AppLogger.info message
# Remove namespace directroy async with delay so
# GitLab has time to remove all projects first
GitlabShellWorker.perform_in(5.minutes, :rm_namespace, new_path)
end
end
def move_dir def move_dir
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(path_was)
if any_project_has_container_registry_tags? if any_project_has_container_registry_tags?
raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry') raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry')
end end
if gitlab_shell.mv_namespace(path_was, path) # Move the namespace directory in all storages paths used by member projects
Gitlab::UploadsTransfer.new.rename_namespace(path_was, path) repository_storage_paths.each do |repository_storage_path|
# Ensure old directory exists before moving it
# If repositories moved successfully we need to gitlab_shell.add_namespace(repository_storage_path, path_was)
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
# So we basically we mute exceptions in next actions # if we cannot move namespace directory we should rollback
begin # db changes in order to prevent out of sync between db and fs
send_update_instructions raise Exception.new('namespace directory cannot be moved')
rescue
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end end
else end
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
raise Exception.new('namespace directory cannot be moved')
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
# So we basically we mute exceptions in next actions
begin
send_update_instructions
rescue
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end end
end end
...@@ -152,4 +138,33 @@ class Namespace < ActiveRecord::Base ...@@ -152,4 +138,33 @@ class Namespace < ActiveRecord::Base
def find_fork_of(project) def find_fork_of(project)
projects.joins(:forked_project_link).find_by('forked_project_links.forked_from_project_id = ?', project.id) projects.joins(:forked_project_link).find_by('forked_project_links.forked_from_project_id = ?', project.id)
end end
private
def repository_storage_paths
# We need to get the storage paths for all the projects, even the ones that are
# pending delete. Unscoping also get rids of the default order, which causes
# problems with SELECT DISTINCT.
Project.unscoped do
projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
end
end
def rm_dir
# Remove the namespace directory in all storages paths used by member projects
@old_repository_storage_paths.each do |repository_storage_path|
# Move namespace directory into trash.
# We will remove it later async
new_path = "#{path}+#{id}+deleted"
if gitlab_shell.mv_namespace(repository_storage_path, path, new_path)
message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
Gitlab::AppLogger.info message
# Remove namespace directroy async with delay so
# GitLab has time to remove all projects first
GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
end
end
end
end end
...@@ -24,8 +24,12 @@ class Project < ActiveRecord::Base ...@@ -24,8 +24,12 @@ class Project < ActiveRecord::Base
default_value_for :wiki_enabled, gitlab_config_features.wiki default_value_for :wiki_enabled, gitlab_config_features.wiki
default_value_for :snippets_enabled, gitlab_config_features.snippets default_value_for :snippets_enabled, gitlab_config_features.snippets
default_value_for :container_registry_enabled, gitlab_config_features.container_registry default_value_for :container_registry_enabled, gitlab_config_features.container_registry
default_value_for(:repository_storage) { current_application_settings.repository_storage }
default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled } default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled }
after_create :ensure_dir_exist
after_save :ensure_dir_exist, if: :namespace_id_changed?
# set last_activity_at to the same as created_at # set last_activity_at to the same as created_at
after_create :set_last_activity_at after_create :set_last_activity_at
def set_last_activity_at def set_last_activity_at
...@@ -165,6 +169,9 @@ class Project < ActiveRecord::Base ...@@ -165,6 +169,9 @@ class Project < ActiveRecord::Base
validate :visibility_level_allowed_by_group validate :visibility_level_allowed_by_group
validate :visibility_level_allowed_as_fork validate :visibility_level_allowed_as_fork
validate :check_wiki_path_conflict validate :check_wiki_path_conflict
validates :repository_storage,
presence: true,
inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
add_authentication_token_field :runners_token add_authentication_token_field :runners_token
before_save :ensure_runners_token before_save :ensure_runners_token
...@@ -376,6 +383,10 @@ class Project < ActiveRecord::Base ...@@ -376,6 +383,10 @@ class Project < ActiveRecord::Base
end end
end end
def repository_storage_path
Gitlab.config.repositories.storages[repository_storage]
end
def team def team
@team ||= ProjectTeam.new(self) @team ||= ProjectTeam.new(self)
end end
...@@ -842,12 +853,12 @@ class Project < ActiveRecord::Base ...@@ -842,12 +853,12 @@ class Project < ActiveRecord::Base
raise Exception.new('Project cannot be renamed, because tags are present in its container registry') raise Exception.new('Project cannot be renamed, because tags are present in its container registry')
end end
if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace) if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to send update instructions to users. # If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository # However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions # So we basically we mute exceptions in next actions
begin begin
gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
send_move_instructions(old_path_with_namespace) send_move_instructions(old_path_with_namespace)
reset_events_cache reset_events_cache
...@@ -988,7 +999,7 @@ class Project < ActiveRecord::Base ...@@ -988,7 +999,7 @@ class Project < ActiveRecord::Base
def create_repository def create_repository
# Forked import is handled asynchronously # Forked import is handled asynchronously
unless forked? unless forked?
if gitlab_shell.add_repository(path_with_namespace) if gitlab_shell.add_repository(repository_storage_path, path_with_namespace)
repository.after_create repository.after_create
true true
else else
...@@ -1140,4 +1151,8 @@ class Project < ActiveRecord::Base ...@@ -1140,4 +1151,8 @@ class Project < ActiveRecord::Base
_, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete)) _, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete))
status.zero? status.zero?
end end
def ensure_dir_exist
gitlab_shell.add_namespace(repository_storage_path, namespace.path)
end
end end
...@@ -159,7 +159,7 @@ class ProjectWiki ...@@ -159,7 +159,7 @@ class ProjectWiki
private private
def init_repo(path_with_namespace) def init_repo(path_with_namespace)
gitlab_shell.add_repository(path_with_namespace) gitlab_shell.add_repository(project.repository_storage_path, path_with_namespace)
end end
def commit_details(action, message = nil, title = nil) def commit_details(action, message = nil, title = nil)
...@@ -173,7 +173,7 @@ class ProjectWiki ...@@ -173,7 +173,7 @@ class ProjectWiki
end end
def path_to_repo def path_to_repo
@path_to_repo ||= File.join(Gitlab.config.gitlab_shell.repos_path, "#{path_with_namespace}.git") @path_to_repo ||= File.join(project.repository_storage_path, "#{path_with_namespace}.git")
end end
def update_project_activity def update_project_activity
......
...@@ -39,7 +39,7 @@ class Repository ...@@ -39,7 +39,7 @@ class Repository
# Return absolute path to repository # Return absolute path to repository
def path_to_repo def path_to_repo
@path_to_repo ||= File.expand_path( @path_to_repo ||= File.expand_path(
File.join(Gitlab.config.gitlab_shell.repos_path, path_with_namespace + ".git") File.join(@project.repository_storage_path, path_with_namespace + ".git")
) )
end end
......
...@@ -51,13 +51,13 @@ module Projects ...@@ -51,13 +51,13 @@ module Projects
return true if params[:skip_repo] == true return true if params[:skip_repo] == true
# There is a possibility project does not have repository or wiki # There is a possibility project does not have repository or wiki
return true unless gitlab_shell.exists?(path + '.git') return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git')
new_path = removal_path(path) new_path = removal_path(path)
if gitlab_shell.mv_repository(path, new_path) if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path)
log_info("Repository \"#{path}\" moved to \"#{new_path}\"") log_info("Repository \"#{path}\" moved to \"#{new_path}\"")
GitlabShellWorker.perform_in(5.minutes, :remove_repository, new_path) GitlabShellWorker.perform_in(5.minutes, :remove_repository, project.repository_storage_path, new_path)
else else
false false
end end
......
...@@ -24,7 +24,7 @@ module Projects ...@@ -24,7 +24,7 @@ module Projects
def execute def execute
raise LeaseTaken unless try_obtain_lease raise LeaseTaken unless try_obtain_lease
GitlabShellOneShotWorker.perform_async(:gc, @project.path_with_namespace) GitlabShellOneShotWorker.perform_async(:gc, @project.repository_storage_path, @project.path_with_namespace)
ensure ensure
Gitlab::Metrics.measure(:reset_pushes_since_gc) do Gitlab::Metrics.measure(:reset_pushes_since_gc) do
@project.update_column(:pushes_since_gc, 0) @project.update_column(:pushes_since_gc, 0)
......
...@@ -42,7 +42,7 @@ module Projects ...@@ -42,7 +42,7 @@ module Projects
def import_repository def import_repository
begin begin
gitlab_shell.import_repository(project.path_with_namespace, project.import_url) gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url)
rescue Gitlab::Shell::Error => e rescue Gitlab::Shell::Error => e
raise Error, "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}" raise Error, "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
end end
......
...@@ -50,12 +50,12 @@ module Projects ...@@ -50,12 +50,12 @@ module Projects
project.send_move_instructions(old_path) project.send_move_instructions(old_path)
# Move main repository # Move main repository
unless gitlab_shell.mv_repository(old_path, new_path) unless gitlab_shell.mv_repository(project.repository_storage_path, old_path, new_path)
raise TransferError.new('Cannot move project') raise TransferError.new('Cannot move project')
end end
# Move wiki repo also if present # Move wiki repo also if present
gitlab_shell.mv_repository("#{old_path}.wiki", "#{new_path}.wiki") gitlab_shell.mv_repository(project.repository_storage_path, "#{old_path}.wiki", "#{new_path}.wiki")
# clear project cached events # clear project cached events
project.reset_events_cache project.reset_events_cache
......
...@@ -310,6 +310,15 @@ ...@@ -310,6 +310,15 @@
.col-sm-10 .col-sm-10
= f.text_field :sentry_dsn, class: 'form-control' = f.text_field :sentry_dsn, class: 'form-control'
%fieldset
%legend Repository Storage
.form-group
= f.label :repository_storage, 'Storage path for new projects', class: 'control-label col-sm-2'
.col-sm-10
= f.select :repository_storage, repository_storage_options_for_select, {}, class: 'form-control'
.help-block
You can manage the repository storage paths in your gitlab.yml configuration file
%fieldset %fieldset
%legend Repository Checks %legend Repository Checks
.form-group .form-group
......
...@@ -10,7 +10,7 @@ ...@@ -10,7 +10,7 @@
.panel-body .panel-body
%pre %pre
:preserve :preserve
#{sanitize_repo_path(@project.import_error)} #{sanitize_repo_path(@project, @project.import_error)}
= form_for @project, url: namespace_project_import_path(@project.namespace, @project), method: :post, html: { class: 'form-horizontal' } do |f| = form_for @project, url: namespace_project_import_path(@project.namespace, @project), method: :post, html: { class: 'form-horizontal' } do |f|
= render "shared/import_form", f: f = render "shared/import_form", f: f
......
...@@ -4,10 +4,10 @@ class PostReceive ...@@ -4,10 +4,10 @@ class PostReceive
sidekiq_options queue: :post_receive sidekiq_options queue: :post_receive
def perform(repo_path, identifier, changes) def perform(repo_path, identifier, changes)
if repo_path.start_with?(Gitlab.config.gitlab_shell.repos_path.to_s) if path = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1].to_s) }
repo_path.gsub!(Gitlab.config.gitlab_shell.repos_path.to_s, "") repo_path.gsub!(path[1].to_s, "")
else else
log("Check gitlab.yml config for correct gitlab_shell.repos_path variable. \"#{Gitlab.config.gitlab_shell.repos_path}\" does not match \"#{repo_path}\"") log("Check gitlab.yml config for correct repositories.storages values. No repository storage path matches \"#{repo_path}\"")
end end
post_received = Gitlab::GitPostReceive.new(repo_path, identifier, changes) post_received = Gitlab::GitPostReceive.new(repo_path, identifier, changes)
......
...@@ -12,7 +12,7 @@ class RepositoryForkWorker ...@@ -12,7 +12,7 @@ class RepositoryForkWorker
return return
end end
result = gitlab_shell.fork_repository(source_path, target_path) result = gitlab_shell.fork_repository(project.repository_storage_path, source_path, target_path)
unless result unless result
logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}") logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}")
project.mark_import_as_failed('The project could not be forked.') project.mark_import_as_failed('The project could not be forked.')
......
...@@ -47,11 +47,13 @@ production: &base ...@@ -47,11 +47,13 @@ production: &base
backup: backup:
path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/) path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/)
repositories:
storages: # REPO PATHS MUST NOT BE A SYMLINK!!!
default: /apps/repositories/
gitlab_shell: gitlab_shell:
path: /apps/gitlab-shell/ path: /apps/gitlab-shell/
# REPOS_PATH MUST NOT BE A SYMLINK!!!
repos_path: /apps/repositories/
hooks_path: /apps/gitlab-shell/hooks/ hooks_path: /apps/gitlab-shell/hooks/
upload_pack: true upload_pack: true
......
...@@ -428,6 +428,13 @@ production: &base ...@@ -428,6 +428,13 @@ production: &base
satellites: satellites:
path: /home/git/gitlab-satellites/ path: /home/git/gitlab-satellites/
## Repositories settings
repositories:
# Paths where repositories can be stored. Give the canonicalized absolute pathname.
# NOTE: REPOS PATHS MUST NOT CONTAIN ANY SYMLINK!!!
storages: # You must have at least a `default` storage path.
default: /home/git/repositories/
## Backup settings ## Backup settings
backup: backup:
path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/) path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/)
...@@ -452,9 +459,6 @@ production: &base ...@@ -452,9 +459,6 @@ production: &base
## GitLab Shell settings ## GitLab Shell settings
gitlab_shell: gitlab_shell:
path: /home/git/gitlab-shell/ path: /home/git/gitlab-shell/
# REPOS_PATH MUST NOT BE A SYMLINK!!!
repos_path: /home/git/repositories/
hooks_path: /home/git/gitlab-shell/hooks/ hooks_path: /home/git/gitlab-shell/hooks/
# File that contains the secret key for verifying access for gitlab-shell. # File that contains the secret key for verifying access for gitlab-shell.
...@@ -528,11 +532,13 @@ test: ...@@ -528,11 +532,13 @@ test:
# user: YOUR_USERNAME # user: YOUR_USERNAME
satellites: satellites:
path: tmp/tests/gitlab-satellites/ path: tmp/tests/gitlab-satellites/
repositories:
storages:
default: tmp/tests/repositories/
backup: backup:
path: tmp/tests/backups path: tmp/tests/backups
gitlab_shell: gitlab_shell:
path: tmp/tests/gitlab-shell/ path: tmp/tests/gitlab-shell/
repos_path: tmp/tests/repositories/
hooks_path: tmp/tests/gitlab-shell/hooks/ hooks_path: tmp/tests/gitlab-shell/hooks/
issues_tracker: issues_tracker:
redmine: redmine:
......
...@@ -304,13 +304,20 @@ Settings.gitlab_shell['hooks_path'] ||= Settings.gitlab['user_home'] + '/gitla ...@@ -304,13 +304,20 @@ Settings.gitlab_shell['hooks_path'] ||= Settings.gitlab['user_home'] + '/gitla
Settings.gitlab_shell['secret_file'] ||= Rails.root.join('.gitlab_shell_secret') Settings.gitlab_shell['secret_file'] ||= Rails.root.join('.gitlab_shell_secret')
Settings.gitlab_shell['receive_pack'] = true if Settings.gitlab_shell['receive_pack'].nil? Settings.gitlab_shell['receive_pack'] = true if Settings.gitlab_shell['receive_pack'].nil?
Settings.gitlab_shell['upload_pack'] = true if Settings.gitlab_shell['upload_pack'].nil? Settings.gitlab_shell['upload_pack'] = true if Settings.gitlab_shell['upload_pack'].nil?
Settings.gitlab_shell['repos_path'] ||= Settings.gitlab['user_home'] + '/repositories/'
Settings.gitlab_shell['ssh_host'] ||= Settings.gitlab.ssh_host Settings.gitlab_shell['ssh_host'] ||= Settings.gitlab.ssh_host
Settings.gitlab_shell['ssh_port'] ||= 22 Settings.gitlab_shell['ssh_port'] ||= 22
Settings.gitlab_shell['ssh_user'] ||= Settings.gitlab.user Settings.gitlab_shell['ssh_user'] ||= Settings.gitlab.user
Settings.gitlab_shell['owner_group'] ||= Settings.gitlab.user Settings.gitlab_shell['owner_group'] ||= Settings.gitlab.user
Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.send(:build_gitlab_shell_ssh_path_prefix) Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.send(:build_gitlab_shell_ssh_path_prefix)
#
# Repositories
#
Settings['repositories'] ||= Settingslogic.new({})
Settings.repositories['storages'] ||= {}
# Setting gitlab_shell.repos_path is DEPRECATED and WILL BE REMOVED in version 9.0
Settings.repositories.storages['default'] ||= Settings.gitlab_shell['repos_path'] || Settings.gitlab['user_home'] + '/repositories/'
# #
# Backup # Backup
# #
......
def storage_name_valid?(name)
!!(name =~ /\A[a-zA-Z0-9\-_]+\z/)
end
def find_parent_path(name, path)
Gitlab.config.repositories.storages.detect do |n, p|
name != n && path.chomp('/').start_with?(p.chomp('/'))
end
end
def error(message)
raise "#{message}. Please fix this in your gitlab.yml before starting GitLab."
end
error('No repository storage path defined') if Gitlab.config.repositories.storages.empty?
Gitlab.config.repositories.storages.each do |name, path|
error("\"#{name}\" is not a valid storage name") unless storage_name_valid?(name)
parent_name, _parent_path = find_parent_path(name, path)
if parent_name
error("#{name} is a nested path of #{parent_name}. Nested paths are not supported for repository storages")
end
end
# Be sure to restart your server when you modify this file. Gitlab::Shell.new.generate_and_link_secret_token
require 'securerandom'
# Your secret key for verifying the gitlab_shell.
secret_file = Gitlab.config.gitlab_shell.secret_file
unless File.exist? secret_file
# Generate a new token of 16 random hexadecimal characters and store it in secret_file.
token = SecureRandom.hex(16)
File.write(secret_file, token)
end
link_path = File.join(Gitlab.config.gitlab_shell.path, '.gitlab_shell_secret')
if File.exist?(Gitlab.config.gitlab_shell.path) && !File.exist?(link_path)
FileUtils.symlink(secret_file, link_path)
end
class AddRepositoryStorageToProjects < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
def up
add_column_with_default(:projects, :repository_storage, :string, default: 'default')
end
def down
remove_column(:projects, :repository_storage)
end
end
class AddRepositoryStorageToApplicationSettings < ActiveRecord::Migration
def change
add_column :application_settings, :repository_storage, :string, default: 'default'
end
end
...@@ -85,6 +85,7 @@ ActiveRecord::Schema.define(version: 20160620115026) do ...@@ -85,6 +85,7 @@ ActiveRecord::Schema.define(version: 20160620115026) do
t.boolean "send_user_confirmation_email", default: false t.boolean "send_user_confirmation_email", default: false
t.integer "container_registry_token_expire_delay", default: 5 t.integer "container_registry_token_expire_delay", default: 5
t.text "after_sign_up_text" t.text "after_sign_up_text"
t.string "repository_storage", default: "default"
end end
create_table "audit_events", force: :cascade do |t| create_table "audit_events", force: :cascade do |t|
...@@ -796,38 +797,39 @@ ActiveRecord::Schema.define(version: 20160620115026) do ...@@ -796,38 +797,39 @@ ActiveRecord::Schema.define(version: 20160620115026) do
t.datetime "created_at" t.datetime "created_at"
t.datetime "updated_at" t.datetime "updated_at"
t.integer "creator_id" t.integer "creator_id"
t.boolean "issues_enabled", default: true, null: false t.boolean "issues_enabled", default: true, null: false
t.boolean "merge_requests_enabled", default: true, null: false t.boolean "merge_requests_enabled", default: true, null: false
t.boolean "wiki_enabled", default: true, null: false t.boolean "wiki_enabled", default: true, null: false
t.integer "namespace_id" t.integer "namespace_id"
t.boolean "snippets_enabled", default: true, null: false t.boolean "snippets_enabled", default: true, null: false
t.datetime "last_activity_at" t.datetime "last_activity_at"
t.string "import_url" t.string "import_url"
t.integer "visibility_level", default: 0, null: false t.integer "visibility_level", default: 0, null: false
t.boolean "archived", default: false, null: false t.boolean "archived", default: false, null: false
t.string "avatar" t.string "avatar"
t.string "import_status" t.string "import_status"
t.float "repository_size", default: 0.0 t.float "repository_size", default: 0.0
t.integer "star_count", default: 0, null: false t.integer "star_count", default: 0, null: false
t.string "import_type" t.string "import_type"
t.string "import_source" t.string "import_source"
t.integer "commit_count", default: 0 t.integer "commit_count", default: 0
t.text "import_error" t.text "import_error"
t.integer "ci_id" t.integer "ci_id"
t.boolean "builds_enabled", default: true, null: false t.boolean "builds_enabled", default: true, null: false
t.boolean "shared_runners_enabled", default: true, null: false t.boolean "shared_runners_enabled", default: true, null: false
t.string "runners_token" t.string "runners_token"
t.string "build_coverage_regex" t.string "build_coverage_regex"
t.boolean "build_allow_git_fetch", default: true, null: false t.boolean "build_allow_git_fetch", default: true, null: false
t.integer "build_timeout", default: 3600, null: false t.integer "build_timeout", default: 3600, null: false
t.boolean "pending_delete", default: false t.boolean "pending_delete", default: false
t.boolean "public_builds", default: true, null: false t.boolean "public_builds", default: true, null: false
t.integer "pushes_since_gc", default: 0 t.integer "pushes_since_gc", default: 0
t.boolean "last_repository_check_failed" t.boolean "last_repository_check_failed"
t.datetime "last_repository_check_at" t.datetime "last_repository_check_at"
t.boolean "container_registry_enabled" t.boolean "container_registry_enabled"
t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false
t.boolean "has_external_issue_tracker" t.boolean "has_external_issue_tracker"
t.string "repository_storage", default: "default", null: false
end end
add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree
......
...@@ -34,6 +34,7 @@ ...@@ -34,6 +34,7 @@
- [Operations](operations/README.md) Keeping GitLab up and running. - [Operations](operations/README.md) Keeping GitLab up and running.
- [Raketasks](raketasks/README.md) Backups, maintenance, automatic webhook setup and the importing of projects. - [Raketasks](raketasks/README.md) Backups, maintenance, automatic webhook setup and the importing of projects.
- [Repository checks](administration/repository_checks.md) Periodic Git repository checks. - [Repository checks](administration/repository_checks.md) Periodic Git repository checks.
- [Repository storages](administration/repository_storages.md) Manage the paths used to store repositories.
- [Security](security/README.md) Learn what you can do to further secure your GitLab instance. - [Security](security/README.md) Learn what you can do to further secure your GitLab instance.
- [System hooks](system_hooks/system_hooks.md) Notifications when users, projects and keys are changed. - [System hooks](system_hooks/system_hooks.md) Notifications when users, projects and keys are changed.
- [Update](update/README.md) Update guides to upgrade your installation. - [Update](update/README.md) Update guides to upgrade your installation.
......
# Repository storages
GitLab allows you to define repository storage paths to enable distribution of
storage load between several mount points.
## For installations from source
Add your repository storage paths in your `gitlab.yml` under repositories -> storages, using key -> value pairs.
>**Notes:**
- You must have at least one storage path called `default`.
- In order for backups to work correctly the storage path must **not** be a
mount point and the GitLab user should have correct permissions for the parent
directory of the path.
## For omnibus installations
Follow the instructions at https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/configuration.md#storing-git-data-in-an-alternative-directory
...@@ -38,7 +38,8 @@ Example response: ...@@ -38,7 +38,8 @@ Example response:
"default_project_visibility" : 0, "default_project_visibility" : 0,
"gravatar_enabled" : true, "gravatar_enabled" : true,
"sign_in_text" : null, "sign_in_text" : null,
"container_registry_token_expire_delay": 5 "container_registry_token_expire_delay": 5,
"repository_storage": "default"
} }
``` ```
...@@ -66,6 +67,7 @@ PUT /application/settings ...@@ -66,6 +67,7 @@ PUT /application/settings
| `user_oauth_applications` | boolean | no | Allow users to register any application to use GitLab as an OAuth provider | | `user_oauth_applications` | boolean | no | Allow users to register any application to use GitLab as an OAuth provider |
| `after_sign_out_path` | string | no | Where to redirect users after logout | | `after_sign_out_path` | string | no | Where to redirect users after logout |
| `container_registry_token_expire_delay` | integer | no | Container Registry token duration in minutes | | `container_registry_token_expire_delay` | integer | no | Container Registry token duration in minutes |
| `repository_storage` | string | no | Storage path for new projects. The value should be the name of one of the repository storage paths defined in your gitlab.yml |
```bash ```bash
curl -X PUT -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v3/application/settings?signup_enabled=false&default_project_visibility=1 curl -X PUT -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v3/application/settings?signup_enabled=false&default_project_visibility=1
...@@ -93,6 +95,7 @@ Example response: ...@@ -93,6 +95,7 @@ Example response:
"restricted_signup_domains": [], "restricted_signup_domains": [],
"user_oauth_applications": true, "user_oauth_applications": true,
"after_sign_out_path": "", "after_sign_out_path": "",
"container_registry_token_expire_delay": 5 "container_registry_token_expire_delay": 5,
"repository_storage": "default"
} }
``` ```
...@@ -14,7 +14,8 @@ ...@@ -14,7 +14,8 @@
- For omnibus-gitlab, it is located at: `/var/opt/gitlab/git-data/repositories` by default, unless you changed - For omnibus-gitlab, it is located at: `/var/opt/gitlab/git-data/repositories` by default, unless you changed
it in the `/etc/gitlab/gitlab.rb` file. it in the `/etc/gitlab/gitlab.rb` file.
- For installations from source, it is usually located at: `/home/git/repositories` or you can see where - For installations from source, it is usually located at: `/home/git/repositories` or you can see where
your repositories are located by looking at `config/gitlab.yml` under the `gitlab_shell => repos_path` entry. your repositories are located by looking at `config/gitlab.yml` under the `repositories => storages` entries
(you'll usually use the `default` storage path to start).
New folder needs to have git user ownership and read/write/execute access for git user and its group: New folder needs to have git user ownership and read/write/execute access for git user and its group:
......
...@@ -376,6 +376,7 @@ module API ...@@ -376,6 +376,7 @@ module API
expose :user_oauth_applications expose :user_oauth_applications
expose :after_sign_out_path expose :after_sign_out_path
expose :container_registry_token_expire_delay expose :container_registry_token_expire_delay
expose :repository_storage
end end
class Release < Grape::Entity class Release < Grape::Entity
......
...@@ -20,6 +20,20 @@ module API ...@@ -20,6 +20,20 @@ module API
@wiki ||= params[:project].end_with?('.wiki') && @wiki ||= params[:project].end_with?('.wiki') &&
!Project.find_with_namespace(params[:project]) !Project.find_with_namespace(params[:project])
end end
def project
@project ||= begin
project_path = params[:project]
# Check for *.wiki repositories.
# Strip out the .wiki from the pathname before finding the
# project. This applies the correct project permissions to
# the wiki repository as well.
project_path.chomp!('.wiki') if wiki?
Project.find_with_namespace(project_path)
end
end
end end
post "/allowed" do post "/allowed" do
...@@ -32,16 +46,6 @@ module API ...@@ -32,16 +46,6 @@ module API
User.find_by(id: params[:user_id]) User.find_by(id: params[:user_id])
end end
project_path = params[:project]
# Check for *.wiki repositories.
# Strip out the .wiki from the pathname before finding the
# project. This applies the correct project permissions to
# the wiki repository as well.
project_path.chomp!('.wiki') if wiki?
project = Project.find_with_namespace(project_path)
access = access =
if wiki? if wiki?
Gitlab::GitAccessWiki.new(actor, project) Gitlab::GitAccessWiki.new(actor, project)
...@@ -49,7 +53,17 @@ module API ...@@ -49,7 +53,17 @@ module API
Gitlab::GitAccess.new(actor, project) Gitlab::GitAccess.new(actor, project)
end end
access.check(params[:action], params[:changes]) access_status = access.check(params[:action], params[:changes])
response = { status: access_status.status, message: access_status.message }
if access_status.status
# Return the repository full path so that gitlab-shell has it when
# handling ssh commands
response[:repository_path] = project.repository.path_to_repo
end
response
end end
# #
......
...@@ -2,8 +2,6 @@ require 'yaml' ...@@ -2,8 +2,6 @@ require 'yaml'
module Backup module Backup
class Repository class Repository
attr_reader :repos_path
def dump def dump
prepare prepare
...@@ -50,10 +48,12 @@ module Backup ...@@ -50,10 +48,12 @@ module Backup
end end
def restore def restore
if File.exists?(repos_path) Gitlab.config.repositories.storages.each do |name, path|
next unless File.exists?(path)
# Move repos dir to 'repositories.old' dir # Move repos dir to 'repositories.old' dir
bk_repos_path = File.join(repos_path, '..', 'repositories.old.' + Time.now.to_i.to_s) bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s)
FileUtils.mv(repos_path, bk_repos_path) FileUtils.mv(path, bk_repos_path)
end end
FileUtils.mkdir_p(repos_path) FileUtils.mkdir_p(repos_path)
...@@ -61,7 +61,7 @@ module Backup ...@@ -61,7 +61,7 @@ module Backup
Project.find_each(batch_size: 1000) do |project| Project.find_each(batch_size: 1000) do |project|
$progress.print " * #{project.path_with_namespace} ... " $progress.print " * #{project.path_with_namespace} ... "
project.namespace.ensure_dir_exist if project.namespace project.ensure_dir_exist
if File.exists?(path_to_bundle(project)) if File.exists?(path_to_bundle(project))
FileUtils.mkdir_p(path_to_repo(project)) FileUtils.mkdir_p(path_to_repo(project))
...@@ -100,8 +100,8 @@ module Backup ...@@ -100,8 +100,8 @@ module Backup
end end
$progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow)
cmd = "#{Gitlab.config.gitlab_shell.path}/bin/create-hooks" cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args
if system(cmd) if system(*cmd)
$progress.puts " [DONE]".color(:green) $progress.puts " [DONE]".color(:green)
else else
puts " [FAILED]".color(:red) puts " [FAILED]".color(:red)
...@@ -120,10 +120,6 @@ module Backup ...@@ -120,10 +120,6 @@ module Backup
File.join(backup_repos_path, project.path_with_namespace + ".bundle") File.join(backup_repos_path, project.path_with_namespace + ".bundle")
end end
def repos_path
Gitlab.config.gitlab_shell.repos_path
end
def backup_repos_path def backup_repos_path
File.join(Gitlab.config.backup.path, "repositories") File.join(Gitlab.config.backup.path, "repositories")
end end
...@@ -139,5 +135,11 @@ module Backup ...@@ -139,5 +135,11 @@ module Backup
def silent def silent
{err: '/dev/null', out: '/dev/null'} {err: '/dev/null', out: '/dev/null'}
end end
private
def repository_storage_paths_args
Gitlab.config.repositories.storages.values
end
end end
end end
require 'securerandom'
module Gitlab module Gitlab
class Shell class Shell
class Error < StandardError; end class Error < StandardError; end
...@@ -18,77 +20,82 @@ module Gitlab ...@@ -18,77 +20,82 @@ module Gitlab
# Init new repository # Init new repository
# #
# storage - project's storage path
# name - project path with namespace # name - project path with namespace
# #
# Ex. # Ex.
# add_repository("gitlab/gitlab-ci") # add_repository("/path/to/storage", "gitlab/gitlab-ci")
# #
def add_repository(name) def add_repository(storage, name)
Gitlab::Utils.system_silent([gitlab_shell_projects_path, Gitlab::Utils.system_silent([gitlab_shell_projects_path,
'add-project', "#{name}.git"]) 'add-project', storage, "#{name}.git"])
end end
# Import repository # Import repository
# #
# storage - project's storage path
# name - project path with namespace # name - project path with namespace
# #
# Ex. # Ex.
# import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git")
# #
def import_repository(name, url) def import_repository(storage, name, url)
output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '900']) output, status = Popen::popen([gitlab_shell_projects_path, 'import-project',
storage, "#{name}.git", url, '900'])
raise Error, output unless status.zero? raise Error, output unless status.zero?
true true
end end
# Move repository # Move repository
# # storage - project's storage path
# path - project path with namespace # path - project path with namespace
# new_path - new project path with namespace # new_path - new project path with namespace
# #
# Ex. # Ex.
# mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new") # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
# #
def mv_repository(path, new_path) def mv_repository(storage, path, new_path)
Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project', Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project',
"#{path}.git", "#{new_path}.git"]) storage, "#{path}.git", "#{new_path}.git"])
end end
# Fork repository to new namespace # Fork repository to new namespace
# # storage - project's storage path
# path - project path with namespace # path - project path with namespace
# fork_namespace - namespace for forked project # fork_namespace - namespace for forked project
# #
# Ex. # Ex.
# fork_repository("gitlab/gitlab-ci", "randx") # fork_repository("/path/to/storage", "gitlab/gitlab-ci", "randx")
# #
def fork_repository(path, fork_namespace) def fork_repository(storage, path, fork_namespace)
Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project', Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project',
"#{path}.git", fork_namespace]) storage, "#{path}.git", fork_namespace])
end end
# Remove repository from file system # Remove repository from file system
# #
# storage - project's storage path
# name - project path with namespace # name - project path with namespace
# #
# Ex. # Ex.
# remove_repository("gitlab/gitlab-ci") # remove_repository("/path/to/storage", "gitlab/gitlab-ci")
# #
def remove_repository(name) def remove_repository(storage, name)
Gitlab::Utils.system_silent([gitlab_shell_projects_path, Gitlab::Utils.system_silent([gitlab_shell_projects_path,
'rm-project', "#{name}.git"]) 'rm-project', storage, "#{name}.git"])
end end
# Gc repository # Gc repository
# #
# storage - project storage path
# path - project path with namespace # path - project path with namespace
# #
# Ex. # Ex.
# gc("gitlab/gitlab-ci") # gc("/path/to/storage", "gitlab/gitlab-ci")
# #
def gc(path) def gc(storage, path)
Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc',
"#{path}.git"]) storage, "#{path}.git"])
end end
# Add new key to gitlab-shell # Add new key to gitlab-shell
...@@ -133,31 +140,31 @@ module Gitlab ...@@ -133,31 +140,31 @@ module Gitlab
# Add empty directory for storing repositories # Add empty directory for storing repositories
# #
# Ex. # Ex.
# add_namespace("gitlab") # add_namespace("/path/to/storage", "gitlab")
# #
def add_namespace(name) def add_namespace(storage, name)
FileUtils.mkdir(full_path(name), mode: 0770) unless exists?(name) FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name)
end end
# Remove directory from repositories storage # Remove directory from repositories storage
# Every repository inside this directory will be removed too # Every repository inside this directory will be removed too
# #
# Ex. # Ex.
# rm_namespace("gitlab") # rm_namespace("/path/to/storage", "gitlab")
# #
def rm_namespace(name) def rm_namespace(storage, name)
FileUtils.rm_r(full_path(name), force: true) FileUtils.rm_r(full_path(storage, name), force: true)
end end
# Move namespace directory inside repositories storage # Move namespace directory inside repositories storage
# #
# Ex. # Ex.
# mv_namespace("gitlab", "gitlabhq") # mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
# #
def mv_namespace(old_name, new_name) def mv_namespace(storage, old_name, new_name)
return false if exists?(new_name) || !exists?(old_name) return false if exists?(storage, new_name) || !exists?(storage, old_name)
FileUtils.mv(full_path(old_name), full_path(new_name)) FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name))
end end
def url_to_repo(path) def url_to_repo(path)
...@@ -176,11 +183,26 @@ module Gitlab ...@@ -176,11 +183,26 @@ module Gitlab
# Check if such directory exists in repositories. # Check if such directory exists in repositories.
# #
# Usage: # Usage:
# exists?('gitlab') # exists?(storage, 'gitlab')
# exists?('gitlab/cookies.git') # exists?(storage, 'gitlab/cookies.git')
# #
def exists?(dir_name) def exists?(storage, dir_name)
File.exist?(full_path(dir_name)) File.exist?(full_path(storage, dir_name))
end
# Create (if necessary) and link the secret token file
def generate_and_link_secret_token
secret_file = Gitlab.config.gitlab_shell.secret_file
unless File.exist? secret_file
# Generate a new token of 16 random hexadecimal characters and store it in secret_file.
token = SecureRandom.hex(16)
File.write(secret_file, token)
end
link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret')
if File.exist?(gitlab_shell_path) && !File.exist?(link_path)
FileUtils.symlink(secret_file, link_path)
end
end end
protected protected
...@@ -193,14 +215,10 @@ module Gitlab ...@@ -193,14 +215,10 @@ module Gitlab
File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}") File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}")
end end
def repos_path def full_path(storage, dir_name)
Gitlab.config.gitlab_shell.repos_path
end
def full_path(dir_name)
raise ArgumentError.new("Directory name can't be blank") if dir_name.blank? raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?
File.join(repos_path, dir_name) File.join(storage, dir_name)
end end
def gitlab_shell_projects_path def gitlab_shell_projects_path
......
...@@ -167,7 +167,7 @@ module Gitlab ...@@ -167,7 +167,7 @@ module Gitlab
def import_wiki def import_wiki
unless project.wiki_enabled? unless project.wiki_enabled?
wiki = WikiFormatter.new(project) wiki = WikiFormatter.new(project)
gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url)
project.update_attribute(:wiki_enabled, true) project.update_attribute(:wiki_enabled, true)
end end
......
...@@ -356,97 +356,108 @@ namespace :gitlab do ...@@ -356,97 +356,108 @@ namespace :gitlab do
######################## ########################
def check_repo_base_exists def check_repo_base_exists
print "Repo base directory exists? ... " puts "Repo base directory exists?"
repo_base_path = Gitlab.config.gitlab_shell.repos_path Gitlab.config.repositories.storages.each do |name, repo_base_path|
print "#{name}... "
if File.exists?(repo_base_path) if File.exists?(repo_base_path)
puts "yes".color(:green) puts "yes".color(:green)
else else
puts "no".color(:red) puts "no".color(:red)
puts "#{repo_base_path} is missing".color(:red) puts "#{repo_base_path} is missing".color(:red)
try_fixing_it( try_fixing_it(
"This should have been created when setting up GitLab Shell.", "This should have been created when setting up GitLab Shell.",
"Make sure it's set correctly in config/gitlab.yml", "Make sure it's set correctly in config/gitlab.yml",
"Make sure GitLab Shell is installed correctly." "Make sure GitLab Shell is installed correctly."
) )
for_more_information( for_more_information(
see_installation_guide_section "GitLab Shell" see_installation_guide_section "GitLab Shell"
) )
fix_and_rerun fix_and_rerun
end
end end
end end
def check_repo_base_is_not_symlink def check_repo_base_is_not_symlink
print "Repo base directory is a symlink? ... " puts "Repo storage directories are symlinks?"
repo_base_path = Gitlab.config.gitlab_shell.repos_path Gitlab.config.repositories.storages.each do |name, repo_base_path|
unless File.exists?(repo_base_path) print "#{name}... "
puts "can't check because of previous errors".color(:magenta)
return
end
unless File.symlink?(repo_base_path) unless File.exists?(repo_base_path)
puts "no".color(:green) puts "can't check because of previous errors".color(:magenta)
else return
puts "yes".color(:red) end
try_fixing_it(
"Make sure it's set to the real directory in config/gitlab.yml" unless File.symlink?(repo_base_path)
) puts "no".color(:green)
fix_and_rerun else
puts "yes".color(:red)
try_fixing_it(
"Make sure it's set to the real directory in config/gitlab.yml"
)
fix_and_rerun
end
end end
end end
def check_repo_base_permissions def check_repo_base_permissions
print "Repo base access is drwxrws---? ... " puts "Repo paths access is drwxrws---?"
repo_base_path = Gitlab.config.gitlab_shell.repos_path Gitlab.config.repositories.storages.each do |name, repo_base_path|
unless File.exists?(repo_base_path) print "#{name}... "
puts "can't check because of previous errors".color(:magenta)
return
end
if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") unless File.exists?(repo_base_path)
puts "yes".color(:green) puts "can't check because of previous errors".color(:magenta)
else return
puts "no".color(:red) end
try_fixing_it(
"sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770")
"sudo chmod -R ug-s #{repo_base_path}", puts "yes".color(:green)
"sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" else
) puts "no".color(:red)
for_more_information( try_fixing_it(
see_installation_guide_section "GitLab Shell" "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}",
) "sudo chmod -R ug-s #{repo_base_path}",
fix_and_rerun "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s"
)
for_more_information(
see_installation_guide_section "GitLab Shell"
)
fix_and_rerun
end
end end
end end
def check_repo_base_user_and_group def check_repo_base_user_and_group
gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user
gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group
print "Repo base owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}? ... " puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?"
repo_base_path = Gitlab.config.gitlab_shell.repos_path Gitlab.config.repositories.storages.each do |name, repo_base_path|
unless File.exists?(repo_base_path) print "#{name}... "
puts "can't check because of previous errors".color(:magenta)
return
end
uid = uid_for(gitlab_shell_ssh_user) unless File.exists?(repo_base_path)
gid = gid_for(gitlab_shell_owner_group) puts "can't check because of previous errors".color(:magenta)
if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid return
puts "yes".color(:green) end
else
puts "no".color(:red) uid = uid_for(gitlab_shell_ssh_user)
puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) gid = gid_for(gitlab_shell_owner_group)
try_fixing_it( if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid
"sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" puts "yes".color(:green)
) else
for_more_information( puts "no".color(:red)
see_installation_guide_section "GitLab Shell" puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue)
) try_fixing_it(
fix_and_rerun "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}"
)
for_more_information(
see_installation_guide_section "GitLab Shell"
)
fix_and_rerun
end
end end
end end
...@@ -473,7 +484,7 @@ namespace :gitlab do ...@@ -473,7 +484,7 @@ namespace :gitlab do
else else
puts "wrong or missing hooks".color(:red) puts "wrong or missing hooks".color(:red)
try_fixing_it( try_fixing_it(
sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')}"), sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"),
'Check the hooks_path in config/gitlab.yml', 'Check the hooks_path in config/gitlab.yml',
'Check your gitlab-shell installation' 'Check your gitlab-shell installation'
) )
...@@ -785,13 +796,13 @@ namespace :gitlab do ...@@ -785,13 +796,13 @@ namespace :gitlab do
namespace :repo do namespace :repo do
desc "GitLab | Check the integrity of the repositories managed by GitLab" desc "GitLab | Check the integrity of the repositories managed by GitLab"
task check: :environment do task check: :environment do
namespace_dirs = Dir.glob( Gitlab.config.repositories.storages.each do |name, path|
File.join(Gitlab.config.gitlab_shell.repos_path, '*') namespace_dirs = Dir.glob(File.join(path, '*'))
)
namespace_dirs.each do |namespace_dir| namespace_dirs.each do |namespace_dir|
repo_dirs = Dir.glob(File.join(namespace_dir, '*')) repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
end
end end
end end
end end
...@@ -799,12 +810,12 @@ namespace :gitlab do ...@@ -799,12 +810,12 @@ namespace :gitlab do
namespace :user do namespace :user do
desc "GitLab | Check the integrity of a specific user's repositories" desc "GitLab | Check the integrity of a specific user's repositories"
task :check_repos, [:username] => :environment do |t, args| task :check_repos, [:username] => :environment do |t, args|
username = args[:username] || prompt("Check repository integrity for which username? ".color(:blue)) username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue))
user = User.find_by(username: username) user = User.find_by(username: username)
if user if user
repo_dirs = user.authorized_projects.map do |p| repo_dirs = user.authorized_projects.map do |p|
File.join( File.join(
Gitlab.config.gitlab_shell.repos_path, p.repository_storage_path,
"#{p.path_with_namespace}.git" "#{p.path_with_namespace}.git"
) )
end end
......
...@@ -5,36 +5,36 @@ namespace :gitlab do ...@@ -5,36 +5,36 @@ namespace :gitlab do
warn_user_is_not_gitlab warn_user_is_not_gitlab
remove_flag = ENV['REMOVE'] remove_flag = ENV['REMOVE']
namespaces = Namespace.pluck(:path) namespaces = Namespace.pluck(:path)
git_base_path = Gitlab.config.gitlab_shell.repos_path Gitlab.config.repositories.storages.each do |name, git_base_path|
all_dirs = Dir.glob(git_base_path + '/*') all_dirs = Dir.glob(git_base_path + '/*')
puts git_base_path.color(:yellow) puts git_base_path.color(:yellow)
puts "Looking for directories to remove... " puts "Looking for directories to remove... "
all_dirs.reject! do |dir| all_dirs.reject! do |dir|
# skip if git repo # skip if git repo
dir =~ /.git$/ dir =~ /.git$/
end end
all_dirs.reject! do |dir| all_dirs.reject! do |dir|
dir_name = File.basename dir dir_name = File.basename dir
# skip if namespace present # skip if namespace present
namespaces.include?(dir_name) namespaces.include?(dir_name)
end end
all_dirs.each do |dir_path| all_dirs.each do |dir_path|
if remove_flag if remove_flag
if FileUtils.rm_rf dir_path if FileUtils.rm_rf dir_path
puts "Removed...#{dir_path}".color(:red) puts "Removed...#{dir_path}".color(:red)
else
puts "Cannot remove #{dir_path}".color(:red)
end
else else
puts "Cannot remove #{dir_path}".color(:red) puts "Can be removed: #{dir_path}".color(:red)
end end
else
puts "Can be removed: #{dir_path}".color(:red)
end end
end end
...@@ -48,20 +48,21 @@ namespace :gitlab do ...@@ -48,20 +48,21 @@ namespace :gitlab do
warn_user_is_not_gitlab warn_user_is_not_gitlab
move_suffix = "+orphaned+#{Time.now.to_i}" move_suffix = "+orphaned+#{Time.now.to_i}"
repo_root = Gitlab.config.gitlab_shell.repos_path Gitlab.config.repositories.storages.each do |name, repo_root|
# Look for global repos (legacy, depth 1) and normal repos (depth 2) # Look for global repos (legacy, depth 1) and normal repos (depth 2)
IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
find.each_line do |path| find.each_line do |path|
path.chomp! path.chomp!
repo_with_namespace = path. repo_with_namespace = path.
sub(repo_root, ''). sub(repo_root, '').
sub(%r{^/*}, ''). sub(%r{^/*}, '').
chomp('.git'). chomp('.git').
chomp('.wiki') chomp('.wiki')
next if Project.find_with_namespace(repo_with_namespace) next if Project.find_with_namespace(repo_with_namespace)
new_path = path + move_suffix new_path = path + move_suffix
puts path.inspect + ' -> ' + new_path.inspect puts path.inspect + ' -> ' + new_path.inspect
File.rename(path, new_path) File.rename(path, new_path)
end
end end
end end
end end
......
...@@ -2,73 +2,73 @@ namespace :gitlab do ...@@ -2,73 +2,73 @@ namespace :gitlab do
namespace :import do namespace :import do
# How to use: # How to use:
# #
# 1. copy the bare repos under the repos_path (commonly /home/git/repositories) # 1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories)
# 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production # 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production
# #
# Notes: # Notes:
# * The project owner will set to the first administator of the system # * The project owner will set to the first administator of the system
# * Existing projects will be skipped # * Existing projects will be skipped
# #
desc "GitLab | Import bare repositories from gitlab_shell -> repos_path into GitLab project instance" desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
task repos: :environment do task repos: :environment do
Gitlab.config.repositories.storages.each do |name, git_base_path|
repos_to_import = Dir.glob(git_base_path + '/**/*.git')
git_base_path = Gitlab.config.gitlab_shell.repos_path repos_to_import.each do |repo_path|
repos_to_import = Dir.glob(git_base_path + '/**/*.git') # strip repo base path
repo_path[0..git_base_path.length] = ''
repos_to_import.each do |repo_path| path = repo_path.sub(/\.git$/, '')
# strip repo base path group_name, name = File.split(path)
repo_path[0..git_base_path.length] = '' group_name = nil if group_name == '.'
path = repo_path.sub(/\.git$/, '') puts "Processing #{repo_path}".color(:yellow)
group_name, name = File.split(path)
group_name = nil if group_name == '.'
puts "Processing #{repo_path}".color(:yellow) if path.end_with?('.wiki')
puts " * Skipping wiki repo"
if path.end_with?('.wiki') next
puts " * Skipping wiki repo" end
next
end
project = Project.find_with_namespace(path) project = Project.find_with_namespace(path)
if project if project
puts " * #{project.name} (#{repo_path}) exists" puts " * #{project.name} (#{repo_path}) exists"
else else
user = User.admins.reorder("id").first user = User.admins.reorder("id").first
project_params = { project_params = {
name: name, name: name,
path: name path: name
} }
# find group namespace # find group namespace
if group_name if group_name
group = Namespace.find_by(path: group_name) group = Namespace.find_by(path: group_name)
# create group namespace # create group namespace
unless group unless group
group = Group.new(:name => group_name) group = Group.new(:name => group_name)
group.path = group_name group.path = group_name
group.owner = user group.owner = user
if group.save if group.save
puts " * Created Group #{group.name} (#{group.id})".color(:green) puts " * Created Group #{group.name} (#{group.id})".color(:green)
else else
puts " * Failed trying to create group #{group.name}".color(:red) puts " * Failed trying to create group #{group.name}".color(:red)
end
end end
# set project group
project_params[:namespace_id] = group.id
end end
# set project group
project_params[:namespace_id] = group.id
end
project = Projects::CreateService.new(user, project_params).execute project = Projects::CreateService.new(user, project_params).execute
if project.persisted? if project.persisted?
puts " * Created #{project.name} (#{repo_path})".color(:green) puts " * Created #{project.name} (#{repo_path})".color(:green)
project.update_repository_size project.update_repository_size
project.update_commit_count project.update_commit_count
else else
puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
puts " Errors: #{project.errors.messages}".color(:red) puts " Errors: #{project.errors.messages}".color(:red)
end
end end
end end
end end
......
...@@ -62,7 +62,10 @@ namespace :gitlab do ...@@ -62,7 +62,10 @@ namespace :gitlab do
puts "" puts ""
puts "GitLab Shell".color(:yellow) puts "GitLab Shell".color(:yellow)
puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}"
puts "Repositories:\t#{Gitlab.config.gitlab_shell.repos_path}" puts "Repository storage paths:"
Gitlab.config.repositories.storages.each do |name, path|
puts "- #{name}: \t#{path}"
end
puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}"
puts "Git:\t\t#{Gitlab.config.git.bin_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}"
......
...@@ -9,7 +9,7 @@ namespace :gitlab do ...@@ -9,7 +9,7 @@ namespace :gitlab do
scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids)
end end
scope.find_each do |project| scope.find_each do |project|
base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) base = File.join(project.repository_storage_path, project.path_with_namespace)
puts base + '.git' puts base + '.git'
puts base + '.wiki.git' puts base + '.wiki.git'
end end
......
...@@ -12,7 +12,6 @@ namespace :gitlab do ...@@ -12,7 +12,6 @@ namespace :gitlab do
gitlab_url = Gitlab.config.gitlab.url gitlab_url = Gitlab.config.gitlab.url
# gitlab-shell requires a / at the end of the url # gitlab-shell requires a / at the end of the url
gitlab_url += '/' unless gitlab_url.end_with?('/') gitlab_url += '/' unless gitlab_url.end_with?('/')
repos_path = Gitlab.config.gitlab_shell.repos_path
target_dir = Gitlab.config.gitlab_shell.path target_dir = Gitlab.config.gitlab_shell.path
# Clone if needed # Clone if needed
...@@ -35,7 +34,6 @@ namespace :gitlab do ...@@ -35,7 +34,6 @@ namespace :gitlab do
user: user, user: user,
gitlab_url: gitlab_url, gitlab_url: gitlab_url,
http_settings: {self_signed_cert: false}.stringify_keys, http_settings: {self_signed_cert: false}.stringify_keys,
repos_path: repos_path,
auth_file: File.join(home_dir, ".ssh", "authorized_keys"), auth_file: File.join(home_dir, ".ssh", "authorized_keys"),
redis: { redis: {
bin: %x{which redis-cli}.chomp, bin: %x{which redis-cli}.chomp,
...@@ -58,10 +56,10 @@ namespace :gitlab do ...@@ -58,10 +56,10 @@ namespace :gitlab do
File.open("config.yml", "w+") {|f| f.puts config.to_yaml} File.open("config.yml", "w+") {|f| f.puts config.to_yaml}
# Launch installation process # Launch installation process
system(*%W(bin/install)) system(*%W(bin/install) + repository_storage_paths_args)
# (Re)create hooks # (Re)create hooks
system(*%W(bin/create-hooks)) system(*%W(bin/create-hooks) + repository_storage_paths_args)
end end
# Required for debian packaging with PKGR: Setup .ssh/environment with # Required for debian packaging with PKGR: Setup .ssh/environment with
...@@ -73,6 +71,8 @@ namespace :gitlab do ...@@ -73,6 +71,8 @@ namespace :gitlab do
File.open(File.join(home_dir, ".ssh", "environment"), "w+") do |f| File.open(File.join(home_dir, ".ssh", "environment"), "w+") do |f|
f.puts "PATH=#{ENV['PATH']}" f.puts "PATH=#{ENV['PATH']}"
end end
Gitlab::Shell.new.generate_and_link_secret_token
end end
desc "GitLab | Setup gitlab-shell" desc "GitLab | Setup gitlab-shell"
...@@ -87,7 +87,8 @@ namespace :gitlab do ...@@ -87,7 +87,8 @@ namespace :gitlab do
if File.exists?(path_to_repo) if File.exists?(path_to_repo)
print '-' print '-'
else else
if Gitlab::Shell.new.add_repository(project.path_with_namespace) if Gitlab::Shell.new.add_repository(project.repository_storage_path,
project.path_with_namespace)
print '.' print '.'
else else
print 'F' print 'F'
...@@ -138,4 +139,3 @@ namespace :gitlab do ...@@ -138,4 +139,3 @@ namespace :gitlab do
system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag}))
end end
end end
...@@ -125,10 +125,16 @@ namespace :gitlab do ...@@ -125,10 +125,16 @@ namespace :gitlab do
end end
def all_repos def all_repos
IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| Gitlab.config.repositories.storages.each do |name, path|
find.each_line do |path| IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
yield path.chomp find.each_line do |path|
yield path.chomp
end
end end
end end
end end
def repository_storage_paths_args
Gitlab.config.repositories.storages.values
end
end end
...@@ -123,11 +123,17 @@ describe ProjectsHelper do ...@@ -123,11 +123,17 @@ describe ProjectsHelper do
end end
describe '#sanitized_import_error' do describe '#sanitized_import_error' do
let(:project) { create(:project) }
before do
allow(project).to receive(:repository_storage_path).and_return('/base/repo/path')
end
it 'removes the repo path' do it 'removes the repo path' do
repo = File.join(Gitlab.config.gitlab_shell.repos_path, '/namespace/test.git') repo = '/base/repo/path/namespace/test.git'
import_error = "Could not clone #{repo}\n" import_error = "Could not clone #{repo}\n"
expect(sanitize_repo_path(import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git') expect(sanitize_repo_path(project, import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git')
end end
end end
end end
require 'spec_helper'
describe '6_validations', lib: true do
context 'with correct settings' do
before do
mock_storages('foo' => '/a/b/c', 'bar' => 'a/b/d')
end
it 'passes through' do
expect { load_validations }.not_to raise_error
end
end
context 'with invalid storage names' do
before do
mock_storages('name with spaces' => '/a/b/c')
end
it 'throws an error' do
expect { load_validations }.to raise_error('"name with spaces" is not a valid storage name. Please fix this in your gitlab.yml before starting GitLab.')
end
end
context 'with nested storage paths' do
before do
mock_storages('foo' => '/a/b/c', 'bar' => '/a/b/c/d')
end
it 'throws an error' do
expect { load_validations }.to raise_error('bar is a nested path of foo. Nested paths are not supported for repository storages. Please fix this in your gitlab.yml before starting GitLab.')
end
end
def mock_storages(storages)
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
def load_validations
load File.join(__dir__, '../../config/initializers/6_validations.rb')
end
end
...@@ -13,9 +13,37 @@ describe Gitlab::Shell, lib: true do ...@@ -13,9 +13,37 @@ describe Gitlab::Shell, lib: true do
it { is_expected.to respond_to :add_repository } it { is_expected.to respond_to :add_repository }
it { is_expected.to respond_to :remove_repository } it { is_expected.to respond_to :remove_repository }
it { is_expected.to respond_to :fork_repository } it { is_expected.to respond_to :fork_repository }
it { is_expected.to respond_to :gc }
it { is_expected.to respond_to :add_namespace }
it { is_expected.to respond_to :rm_namespace }
it { is_expected.to respond_to :mv_namespace }
it { is_expected.to respond_to :exists? }
it { expect(gitlab_shell.url_to_repo('diaspora')).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + "diaspora.git") } it { expect(gitlab_shell.url_to_repo('diaspora')).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + "diaspora.git") }
describe 'generate_and_link_secret_token' do
let(:secret_file) { 'tmp/tests/.secret_shell_test' }
let(:link_file) { 'tmp/tests/shell-secret-test/.gitlab_shell_secret' }
before do
allow(Gitlab.config.gitlab_shell).to receive(:path).and_return('tmp/tests/shell-secret-test')
allow(Gitlab.config.gitlab_shell).to receive(:secret_file).and_return(secret_file)
FileUtils.mkdir('tmp/tests/shell-secret-test')
gitlab_shell.generate_and_link_secret_token
end
after do
FileUtils.rm_rf('tmp/tests/shell-secret-test')
FileUtils.rm_rf(secret_file)
end
it 'creates and links the secret token file' do
expect(File.exist?(secret_file)).to be(true)
expect(File.symlink?(link_file)).to be(true)
expect(File.readlink(link_file)).to eq(secret_file)
end
end
describe Gitlab::Shell::KeyAdder, lib: true do describe Gitlab::Shell::KeyAdder, lib: true do
describe '#add_key' do describe '#add_key' do
it 'normalizes space characters in the key' do it 'normalizes space characters in the key' do
......
...@@ -40,6 +40,16 @@ describe ApplicationSetting, models: true do ...@@ -40,6 +40,16 @@ describe ApplicationSetting, models: true do
it_behaves_like 'an object with email-formated attributes', :admin_notification_email do it_behaves_like 'an object with email-formated attributes', :admin_notification_email do
subject { setting } subject { setting }
end end
context 'repository storages inclussion' do
before do
storages = { 'custom' => 'tmp/tests/custom_repositories' }
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
it { is_expected.to allow_value('custom').for(:repository_storage) }
it { is_expected.not_to allow_value('alternative').for(:repository_storage) }
end
end end
context 'restricted signup domains' do context 'restricted signup domains' do
......
...@@ -57,6 +57,7 @@ describe Namespace, models: true do ...@@ -57,6 +57,7 @@ describe Namespace, models: true do
describe :move_dir do describe :move_dir do
before do before do
@namespace = create :namespace @namespace = create :namespace
@project = create :project, namespace: @namespace
allow(@namespace).to receive(:path_changed?).and_return(true) allow(@namespace).to receive(:path_changed?).and_return(true)
end end
...@@ -87,8 +88,13 @@ describe Namespace, models: true do ...@@ -87,8 +88,13 @@ describe Namespace, models: true do
end end
describe :rm_dir do describe :rm_dir do
it "should remove dir" do let!(:project) { create(:project, namespace: namespace) }
expect(namespace.rm_dir).to be_truthy let!(:path) { File.join(Gitlab.config.repositories.storages.default, namespace.path) }
before { namespace.destroy }
it "should remove its dirs when deleted" do
expect(File.exist?(path)).to be(false)
end end
end end
......
...@@ -56,6 +56,7 @@ describe Project, models: true do ...@@ -56,6 +56,7 @@ describe Project, models: true do
it { is_expected.to validate_length_of(:description).is_within(0..2000) } it { is_expected.to validate_length_of(:description).is_within(0..2000) }
it { is_expected.to validate_presence_of(:creator) } it { is_expected.to validate_presence_of(:creator) }
it { is_expected.to validate_presence_of(:namespace) } it { is_expected.to validate_presence_of(:namespace) }
it { is_expected.to validate_presence_of(:repository_storage) }
it 'should not allow new projects beyond user limits' do it 'should not allow new projects beyond user limits' do
project2 = build(:project) project2 = build(:project)
...@@ -84,6 +85,20 @@ describe Project, models: true do ...@@ -84,6 +85,20 @@ describe Project, models: true do
end end
end end
end end
context 'repository storages inclussion' do
let(:project2) { build(:project, repository_storage: 'missing') }
before do
storages = { 'custom' => 'tmp/tests/custom_repositories' }
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
it "should not allow repository storages that don't match a label in the configuration" do
expect(project2).not_to be_valid
expect(project2.errors[:repository_storage].first).to match(/is not included in the list/)
end
end
end end
describe 'default_scope' do describe 'default_scope' do
...@@ -131,6 +146,24 @@ describe Project, models: true do ...@@ -131,6 +146,24 @@ describe Project, models: true do
end end
end end
describe '#repository_storage_path' do
let(:project) { create(:project, repository_storage: 'custom') }
before do
FileUtils.mkdir('tmp/tests/custom_repositories')
storages = { 'custom' => 'tmp/tests/custom_repositories' }
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
after do
FileUtils.rm_rf('tmp/tests/custom_repositories')
end
it 'returns the repository storage path' do
expect(project.repository_storage_path).to eq('tmp/tests/custom_repositories')
end
end
it 'should return valid url to repo' do it 'should return valid url to repo' do
project = Project.new(path: 'somewhere') project = Project.new(path: 'somewhere')
expect(project.url_to_repo).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + 'somewhere.git') expect(project.url_to_repo).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + 'somewhere.git')
...@@ -574,6 +607,21 @@ describe Project, models: true do ...@@ -574,6 +607,21 @@ describe Project, models: true do
end end
end end
context 'repository storage by default' do
let(:project) { create(:empty_project) }
subject { project.repository_storage }
before do
storages = { 'alternative_storage' => '/some/path' }
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
stub_application_setting(repository_storage: 'alternative_storage')
allow_any_instance_of(Project).to receive(:ensure_dir_exist).and_return(true)
end
it { is_expected.to eq('alternative_storage') }
end
context 'shared runners by default' do context 'shared runners by default' do
let(:project) { create(:empty_project) } let(:project) { create(:empty_project) }
...@@ -729,12 +777,12 @@ describe Project, models: true do ...@@ -729,12 +777,12 @@ describe Project, models: true do
expect(gitlab_shell).to receive(:mv_repository). expect(gitlab_shell).to receive(:mv_repository).
ordered. ordered.
with("#{ns}/foo", "#{ns}/#{project.path}"). with(project.repository_storage_path, "#{ns}/foo", "#{ns}/#{project.path}").
and_return(true) and_return(true)
expect(gitlab_shell).to receive(:mv_repository). expect(gitlab_shell).to receive(:mv_repository).
ordered. ordered.
with("#{ns}/foo.wiki", "#{ns}/#{project.path}.wiki"). with(project.repository_storage_path, "#{ns}/foo.wiki", "#{ns}/#{project.path}.wiki").
and_return(true) and_return(true)
expect_any_instance_of(SystemHooksService). expect_any_instance_of(SystemHooksService).
...@@ -826,7 +874,7 @@ describe Project, models: true do ...@@ -826,7 +874,7 @@ describe Project, models: true do
context 'using a regular repository' do context 'using a regular repository' do
it 'creates the repository' do it 'creates the repository' do
expect(shell).to receive(:add_repository). expect(shell).to receive(:add_repository).
with(project.path_with_namespace). with(project.repository_storage_path, project.path_with_namespace).
and_return(true) and_return(true)
expect(project.repository).to receive(:after_create) expect(project.repository).to receive(:after_create)
...@@ -836,7 +884,7 @@ describe Project, models: true do ...@@ -836,7 +884,7 @@ describe Project, models: true do
it 'adds an error if the repository could not be created' do it 'adds an error if the repository could not be created' do
expect(shell).to receive(:add_repository). expect(shell).to receive(:add_repository).
with(project.path_with_namespace). with(project.repository_storage_path, project.path_with_namespace).
and_return(false) and_return(false)
expect(project.repository).not_to receive(:after_create) expect(project.repository).not_to receive(:after_create)
......
...@@ -72,6 +72,7 @@ describe API::API, api: true do ...@@ -72,6 +72,7 @@ describe API::API, api: true do
expect(response).to have_http_status(200) expect(response).to have_http_status(200)
expect(json_response["status"]).to be_truthy expect(json_response["status"]).to be_truthy
expect(json_response["repository_path"]).to eq(project.repository.path_to_repo)
end end
end end
...@@ -81,6 +82,7 @@ describe API::API, api: true do ...@@ -81,6 +82,7 @@ describe API::API, api: true do
expect(response).to have_http_status(200) expect(response).to have_http_status(200)
expect(json_response["status"]).to be_truthy expect(json_response["status"]).to be_truthy
expect(json_response["repository_path"]).to eq(project.repository.path_to_repo)
end end
end end
end end
......
...@@ -14,16 +14,23 @@ describe API::API, 'Settings', api: true do ...@@ -14,16 +14,23 @@ describe API::API, 'Settings', api: true do
expect(json_response).to be_an Hash expect(json_response).to be_an Hash
expect(json_response['default_projects_limit']).to eq(42) expect(json_response['default_projects_limit']).to eq(42)
expect(json_response['signin_enabled']).to be_truthy expect(json_response['signin_enabled']).to be_truthy
expect(json_response['repository_storage']).to eq('default')
end end
end end
describe "PUT /application/settings" do describe "PUT /application/settings" do
before do
storages = { 'custom' => 'tmp/tests/custom_repositories' }
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
it "should update application settings" do it "should update application settings" do
put api("/application/settings", admin), put api("/application/settings", admin),
default_projects_limit: 3, signin_enabled: false default_projects_limit: 3, signin_enabled: false, repository_storage: 'custom'
expect(response).to have_http_status(200) expect(response).to have_http_status(200)
expect(json_response['default_projects_limit']).to eq(3) expect(json_response['default_projects_limit']).to eq(3)
expect(json_response['signin_enabled']).to be_falsey expect(json_response['signin_enabled']).to be_falsey
expect(json_response['repository_storage']).to eq('custom')
end end
end end
end end
...@@ -23,8 +23,8 @@ describe DestroyGroupService, services: true do ...@@ -23,8 +23,8 @@ describe DestroyGroupService, services: true do
Sidekiq::Testing.inline! { destroy_group(group, user) } Sidekiq::Testing.inline! { destroy_group(group, user) }
end end
it { expect(gitlab_shell.exists?(group.path)).to be_falsey } it { expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_falsey }
it { expect(gitlab_shell.exists?(remove_path)).to be_falsey } it { expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_falsey }
end end
context 'Sidekiq fake' do context 'Sidekiq fake' do
...@@ -33,8 +33,8 @@ describe DestroyGroupService, services: true do ...@@ -33,8 +33,8 @@ describe DestroyGroupService, services: true do
Sidekiq::Testing.fake! { destroy_group(group, user) } Sidekiq::Testing.fake! { destroy_group(group, user) }
end end
it { expect(gitlab_shell.exists?(group.path)).to be_falsey } it { expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_falsey }
it { expect(gitlab_shell.exists?(remove_path)).to be_truthy } it { expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_truthy }
end end
end end
......
...@@ -12,7 +12,7 @@ describe Projects::HousekeepingService do ...@@ -12,7 +12,7 @@ describe Projects::HousekeepingService do
it 'enqueues a sidekiq job' do it 'enqueues a sidekiq job' do
expect(subject).to receive(:try_obtain_lease).and_return(true) expect(subject).to receive(:try_obtain_lease).and_return(true)
expect(GitlabShellOneShotWorker).to receive(:perform_async).with(:gc, project.path_with_namespace) expect(GitlabShellOneShotWorker).to receive(:perform_async).with(:gc, project.repository_storage_path, project.path_with_namespace)
subject.execute subject.execute
expect(project.pushes_since_gc).to eq(0) expect(project.pushes_since_gc).to eq(0)
......
...@@ -36,7 +36,7 @@ describe Projects::ImportService, services: true do ...@@ -36,7 +36,7 @@ describe Projects::ImportService, services: true do
end end
it 'succeeds if repository import is successfully' do it 'succeeds if repository import is successfully' do
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
result = subject.execute result = subject.execute
...@@ -44,7 +44,7 @@ describe Projects::ImportService, services: true do ...@@ -44,7 +44,7 @@ describe Projects::ImportService, services: true do
end end
it 'fails if repository import fails' do it 'fails if repository import fails' do
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_raise(Gitlab::Shell::Error.new('Failed to import the repository')) expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_raise(Gitlab::Shell::Error.new('Failed to import the repository'))
result = subject.execute result = subject.execute
...@@ -64,7 +64,7 @@ describe Projects::ImportService, services: true do ...@@ -64,7 +64,7 @@ describe Projects::ImportService, services: true do
end end
it 'succeeds if importer succeeds' do it 'succeeds if importer succeeds' do
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(true) expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(true)
result = subject.execute result = subject.execute
...@@ -74,7 +74,7 @@ describe Projects::ImportService, services: true do ...@@ -74,7 +74,7 @@ describe Projects::ImportService, services: true do
it 'flushes various caches' do it 'flushes various caches' do
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository). expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).
with(project.path_with_namespace, project.import_url). with(project.repository_storage_path, project.path_with_namespace, project.import_url).
and_return(true) and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute). expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).
...@@ -90,7 +90,7 @@ describe Projects::ImportService, services: true do ...@@ -90,7 +90,7 @@ describe Projects::ImportService, services: true do
end end
it 'fails if importer fails' do it 'fails if importer fails' do
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(false) expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(false)
result = subject.execute result = subject.execute
...@@ -100,7 +100,7 @@ describe Projects::ImportService, services: true do ...@@ -100,7 +100,7 @@ describe Projects::ImportService, services: true do
end end
it 'fails if importer raise an error' do it 'fails if importer raise an error' do
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_raise(Projects::ImportService::Error.new('Github: failed to connect API')) expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_raise(Projects::ImportService::Error.new('Github: failed to connect API'))
result = subject.execute result = subject.execute
......
...@@ -80,8 +80,9 @@ module TestEnv ...@@ -80,8 +80,9 @@ module TestEnv
end end
def setup_gitlab_shell def setup_gitlab_shell
unless File.directory?(Rails.root.join(*%w(tmp tests gitlab-shell))) unless File.directory?(Gitlab.config.gitlab_shell.path)
`rake gitlab:shell:install` # TODO: Remove `[shards]` when gitlab-shell v3.1.0 is published
`rake gitlab:shell:install[shards]`
end end
end end
...@@ -127,14 +128,14 @@ module TestEnv ...@@ -127,14 +128,14 @@ module TestEnv
def copy_repo(project) def copy_repo(project)
base_repo_path = File.expand_path(factory_repo_path_bare) base_repo_path = File.expand_path(factory_repo_path_bare)
target_repo_path = File.expand_path(repos_path + "/#{project.namespace.path}/#{project.path}.git") target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.namespace.path}/#{project.path}.git")
FileUtils.mkdir_p(target_repo_path) FileUtils.mkdir_p(target_repo_path)
FileUtils.cp_r("#{base_repo_path}/.", target_repo_path) FileUtils.cp_r("#{base_repo_path}/.", target_repo_path)
FileUtils.chmod_R 0755, target_repo_path FileUtils.chmod_R 0755, target_repo_path
end end
def repos_path def repos_path
Gitlab.config.gitlab_shell.repos_path Gitlab.config.repositories.storages.default
end end
def backup_path def backup_path
...@@ -143,7 +144,7 @@ module TestEnv ...@@ -143,7 +144,7 @@ module TestEnv
def copy_forked_repo_with_submodules(project) def copy_forked_repo_with_submodules(project)
base_repo_path = File.expand_path(forked_repo_path_bare) base_repo_path = File.expand_path(forked_repo_path_bare)
target_repo_path = File.expand_path(repos_path + "/#{project.namespace.path}/#{project.path}.git") target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.namespace.path}/#{project.path}.git")
FileUtils.mkdir_p(target_repo_path) FileUtils.mkdir_p(target_repo_path)
FileUtils.cp_r("#{base_repo_path}/.", target_repo_path) FileUtils.cp_r("#{base_repo_path}/.", target_repo_path)
FileUtils.chmod_R 0755, target_repo_path FileUtils.chmod_R 0755, target_repo_path
......
...@@ -98,67 +98,107 @@ describe 'gitlab:app namespace rake task' do ...@@ -98,67 +98,107 @@ describe 'gitlab:app namespace rake task' do
@backup_tar = tars_glob.first @backup_tar = tars_glob.first
end end
before do context 'tar creation' do
create_backup before do
end create_backup
end
after do
FileUtils.rm(@backup_tar)
end
context 'archive file permissions' do after do
it 'should set correct permissions on the tar file' do FileUtils.rm(@backup_tar)
expect(File.exist?(@backup_tar)).to be_truthy
expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100600')
end end
context 'with custom archive_permissions' do context 'archive file permissions' do
before do it 'should set correct permissions on the tar file' do
allow(Gitlab.config.backup).to receive(:archive_permissions).and_return(0651) expect(File.exist?(@backup_tar)).to be_truthy
# We created a backup in a before(:all) so it got the default permissions. expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100600')
# We now need to do some work to create a _new_ backup file using our stub.
FileUtils.rm(@backup_tar)
create_backup
end end
it 'uses the custom permissions' do context 'with custom archive_permissions' do
expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100651') before do
allow(Gitlab.config.backup).to receive(:archive_permissions).and_return(0651)
# We created a backup in a before(:all) so it got the default permissions.
# We now need to do some work to create a _new_ backup file using our stub.
FileUtils.rm(@backup_tar)
create_backup
end
it 'uses the custom permissions' do
expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100651')
end
end end
end end
end
it 'should set correct permissions on the tar contents' do it 'should set correct permissions on the tar contents' do
tar_contents, exit_status = Gitlab::Popen.popen( tar_contents, exit_status = Gitlab::Popen.popen(
%W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz registry.tar.gz} %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz registry.tar.gz}
) )
expect(exit_status).to eq(0) expect(exit_status).to eq(0)
expect(tar_contents).to match('db/') expect(tar_contents).to match('db/')
expect(tar_contents).to match('uploads.tar.gz') expect(tar_contents).to match('uploads.tar.gz')
expect(tar_contents).to match('repositories/') expect(tar_contents).to match('repositories/')
expect(tar_contents).to match('builds.tar.gz') expect(tar_contents).to match('builds.tar.gz')
expect(tar_contents).to match('artifacts.tar.gz') expect(tar_contents).to match('artifacts.tar.gz')
expect(tar_contents).to match('lfs.tar.gz') expect(tar_contents).to match('lfs.tar.gz')
expect(tar_contents).to match('registry.tar.gz') expect(tar_contents).to match('registry.tar.gz')
expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz|registry.tar.gz)\/$/) expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz|registry.tar.gz)\/$/)
end end
it 'should delete temp directories' do it 'should delete temp directories' do
temp_dirs = Dir.glob( temp_dirs = Dir.glob(
File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,lfs,registry}') File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,lfs,registry}')
) )
expect(temp_dirs).to be_empty
end
expect(temp_dirs).to be_empty context 'registry disabled' do
let(:enable_registry) { false }
it 'should not create registry.tar.gz' do
tar_contents, exit_status = Gitlab::Popen.popen(
%W{tar -tvf #{@backup_tar}}
)
expect(exit_status).to eq(0)
expect(tar_contents).not_to match('registry.tar.gz')
end
end
end end
context 'registry disabled' do context 'multiple repository storages' do
let(:enable_registry) { false } let(:project_a) { create(:project, repository_storage: 'default') }
let(:project_b) { create(:project, repository_storage: 'custom') }
before do
FileUtils.mkdir('tmp/tests/default_storage')
FileUtils.mkdir('tmp/tests/custom_storage')
storages = {
'default' => 'tmp/tests/default_storage',
'custom' => 'tmp/tests/custom_storage'
}
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
# Create the projects now, after mocking the settings but before doing the backup
project_a
project_b
# We only need a backup of the repositories for this test
ENV["SKIP"] = "db,uploads,builds,artifacts,lfs,registry"
create_backup
end
after do
FileUtils.rm_rf('tmp/tests/default_storage')
FileUtils.rm_rf('tmp/tests/custom_storage')
FileUtils.rm(@backup_tar)
end
it 'should not create registry.tar.gz' do it 'should include repositories in all repository storages' do
tar_contents, exit_status = Gitlab::Popen.popen( tar_contents, exit_status = Gitlab::Popen.popen(
%W{tar -tvf #{@backup_tar}} %W{tar -tvf #{@backup_tar} repositories}
) )
expect(exit_status).to eq(0) expect(exit_status).to eq(0)
expect(tar_contents).not_to match('registry.tar.gz') expect(tar_contents).to match("repositories/#{project_a.path_with_namespace}.bundle")
expect(tar_contents).to match("repositories/#{project_b.path_with_namespace}.bundle")
end end
end end
end # backup_create task end # backup_create task
......
...@@ -91,6 +91,6 @@ describe PostReceive do ...@@ -91,6 +91,6 @@ describe PostReceive do
end end
def pwd(project) def pwd(project)
File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) File.join(Gitlab.config.repositories.storages.default, project.path_with_namespace)
end end
end end
...@@ -14,6 +14,7 @@ describe RepositoryForkWorker do ...@@ -14,6 +14,7 @@ describe RepositoryForkWorker do
describe "#perform" do describe "#perform" do
it "creates a new repository from a fork" do it "creates a new repository from a fork" do
expect(shell).to receive(:fork_repository).with( expect(shell).to receive(:fork_repository).with(
project.repository_storage_path,
project.path_with_namespace, project.path_with_namespace,
fork_project.namespace.path fork_project.namespace.path
).and_return(true) ).and_return(true)
...@@ -25,9 +26,11 @@ describe RepositoryForkWorker do ...@@ -25,9 +26,11 @@ describe RepositoryForkWorker do
end end
it 'flushes various caches' do it 'flushes various caches' do
expect(shell).to receive(:fork_repository). expect(shell).to receive(:fork_repository).with(
with(project.path_with_namespace, fork_project.namespace.path). project.repository_storage_path,
and_return(true) project.path_with_namespace,
fork_project.namespace.path
).and_return(true)
expect_any_instance_of(Repository).to receive(:expire_emptiness_caches). expect_any_instance_of(Repository).to receive(:expire_emptiness_caches).
and_call_original and_call_original
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment