Commit 1a68a0a6 authored by Yorick Peterse's avatar Yorick Peterse

Merge branch 'shards' into 'master'

Implement multiple repository mount points

See merge request !4578
parents 2efee5f6 1735fdd4
......@@ -3,6 +3,8 @@ Please view this file on the master branch, on stable branches it's out of date.
v 8.10.0 (unreleased)
- Fix commit builds API, return all builds for all pipelines for given commit. !4849
- Replace Haml with Hamlit to make view rendering faster. !3666
- Refactor repository paths handling to allow multiple git mount points
- Add Application Setting to configure default Repository Path for new projects
- Wrap code blocks on Activies and Todos page. !4783 (winniehell)
- Align flash messages with left side of page content !4959 (winniehell)
- Display last commit of deleted branch in push events !4699 (winniehell)
......
......@@ -109,6 +109,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
:metrics_packet_size,
:send_user_confirmation_email,
:container_registry_token_expire_delay,
:repository_storage,
restricted_visibility_levels: [],
import_sources: [],
disabled_oauth_sign_in_sources: []
......
......@@ -78,4 +78,12 @@ module ApplicationSettingsHelper
end
end
end
def repository_storage_options_for_select
options = Gitlab.config.repositories.storages.map do |name, path|
["#{name} - #{path}", name]
end
options_for_select(options, @application_setting.repository_storage)
end
end
......@@ -327,9 +327,9 @@ module ProjectsHelper
end
end
def sanitize_repo_path(message)
def sanitize_repo_path(project, message)
return '' unless message.present?
message.strip.gsub(Gitlab.config.gitlab_shell.repos_path.chomp('/'), "[REPOS PATH]")
message.strip.gsub(project.repository_storage_path.chomp('/'), "[REPOS PATH]")
end
end
......@@ -55,6 +55,10 @@ class ApplicationSetting < ActiveRecord::Base
presence: true,
numericality: { only_integer: true, greater_than: 0 }
validates :repository_storage,
presence: true,
inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
validates_each :restricted_visibility_levels do |record, attr, value|
unless value.nil?
value.each do |level|
......@@ -134,6 +138,7 @@ class ApplicationSetting < ActiveRecord::Base
disabled_oauth_sign_in_sources: [],
send_user_confirmation_email: false,
container_registry_token_expire_delay: 5,
repository_storage: 'default',
)
end
......
......@@ -21,8 +21,10 @@ class Namespace < ActiveRecord::Base
delegate :name, to: :owner, allow_nil: true, prefix: true
after_create :ensure_dir_exist
after_update :move_dir, if: :path_changed?
# Save the storage paths before the projects are destroyed to use them on after destroy
before_destroy(prepend: true) { @old_repository_storage_paths = repository_storage_paths }
after_destroy :rm_dir
scope :root, -> { where('type IS NULL') }
......@@ -87,51 +89,35 @@ class Namespace < ActiveRecord::Base
owner_name
end
def ensure_dir_exist
gitlab_shell.add_namespace(path)
end
def rm_dir
# Move namespace directory into trash.
# We will remove it later async
new_path = "#{path}+#{id}+deleted"
if gitlab_shell.mv_namespace(path, new_path)
message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
Gitlab::AppLogger.info message
# Remove namespace directroy async with delay so
# GitLab has time to remove all projects first
GitlabShellWorker.perform_in(5.minutes, :rm_namespace, new_path)
end
end
def move_dir
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(path_was)
if any_project_has_container_registry_tags?
raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry')
end
if gitlab_shell.mv_namespace(path_was, path)
Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
# So we basically we mute exceptions in next actions
begin
send_update_instructions
rescue
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
# Move the namespace directory in all storages paths used by member projects
repository_storage_paths.each do |repository_storage_path|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, path_was)
unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise Exception.new('namespace directory cannot be moved')
end
else
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise Exception.new('namespace directory cannot be moved')
end
Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
# So we basically we mute exceptions in next actions
begin
send_update_instructions
rescue
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
end
......@@ -152,4 +138,33 @@ class Namespace < ActiveRecord::Base
def find_fork_of(project)
projects.joins(:forked_project_link).find_by('forked_project_links.forked_from_project_id = ?', project.id)
end
private
def repository_storage_paths
# We need to get the storage paths for all the projects, even the ones that are
# pending delete. Unscoping also get rids of the default order, which causes
# problems with SELECT DISTINCT.
Project.unscoped do
projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
end
end
def rm_dir
# Remove the namespace directory in all storages paths used by member projects
@old_repository_storage_paths.each do |repository_storage_path|
# Move namespace directory into trash.
# We will remove it later async
new_path = "#{path}+#{id}+deleted"
if gitlab_shell.mv_namespace(repository_storage_path, path, new_path)
message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
Gitlab::AppLogger.info message
# Remove namespace directroy async with delay so
# GitLab has time to remove all projects first
GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
end
end
end
end
......@@ -24,8 +24,12 @@ class Project < ActiveRecord::Base
default_value_for :wiki_enabled, gitlab_config_features.wiki
default_value_for :snippets_enabled, gitlab_config_features.snippets
default_value_for :container_registry_enabled, gitlab_config_features.container_registry
default_value_for(:repository_storage) { current_application_settings.repository_storage }
default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled }
after_create :ensure_dir_exist
after_save :ensure_dir_exist, if: :namespace_id_changed?
# set last_activity_at to the same as created_at
after_create :set_last_activity_at
def set_last_activity_at
......@@ -165,6 +169,9 @@ class Project < ActiveRecord::Base
validate :visibility_level_allowed_by_group
validate :visibility_level_allowed_as_fork
validate :check_wiki_path_conflict
validates :repository_storage,
presence: true,
inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
add_authentication_token_field :runners_token
before_save :ensure_runners_token
......@@ -376,6 +383,10 @@ class Project < ActiveRecord::Base
end
end
def repository_storage_path
Gitlab.config.repositories.storages[repository_storage]
end
def team
@team ||= ProjectTeam.new(self)
end
......@@ -842,12 +853,12 @@ class Project < ActiveRecord::Base
raise Exception.new('Project cannot be renamed, because tags are present in its container registry')
end
if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace)
if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
send_move_instructions(old_path_with_namespace)
reset_events_cache
......@@ -988,7 +999,7 @@ class Project < ActiveRecord::Base
def create_repository
# Forked import is handled asynchronously
unless forked?
if gitlab_shell.add_repository(path_with_namespace)
if gitlab_shell.add_repository(repository_storage_path, path_with_namespace)
repository.after_create
true
else
......@@ -1140,4 +1151,8 @@ class Project < ActiveRecord::Base
_, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete))
status.zero?
end
def ensure_dir_exist
gitlab_shell.add_namespace(repository_storage_path, namespace.path)
end
end
......@@ -159,7 +159,7 @@ class ProjectWiki
private
def init_repo(path_with_namespace)
gitlab_shell.add_repository(path_with_namespace)
gitlab_shell.add_repository(project.repository_storage_path, path_with_namespace)
end
def commit_details(action, message = nil, title = nil)
......@@ -173,7 +173,7 @@ class ProjectWiki
end
def path_to_repo
@path_to_repo ||= File.join(Gitlab.config.gitlab_shell.repos_path, "#{path_with_namespace}.git")
@path_to_repo ||= File.join(project.repository_storage_path, "#{path_with_namespace}.git")
end
def update_project_activity
......
......@@ -39,7 +39,7 @@ class Repository
# Return absolute path to repository
def path_to_repo
@path_to_repo ||= File.expand_path(
File.join(Gitlab.config.gitlab_shell.repos_path, path_with_namespace + ".git")
File.join(@project.repository_storage_path, path_with_namespace + ".git")
)
end
......
......@@ -51,13 +51,13 @@ module Projects
return true if params[:skip_repo] == true
# There is a possibility project does not have repository or wiki
return true unless gitlab_shell.exists?(path + '.git')
return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git')
new_path = removal_path(path)
if gitlab_shell.mv_repository(path, new_path)
if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path)
log_info("Repository \"#{path}\" moved to \"#{new_path}\"")
GitlabShellWorker.perform_in(5.minutes, :remove_repository, new_path)
GitlabShellWorker.perform_in(5.minutes, :remove_repository, project.repository_storage_path, new_path)
else
false
end
......
......@@ -24,7 +24,7 @@ module Projects
def execute
raise LeaseTaken unless try_obtain_lease
GitlabShellOneShotWorker.perform_async(:gc, @project.path_with_namespace)
GitlabShellOneShotWorker.perform_async(:gc, @project.repository_storage_path, @project.path_with_namespace)
ensure
Gitlab::Metrics.measure(:reset_pushes_since_gc) do
@project.update_column(:pushes_since_gc, 0)
......
......@@ -42,7 +42,7 @@ module Projects
def import_repository
begin
gitlab_shell.import_repository(project.path_with_namespace, project.import_url)
gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url)
rescue Gitlab::Shell::Error => e
raise Error, "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
end
......
......@@ -50,12 +50,12 @@ module Projects
project.send_move_instructions(old_path)
# Move main repository
unless gitlab_shell.mv_repository(old_path, new_path)
unless gitlab_shell.mv_repository(project.repository_storage_path, old_path, new_path)
raise TransferError.new('Cannot move project')
end
# Move wiki repo also if present
gitlab_shell.mv_repository("#{old_path}.wiki", "#{new_path}.wiki")
gitlab_shell.mv_repository(project.repository_storage_path, "#{old_path}.wiki", "#{new_path}.wiki")
# clear project cached events
project.reset_events_cache
......
......@@ -310,6 +310,15 @@
.col-sm-10
= f.text_field :sentry_dsn, class: 'form-control'
%fieldset
%legend Repository Storage
.form-group
= f.label :repository_storage, 'Storage path for new projects', class: 'control-label col-sm-2'
.col-sm-10
= f.select :repository_storage, repository_storage_options_for_select, {}, class: 'form-control'
.help-block
You can manage the repository storage paths in your gitlab.yml configuration file
%fieldset
%legend Repository Checks
.form-group
......
......@@ -10,7 +10,7 @@
.panel-body
%pre
:preserve
#{sanitize_repo_path(@project.import_error)}
#{sanitize_repo_path(@project, @project.import_error)}
= form_for @project, url: namespace_project_import_path(@project.namespace, @project), method: :post, html: { class: 'form-horizontal' } do |f|
= render "shared/import_form", f: f
......
......@@ -4,10 +4,10 @@ class PostReceive
sidekiq_options queue: :post_receive
def perform(repo_path, identifier, changes)
if repo_path.start_with?(Gitlab.config.gitlab_shell.repos_path.to_s)
repo_path.gsub!(Gitlab.config.gitlab_shell.repos_path.to_s, "")
if path = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1].to_s) }
repo_path.gsub!(path[1].to_s, "")
else
log("Check gitlab.yml config for correct gitlab_shell.repos_path variable. \"#{Gitlab.config.gitlab_shell.repos_path}\" does not match \"#{repo_path}\"")
log("Check gitlab.yml config for correct repositories.storages values. No repository storage path matches \"#{repo_path}\"")
end
post_received = Gitlab::GitPostReceive.new(repo_path, identifier, changes)
......
......@@ -12,7 +12,7 @@ class RepositoryForkWorker
return
end
result = gitlab_shell.fork_repository(source_path, target_path)
result = gitlab_shell.fork_repository(project.repository_storage_path, source_path, target_path)
unless result
logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}")
project.mark_import_as_failed('The project could not be forked.')
......
......@@ -47,11 +47,13 @@ production: &base
backup:
path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/)
repositories:
storages: # REPO PATHS MUST NOT BE A SYMLINK!!!
default: /apps/repositories/
gitlab_shell:
path: /apps/gitlab-shell/
# REPOS_PATH MUST NOT BE A SYMLINK!!!
repos_path: /apps/repositories/
hooks_path: /apps/gitlab-shell/hooks/
upload_pack: true
......
......@@ -428,6 +428,13 @@ production: &base
satellites:
path: /home/git/gitlab-satellites/
## Repositories settings
repositories:
# Paths where repositories can be stored. Give the canonicalized absolute pathname.
# NOTE: REPOS PATHS MUST NOT CONTAIN ANY SYMLINK!!!
storages: # You must have at least a `default` storage path.
default: /home/git/repositories/
## Backup settings
backup:
path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/)
......@@ -452,9 +459,6 @@ production: &base
## GitLab Shell settings
gitlab_shell:
path: /home/git/gitlab-shell/
# REPOS_PATH MUST NOT BE A SYMLINK!!!
repos_path: /home/git/repositories/
hooks_path: /home/git/gitlab-shell/hooks/
# File that contains the secret key for verifying access for gitlab-shell.
......@@ -528,11 +532,13 @@ test:
# user: YOUR_USERNAME
satellites:
path: tmp/tests/gitlab-satellites/
repositories:
storages:
default: tmp/tests/repositories/
backup:
path: tmp/tests/backups
gitlab_shell:
path: tmp/tests/gitlab-shell/
repos_path: tmp/tests/repositories/
hooks_path: tmp/tests/gitlab-shell/hooks/
issues_tracker:
redmine:
......
......@@ -304,13 +304,20 @@ Settings.gitlab_shell['hooks_path'] ||= Settings.gitlab['user_home'] + '/gitla
Settings.gitlab_shell['secret_file'] ||= Rails.root.join('.gitlab_shell_secret')
Settings.gitlab_shell['receive_pack'] = true if Settings.gitlab_shell['receive_pack'].nil?
Settings.gitlab_shell['upload_pack'] = true if Settings.gitlab_shell['upload_pack'].nil?
Settings.gitlab_shell['repos_path'] ||= Settings.gitlab['user_home'] + '/repositories/'
Settings.gitlab_shell['ssh_host'] ||= Settings.gitlab.ssh_host
Settings.gitlab_shell['ssh_port'] ||= 22
Settings.gitlab_shell['ssh_user'] ||= Settings.gitlab.user
Settings.gitlab_shell['owner_group'] ||= Settings.gitlab.user
Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.send(:build_gitlab_shell_ssh_path_prefix)
#
# Repositories
#
Settings['repositories'] ||= Settingslogic.new({})
Settings.repositories['storages'] ||= {}
# Setting gitlab_shell.repos_path is DEPRECATED and WILL BE REMOVED in version 9.0
Settings.repositories.storages['default'] ||= Settings.gitlab_shell['repos_path'] || Settings.gitlab['user_home'] + '/repositories/'
#
# Backup
#
......
def storage_name_valid?(name)
!!(name =~ /\A[a-zA-Z0-9\-_]+\z/)
end
def find_parent_path(name, path)
Gitlab.config.repositories.storages.detect do |n, p|
name != n && path.chomp('/').start_with?(p.chomp('/'))
end
end
def error(message)
raise "#{message}. Please fix this in your gitlab.yml before starting GitLab."
end
error('No repository storage path defined') if Gitlab.config.repositories.storages.empty?
Gitlab.config.repositories.storages.each do |name, path|
error("\"#{name}\" is not a valid storage name") unless storage_name_valid?(name)
parent_name, _parent_path = find_parent_path(name, path)
if parent_name
error("#{name} is a nested path of #{parent_name}. Nested paths are not supported for repository storages")
end
end
# Be sure to restart your server when you modify this file.
require 'securerandom'
# Your secret key for verifying the gitlab_shell.
secret_file = Gitlab.config.gitlab_shell.secret_file
unless File.exist? secret_file
# Generate a new token of 16 random hexadecimal characters and store it in secret_file.
token = SecureRandom.hex(16)
File.write(secret_file, token)
end
link_path = File.join(Gitlab.config.gitlab_shell.path, '.gitlab_shell_secret')
if File.exist?(Gitlab.config.gitlab_shell.path) && !File.exist?(link_path)
FileUtils.symlink(secret_file, link_path)
end
Gitlab::Shell.new.generate_and_link_secret_token
class AddRepositoryStorageToProjects < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
def up
add_column_with_default(:projects, :repository_storage, :string, default: 'default')
end
def down
remove_column(:projects, :repository_storage)
end
end
class AddRepositoryStorageToApplicationSettings < ActiveRecord::Migration
def change
add_column :application_settings, :repository_storage, :string, default: 'default'
end
end
......@@ -85,6 +85,7 @@ ActiveRecord::Schema.define(version: 20160620115026) do
t.boolean "send_user_confirmation_email", default: false
t.integer "container_registry_token_expire_delay", default: 5
t.text "after_sign_up_text"
t.string "repository_storage", default: "default"
end
create_table "audit_events", force: :cascade do |t|
......@@ -796,38 +797,39 @@ ActiveRecord::Schema.define(version: 20160620115026) do
t.datetime "created_at"
t.datetime "updated_at"
t.integer "creator_id"
t.boolean "issues_enabled", default: true, null: false
t.boolean "merge_requests_enabled", default: true, null: false
t.boolean "wiki_enabled", default: true, null: false
t.boolean "issues_enabled", default: true, null: false
t.boolean "merge_requests_enabled", default: true, null: false
t.boolean "wiki_enabled", default: true, null: false
t.integer "namespace_id"
t.boolean "snippets_enabled", default: true, null: false
t.boolean "snippets_enabled", default: true, null: false
t.datetime "last_activity_at"
t.string "import_url"
t.integer "visibility_level", default: 0, null: false
t.boolean "archived", default: false, null: false
t.integer "visibility_level", default: 0, null: false
t.boolean "archived", default: false, null: false
t.string "avatar"
t.string "import_status"
t.float "repository_size", default: 0.0
t.integer "star_count", default: 0, null: false
t.integer "star_count", default: 0, null: false
t.string "import_type"
t.string "import_source"
t.integer "commit_count", default: 0
t.text "import_error"
t.integer "ci_id"
t.boolean "builds_enabled", default: true, null: false
t.boolean "shared_runners_enabled", default: true, null: false
t.boolean "builds_enabled", default: true, null: false
t.boolean "shared_runners_enabled", default: true, null: false
t.string "runners_token"
t.string "build_coverage_regex"
t.boolean "build_allow_git_fetch", default: true, null: false
t.integer "build_timeout", default: 3600, null: false
t.boolean "build_allow_git_fetch", default: true, null: false
t.integer "build_timeout", default: 3600, null: false
t.boolean "pending_delete", default: false
t.boolean "public_builds", default: true, null: false
t.boolean "public_builds", default: true, null: false
t.integer "pushes_since_gc", default: 0
t.boolean "last_repository_check_failed"
t.datetime "last_repository_check_at"
t.boolean "container_registry_enabled"
t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false
t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false
t.boolean "has_external_issue_tracker"
t.string "repository_storage", default: "default", null: false
end
add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree
......
......@@ -34,6 +34,7 @@
- [Operations](operations/README.md) Keeping GitLab up and running.
- [Raketasks](raketasks/README.md) Backups, maintenance, automatic webhook setup and the importing of projects.
- [Repository checks](administration/repository_checks.md) Periodic Git repository checks.
- [Repository storages](administration/repository_storages.md) Manage the paths used to store repositories.
- [Security](security/README.md) Learn what you can do to further secure your GitLab instance.
- [System hooks](system_hooks/system_hooks.md) Notifications when users, projects and keys are changed.
- [Update](update/README.md) Update guides to upgrade your installation.
......
# Repository storages
GitLab allows you to define repository storage paths to enable distribution of
storage load between several mount points.
## For installations from source
Add your repository storage paths in your `gitlab.yml` under repositories -> storages, using key -> value pairs.
>**Notes:**
- You must have at least one storage path called `default`.
- In order for backups to work correctly the storage path must **not** be a
mount point and the GitLab user should have correct permissions for the parent
directory of the path.
## For omnibus installations
Follow the instructions at https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/configuration.md#storing-git-data-in-an-alternative-directory
......@@ -38,7 +38,8 @@ Example response:
"default_project_visibility" : 0,
"gravatar_enabled" : true,
"sign_in_text" : null,
"container_registry_token_expire_delay": 5
"container_registry_token_expire_delay": 5,
"repository_storage": "default"
}
```
......@@ -66,6 +67,7 @@ PUT /application/settings
| `user_oauth_applications` | boolean | no | Allow users to register any application to use GitLab as an OAuth provider |
| `after_sign_out_path` | string | no | Where to redirect users after logout |
| `container_registry_token_expire_delay` | integer | no | Container Registry token duration in minutes |
| `repository_storage` | string | no | Storage path for new projects. The value should be the name of one of the repository storage paths defined in your gitlab.yml |
```bash
curl -X PUT -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v3/application/settings?signup_enabled=false&default_project_visibility=1
......@@ -93,6 +95,7 @@ Example response:
"restricted_signup_domains": [],
"user_oauth_applications": true,
"after_sign_out_path": "",
"container_registry_token_expire_delay": 5
"container_registry_token_expire_delay": 5,
"repository_storage": "default"
}
```
......@@ -14,7 +14,8 @@
- For omnibus-gitlab, it is located at: `/var/opt/gitlab/git-data/repositories` by default, unless you changed
it in the `/etc/gitlab/gitlab.rb` file.
- For installations from source, it is usually located at: `/home/git/repositories` or you can see where
your repositories are located by looking at `config/gitlab.yml` under the `gitlab_shell => repos_path` entry.
your repositories are located by looking at `config/gitlab.yml` under the `repositories => storages` entries
(you'll usually use the `default` storage path to start).
New folder needs to have git user ownership and read/write/execute access for git user and its group:
......
......@@ -376,6 +376,7 @@ module API
expose :user_oauth_applications
expose :after_sign_out_path
expose :container_registry_token_expire_delay
expose :repository_storage
end
class Release < Grape::Entity
......
......@@ -20,6 +20,20 @@ module API
@wiki ||= params[:project].end_with?('.wiki') &&
!Project.find_with_namespace(params[:project])
end
def project
@project ||= begin
project_path = params[:project]
# Check for *.wiki repositories.
# Strip out the .wiki from the pathname before finding the
# project. This applies the correct project permissions to
# the wiki repository as well.
project_path.chomp!('.wiki') if wiki?
Project.find_with_namespace(project_path)
end
end
end
post "/allowed" do
......@@ -32,16 +46,6 @@ module API
User.find_by(id: params[:user_id])
end
project_path = params[:project]
# Check for *.wiki repositories.
# Strip out the .wiki from the pathname before finding the
# project. This applies the correct project permissions to
# the wiki repository as well.
project_path.chomp!('.wiki') if wiki?
project = Project.find_with_namespace(project_path)
access =
if wiki?
Gitlab::GitAccessWiki.new(actor, project)
......@@ -49,7 +53,17 @@ module API
Gitlab::GitAccess.new(actor, project)
end
access.check(params[:action], params[:changes])
access_status = access.check(params[:action], params[:changes])
response = { status: access_status.status, message: access_status.message }
if access_status.status
# Return the repository full path so that gitlab-shell has it when
# handling ssh commands
response[:repository_path] = project.repository.path_to_repo
end
response
end
#
......
......@@ -2,8 +2,6 @@ require 'yaml'
module Backup
class Repository
attr_reader :repos_path
def dump
prepare
......@@ -50,10 +48,12 @@ module Backup
end
def restore
if File.exists?(repos_path)
Gitlab.config.repositories.storages.each do |name, path|
next unless File.exists?(path)
# Move repos dir to 'repositories.old' dir
bk_repos_path = File.join(repos_path, '..', 'repositories.old.' + Time.now.to_i.to_s)
FileUtils.mv(repos_path, bk_repos_path)
bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s)
FileUtils.mv(path, bk_repos_path)
end
FileUtils.mkdir_p(repos_path)
......@@ -61,7 +61,7 @@ module Backup
Project.find_each(batch_size: 1000) do |project|
$progress.print " * #{project.path_with_namespace} ... "
project.namespace.ensure_dir_exist if project.namespace
project.ensure_dir_exist
if File.exists?(path_to_bundle(project))
FileUtils.mkdir_p(path_to_repo(project))
......@@ -100,8 +100,8 @@ module Backup
end
$progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow)
cmd = "#{Gitlab.config.gitlab_shell.path}/bin/create-hooks"
if system(cmd)
cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args
if system(*cmd)
$progress.puts " [DONE]".color(:green)
else
puts " [FAILED]".color(:red)
......@@ -120,10 +120,6 @@ module Backup
File.join(backup_repos_path, project.path_with_namespace + ".bundle")
end
def repos_path
Gitlab.config.gitlab_shell.repos_path
end
def backup_repos_path
File.join(Gitlab.config.backup.path, "repositories")
end
......@@ -139,5 +135,11 @@ module Backup
def silent
{err: '/dev/null', out: '/dev/null'}
end
private
def repository_storage_paths_args
Gitlab.config.repositories.storages.values
end
end
end
require 'securerandom'
module Gitlab
class Shell
class Error < StandardError; end
......@@ -18,77 +20,82 @@ module Gitlab
# Init new repository
#
# storage - project's storage path
# name - project path with namespace
#
# Ex.
# add_repository("gitlab/gitlab-ci")
# add_repository("/path/to/storage", "gitlab/gitlab-ci")
#
def add_repository(name)
def add_repository(storage, name)
Gitlab::Utils.system_silent([gitlab_shell_projects_path,
'add-project', "#{name}.git"])
'add-project', storage, "#{name}.git"])
end
# Import repository
#
# storage - project's storage path
# name - project path with namespace
#
# Ex.
# import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git")
# import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git")
#
def import_repository(name, url)
output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '900'])
def import_repository(storage, name, url)
output, status = Popen::popen([gitlab_shell_projects_path, 'import-project',
storage, "#{name}.git", url, '900'])
raise Error, output unless status.zero?
true
end
# Move repository
#
# storage - project's storage path
# path - project path with namespace
# new_path - new project path with namespace
#
# Ex.
# mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new")
# mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
#
def mv_repository(path, new_path)
def mv_repository(storage, path, new_path)
Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project',
"#{path}.git", "#{new_path}.git"])
storage, "#{path}.git", "#{new_path}.git"])
end
# Fork repository to new namespace
#
# storage - project's storage path
# path - project path with namespace
# fork_namespace - namespace for forked project
#
# Ex.
# fork_repository("gitlab/gitlab-ci", "randx")
# fork_repository("/path/to/storage", "gitlab/gitlab-ci", "randx")
#
def fork_repository(path, fork_namespace)
def fork_repository(storage, path, fork_namespace)
Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project',
"#{path}.git", fork_namespace])
storage, "#{path}.git", fork_namespace])
end
# Remove repository from file system
#
# storage - project's storage path
# name - project path with namespace
#
# Ex.
# remove_repository("gitlab/gitlab-ci")
# remove_repository("/path/to/storage", "gitlab/gitlab-ci")
#
def remove_repository(name)
def remove_repository(storage, name)
Gitlab::Utils.system_silent([gitlab_shell_projects_path,
'rm-project', "#{name}.git"])
'rm-project', storage, "#{name}.git"])
end
# Gc repository
#
# storage - project storage path
# path - project path with namespace
#
# Ex.
# gc("gitlab/gitlab-ci")
# gc("/path/to/storage", "gitlab/gitlab-ci")
#
def gc(path)
def gc(storage, path)
Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc',
"#{path}.git"])
storage, "#{path}.git"])
end
# Add new key to gitlab-shell
......@@ -133,31 +140,31 @@ module Gitlab
# Add empty directory for storing repositories
#
# Ex.
# add_namespace("gitlab")
# add_namespace("/path/to/storage", "gitlab")
#
def add_namespace(name)
FileUtils.mkdir(full_path(name), mode: 0770) unless exists?(name)
def add_namespace(storage, name)
FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name)
end
# Remove directory from repositories storage
# Every repository inside this directory will be removed too
#
# Ex.
# rm_namespace("gitlab")
# rm_namespace("/path/to/storage", "gitlab")
#
def rm_namespace(name)
FileUtils.rm_r(full_path(name), force: true)
def rm_namespace(storage, name)
FileUtils.rm_r(full_path(storage, name), force: true)
end
# Move namespace directory inside repositories storage
#
# Ex.
# mv_namespace("gitlab", "gitlabhq")
# mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
#
def mv_namespace(old_name, new_name)
return false if exists?(new_name) || !exists?(old_name)
def mv_namespace(storage, old_name, new_name)
return false if exists?(storage, new_name) || !exists?(storage, old_name)
FileUtils.mv(full_path(old_name), full_path(new_name))
FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name))
end
def url_to_repo(path)
......@@ -176,11 +183,26 @@ module Gitlab
# Check if such directory exists in repositories.
#
# Usage:
# exists?('gitlab')
# exists?('gitlab/cookies.git')
# exists?(storage, 'gitlab')
# exists?(storage, 'gitlab/cookies.git')
#
def exists?(dir_name)
File.exist?(full_path(dir_name))
def exists?(storage, dir_name)
File.exist?(full_path(storage, dir_name))
end
# Create (if necessary) and link the secret token file
def generate_and_link_secret_token
secret_file = Gitlab.config.gitlab_shell.secret_file
unless File.exist? secret_file
# Generate a new token of 16 random hexadecimal characters and store it in secret_file.
token = SecureRandom.hex(16)
File.write(secret_file, token)
end
link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret')
if File.exist?(gitlab_shell_path) && !File.exist?(link_path)
FileUtils.symlink(secret_file, link_path)
end
end
protected
......@@ -193,14 +215,10 @@ module Gitlab
File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}")
end
def repos_path
Gitlab.config.gitlab_shell.repos_path
end
def full_path(dir_name)
def full_path(storage, dir_name)
raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?
File.join(repos_path, dir_name)
File.join(storage, dir_name)
end
def gitlab_shell_projects_path
......
......@@ -167,7 +167,7 @@ module Gitlab
def import_wiki
unless project.wiki_enabled?
wiki = WikiFormatter.new(project)
gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url)
gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url)
project.update_attribute(:wiki_enabled, true)
end
......
......@@ -356,97 +356,108 @@ namespace :gitlab do
########################
def check_repo_base_exists
print "Repo base directory exists? ... "
puts "Repo base directory exists?"
repo_base_path = Gitlab.config.gitlab_shell.repos_path
Gitlab.config.repositories.storages.each do |name, repo_base_path|
print "#{name}... "
if File.exists?(repo_base_path)
puts "yes".color(:green)
else
puts "no".color(:red)
puts "#{repo_base_path} is missing".color(:red)
try_fixing_it(
"This should have been created when setting up GitLab Shell.",
"Make sure it's set correctly in config/gitlab.yml",
"Make sure GitLab Shell is installed correctly."
)
for_more_information(
see_installation_guide_section "GitLab Shell"
)
fix_and_rerun
if File.exists?(repo_base_path)
puts "yes".color(:green)
else
puts "no".color(:red)
puts "#{repo_base_path} is missing".color(:red)
try_fixing_it(
"This should have been created when setting up GitLab Shell.",
"Make sure it's set correctly in config/gitlab.yml",
"Make sure GitLab Shell is installed correctly."
)
for_more_information(
see_installation_guide_section "GitLab Shell"
)
fix_and_rerun
end
end
end
def check_repo_base_is_not_symlink
print "Repo base directory is a symlink? ... "
puts "Repo storage directories are symlinks?"
repo_base_path = Gitlab.config.gitlab_shell.repos_path
unless File.exists?(repo_base_path)
puts "can't check because of previous errors".color(:magenta)
return
end
Gitlab.config.repositories.storages.each do |name, repo_base_path|
print "#{name}... "
unless File.symlink?(repo_base_path)
puts "no".color(:green)
else
puts "yes".color(:red)
try_fixing_it(
"Make sure it's set to the real directory in config/gitlab.yml"
)
fix_and_rerun
unless File.exists?(repo_base_path)
puts "can't check because of previous errors".color(:magenta)
return
end
unless File.symlink?(repo_base_path)
puts "no".color(:green)
else
puts "yes".color(:red)
try_fixing_it(
"Make sure it's set to the real directory in config/gitlab.yml"
)
fix_and_rerun
end
end
end
def check_repo_base_permissions
print "Repo base access is drwxrws---? ... "
puts "Repo paths access is drwxrws---?"
repo_base_path = Gitlab.config.gitlab_shell.repos_path
unless File.exists?(repo_base_path)
puts "can't check because of previous errors".color(:magenta)
return
end
Gitlab.config.repositories.storages.each do |name, repo_base_path|
print "#{name}... "
if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770")
puts "yes".color(:green)
else
puts "no".color(:red)
try_fixing_it(
"sudo chmod -R ug+rwX,o-rwx #{repo_base_path}",
"sudo chmod -R ug-s #{repo_base_path}",
"sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s"
)
for_more_information(
see_installation_guide_section "GitLab Shell"
)
fix_and_rerun
unless File.exists?(repo_base_path)
puts "can't check because of previous errors".color(:magenta)
return
end
if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770")
puts "yes".color(:green)
else
puts "no".color(:red)
try_fixing_it(
"sudo chmod -R ug+rwX,o-rwx #{repo_base_path}",
"sudo chmod -R ug-s #{repo_base_path}",
"sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s"
)
for_more_information(
see_installation_guide_section "GitLab Shell"
)
fix_and_rerun
end
end
end
def check_repo_base_user_and_group
gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user
gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group
print "Repo base owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}? ... "
puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?"
repo_base_path = Gitlab.config.gitlab_shell.repos_path
unless File.exists?(repo_base_path)
puts "can't check because of previous errors".color(:magenta)
return
end
Gitlab.config.repositories.storages.each do |name, repo_base_path|
print "#{name}... "
uid = uid_for(gitlab_shell_ssh_user)
gid = gid_for(gitlab_shell_owner_group)
if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid
puts "yes".color(:green)
else
puts "no".color(:red)
puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue)
try_fixing_it(
"sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}"
)
for_more_information(
see_installation_guide_section "GitLab Shell"
)
fix_and_rerun
unless File.exists?(repo_base_path)
puts "can't check because of previous errors".color(:magenta)
return
end
uid = uid_for(gitlab_shell_ssh_user)
gid = gid_for(gitlab_shell_owner_group)
if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid
puts "yes".color(:green)
else
puts "no".color(:red)
puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue)
try_fixing_it(
"sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}"
)
for_more_information(
see_installation_guide_section "GitLab Shell"
)
fix_and_rerun
end
end
end
......@@ -473,7 +484,7 @@ namespace :gitlab do
else
puts "wrong or missing hooks".color(:red)
try_fixing_it(
sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')}"),
sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"),
'Check the hooks_path in config/gitlab.yml',
'Check your gitlab-shell installation'
)
......@@ -785,13 +796,13 @@ namespace :gitlab do
namespace :repo do
desc "GitLab | Check the integrity of the repositories managed by GitLab"
task check: :environment do
namespace_dirs = Dir.glob(
File.join(Gitlab.config.gitlab_shell.repos_path, '*')
)
Gitlab.config.repositories.storages.each do |name, path|
namespace_dirs = Dir.glob(File.join(path, '*'))
namespace_dirs.each do |namespace_dir|
repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
namespace_dirs.each do |namespace_dir|
repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
end
end
end
end
......@@ -799,12 +810,12 @@ namespace :gitlab do
namespace :user do
desc "GitLab | Check the integrity of a specific user's repositories"
task :check_repos, [:username] => :environment do |t, args|
username = args[:username] || prompt("Check repository integrity for which username? ".color(:blue))
username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue))
user = User.find_by(username: username)
if user
repo_dirs = user.authorized_projects.map do |p|
File.join(
Gitlab.config.gitlab_shell.repos_path,
p.repository_storage_path,
"#{p.path_with_namespace}.git"
)
end
......
......@@ -5,36 +5,36 @@ namespace :gitlab do
warn_user_is_not_gitlab
remove_flag = ENV['REMOVE']
namespaces = Namespace.pluck(:path)
git_base_path = Gitlab.config.gitlab_shell.repos_path
all_dirs = Dir.glob(git_base_path + '/*')
Gitlab.config.repositories.storages.each do |name, git_base_path|
all_dirs = Dir.glob(git_base_path + '/*')
puts git_base_path.color(:yellow)
puts "Looking for directories to remove... "
puts git_base_path.color(:yellow)
puts "Looking for directories to remove... "
all_dirs.reject! do |dir|
# skip if git repo
dir =~ /.git$/
end
all_dirs.reject! do |dir|
# skip if git repo
dir =~ /.git$/
end
all_dirs.reject! do |dir|
dir_name = File.basename dir
all_dirs.reject! do |dir|
dir_name = File.basename dir
# skip if namespace present
namespaces.include?(dir_name)
end
# skip if namespace present
namespaces.include?(dir_name)
end
all_dirs.each do |dir_path|
all_dirs.each do |dir_path|
if remove_flag
if FileUtils.rm_rf dir_path
puts "Removed...#{dir_path}".color(:red)
if remove_flag
if FileUtils.rm_rf dir_path
puts "Removed...#{dir_path}".color(:red)
else
puts "Cannot remove #{dir_path}".color(:red)
end
else
puts "Cannot remove #{dir_path}".color(:red)
puts "Can be removed: #{dir_path}".color(:red)
end
else
puts "Can be removed: #{dir_path}".color(:red)
end
end
......@@ -48,20 +48,21 @@ namespace :gitlab do
warn_user_is_not_gitlab
move_suffix = "+orphaned+#{Time.now.to_i}"
repo_root = Gitlab.config.gitlab_shell.repos_path
# Look for global repos (legacy, depth 1) and normal repos (depth 2)
IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
find.each_line do |path|
path.chomp!
repo_with_namespace = path.
sub(repo_root, '').
sub(%r{^/*}, '').
chomp('.git').
chomp('.wiki')
next if Project.find_with_namespace(repo_with_namespace)
new_path = path + move_suffix
puts path.inspect + ' -> ' + new_path.inspect
File.rename(path, new_path)
Gitlab.config.repositories.storages.each do |name, repo_root|
# Look for global repos (legacy, depth 1) and normal repos (depth 2)
IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
find.each_line do |path|
path.chomp!
repo_with_namespace = path.
sub(repo_root, '').
sub(%r{^/*}, '').
chomp('.git').
chomp('.wiki')
next if Project.find_with_namespace(repo_with_namespace)
new_path = path + move_suffix
puts path.inspect + ' -> ' + new_path.inspect
File.rename(path, new_path)
end
end
end
end
......
......@@ -2,73 +2,73 @@ namespace :gitlab do
namespace :import do
# How to use:
#
# 1. copy the bare repos under the repos_path (commonly /home/git/repositories)
# 1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories)
# 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production
#
# Notes:
# * The project owner will set to the first administator of the system
# * Existing projects will be skipped
#
desc "GitLab | Import bare repositories from gitlab_shell -> repos_path into GitLab project instance"
desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
task repos: :environment do
Gitlab.config.repositories.storages.each do |name, git_base_path|
repos_to_import = Dir.glob(git_base_path + '/**/*.git')
git_base_path = Gitlab.config.gitlab_shell.repos_path
repos_to_import = Dir.glob(git_base_path + '/**/*.git')
repos_to_import.each do |repo_path|
# strip repo base path
repo_path[0..git_base_path.length] = ''
repos_to_import.each do |repo_path|
# strip repo base path
repo_path[0..git_base_path.length] = ''
path = repo_path.sub(/\.git$/, '')
group_name, name = File.split(path)
group_name = nil if group_name == '.'
path = repo_path.sub(/\.git$/, '')
group_name, name = File.split(path)
group_name = nil if group_name == '.'
puts "Processing #{repo_path}".color(:yellow)
puts "Processing #{repo_path}".color(:yellow)
if path.end_with?('.wiki')
puts " * Skipping wiki repo"
next
end
if path.end_with?('.wiki')
puts " * Skipping wiki repo"
next
end
project = Project.find_with_namespace(path)
project = Project.find_with_namespace(path)
if project
puts " * #{project.name} (#{repo_path}) exists"
else
user = User.admins.reorder("id").first
if project
puts " * #{project.name} (#{repo_path}) exists"
else
user = User.admins.reorder("id").first
project_params = {
name: name,
path: name
}
project_params = {
name: name,
path: name
}
# find group namespace
if group_name
group = Namespace.find_by(path: group_name)
# create group namespace
unless group
group = Group.new(:name => group_name)
group.path = group_name
group.owner = user
if group.save
puts " * Created Group #{group.name} (#{group.id})".color(:green)
else
puts " * Failed trying to create group #{group.name}".color(:red)
# find group namespace
if group_name
group = Namespace.find_by(path: group_name)
# create group namespace
unless group
group = Group.new(:name => group_name)
group.path = group_name
group.owner = user
if group.save
puts " * Created Group #{group.name} (#{group.id})".color(:green)
else
puts " * Failed trying to create group #{group.name}".color(:red)
end
end
# set project group
project_params[:namespace_id] = group.id
end
# set project group
project_params[:namespace_id] = group.id
end
project = Projects::CreateService.new(user, project_params).execute
project = Projects::CreateService.new(user, project_params).execute
if project.persisted?
puts " * Created #{project.name} (#{repo_path})".color(:green)
project.update_repository_size
project.update_commit_count
else
puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
puts " Errors: #{project.errors.messages}".color(:red)
if project.persisted?
puts " * Created #{project.name} (#{repo_path})".color(:green)
project.update_repository_size
project.update_commit_count
else
puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
puts " Errors: #{project.errors.messages}".color(:red)
end
end
end
end
......
......@@ -62,7 +62,10 @@ namespace :gitlab do
puts ""
puts "GitLab Shell".color(:yellow)
puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}"
puts "Repositories:\t#{Gitlab.config.gitlab_shell.repos_path}"
puts "Repository storage paths:"
Gitlab.config.repositories.storages.each do |name, path|
puts "- #{name}: \t#{path}"
end
puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}"
puts "Git:\t\t#{Gitlab.config.git.bin_path}"
......
......@@ -9,7 +9,7 @@ namespace :gitlab do
scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids)
end
scope.find_each do |project|
base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace)
base = File.join(project.repository_storage_path, project.path_with_namespace)
puts base + '.git'
puts base + '.wiki.git'
end
......
......@@ -12,7 +12,6 @@ namespace :gitlab do
gitlab_url = Gitlab.config.gitlab.url
# gitlab-shell requires a / at the end of the url
gitlab_url += '/' unless gitlab_url.end_with?('/')
repos_path = Gitlab.config.gitlab_shell.repos_path
target_dir = Gitlab.config.gitlab_shell.path
# Clone if needed
......@@ -35,7 +34,6 @@ namespace :gitlab do
user: user,
gitlab_url: gitlab_url,
http_settings: {self_signed_cert: false}.stringify_keys,
repos_path: repos_path,
auth_file: File.join(home_dir, ".ssh", "authorized_keys"),
redis: {
bin: %x{which redis-cli}.chomp,
......@@ -58,10 +56,10 @@ namespace :gitlab do
File.open("config.yml", "w+") {|f| f.puts config.to_yaml}
# Launch installation process
system(*%W(bin/install))
system(*%W(bin/install) + repository_storage_paths_args)
# (Re)create hooks
system(*%W(bin/create-hooks))
system(*%W(bin/create-hooks) + repository_storage_paths_args)
end
# Required for debian packaging with PKGR: Setup .ssh/environment with
......@@ -73,6 +71,8 @@ namespace :gitlab do
File.open(File.join(home_dir, ".ssh", "environment"), "w+") do |f|
f.puts "PATH=#{ENV['PATH']}"
end
Gitlab::Shell.new.generate_and_link_secret_token
end
desc "GitLab | Setup gitlab-shell"
......@@ -87,7 +87,8 @@ namespace :gitlab do
if File.exists?(path_to_repo)
print '-'
else
if Gitlab::Shell.new.add_repository(project.path_with_namespace)
if Gitlab::Shell.new.add_repository(project.repository_storage_path,
project.path_with_namespace)
print '.'
else
print 'F'
......@@ -138,4 +139,3 @@ namespace :gitlab do
system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag}))
end
end
......@@ -125,10 +125,16 @@ namespace :gitlab do
end
def all_repos
IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
find.each_line do |path|
yield path.chomp
Gitlab.config.repositories.storages.each do |name, path|
IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
find.each_line do |path|
yield path.chomp
end
end
end
end
def repository_storage_paths_args
Gitlab.config.repositories.storages.values
end
end
......@@ -123,11 +123,17 @@ describe ProjectsHelper do
end
describe '#sanitized_import_error' do
let(:project) { create(:project) }
before do
allow(project).to receive(:repository_storage_path).and_return('/base/repo/path')
end
it 'removes the repo path' do
repo = File.join(Gitlab.config.gitlab_shell.repos_path, '/namespace/test.git')
repo = '/base/repo/path/namespace/test.git'
import_error = "Could not clone #{repo}\n"
expect(sanitize_repo_path(import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git')
expect(sanitize_repo_path(project, import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git')
end
end
end
require 'spec_helper'
describe '6_validations', lib: true do
context 'with correct settings' do
before do
mock_storages('foo' => '/a/b/c', 'bar' => 'a/b/d')
end
it 'passes through' do
expect { load_validations }.not_to raise_error
end
end
context 'with invalid storage names' do
before do
mock_storages('name with spaces' => '/a/b/c')
end
it 'throws an error' do
expect { load_validations }.to raise_error('"name with spaces" is not a valid storage name. Please fix this in your gitlab.yml before starting GitLab.')
end
end
context 'with nested storage paths' do
before do
mock_storages('foo' => '/a/b/c', 'bar' => '/a/b/c/d')
end
it 'throws an error' do
expect { load_validations }.to raise_error('bar is a nested path of foo. Nested paths are not supported for repository storages. Please fix this in your gitlab.yml before starting GitLab.')
end
end
def mock_storages(storages)
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
def load_validations
load File.join(__dir__, '../../config/initializers/6_validations.rb')
end
end
......@@ -13,9 +13,37 @@ describe Gitlab::Shell, lib: true do
it { is_expected.to respond_to :add_repository }
it { is_expected.to respond_to :remove_repository }
it { is_expected.to respond_to :fork_repository }
it { is_expected.to respond_to :gc }
it { is_expected.to respond_to :add_namespace }
it { is_expected.to respond_to :rm_namespace }
it { is_expected.to respond_to :mv_namespace }
it { is_expected.to respond_to :exists? }
it { expect(gitlab_shell.url_to_repo('diaspora')).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + "diaspora.git") }
describe 'generate_and_link_secret_token' do
let(:secret_file) { 'tmp/tests/.secret_shell_test' }
let(:link_file) { 'tmp/tests/shell-secret-test/.gitlab_shell_secret' }
before do
allow(Gitlab.config.gitlab_shell).to receive(:path).and_return('tmp/tests/shell-secret-test')
allow(Gitlab.config.gitlab_shell).to receive(:secret_file).and_return(secret_file)
FileUtils.mkdir('tmp/tests/shell-secret-test')
gitlab_shell.generate_and_link_secret_token
end
after do
FileUtils.rm_rf('tmp/tests/shell-secret-test')
FileUtils.rm_rf(secret_file)
end
it 'creates and links the secret token file' do
expect(File.exist?(secret_file)).to be(true)
expect(File.symlink?(link_file)).to be(true)
expect(File.readlink(link_file)).to eq(secret_file)
end
end
describe Gitlab::Shell::KeyAdder, lib: true do
describe '#add_key' do
it 'normalizes space characters in the key' do
......
......@@ -40,6 +40,16 @@ describe ApplicationSetting, models: true do
it_behaves_like 'an object with email-formated attributes', :admin_notification_email do
subject { setting }
end
context 'repository storages inclussion' do
before do
storages = { 'custom' => 'tmp/tests/custom_repositories' }
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
it { is_expected.to allow_value('custom').for(:repository_storage) }
it { is_expected.not_to allow_value('alternative').for(:repository_storage) }
end
end
context 'restricted signup domains' do
......
......@@ -57,6 +57,7 @@ describe Namespace, models: true do
describe :move_dir do
before do
@namespace = create :namespace
@project = create :project, namespace: @namespace
allow(@namespace).to receive(:path_changed?).and_return(true)
end
......@@ -87,8 +88,13 @@ describe Namespace, models: true do
end
describe :rm_dir do
it "should remove dir" do
expect(namespace.rm_dir).to be_truthy
let!(:project) { create(:project, namespace: namespace) }
let!(:path) { File.join(Gitlab.config.repositories.storages.default, namespace.path) }
before { namespace.destroy }
it "should remove its dirs when deleted" do
expect(File.exist?(path)).to be(false)
end
end
......
......@@ -56,6 +56,7 @@ describe Project, models: true do
it { is_expected.to validate_length_of(:description).is_within(0..2000) }
it { is_expected.to validate_presence_of(:creator) }
it { is_expected.to validate_presence_of(:namespace) }
it { is_expected.to validate_presence_of(:repository_storage) }
it 'should not allow new projects beyond user limits' do
project2 = build(:project)
......@@ -84,6 +85,20 @@ describe Project, models: true do
end
end
end
context 'repository storages inclussion' do
let(:project2) { build(:project, repository_storage: 'missing') }
before do
storages = { 'custom' => 'tmp/tests/custom_repositories' }
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
it "should not allow repository storages that don't match a label in the configuration" do
expect(project2).not_to be_valid
expect(project2.errors[:repository_storage].first).to match(/is not included in the list/)
end
end
end
describe 'default_scope' do
......@@ -131,6 +146,24 @@ describe Project, models: true do
end
end
describe '#repository_storage_path' do
let(:project) { create(:project, repository_storage: 'custom') }
before do
FileUtils.mkdir('tmp/tests/custom_repositories')
storages = { 'custom' => 'tmp/tests/custom_repositories' }
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
after do
FileUtils.rm_rf('tmp/tests/custom_repositories')
end
it 'returns the repository storage path' do
expect(project.repository_storage_path).to eq('tmp/tests/custom_repositories')
end
end
it 'should return valid url to repo' do
project = Project.new(path: 'somewhere')
expect(project.url_to_repo).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + 'somewhere.git')
......@@ -574,6 +607,21 @@ describe Project, models: true do
end
end
context 'repository storage by default' do
let(:project) { create(:empty_project) }
subject { project.repository_storage }
before do
storages = { 'alternative_storage' => '/some/path' }
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
stub_application_setting(repository_storage: 'alternative_storage')
allow_any_instance_of(Project).to receive(:ensure_dir_exist).and_return(true)
end
it { is_expected.to eq('alternative_storage') }
end
context 'shared runners by default' do
let(:project) { create(:empty_project) }
......@@ -729,12 +777,12 @@ describe Project, models: true do
expect(gitlab_shell).to receive(:mv_repository).
ordered.
with("#{ns}/foo", "#{ns}/#{project.path}").
with(project.repository_storage_path, "#{ns}/foo", "#{ns}/#{project.path}").
and_return(true)
expect(gitlab_shell).to receive(:mv_repository).
ordered.
with("#{ns}/foo.wiki", "#{ns}/#{project.path}.wiki").
with(project.repository_storage_path, "#{ns}/foo.wiki", "#{ns}/#{project.path}.wiki").
and_return(true)
expect_any_instance_of(SystemHooksService).
......@@ -826,7 +874,7 @@ describe Project, models: true do
context 'using a regular repository' do
it 'creates the repository' do
expect(shell).to receive(:add_repository).
with(project.path_with_namespace).
with(project.repository_storage_path, project.path_with_namespace).
and_return(true)
expect(project.repository).to receive(:after_create)
......@@ -836,7 +884,7 @@ describe Project, models: true do
it 'adds an error if the repository could not be created' do
expect(shell).to receive(:add_repository).
with(project.path_with_namespace).
with(project.repository_storage_path, project.path_with_namespace).
and_return(false)
expect(project.repository).not_to receive(:after_create)
......
......@@ -72,6 +72,7 @@ describe API::API, api: true do
expect(response).to have_http_status(200)
expect(json_response["status"]).to be_truthy
expect(json_response["repository_path"]).to eq(project.repository.path_to_repo)
end
end
......@@ -81,6 +82,7 @@ describe API::API, api: true do
expect(response).to have_http_status(200)
expect(json_response["status"]).to be_truthy
expect(json_response["repository_path"]).to eq(project.repository.path_to_repo)
end
end
end
......
......@@ -14,16 +14,23 @@ describe API::API, 'Settings', api: true do
expect(json_response).to be_an Hash
expect(json_response['default_projects_limit']).to eq(42)
expect(json_response['signin_enabled']).to be_truthy
expect(json_response['repository_storage']).to eq('default')
end
end
describe "PUT /application/settings" do
before do
storages = { 'custom' => 'tmp/tests/custom_repositories' }
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
it "should update application settings" do
put api("/application/settings", admin),
default_projects_limit: 3, signin_enabled: false
default_projects_limit: 3, signin_enabled: false, repository_storage: 'custom'
expect(response).to have_http_status(200)
expect(json_response['default_projects_limit']).to eq(3)
expect(json_response['signin_enabled']).to be_falsey
expect(json_response['repository_storage']).to eq('custom')
end
end
end
......@@ -23,8 +23,8 @@ describe DestroyGroupService, services: true do
Sidekiq::Testing.inline! { destroy_group(group, user) }
end
it { expect(gitlab_shell.exists?(group.path)).to be_falsey }
it { expect(gitlab_shell.exists?(remove_path)).to be_falsey }
it { expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_falsey }
it { expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_falsey }
end
context 'Sidekiq fake' do
......@@ -33,8 +33,8 @@ describe DestroyGroupService, services: true do
Sidekiq::Testing.fake! { destroy_group(group, user) }
end
it { expect(gitlab_shell.exists?(group.path)).to be_falsey }
it { expect(gitlab_shell.exists?(remove_path)).to be_truthy }
it { expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_falsey }
it { expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_truthy }
end
end
......
......@@ -12,7 +12,7 @@ describe Projects::HousekeepingService do
it 'enqueues a sidekiq job' do
expect(subject).to receive(:try_obtain_lease).and_return(true)
expect(GitlabShellOneShotWorker).to receive(:perform_async).with(:gc, project.path_with_namespace)
expect(GitlabShellOneShotWorker).to receive(:perform_async).with(:gc, project.repository_storage_path, project.path_with_namespace)
subject.execute
expect(project.pushes_since_gc).to eq(0)
......
......@@ -36,7 +36,7 @@ describe Projects::ImportService, services: true do
end
it 'succeeds if repository import is successfully' do
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
result = subject.execute
......@@ -44,7 +44,7 @@ describe Projects::ImportService, services: true do
end
it 'fails if repository import fails' do
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_raise(Gitlab::Shell::Error.new('Failed to import the repository'))
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_raise(Gitlab::Shell::Error.new('Failed to import the repository'))
result = subject.execute
......@@ -64,7 +64,7 @@ describe Projects::ImportService, services: true do
end
it 'succeeds if importer succeeds' do
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(true)
result = subject.execute
......@@ -74,7 +74,7 @@ describe Projects::ImportService, services: true do
it 'flushes various caches' do
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).
with(project.path_with_namespace, project.import_url).
with(project.repository_storage_path, project.path_with_namespace, project.import_url).
and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).
......@@ -90,7 +90,7 @@ describe Projects::ImportService, services: true do
end
it 'fails if importer fails' do
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(false)
result = subject.execute
......@@ -100,7 +100,7 @@ describe Projects::ImportService, services: true do
end
it 'fails if importer raise an error' do
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true)
expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_raise(Projects::ImportService::Error.new('Github: failed to connect API'))
result = subject.execute
......
......@@ -80,8 +80,9 @@ module TestEnv
end
def setup_gitlab_shell
unless File.directory?(Rails.root.join(*%w(tmp tests gitlab-shell)))
`rake gitlab:shell:install`
unless File.directory?(Gitlab.config.gitlab_shell.path)
# TODO: Remove `[shards]` when gitlab-shell v3.1.0 is published
`rake gitlab:shell:install[shards]`
end
end
......@@ -127,14 +128,14 @@ module TestEnv
def copy_repo(project)
base_repo_path = File.expand_path(factory_repo_path_bare)
target_repo_path = File.expand_path(repos_path + "/#{project.namespace.path}/#{project.path}.git")
target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.namespace.path}/#{project.path}.git")
FileUtils.mkdir_p(target_repo_path)
FileUtils.cp_r("#{base_repo_path}/.", target_repo_path)
FileUtils.chmod_R 0755, target_repo_path
end
def repos_path
Gitlab.config.gitlab_shell.repos_path
Gitlab.config.repositories.storages.default
end
def backup_path
......@@ -143,7 +144,7 @@ module TestEnv
def copy_forked_repo_with_submodules(project)
base_repo_path = File.expand_path(forked_repo_path_bare)
target_repo_path = File.expand_path(repos_path + "/#{project.namespace.path}/#{project.path}.git")
target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.namespace.path}/#{project.path}.git")
FileUtils.mkdir_p(target_repo_path)
FileUtils.cp_r("#{base_repo_path}/.", target_repo_path)
FileUtils.chmod_R 0755, target_repo_path
......
......@@ -98,67 +98,107 @@ describe 'gitlab:app namespace rake task' do
@backup_tar = tars_glob.first
end
before do
create_backup
end
after do
FileUtils.rm(@backup_tar)
end
context 'tar creation' do
before do
create_backup
end
context 'archive file permissions' do
it 'should set correct permissions on the tar file' do
expect(File.exist?(@backup_tar)).to be_truthy
expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100600')
after do
FileUtils.rm(@backup_tar)
end
context 'with custom archive_permissions' do
before do
allow(Gitlab.config.backup).to receive(:archive_permissions).and_return(0651)
# We created a backup in a before(:all) so it got the default permissions.
# We now need to do some work to create a _new_ backup file using our stub.
FileUtils.rm(@backup_tar)
create_backup
context 'archive file permissions' do
it 'should set correct permissions on the tar file' do
expect(File.exist?(@backup_tar)).to be_truthy
expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100600')
end
it 'uses the custom permissions' do
expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100651')
context 'with custom archive_permissions' do
before do
allow(Gitlab.config.backup).to receive(:archive_permissions).and_return(0651)
# We created a backup in a before(:all) so it got the default permissions.
# We now need to do some work to create a _new_ backup file using our stub.
FileUtils.rm(@backup_tar)
create_backup
end
it 'uses the custom permissions' do
expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100651')
end
end
end
end
it 'should set correct permissions on the tar contents' do
tar_contents, exit_status = Gitlab::Popen.popen(
%W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz registry.tar.gz}
)
expect(exit_status).to eq(0)
expect(tar_contents).to match('db/')
expect(tar_contents).to match('uploads.tar.gz')
expect(tar_contents).to match('repositories/')
expect(tar_contents).to match('builds.tar.gz')
expect(tar_contents).to match('artifacts.tar.gz')
expect(tar_contents).to match('lfs.tar.gz')
expect(tar_contents).to match('registry.tar.gz')
expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz|registry.tar.gz)\/$/)
end
it 'should set correct permissions on the tar contents' do
tar_contents, exit_status = Gitlab::Popen.popen(
%W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz registry.tar.gz}
)
expect(exit_status).to eq(0)
expect(tar_contents).to match('db/')
expect(tar_contents).to match('uploads.tar.gz')
expect(tar_contents).to match('repositories/')
expect(tar_contents).to match('builds.tar.gz')
expect(tar_contents).to match('artifacts.tar.gz')
expect(tar_contents).to match('lfs.tar.gz')
expect(tar_contents).to match('registry.tar.gz')
expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz|registry.tar.gz)\/$/)
end
it 'should delete temp directories' do
temp_dirs = Dir.glob(
File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,lfs,registry}')
)
it 'should delete temp directories' do
temp_dirs = Dir.glob(
File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,lfs,registry}')
)
expect(temp_dirs).to be_empty
end
expect(temp_dirs).to be_empty
context 'registry disabled' do
let(:enable_registry) { false }
it 'should not create registry.tar.gz' do
tar_contents, exit_status = Gitlab::Popen.popen(
%W{tar -tvf #{@backup_tar}}
)
expect(exit_status).to eq(0)
expect(tar_contents).not_to match('registry.tar.gz')
end
end
end
context 'registry disabled' do
let(:enable_registry) { false }
context 'multiple repository storages' do
let(:project_a) { create(:project, repository_storage: 'default') }
let(:project_b) { create(:project, repository_storage: 'custom') }
before do
FileUtils.mkdir('tmp/tests/default_storage')
FileUtils.mkdir('tmp/tests/custom_storage')
storages = {
'default' => 'tmp/tests/default_storage',
'custom' => 'tmp/tests/custom_storage'
}
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
# Create the projects now, after mocking the settings but before doing the backup
project_a
project_b
# We only need a backup of the repositories for this test
ENV["SKIP"] = "db,uploads,builds,artifacts,lfs,registry"
create_backup
end
after do
FileUtils.rm_rf('tmp/tests/default_storage')
FileUtils.rm_rf('tmp/tests/custom_storage')
FileUtils.rm(@backup_tar)
end
it 'should not create registry.tar.gz' do
it 'should include repositories in all repository storages' do
tar_contents, exit_status = Gitlab::Popen.popen(
%W{tar -tvf #{@backup_tar}}
%W{tar -tvf #{@backup_tar} repositories}
)
expect(exit_status).to eq(0)
expect(tar_contents).not_to match('registry.tar.gz')
expect(tar_contents).to match("repositories/#{project_a.path_with_namespace}.bundle")
expect(tar_contents).to match("repositories/#{project_b.path_with_namespace}.bundle")
end
end
end # backup_create task
......
......@@ -91,6 +91,6 @@ describe PostReceive do
end
def pwd(project)
File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace)
File.join(Gitlab.config.repositories.storages.default, project.path_with_namespace)
end
end
......@@ -14,6 +14,7 @@ describe RepositoryForkWorker do
describe "#perform" do
it "creates a new repository from a fork" do
expect(shell).to receive(:fork_repository).with(
project.repository_storage_path,
project.path_with_namespace,
fork_project.namespace.path
).and_return(true)
......@@ -25,9 +26,11 @@ describe RepositoryForkWorker do
end
it 'flushes various caches' do
expect(shell).to receive(:fork_repository).
with(project.path_with_namespace, fork_project.namespace.path).
and_return(true)
expect(shell).to receive(:fork_repository).with(
project.repository_storage_path,
project.path_with_namespace,
fork_project.namespace.path
).and_return(true)
expect_any_instance_of(Repository).to receive(:expire_emptiness_caches).
and_call_original
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment