Commit a5db7f54 authored by Sean McGivern's avatar Sean McGivern

Merge branch '28447-hybrid-repository-storages' into 'master'

Update storage settings to allow extra values per shard

See merge request !9597
parents ba3ce6bd 0b9d56f9
......@@ -81,8 +81,8 @@ module ApplicationSettingsHelper
end
def repository_storages_options_for_select
options = Gitlab.config.repositories.storages.map do |name, path|
["#{name} - #{path}", name]
options = Gitlab.config.repositories.storages.map do |name, storage|
["#{name} - #{storage['path']}", name]
end
options_for_select(options, @application_setting.repository_storages)
......
......@@ -392,7 +392,7 @@ class Project < ActiveRecord::Base
end
def repository_storage_path
Gitlab.config.repositories.storages[repository_storage]
Gitlab.config.repositories.storages[repository_storage]['path']
end
def team
......
......@@ -50,10 +50,6 @@ class Repository
end
end
def self.storages
Gitlab.config.repositories.storages
end
def initialize(path_with_namespace, project)
@path_with_namespace = path_with_namespace
@project = project
......
......@@ -3,8 +3,8 @@ class PostReceive
include DedicatedSidekiqQueue
def perform(repo_path, identifier, changes)
if path = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1].to_s) }
repo_path.gsub!(path[1].to_s, "")
if repository_storage = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1]['path'].to_s) }
repo_path.gsub!(repository_storage[1]['path'].to_s, "")
else
log("Check gitlab.yml config for correct repositories.storages values. No repository storage path matches \"#{repo_path}\"")
end
......
---
title: Update storage settings to allow extra values per repository storage
merge_request: 9597
author:
......@@ -461,7 +461,8 @@ production: &base
# gitlab-shell invokes Dir.pwd inside the repository path and that results
# real path not the symlink.
storages: # You must have at least a `default` storage path.
default: /home/git/repositories/
default:
path: /home/git/repositories/
## Backup settings
backup:
......@@ -574,7 +575,8 @@ test:
path: tmp/tests/gitlab-satellites/
repositories:
storages:
default: tmp/tests/repositories/
default:
path: tmp/tests/repositories/
backup:
path: tmp/tests/backups
gitlab_shell:
......
......@@ -366,8 +366,13 @@ Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.send(:build_gitlab_shell_s
#
Settings['repositories'] ||= Settingslogic.new({})
Settings.repositories['storages'] ||= {}
# Setting gitlab_shell.repos_path is DEPRECATED and WILL BE REMOVED in version 9.0
Settings.repositories.storages['default'] ||= Settings.gitlab_shell['repos_path'] || Settings.gitlab['user_home'] + '/repositories/'
unless Settings.repositories.storages['default']
Settings.repositories.storages['default'] ||= {}
# We set the path only if the default storage doesn't exist, in case it exists
# but follows the pre-9.0 configuration structure. `6_validations.rb` initializer
# will validate all storages and throw a relevant error to the user if necessary.
Settings.repositories.storages['default']['path'] ||= Settings.gitlab['user_home'] + '/repositories/'
end
#
# The repository_downloads_path is used to remove outdated repository
......@@ -376,11 +381,11 @@ Settings.repositories.storages['default'] ||= Settings.gitlab_shell['repos_path'
# data-integrity issue. In this case, we sets it to the default
# repository_downloads_path value.
#
repositories_storages_path = Settings.repositories.storages.values
repositories_storages = Settings.repositories.storages.values
repository_downloads_path = Settings.gitlab['repository_downloads_path'].to_s.gsub(/\/$/, '')
repository_downloads_full_path = File.expand_path(repository_downloads_path, Settings.gitlab['user_home'])
if repository_downloads_path.blank? || repositories_storages_path.any? { |path| [repository_downloads_path, repository_downloads_full_path].include?(path.gsub(/\/$/, '')) }
if repository_downloads_path.blank? || repositories_storages.any? { |rs| [repository_downloads_path, repository_downloads_full_path].include?(rs['path'].gsub(/\/$/, '')) }
Settings.gitlab['repository_downloads_path'] = File.join(Settings.shared['path'], 'cache/archive')
end
......
......@@ -4,8 +4,8 @@ end
def find_parent_path(name, path)
parent = Pathname.new(path).realpath.parent
Gitlab.config.repositories.storages.detect do |n, p|
name != n && Pathname.new(p).realpath == parent
Gitlab.config.repositories.storages.detect do |n, rs|
name != n && Pathname.new(rs['path']).realpath == parent
end
end
......@@ -16,10 +16,22 @@ end
def validate_storages
storage_validation_error('No repository storage path defined') if Gitlab.config.repositories.storages.empty?
Gitlab.config.repositories.storages.each do |name, path|
Gitlab.config.repositories.storages.each do |name, repository_storage|
storage_validation_error("\"#{name}\" is not a valid storage name") unless storage_name_valid?(name)
parent_name, _parent_path = find_parent_path(name, path)
if repository_storage.is_a?(String)
error = "#{name} is not a valid storage, because it has no `path` key. " \
"It may be configured as:\n\n#{name}:\n path: #{repository_storage}\n\n" \
"Refer to gitlab.yml.example for an updated example"
storage_validation_error(error)
end
if !repository_storage.is_a?(Hash) || repository_storage['path'].nil?
storage_validation_error("#{name} is not a valid storage, because it has no `path` key. Refer to gitlab.yml.example for an updated example")
end
parent_name, _parent_path = find_parent_path(name, repository_storage['path'])
if parent_name
storage_validation_error("#{name} is a nested path of #{parent_name}. Nested paths are not supported for repository storages")
end
......
......@@ -8,7 +8,7 @@ class MigrateRepoSize < ActiveRecord::Migration
project_data.each do |project|
id = project['id']
namespace_path = project['namespace_path'] || ''
repos_path = Gitlab.config.gitlab_shell['repos_path'] || Gitlab.config.repositories.storages.default
repos_path = Gitlab.config.gitlab_shell['repos_path'] || Gitlab.config.repositories.storages.default['path']
path = File.join(repos_path, namespace_path, project['project_path'] + '.git')
begin
......
......@@ -12,7 +12,7 @@ class MigrateProcessCommitWorkerJobs < ActiveRecord::Migration
end
def repository_storage_path
Gitlab.config.repositories.storages[repository_storage]
Gitlab.config.repositories.storages[repository_storage]['path']
end
def repository_path
......
......@@ -60,7 +60,7 @@ class RemoveDotGitFromGroupNames < ActiveRecord::Migration
def move_namespace(group_id, path_was, path)
repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row|
Gitlab.config.repositories.storages[row['repository_storage']]
Gitlab.config.repositories.storages[row['repository_storage']]['path']
end.compact
# Move the namespace directory in all storages paths used by member projects
......
......@@ -71,7 +71,7 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
route_exists = route_exists?(path)
Gitlab.config.repositories.storages.each_value do |storage|
if route_exists || path_exists?(path, storage)
if route_exists || path_exists?(path, storage['path'])
counter += 1
path = "#{base}#{counter}"
......@@ -84,7 +84,7 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
def move_namespace(namespace_id, path_was, path)
repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row|
Gitlab.config.repositories.storages[row['repository_storage']]
Gitlab.config.repositories.storages[row['repository_storage']]['path']
end.compact
# Move the namespace directory in all storages paths used by member projects
......
......@@ -52,9 +52,12 @@ respectively.
# Paths where repositories can be stored. Give the canonicalized absolute pathname.
# NOTE: REPOS PATHS MUST NOT CONTAIN ANY SYMLINK!!!
storages: # You must have at least a 'default' storage path.
default: /home/git/repositories
nfs: /mnt/nfs/repositories
cephfs: /mnt/cephfs/repositories
default:
path: /home/git/repositories
nfs:
path: /mnt/nfs/repositories
cephfs:
path: /mnt/cephfs/repositories
```
1. [Restart GitLab] for the changes to take effect.
......@@ -75,9 +78,9 @@ working, you can remove the `repos_path` line.
```ruby
git_data_dirs({
"default" => "/var/opt/gitlab/git-data",
"nfs" => "/mnt/nfs/git-data",
"cephfs" => "/mnt/cephfs/git-data"
"default" => { "path" => "/var/opt/gitlab/git-data" },
"nfs" => { "path" => "/mnt/nfs/git-data" },
"cephfs" => { "path" => "/mnt/cephfs/git-data" }
})
```
......
#### Configuration changes for repository storages
This version introduces a new configuration structure for repository storages.
Update your current configuration as follows, replacing with your storages names and paths:
**For installations from source**
1. Update your `gitlab.yml`, from
```yaml
repositories:
storages: # You must have at least a 'default' storage path.
default: /home/git/repositories
nfs: /mnt/nfs/repositories
cephfs: /mnt/cephfs/repositories
```
to
```yaml
repositories:
storages: # You must have at least a 'default' storage path.
default:
path: /home/git/repositories
nfs:
path: /mnt/nfs/repositories
cephfs:
path: /mnt/cephfs/repositories
```
**For Omnibus installations**
1. Upate your `/etc/gitlab/gitlab.rb`, from
```ruby
git_data_dirs({
"default" => "/var/opt/gitlab/git-data",
"nfs" => "/mnt/nfs/git-data",
"cephfs" => "/mnt/cephfs/git-data"
})
```
to
```ruby
git_data_dirs({
"default" => { "path" => "/var/opt/gitlab/git-data" },
"nfs" => { "path" => "/mnt/nfs/git-data" },
"cephfs" => { "path" => "/mnt/cephfs/git-data" }
})
```
#### Git configuration
Configure Git to generate packfile bitmaps (introduced in Git 2.0) on
the GitLab server during `git gc`.
```sh
cd /home/git/gitlab
sudo -u git -H git config --global repack.writeBitmaps true
```
#### Nginx configuration
Ensure you're still up-to-date with the latest NGINX configuration changes:
......
......@@ -9,11 +9,11 @@ module API
# In addition, they may have a '.git' extension and multiple namespaces
#
# Transform all these cases to 'namespace/project'
def clean_project_path(project_path, storage_paths = Repository.storages.values)
def clean_project_path(project_path, storages = Gitlab.config.repositories.storages.values)
project_path = project_path.sub(/\.git\z/, '')
storage_paths.each do |storage_path|
storage_path = File.expand_path(storage_path)
storages.each do |storage|
storage_path = File.expand_path(storage['path'])
if project_path.start_with?(storage_path)
project_path = project_path.sub(storage_path, '')
......
......@@ -68,7 +68,8 @@ module Backup
end
def restore
Gitlab.config.repositories.storages.each do |name, path|
Gitlab.config.repositories.storages.each do |name, repository_storage|
path = repository_storage['path']
next unless File.exist?(path)
# Move repos dir to 'repositories.old' dir
......@@ -199,7 +200,7 @@ module Backup
private
def repository_storage_paths_args
Gitlab.config.repositories.storages.values
Gitlab.config.repositories.storages.values.map { |rs| rs['path'] }
end
end
end
......@@ -354,7 +354,8 @@ namespace :gitlab do
def check_repo_base_exists
puts "Repo base directory exists?"
Gitlab.config.repositories.storages.each do |name, repo_base_path|
Gitlab.config.repositories.storages.each do |name, repository_storage|
repo_base_path = repository_storage['path']
print "#{name}... "
if File.exist?(repo_base_path)
......@@ -378,7 +379,8 @@ namespace :gitlab do
def check_repo_base_is_not_symlink
puts "Repo storage directories are symlinks?"
Gitlab.config.repositories.storages.each do |name, repo_base_path|
Gitlab.config.repositories.storages.each do |name, repository_storage|
repo_base_path = repository_storage['path']
print "#{name}... "
unless File.exist?(repo_base_path)
......@@ -401,7 +403,8 @@ namespace :gitlab do
def check_repo_base_permissions
puts "Repo paths access is drwxrws---?"
Gitlab.config.repositories.storages.each do |name, repo_base_path|
Gitlab.config.repositories.storages.each do |name, repository_storage|
repo_base_path = repository_storage['path']
print "#{name}... "
unless File.exist?(repo_base_path)
......@@ -431,7 +434,8 @@ namespace :gitlab do
gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group
puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?"
Gitlab.config.repositories.storages.each do |name, repo_base_path|
Gitlab.config.repositories.storages.each do |name, repository_storage|
repo_base_path = repository_storage['path']
print "#{name}... "
unless File.exist?(repo_base_path)
......@@ -810,8 +814,8 @@ namespace :gitlab do
namespace :repo do
desc "GitLab | Check the integrity of the repositories managed by GitLab"
task check: :environment do
Gitlab.config.repositories.storages.each do |name, path|
namespace_dirs = Dir.glob(File.join(path, '*'))
Gitlab.config.repositories.storages.each do |name, repository_storage|
namespace_dirs = Dir.glob(File.join(repository_storage['path'], '*'))
namespace_dirs.each do |namespace_dir|
repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
......
......@@ -6,7 +6,8 @@ namespace :gitlab do
remove_flag = ENV['REMOVE']
namespaces = Namespace.pluck(:path)
Gitlab.config.repositories.storages.each do |name, git_base_path|
Gitlab.config.repositories.storages.each do |name, repository_storage|
git_base_path = repository_storage['path']
all_dirs = Dir.glob(git_base_path + '/*')
puts git_base_path.color(:yellow)
......@@ -47,7 +48,8 @@ namespace :gitlab do
warn_user_is_not_gitlab
move_suffix = "+orphaned+#{Time.now.to_i}"
Gitlab.config.repositories.storages.each do |name, repo_root|
Gitlab.config.repositories.storages.each do |name, repository_storage|
repo_root = repository_storage['path']
# Look for global repos (legacy, depth 1) and normal repos (depth 2)
IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
find.each_line do |path|
......
......@@ -11,7 +11,8 @@ namespace :gitlab do
#
desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
task repos: :environment do
Gitlab.config.repositories.storages.each do |name, git_base_path|
Gitlab.config.repositories.storages.each_value do |repository_storage|
git_base_path = repository_storage['path']
repos_to_import = Dir.glob(git_base_path + '/**/*.git')
repos_to_import.each do |repo_path|
......
......@@ -65,8 +65,8 @@ namespace :gitlab do
puts "GitLab Shell".color(:yellow)
puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}"
puts "Repository storage paths:"
Gitlab.config.repositories.storages.each do |name, path|
puts "- #{name}: \t#{path}"
Gitlab.config.repositories.storages.each do |name, repository_storage|
puts "- #{name}: \t#{repository_storage['path']}"
end
puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}"
puts "Git:\t\t#{Gitlab.config.git.bin_path}"
......
......@@ -130,8 +130,8 @@ module Gitlab
end
def all_repos
Gitlab.config.repositories.storages.each do |name, path|
IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
Gitlab.config.repositories.storages.each_value do |repository_storage|
IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
find.each_line do |path|
yield path.chomp
end
......@@ -140,7 +140,7 @@ module Gitlab
end
def repository_storage_paths_args
Gitlab.config.repositories.storages.values
Gitlab.config.repositories.storages.values.map { |rs| rs['path'] }
end
def user_home
......
......@@ -14,7 +14,7 @@ describe '6_validations', lib: true do
context 'with correct settings' do
before do
mock_storages('foo' => 'tmp/tests/paths/a/b/c', 'bar' => 'tmp/tests/paths/a/b/d')
mock_storages('foo' => { 'path' => 'tmp/tests/paths/a/b/c' }, 'bar' => { 'path' => 'tmp/tests/paths/a/b/d' })
end
it 'passes through' do
......@@ -24,7 +24,7 @@ describe '6_validations', lib: true do
context 'with invalid storage names' do
before do
mock_storages('name with spaces' => 'tmp/tests/paths/a/b/c')
mock_storages('name with spaces' => { 'path' => 'tmp/tests/paths/a/b/c' })
end
it 'throws an error' do
......@@ -34,7 +34,7 @@ describe '6_validations', lib: true do
context 'with nested storage paths' do
before do
mock_storages('foo' => 'tmp/tests/paths/a/b/c', 'bar' => 'tmp/tests/paths/a/b/c/d')
mock_storages('foo' => { 'path' => 'tmp/tests/paths/a/b/c' }, 'bar' => { 'path' => 'tmp/tests/paths/a/b/c/d' })
end
it 'throws an error' do
......@@ -44,7 +44,7 @@ describe '6_validations', lib: true do
context 'with similar but un-nested storage paths' do
before do
mock_storages('foo' => 'tmp/tests/paths/a/b/c', 'bar' => 'tmp/tests/paths/a/b/c2')
mock_storages('foo' => { 'path' => 'tmp/tests/paths/a/b/c' }, 'bar' => { 'path' => 'tmp/tests/paths/a/b/c2' })
end
it 'passes through' do
......@@ -52,6 +52,26 @@ describe '6_validations', lib: true do
end
end
context 'with incomplete settings' do
before do
mock_storages('foo' => {})
end
it 'throws an error suggesting the user to update its settings' do
expect { validate_storages }.to raise_error('foo is not a valid storage, because it has no `path` key. Refer to gitlab.yml.example for an updated example. Please fix this in your gitlab.yml before starting GitLab.')
end
end
context 'with deprecated settings structure' do
before do
mock_storages('foo' => 'tmp/tests/paths/a/b/c')
end
it 'throws an error suggesting the user to update its settings' do
expect { validate_storages }.to raise_error("foo is not a valid storage, because it has no `path` key. It may be configured as:\n\nfoo:\n path: tmp/tests/paths/a/b/c\n\nRefer to gitlab.yml.example for an updated example. Please fix this in your gitlab.yml before starting GitLab.")
end
end
def mock_storages(storages)
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
......
......@@ -165,7 +165,7 @@ describe Namespace, models: true do
describe :rm_dir do
let!(:project) { create(:empty_project, namespace: namespace) }
let!(:path) { File.join(Gitlab.config.repositories.storages.default, namespace.full_path) }
let!(:path) { File.join(Gitlab.config.repositories.storages.default['path'], namespace.full_path) }
it "removes its dirs when deleted" do
namespace.destroy
......
......@@ -179,7 +179,7 @@ describe Project, models: true do
let(:project2) { build(:empty_project, repository_storage: 'missing') }
before do
storages = { 'custom' => 'tmp/tests/custom_repositories' }
storages = { 'custom' => { 'path' => 'tmp/tests/custom_repositories' } }
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
......@@ -381,7 +381,7 @@ describe Project, models: true do
before do
FileUtils.mkdir('tmp/tests/custom_repositories')
storages = { 'custom' => 'tmp/tests/custom_repositories' }
storages = { 'custom' => { 'path' => 'tmp/tests/custom_repositories' } }
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
......@@ -947,8 +947,8 @@ describe Project, models: true do
before do
storages = {
'default' => 'tmp/tests/repositories',
'picked' => 'tmp/tests/repositories',
'default' => { 'path' => 'tmp/tests/repositories' },
'picked' => { 'path' => 'tmp/tests/repositories' },
}
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
......
......@@ -21,7 +21,7 @@ describe ::API::Helpers::InternalHelpers do
# Relative and absolute storage paths, with and without trailing /
['.', './', Dir.pwd, Dir.pwd + '/'].each do |storage_path|
context "storage path is #{storage_path}" do
subject { clean_project_path(project_path, [storage_path]) }
subject { clean_project_path(project_path, [{ 'path' => storage_path }]) }
it { is_expected.to eq(expected) }
end
......
......@@ -143,7 +143,7 @@ module TestEnv
end
def repos_path
Gitlab.config.repositories.storages.default
Gitlab.config.repositories.storages.default['path']
end
def backup_path
......
......@@ -227,8 +227,8 @@ describe 'gitlab:app namespace rake task' do
FileUtils.mkdir('tmp/tests/default_storage')
FileUtils.mkdir('tmp/tests/custom_storage')
storages = {
'default' => 'tmp/tests/default_storage',
'custom' => 'tmp/tests/custom_storage'
'default' => { 'path' => 'tmp/tests/default_storage' },
'custom' => { 'path' => 'tmp/tests/custom_storage' }
}
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
......
......@@ -105,6 +105,6 @@ describe PostReceive do
end
def pwd(project)
File.join(Gitlab.config.repositories.storages.default, project.path_with_namespace)
File.join(Gitlab.config.repositories.storages.default['path'], project.path_with_namespace)
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment