diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index 2cd87895c5597b37137db549b88456694679ff10..015a41b6e82e9b6ebfcfa189feba8e12d01b6c2e 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -3,20 +3,16 @@ class PostReceive include DedicatedSidekiqQueue def perform(repo_path, identifier, changes) - if repository_storage = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1]['path'].to_s) } - repo_path.gsub!(repository_storage[1]['path'].to_s, "") - else - log("Check gitlab.yml config for correct repositories.storages values. No repository storage path matches \"#{repo_path}\"") - end + repo_relative_path = Gitlab::RepoPath.strip_storage_path(repo_path) changes = Base64.decode64(changes) unless changes.include?(' ') # Use Sidekiq.logger so arguments can be correlated with execution # time and thread ID's. Sidekiq.logger.info "changes: #{changes.inspect}" if ENV['SIDEKIQ_LOG_ARGUMENTS'] - post_received = Gitlab::GitPostReceive.new(repo_path, identifier, changes) + post_received = Gitlab::GitPostReceive.new(repo_relative_path, identifier, changes) if post_received.project.nil? - log("Triggered hook for non-existing project with full path \"#{repo_path}\"") + log("Triggered hook for non-existing project with full path \"#{repo_relative_path}\"") return false end @@ -25,7 +21,7 @@ class PostReceive elsif post_received.regular_project? process_project_changes(post_received) else - log("Triggered hook for unidentifiable repository type with full path \"#{repo_path}\"") + log("Triggered hook for unidentifiable repository type with full path \"#{repo_relative_path}\"") false end end diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 3747baf4c3b8fc40c74e4e84c37385b7aea2d52a..d89e25c09596fb3958f0d09e3e1ea2d9f5e13ce8 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -443,14 +443,10 @@ production: &base # Gitaly settings gitaly: - # The socket_path setting is optional and obsolete. When this is set - # GitLab assumes it can reach a Gitaly services via a Unix socket at - # this path. When this is commented out GitLab will not use Gitaly. - # - # This setting is obsolete because we expect it to be moved under - # repositories/storages in GitLab 9.1. - # - # socket_path: tmp/sockets/private/gitaly.socket + # This setting controls whether GitLab uses Gitaly (new component + # introduced in 9.0). Eventually Gitaly use will become mandatory and + # this option will disappear. + enabled: false # # 4. Advanced settings @@ -465,6 +461,7 @@ production: &base storages: # You must have at least a `default` storage path. default: path: /home/git/repositories/ + gitaly_address: unix:/home/git/gitlab/tmp/sockets/private/gitaly.socket ## Backup settings backup: @@ -577,6 +574,9 @@ test: storages: default: path: tmp/tests/repositories/ + gitaly_address: unix:<%= Rails.root.join('tmp/sockets/private/gitaly.socket') %> + gitaly: + enabled: false backup: path: tmp/tests/backups gitlab_shell: diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 62020fa9a75a3e32568eb03a0579cb8d7bba89d8..e8fef0000c1b7894343cc51c42e05f6998cda79c 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -79,6 +79,10 @@ class Settings < Settingslogic value end + def absolute(path) + File.expand_path(path, Rails.root) + end + private def base_url(config) @@ -178,7 +182,7 @@ if github_settings end Settings['shared'] ||= Settingslogic.new({}) -Settings.shared['path'] = File.expand_path(Settings.shared['path'] || "shared", Rails.root) +Settings.shared['path'] = Settings.absolute(Settings.shared['path'] || "shared") Settings['issues_tracker'] ||= {} @@ -237,7 +241,7 @@ Settings['gitlab_ci'] ||= Settingslogic.new({}) Settings.gitlab_ci['shared_runners_enabled'] = true if Settings.gitlab_ci['shared_runners_enabled'].nil? Settings.gitlab_ci['all_broken_builds'] = true if Settings.gitlab_ci['all_broken_builds'].nil? Settings.gitlab_ci['add_pusher'] = false if Settings.gitlab_ci['add_pusher'].nil? -Settings.gitlab_ci['builds_path'] = File.expand_path(Settings.gitlab_ci['builds_path'] || "builds/", Rails.root) +Settings.gitlab_ci['builds_path'] = Settings.absolute(Settings.gitlab_ci['builds_path'] || "builds/") Settings.gitlab_ci['url'] ||= Settings.send(:build_gitlab_ci_url) # @@ -251,7 +255,7 @@ Settings.incoming_email['enabled'] = false if Settings.incoming_email['enabled'] # Settings['artifacts'] ||= Settingslogic.new({}) Settings.artifacts['enabled'] = true if Settings.artifacts['enabled'].nil? -Settings.artifacts['path'] = File.expand_path(Settings.artifacts['path'] || File.join(Settings.shared['path'], "artifacts"), Rails.root) +Settings.artifacts['path'] = Settings.absolute(Settings.artifacts['path'] || File.join(Settings.shared['path'], "artifacts")) Settings.artifacts['max_size'] ||= 100 # in megabytes # @@ -265,14 +269,14 @@ Settings.registry['api_url'] ||= "http://localhost:5000/" Settings.registry['key'] ||= nil Settings.registry['issuer'] ||= nil Settings.registry['host_port'] ||= [Settings.registry['host'], Settings.registry['port']].compact.join(':') -Settings.registry['path'] = File.expand_path(Settings.registry['path'] || File.join(Settings.shared['path'], 'registry'), Rails.root) +Settings.registry['path'] = Settings.absolute(Settings.registry['path'] || File.join(Settings.shared['path'], 'registry')) # # Pages # Settings['pages'] ||= Settingslogic.new({}) Settings.pages['enabled'] = false if Settings.pages['enabled'].nil? -Settings.pages['path'] = File.expand_path(Settings.pages['path'] || File.join(Settings.shared['path'], "pages"), Rails.root) +Settings.pages['path'] = Settings.absolute(Settings.pages['path'] || File.join(Settings.shared['path'], "pages")) Settings.pages['https'] = false if Settings.pages['https'].nil? Settings.pages['host'] ||= "example.com" Settings.pages['port'] ||= Settings.pages.https ? 443 : 80 @@ -286,7 +290,7 @@ Settings.pages['external_https'] ||= false unless Settings.pages['external_http # Settings['lfs'] ||= Settingslogic.new({}) Settings.lfs['enabled'] = true if Settings.lfs['enabled'].nil? -Settings.lfs['storage_path'] = File.expand_path(Settings.lfs['storage_path'] || File.join(Settings.shared['path'], "lfs-objects"), Rails.root) +Settings.lfs['storage_path'] = Settings.absolute(Settings.lfs['storage_path'] || File.join(Settings.shared['path'], "lfs-objects")) # # Mattermost @@ -350,8 +354,8 @@ Settings.cron_jobs['remove_unreferenced_lfs_objects_worker']['job_class'] = 'Rem # GitLab Shell # Settings['gitlab_shell'] ||= Settingslogic.new({}) -Settings.gitlab_shell['path'] ||= Settings.gitlab['user_home'] + '/gitlab-shell/' -Settings.gitlab_shell['hooks_path'] ||= Settings.gitlab['user_home'] + '/gitlab-shell/hooks/' +Settings.gitlab_shell['path'] = Settings.absolute(Settings.gitlab_shell['path'] || Settings.gitlab['user_home'] + '/gitlab-shell/') +Settings.gitlab_shell['hooks_path'] = Settings.absolute(Settings.gitlab_shell['hooks_path'] || Settings.gitlab['user_home'] + '/gitlab-shell/hooks/') Settings.gitlab_shell['secret_file'] ||= Rails.root.join('.gitlab_shell_secret') Settings.gitlab_shell['receive_pack'] = true if Settings.gitlab_shell['receive_pack'].nil? Settings.gitlab_shell['upload_pack'] = true if Settings.gitlab_shell['upload_pack'].nil? @@ -374,6 +378,11 @@ unless Settings.repositories.storages['default'] Settings.repositories.storages['default']['path'] ||= Settings.gitlab['user_home'] + '/repositories/' end +Settings.repositories.storages.values.each do |storage| + # Expand relative paths + storage['path'] = Settings.absolute(storage['path']) +end + # # The repository_downloads_path is used to remove outdated repository # archives, if someone has it configured incorrectly, and it points @@ -395,7 +404,7 @@ end Settings['backup'] ||= Settingslogic.new({}) Settings.backup['keep_time'] ||= 0 Settings.backup['pg_schema'] = nil -Settings.backup['path'] = File.expand_path(Settings.backup['path'] || "tmp/backups/", Rails.root) +Settings.backup['path'] = Settings.absolute(Settings.backup['path'] || "tmp/backups/") Settings.backup['archive_permissions'] ||= 0600 Settings.backup['upload'] ||= Settingslogic.new({ 'remote_directory' => nil, 'connection' => nil }) # Convert upload connection settings to use symbol keys, to make Fog happy @@ -418,7 +427,7 @@ Settings.git['timeout'] ||= 10 # least. This setting is fed to 'rm -rf' in # db/migrate/20151023144219_remove_satellites.rb Settings['satellites'] ||= Settingslogic.new({}) -Settings.satellites['path'] = File.expand_path(Settings.satellites['path'] || "tmp/repo_satellites/", Rails.root) +Settings.satellites['path'] = Settings.absolute(Settings.satellites['path'] || "tmp/repo_satellites/") # # Extra customization @@ -440,7 +449,7 @@ Settings.rack_attack.git_basic_auth['bantime'] ||= 1.hour # Gitaly # Settings['gitaly'] ||= Settingslogic.new({}) -Settings.gitaly['socket_path'] ||= ENV['GITALY_SOCKET_PATH'] +Settings.gitaly['enabled'] ||= false # # Webpack settings diff --git a/config/initializers/8_gitaly.rb b/config/initializers/8_gitaly.rb index 07dd30f0a24897e8390672ac56cc369b3d377946..69c0a91d6f0381713237d4f07db20373723b0fb2 100644 --- a/config/initializers/8_gitaly.rb +++ b/config/initializers/8_gitaly.rb @@ -1,2 +1,18 @@ -# Make sure we initialize a Gitaly channel before Sidekiq starts multi-threaded execution. -Gitlab::GitalyClient.channel unless Rails.env.test? +require 'uri' + +# Make sure we initialize our Gitaly channels before Sidekiq starts multi-threaded execution. +if Gitlab.config.gitaly.enabled || Rails.env.test? + Gitlab.config.repositories.storages.each do |name, params| + address = params['gitaly_address'] + + unless address.present? + raise "storage #{name.inspect} is missing a gitaly_address" + end + + unless URI(address).scheme == 'unix' + raise "Unsupported Gitaly address: #{address.inspect}" + end + + Gitlab::GitalyClient.configure_channel(name, address) + end +end diff --git a/doc/install/installation.md b/doc/install/installation.md index a6b10176450c86d9345267d75e8c3db7a0ddb1c9..a2248a384351b6ab09eb3207e57c58dcf3832b5b 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -477,12 +477,12 @@ with setting up Gitaly until you upgrade to GitLab 9.1 or later. # Enable Gitaly in the init script echo 'gitaly_enabled=true' | sudo tee -a /etc/default/gitlab -Next, edit `/home/git/gitlab/config/gitlab.yml` and make sure `socket_path` in +Next, edit `/home/git/gitlab/config/gitlab.yml` and make sure `enabled: true` in the `gitaly:` section is uncommented. # <- gitlab.yml indentation starts here gitaly: - socket_path: tmp/sockets/private/gitaly.socket + enabled: true For more information about configuring Gitaly see [doc/administration/gitaly](../administration/gitaly). diff --git a/doc/update/9.0-to-9.1.md b/doc/update/9.0-to-9.1.md new file mode 100644 index 0000000000000000000000000000000000000000..53cddb3f290953040eb0d7a3fff27ff74eb4a6d4 --- /dev/null +++ b/doc/update/9.0-to-9.1.md @@ -0,0 +1,366 @@ +# From 9.0 to 9.1 + +** TODO: ** + +# TODO clean out 9.0-specific stuff + +Make sure you view this update guide from the tag (version) of GitLab you would +like to install. In most cases this should be the highest numbered production +tag (without rc in it). You can select the tag in the version dropdown at the +top left corner of GitLab (below the menu bar). + +If the highest number stable branch is unclear please check the +[GitLab Blog](https://about.gitlab.com/blog/archives.html) for installation +guide links by version. + +### 1. Stop server + +```bash +sudo service gitlab stop +``` + +### 2. Backup + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production +``` + +### 3. Update Ruby + +NOTE: GitLab 9.0 and higher only support Ruby 2.3.x and dropped support for Ruby 2.1.x. Be +sure to upgrade your interpreter if necessary. + +You can check which version you are running with `ruby -v`. + +Download and compile Ruby: + +```bash +mkdir /tmp/ruby && cd /tmp/ruby +curl --remote-name --progress https://cache.ruby-lang.org/pub/ruby/2.3/ruby-2.3.3.tar.gz +echo '1014ee699071aa2ddd501907d18cbe15399c997d ruby-2.3.3.tar.gz' | shasum -c - && tar xzf ruby-2.3.3.tar.gz +cd ruby-2.3.3 +./configure --disable-install-rdoc +make +sudo make install +``` + +Install Bundler: + +```bash +sudo gem install bundler --no-ri --no-rdoc +``` + +### 4. Update Node + +GitLab now runs [webpack](http://webpack.js.org) to compile frontend assets and +it has a minimum requirement of node v4.3.0. + +You can check which version you are running with `node -v`. If you are running +a version older than `v4.3.0` you will need to update to a newer version. You +can find instructions to install from community maintained packages or compile +from source at the nodejs.org website. + +<https://nodejs.org/en/download/> + + +Since 8.17, GitLab requires the use of yarn `>= v0.17.0` to manage +JavaScript dependencies. + +```bash +curl --location https://yarnpkg.com/install.sh | bash - +``` + +More information can be found on the [yarn website](https://yarnpkg.com/en/docs/install). + +### 5. Get latest code + +```bash +cd /home/git/gitlab + +sudo -u git -H git fetch --all +sudo -u git -H git checkout -- db/schema.rb # local changes will be restored automatically +``` + +For GitLab Community Edition: + +```bash +cd /home/git/gitlab + +sudo -u git -H git checkout 9-1-stable +``` + +OR + +For GitLab Enterprise Edition: + +```bash +cd /home/git/gitlab + +sudo -u git -H git checkout 9-1-stable-ee +``` + +### 6. Update gitlab-shell + +```bash +cd /home/git/gitlab-shell + +sudo -u git -H git fetch --all --tags +sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_SHELL_VERSION) +``` + +### 7. Update gitlab-workhorse + +Install and compile gitlab-workhorse. This requires +[Go 1.5](https://golang.org/dl) which should already be on your system from +GitLab 8.1. GitLab-Workhorse uses [GNU Make](https://www.gnu.org/software/make/). +If you are not using Linux you may have to run `gmake` instead of +`make` below. + +```bash +cd /home/git/gitlab-workhorse + +sudo -u git -H git fetch --all --tags +sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_WORKHORSE_VERSION) +sudo -u git -H make +``` + +### 8. Update configuration files + +#### New configuration options for `gitlab.yml` + +There might be configuration options available for [`gitlab.yml`][yaml]. View them with the command below and apply them manually to your current `gitlab.yml`: + +```sh +cd /home/git/gitlab + +git diff origin/9-0-stable:config/gitlab.yml.example origin/9-1-stable:config/gitlab.yml.example +``` + +#### Configuration changes for repository storages + +This version introduces a new configuration structure for repository storages. +Update your current configuration as follows, replacing with your storages names and paths: + +**For installations from source** + +1. Update your `gitlab.yml`, from + + ```yaml + repositories: + storages: # You must have at least a 'default' storage path. + default: /home/git/repositories + nfs: /mnt/nfs/repositories + cephfs: /mnt/cephfs/repositories + ``` + + to + + ```yaml + repositories: + storages: # You must have at least a 'default' storage path. + default: + path: /home/git/repositories + nfs: + path: /mnt/nfs/repositories + cephfs: + path: /mnt/cephfs/repositories + ``` + +**For Omnibus installations** + +1. Update your `/etc/gitlab/gitlab.rb`, from + + ```ruby + git_data_dirs({ + "default" => "/var/opt/gitlab/git-data", + "nfs" => "/mnt/nfs/git-data", + "cephfs" => "/mnt/cephfs/git-data" + }) + ``` + + to + + ```ruby + git_data_dirs({ + "default" => { "path" => "/var/opt/gitlab/git-data" }, + "nfs" => { "path" => "/mnt/nfs/git-data" }, + "cephfs" => { "path" => "/mnt/cephfs/git-data" } + }) + ``` + +#### Git configuration + +Configure Git to generate packfile bitmaps (introduced in Git 2.0) on +the GitLab server during `git gc`. + +```sh +cd /home/git/gitlab + +sudo -u git -H git config --global repack.writeBitmaps true +``` + +#### Nginx configuration + +Ensure you're still up-to-date with the latest NGINX configuration changes: + +```sh +cd /home/git/gitlab + +# For HTTPS configurations +git diff origin/9-0-stable:lib/support/nginx/gitlab-ssl origin/9-1-stable:lib/support/nginx/gitlab-ssl + +# For HTTP configurations +git diff origin/9-0-stable:lib/support/nginx/gitlab origin/9-1-stable:lib/support/nginx/gitlab +``` + +If you are using Strict-Transport-Security in your installation to continue using it you must enable it in your Nginx +configuration as GitLab application no longer handles setting it. + +If you are using Apache instead of NGINX please see the updated [Apache templates]. +Also note that because Apache does not support upstreams behind Unix sockets you +will need to let gitlab-workhorse listen on a TCP port. You can do this +via [/etc/default/gitlab]. + +[Apache templates]: https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache +[/etc/default/gitlab]: https://gitlab.com/gitlab-org/gitlab-ce/blob/9-1-stable/lib/support/init.d/gitlab.default.example#L38 + +#### SMTP configuration + +If you're installing from source and use SMTP to deliver mail, you will need to add the following line +to config/initializers/smtp_settings.rb: + +```ruby +ActionMailer::Base.delivery_method = :smtp +``` + +See [smtp_settings.rb.sample] as an example. + +[smtp_settings.rb.sample]: https://gitlab.com/gitlab-org/gitlab-ce/blob/9-0-stable/config/initializers/smtp_settings.rb.sample#L13 + +#### Init script + +There might be new configuration options available for [`gitlab.default.example`][gl-example]. View them with the command below and apply them manually to your current `/etc/default/gitlab`: + +```sh +cd /home/git/gitlab + +git diff origin/9-0-stable:lib/support/init.d/gitlab.default.example origin/9-1-stable:lib/support/init.d/gitlab.default.example +``` + +Ensure you're still up-to-date with the latest init script changes: + +```bash +cd /home/git/gitlab + +sudo cp lib/support/init.d/gitlab /etc/init.d/gitlab +``` + +For Ubuntu 16.04.1 LTS: + +```bash +sudo systemctl daemon-reload +``` + +### 9. Install libs, migrations, etc. + +```bash +cd /home/git/gitlab + +# MySQL installations (note: the line below states '--without postgres') +sudo -u git -H bundle install --without postgres development test --deployment + +# PostgreSQL installations (note: the line below states '--without mysql') +sudo -u git -H bundle install --without mysql development test --deployment + +# Optional: clean up old gems +sudo -u git -H bundle clean + +# Run database migrations +sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production + +# Update node dependencies and recompile assets +sudo -u git -H bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile RAILS_ENV=production NODE_ENV=production + +# Clean up cache +sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production +``` + +**MySQL installations**: Run through the `MySQL strings limits` and `Tables and data conversion to utf8mb4` [tasks](../install/database_mysql.md). + +### 10. Optional: install Gitaly + +Gitaly is still an optional component of GitLab. If you want to save time +during your 9.1 upgrade **you can skip this step**. + +If you have not yet set up Gitaly then follow [Gitaly section of the installation +guide](../install/installation.md#install-gitaly). + +If you installed Gitaly in GitLab 9.0 you need to make some changes in gitlab.yml. + +Look for `socket_path:` the `gitaly:` section. Its value is usually +`/home/git/gitlab/tmp/sockets/private/gitaly.socket`. Note what socket +path your gitlab.yml is using. Now go to the `repositories:` section, +and for each entry under `storages:`, add a `gitaly_address:` based on +the socket path, but with `unix:` in front. + +```yaml + repositories: + storages: + default: + path: /home/git/repositories + gitaly_address: unix:/home/git/gitlab/tmp/sockets/private/gitaly.socket + other_storage: + path: /home/git/other-repositories + gitaly_address: unix:/home/git/gitlab/tmp/sockets/private/gitaly.socket +``` + +Each entry under `storages:` should use the same `gitaly_address`. + +### 11. Start application + +```bash +sudo service gitlab start +sudo service nginx restart +``` + +### 12. Check application status + +Check if GitLab and its environment are configured correctly: + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:env:info RAILS_ENV=production +``` + +To make sure you didn't miss anything run a more thorough check: + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production +``` + +If all items are green, then congratulations, the upgrade is complete! + +## Things went south? Revert to previous version (9.0) + +### 1. Revert the code to the previous version + +Follow the [upgrade guide from 8.17 to 9.0](8.17-to-9.0.md), except for the +database migration (the backup is already migrated to the previous version). + +### 2. Restore from the backup + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:backup:restore RAILS_ENV=production +``` + +If you have more than one backup `*.tar` file(s) please add `BACKUP=timestamp_of_backup` to the command above. + +[yaml]: https://gitlab.com/gitlab-org/gitlab-ce/blob/9-1-stable/config/gitlab.yml.example +[gl-example]: https://gitlab.com/gitlab-org/gitlab-ce/blob/9-1-stable/lib/support/init.d/gitlab.default.example diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 7eed93aba0060140e14e8cc7c7526cf9180ee8d7..523f38d129ec4fae4f78728ff8ab90a9331e0b25 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -139,7 +139,7 @@ module API return unless Gitlab::GitalyClient.enabled? begin - Gitlab::GitalyClient::Notifications.new.post_receive(params[:repo_path]) + Gitlab::GitalyClient::Notifications.new(params[:repo_path]).post_receive rescue GRPC::Unavailable => e render_api_error(e, 500) end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index cd745d35e7cfd0ed5f78e4be9d89ef40ab617fde..6b29600a75133a3d350c7028e18161f6853b6eaf 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -182,7 +182,9 @@ module Backup dir_entries = Dir.entries(path) - yield('custom_hooks') if dir_entries.include?('custom_hooks') + if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar') + yield('custom_hooks') + end end def prepare diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 1ce47ef2b05d6141c1a19061fc92358cb4486e9e..a0dbe0a8c110834707a41b3cd40962f4286ed0d0 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -4,28 +4,30 @@ module Gitlab module GitalyClient SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze - def self.gitaly_address - if Gitlab.config.gitaly.socket_path - "unix://#{Gitlab.config.gitaly.socket_path}" - end + def self.configure_channel(storage, address) + @addresses ||= {} + @addresses[storage] = address + @channels ||= {} + @channels[storage] = new_channel(address) + end + + def self.new_channel(address) + # NOTE: Gitaly currently runs on a Unix socket, so permissions are + # handled using the file system and no additional authentication is + # required (therefore the :this_channel_is_insecure flag) + GRPC::Core::Channel.new(address, {}, :this_channel_is_insecure) end - def self.channel - return @channel if defined?(@channel) + def self.get_channel(storage) + @channels[storage] + end - @channel = - if enabled? - # NOTE: Gitaly currently runs on a Unix socket, so permissions are - # handled using the file system and no additional authentication is - # required (therefore the :this_channel_is_insecure flag) - GRPC::Core::Channel.new(gitaly_address, {}, :this_channel_is_insecure) - else - nil - end + def self.get_address(storage) + @addresses[storage] end def self.enabled? - gitaly_address.present? + Gitlab.config.gitaly.enabled end def self.feature_enabled?(feature) diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index 525b8d680e90fc0f2b00db1191b850a0fb207bba..9c714a3ee45efff0cbe1416bb48a0f12cc567852 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -7,8 +7,10 @@ module Gitlab class << self def diff_from_parent(commit, options = {}) - stub = Gitaly::Diff::Stub.new(nil, nil, channel_override: GitalyClient.channel) - repo = Gitaly::Repository.new(path: commit.project.repository.path_to_repo) + project = commit.project + channel = GitalyClient.get_channel(project.repository_storage) + stub = Gitaly::Diff::Stub.new(nil, nil, channel_override: channel) + repo = Gitaly::Repository.new(path: project.repository.path_to_repo) parent = commit.parents[0] parent_id = parent ? parent.id : EMPTY_TREE_ID request = Gitaly::CommitDiffRequest.new( diff --git a/lib/gitlab/gitaly_client/notifications.rb b/lib/gitlab/gitaly_client/notifications.rb index b827a56207f6bb5e9b32704b39c4a71fbbe28f6e..cbfb129c0020bebcbe6c865ab765e98ffa8cc5f8 100644 --- a/lib/gitlab/gitaly_client/notifications.rb +++ b/lib/gitlab/gitaly_client/notifications.rb @@ -3,14 +3,19 @@ module Gitlab class Notifications attr_accessor :stub - def initialize - @stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: GitalyClient.channel) + def initialize(repo_path) + full_path = Gitlab::RepoPath.strip_storage_path(repo_path). + sub(/\.git\z/, '').sub(/\.wiki\z/, '') + @project = Project.find_by_full_path(full_path) + + channel = GitalyClient.get_channel(@project.repository_storage) + @stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: channel) end - def post_receive(repo_path) - repository = Gitaly::Repository.new(path: repo_path) + def post_receive + repository = Gitaly::Repository.new(path: @project.repository.path_to_repo) request = Gitaly::PostReceiveRequest.new(repository: repository) - stub.post_receive(request) + @stub.post_receive(request) end end end diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb new file mode 100644 index 0000000000000000000000000000000000000000..4b1d828c45ca33acb8679ab59a0cf5a5d5468443 --- /dev/null +++ b/lib/gitlab/repo_path.rb @@ -0,0 +1,23 @@ +module Gitlab + module RepoPath + NotFoundError = Class.new(StandardError) + + def self.strip_storage_path(repo_path) + result = nil + + Gitlab.config.repositories.storages.values.each do |params| + storage_path = params['path'] + if repo_path.start_with?(storage_path) + result = repo_path.sub(storage_path, '') + break + end + end + + if result.nil? + raise NotFoundError.new("No known storage path matches #{repo_path.inspect}") + end + + result.sub(/\A\/*/, '') + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index eae1a0abf06baced27e1e6945179ca5b5b1d2f6f..6fe85af3c30d5143d489f534d6e45e065f118db4 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -1,6 +1,7 @@ require 'base64' require 'json' require 'securerandom' +require 'uri' module Gitlab class Workhorse @@ -21,10 +22,10 @@ module Gitlab RepoPath: repository.path_to_repo, } - params.merge!( - GitalySocketPath: Gitlab.config.gitaly.socket_path, - GitalyResourcePath: "/projects/#{repository.project.id}/git-http/info-refs", - ) if Gitlab.config.gitaly.socket_path.present? + if Gitlab.config.gitaly.enabled + address = Gitlab::GitalyClient.get_address(repository.project.repository_storage) + params[:GitalySocketPath] = URI(address).path + end params end diff --git a/spec/lib/gitlab/gitaly_client/notifications_spec.rb b/spec/lib/gitlab/gitaly_client/notifications_spec.rb index a6252c99aa14819c2d6c3bc25c6c63e268060095..bb5d93994ad45f7beddf74ad9b8ff3164387a5ed 100644 --- a/spec/lib/gitlab/gitaly_client/notifications_spec.rb +++ b/spec/lib/gitlab/gitaly_client/notifications_spec.rb @@ -1,20 +1,13 @@ require 'spec_helper' describe Gitlab::GitalyClient::Notifications do - let(:client) { Gitlab::GitalyClient::Notifications.new } - - before do - allow(Gitlab.config.gitaly).to receive(:socket_path).and_return('path/to/gitaly.socket') - end - describe '#post_receive' do - let(:repo_path) { '/path/to/my_repo.git' } - it 'sends a post_receive message' do + repo_path = create(:empty_project).repository.path_to_repo expect_any_instance_of(Gitaly::Notifications::Stub). to receive(:post_receive).with(post_receive_request_with_repo_path(repo_path)) - client.post_receive(repo_path) + described_class.new(repo_path).post_receive end end end diff --git a/spec/lib/gitlab/repo_path_spec.rb b/spec/lib/gitlab/repo_path_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..0fb5d7646f220ac3e0f444629b7603779d1468ed --- /dev/null +++ b/spec/lib/gitlab/repo_path_spec.rb @@ -0,0 +1,22 @@ +require 'spec_helper' + +describe ::Gitlab::RepoPath do + describe '.strip_storage_path' do + before do + allow(Gitlab.config.repositories).to receive(:storages).and_return({ + 'storage1' => { 'path' => '/foo' }, + 'storage2' => { 'path' => '/bar' }, + }) + end + + it 'strips the storage path' do + expect(described_class.strip_storage_path('/bar/foo/qux/baz.git')).to eq('foo/qux/baz.git') + end + + it 'raises NotFoundError if no storage matches the path' do + expect { described_class.strip_storage_path('/doesnotexist/foo.git') }.to raise_error( + described_class::NotFoundError + ) + end + end +end diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index 8e5e8288c49b44a121e6f7ae503d7e60b9202599..535c96eeee9aa12ac8a2d2b82f28cb1c7223bc9f 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -184,18 +184,14 @@ describe Gitlab::Workhorse, lib: true do it { expect(subject).to eq({ GL_ID: "user-#{user.id}", RepoPath: repository.path_to_repo }) } - context 'when Gitaly socket path is present' do - let(:gitaly_socket_path) { '/tmp/gitaly.sock' } - + context 'when Gitaly is enabled' do before do - allow(Gitlab.config.gitaly).to receive(:socket_path).and_return(gitaly_socket_path) + allow(Gitlab.config.gitaly).to receive(:enabled).and_return(true) end it 'includes Gitaly params in the returned value' do - expect(subject).to include({ - GitalyResourcePath: "/projects/#{repository.project.id}/git-http/info-refs", - GitalySocketPath: gitaly_socket_path, - }) + gitaly_socket_path = URI(Gitlab::GitalyClient.get_address('default')).path + expect(subject).to include({ GitalySocketPath: gitaly_socket_path }) end end end diff --git a/spec/requests/api/internal_spec.rb b/spec/requests/api/internal_spec.rb index 63ec00cdf0491969b34583161162a7cc848a9974..eed45d37444440aa19d6517301da7ff486aa8469 100644 --- a/spec/requests/api/internal_spec.rb +++ b/spec/requests/api/internal_spec.rb @@ -424,12 +424,12 @@ describe API::Internal, api: true do end before do - allow(Gitlab.config.gitaly).to receive(:socket_path).and_return('path/to/gitaly.socket') + allow(Gitlab.config.gitaly).to receive(:enabled).and_return(true) end it "calls the Gitaly client if it's enabled" do expect_any_instance_of(Gitlab::GitalyClient::Notifications). - to receive(:post_receive).with(project.repository.path) + to receive(:post_receive) post api("/internal/notify_post_receive"), valid_params @@ -438,7 +438,7 @@ describe API::Internal, api: true do it "returns 500 if the gitaly call fails" do expect_any_instance_of(Gitlab::GitalyClient::Notifications). - to receive(:post_receive).with(project.repository.path).and_raise(GRPC::Unavailable) + to receive(:post_receive).and_raise(GRPC::Unavailable) post api("/internal/notify_post_receive"), valid_params diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 10458966cb955cdb903da4b83ea68340e06130ae..665c8fe556aa80a28c9c56d6b1edb439b36849e2 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -81,6 +81,10 @@ describe 'gitlab:app namespace rake task' do end # backup_restore task describe 'backup' do + before(:all) do + ENV['force'] = 'yes' + end + def tars_glob Dir.glob(File.join(Gitlab.config.backup.path, '*_gitlab_backup.tar')) end @@ -88,6 +92,9 @@ describe 'gitlab:app namespace rake task' do def create_backup FileUtils.rm tars_glob + # This reconnect makes our project fixture disappear, breaking the restore. Stub it out. + allow(ActiveRecord::Base.connection).to receive(:reconnect!) + # Redirect STDOUT and run the rake task orig_stdout = $stdout $stdout = StringIO.new @@ -119,9 +126,6 @@ describe 'gitlab:app namespace rake task' do FileUtils.mkdir_p(path) FileUtils.touch(File.join(path, "dummy.txt")) - # We need to use the full path instead of the relative one - allow(Gitlab.config.gitlab_shell).to receive(:path).and_return(File.expand_path(Gitlab.config.gitlab_shell.path, Rails.root.to_s)) - ENV["SKIP"] = "db" create_backup end @@ -227,8 +231,8 @@ describe 'gitlab:app namespace rake task' do FileUtils.mkdir('tmp/tests/default_storage') FileUtils.mkdir('tmp/tests/custom_storage') storages = { - 'default' => { 'path' => 'tmp/tests/default_storage' }, - 'custom' => { 'path' => 'tmp/tests/custom_storage' } + 'default' => { 'path' => Settings.absolute('tmp/tests/default_storage') }, + 'custom' => { 'path' => Settings.absolute('tmp/tests/custom_storage') } } allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)