Merge branch 'test-with-local-gitaly' into 'master'

Allow local tests to use a modified Gitaly

Closes gitaly#833

See merge request gitlab-org/gitlab-ce!16220

doc/development/gitaly.md

@@ -97,6 +97,29 @@ describe 'Gitaly Request count tests' do
 end
 ```
 
+## Running tests with a locally modified version of Gitaly
+
+Normally, gitlab-ce/ee tests use a local clone of Gitaly in `tmp/tests/gitaly`
+pinned at the version specified in GITALY_SERVER_VERSION. If you want
+to run tests locally against a modified version of Gitaly you can
+replace `tmp/tests/gitaly` with a symlink.
+
+```shell
+rm -rf tmp/tests/gitaly
+ln -s /path/to/gitaly tmp/tests/gitaly
+```
+
+Make sure you run `make` in your local Gitaly directory before running
+tests. Otherwise, Gitaly will fail to boot.
+
+If you make changes to your local Gitaly in between test runs you need
+to manually run `make` again.
+
+Note that CI tests will not use your locally modified version of
+Gitaly. To use a custom Gitaly version in CI you need to update
+GITALY_SERVER_VERSION. You can use the format `=revision` to use a
+non-tagged commit from https://gitlab.com/gitlab-org/gitaly in CI.
+
 ---
 
 [Return to Development documentation](README.md)

lib/gitlab/setup_helper.rb

+module Gitlab
+  module SetupHelper
+    class << self
+      # We cannot create config.toml files for all possible Gitaly configuations.
+      # For instance, if Gitaly is running on another machine then it makes no
+      # sense to write a config.toml file on the current machine. This method will
+      # only generate a configuration for the most common and simplest case: when
+      # we have exactly one Gitaly process and we are sure it is running locally
+      # because it uses a Unix socket.
+      # For development and testing purposes, an extra storage is added to gitaly,
+      # which is not known to Rails, but must be explicitly stubbed.
+      def gitaly_configuration_toml(gitaly_dir, gitaly_ruby: true)
+        storages = []
+        address = nil
+
+        Gitlab.config.repositories.storages.each do |key, val|
+          if address
+            if address != val['gitaly_address']
+              raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address."
+            end
+          elsif URI(val['gitaly_address']).scheme != 'unix'
+            raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses."
+          else
+            address = val['gitaly_address']
+          end
+
+          storages << { name: key, path: val['path'] }
+        end
+
+        if Rails.env.test?
+          storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s }
+        end
+
+        config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages }
+        config[:auth] = { token: 'secret' } if Rails.env.test?
+        config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby
+        config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
+        config[:bin_dir] = Gitlab.config.gitaly.client_path
+
+        TOML.dump(config)
+      end
+
+      # rubocop:disable Rails/Output
+      def create_gitaly_configuration(dir, force: false)
+        config_path = File.join(dir, 'config.toml')
+        FileUtils.rm_f(config_path) if force
+
+        File.open(config_path, File::WRONLY | File::CREAT | File::EXCL) do |f|
+          f.puts gitaly_configuration_toml(dir)
+        end
+      rescue Errno::EEXIST
+        puts "Skipping config.toml generation:"
+        puts "A configuration file already exists."
+      rescue ArgumentError => e
+        puts "Skipping config.toml generation:"
+        puts e.message
+      end
+      # rubocop:enable Rails/Output
+    end
+  end
+end

lib/tasks/gitlab/gitaly.rake

@@ -21,8 +21,8 @@ namespace :gitlab do
 
       command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test?
 
+      Gitlab::SetupHelper.create_gitaly_configuration(args.dir)
       Dir.chdir(args.dir) do
-        create_gitaly_configuration
         # In CI we run scripts/gitaly-test-build instead of this command
         unless ENV['CI'].present?
           Bundler.with_original_env { run_command!(command) }
@@ -39,60 +39,7 @@ namespace :gitlab do
 
       # Exclude gitaly-ruby configuration because that depends on the gitaly
       # installation directory.
-      puts gitaly_configuration_toml(gitaly_ruby: false)
-    end
-
-    private
-
-    # We cannot create config.toml files for all possible Gitaly configuations.
-    # For instance, if Gitaly is running on another machine then it makes no
-    # sense to write a config.toml file on the current machine. This method will
-    # only generate a configuration for the most common and simplest case: when
-    # we have exactly one Gitaly process and we are sure it is running locally
-    # because it uses a Unix socket.
-    # For development and testing purposes, an extra storage is added to gitaly,
-    # which is not known to Rails, but must be explicitly stubbed.
-    def gitaly_configuration_toml(gitaly_ruby: true)
-      storages = []
-      address = nil
-
-      Gitlab.config.repositories.storages.each do |key, val|
-        if address
-          if address != val['gitaly_address']
-            raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address."
-          end
-        elsif URI(val['gitaly_address']).scheme != 'unix'
-          raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses."
-        else
-          address = val['gitaly_address']
-        end
-
-        storages << { name: key, path: val['path'] }
-      end
-
-      if Rails.env.test?
-        storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s }
-      end
-
-      config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages }
-      config[:auth] = { token: 'secret' } if Rails.env.test?
-      config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby
-      config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
-      config[:bin_dir] = Gitlab.config.gitaly.client_path
-
-      TOML.dump(config)
-    end
-
-    def create_gitaly_configuration
-      File.open("config.toml", File::WRONLY | File::CREAT | File::EXCL) do |f|
-        f.puts gitaly_configuration_toml
-      end
-    rescue Errno::EEXIST
-      puts "Skipping config.toml generation:"
-      puts "A configuration file already exists."
-    rescue ArgumentError => e
-      puts "Skipping config.toml generation:"
-      puts e.message
+      puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false)
     end
   end
 end

spec/support/test_env.rb

 require 'rspec/mocks'
+require 'toml'
 
 module TestEnv
   extend self
@@ -147,6 +148,9 @@ module TestEnv
       version: Gitlab::GitalyClient.expected_server_version,
       task: "gitlab:gitaly:install[#{gitaly_dir}]") do
 
+      # Always re-create config, in case it's outdated. This is fast anyway.
+      Gitlab::SetupHelper.create_gitaly_configuration(gitaly_dir, force: true)
+
       start_gitaly(gitaly_dir)
     end
   end
@@ -347,6 +351,9 @@ module TestEnv
   end
 
   def component_needs_update?(component_folder, expected_version)
+    # Allow local overrides of the component for tests during development
+    return false if Rails.env.test? && File.symlink?(component_folder)
+
     version = File.read(File.join(component_folder, 'VERSION')).strip
 
     # Notice that this will always yield true when using branch versions