Merge branch 'gitaly-mandatory-20180709-jv' into 'master'

Use Gitaly for fetches and creating bundles

Closes gitaly#387, gitaly#874, gitaly#750, and gitaly#665

See merge request gitlab-org/gitlab-ce!20490

lib/gitlab/git/repository.rb

@@ -904,12 +904,8 @@ module Gitlab
       end
 
       def fetch_source_branch!(source_repository, source_branch, local_ref)
-        Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled|
-          if is_enabled
+        wrapped_gitaly_errors do
           gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref)
-          else
-            rugged_fetch_source_branch(source_repository, source_branch, local_ref)
-          end
         end
       end
 
@@ -1064,12 +1060,8 @@ module Gitlab
       end
 
       def bundle_to_disk(save_path)
-        gitaly_migrate(:bundle_to_disk) do |is_enabled|
-          if is_enabled
+        wrapped_gitaly_errors do
           gitaly_repository_client.create_bundle(save_path)
-          else
-            run_git!(%W(bundle create #{save_path} --all))
-          end
         end
 
         true

lib/gitlab/shell.rb

@@ -92,21 +92,13 @@ module Gitlab
     # Ex.
     #   import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git")
     #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874
     def import_repository(storage, name, url)
       if url.start_with?('.', '/')
         raise Error.new("don't use disk paths with import_repository: #{url.inspect}")
       end
 
       relative_path = "#{name}.git"
-      cmd = gitaly_migrate(:import_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-        if is_enabled
-          GitalyGitlabProjects.new(storage, relative_path)
-        else
-          # The timeout ensures the subprocess won't hang forever
-          gitlab_projects(storage, relative_path)
-        end
-      end
+      cmd = GitalyGitlabProjects.new(storage, relative_path)
 
       success = cmd.import_project(url, git_timeout)
       raise Error, cmd.output unless success
@@ -126,12 +118,8 @@ module Gitlab
     #   fetch_remote(my_repo, "upstream")
     #
     def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true)
-      gitaly_migrate(:fetch_remote) do |is_enabled|
-        if is_enabled
+      wrapped_gitaly_errors do
         repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune)
-        else
-          local_fetch_remote(repository.storage, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune)
-        end
       end
     end
 
@@ -389,28 +377,6 @@ module Gitlab
       )
     end
 
-    def local_fetch_remote(storage_name, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true)
-      vars = { force: forced, tags: !no_tags, prune: prune }
-
-      if ssh_auth&.ssh_import?
-        if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
-          vars[:ssh_key] = ssh_auth.ssh_private_key
-        end
-
-        if ssh_auth.ssh_known_hosts.present?
-          vars[:known_hosts] = ssh_auth.ssh_known_hosts
-        end
-      end
-
-      cmd = gitlab_projects(storage_name, repository_relative_path)
-
-      success = cmd.fetch_remote(remote, git_timeout, vars)
-
-      raise Error, cmd.output unless success
-
-      success
-    end
-
     def gitlab_shell_fast_execute(cmd)
       output, status = gitlab_shell_fast_execute_helper(cmd)
 
@@ -440,10 +406,6 @@ module Gitlab
       Gitlab.config.gitlab_shell.git_timeout
     end
 
-    def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
-      wrapped_gitaly_errors { Gitlab::GitalyClient.migrate(method, status: status, &block) }
-    end
-
     def wrapped_gitaly_errors
       yield
     rescue GRPC::NotFound, GRPC::BadStatus => e

spec/lib/gitlab/git/repository_spec.rb

@@ -1716,7 +1716,6 @@ describe Gitlab::Git::Repository, seed_helper: true do
   end
 
   describe '#fetch_source_branch!' do
-    shared_examples '#fetch_source_branch!' do
     let(:local_ref) { 'refs/merge-requests/1/head' }
     let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') }
     let(:source_repository) { Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') }
@@ -1765,13 +1764,6 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
-    it_behaves_like '#fetch_source_branch!'
-
-    context 'without gitaly', :skip_gitaly_mock do
-      it_behaves_like '#fetch_source_branch!'
-    end
-  end
-
   describe '#rm_branch' do
     shared_examples "user deleting a branch" do
       let(:project) { create(:project, :repository) }

spec/lib/gitlab/shell_spec.rb

@@ -403,7 +403,6 @@ describe Gitlab::Shell do
     end
 
     describe '#create_repository' do
-      shared_examples '#create_repository' do
       let(:repository_storage) { 'default' }
       let(:repository_storage_path) do
         Gitlab::GitalyClient::StorageSettings.allow_disk_access do
@@ -437,15 +436,6 @@ describe Gitlab::Shell do
       end
     end
 
-      context 'with gitaly' do
-        it_behaves_like '#create_repository'
-      end
-
-      context 'without gitaly', :skip_gitaly_mock do
-        it_behaves_like '#create_repository'
-      end
-    end
-
     describe '#remove_repository' do
       let!(:project) { create(:project, :repository, :legacy_storage) }
       let(:disk_path) { "#{project.disk_path}.git" }
@@ -513,22 +503,12 @@ describe Gitlab::Shell do
       end
     end
 
-    shared_examples 'fetch_remote' do |gitaly_on|
+    describe '#fetch_remote' do
       def fetch_remote(ssh_auth = nil, prune = true)
         gitlab_shell.fetch_remote(repository.raw_repository, 'remote-name', ssh_auth: ssh_auth, prune: prune)
       end
 
-      def expect_gitlab_projects(fail = false, options = {})
-        expect(gitlab_projects).to receive(:fetch_remote).with(
-          'remote-name',
-          timeout,
-          options
-        ).and_return(!fail)
-
-        allow(gitlab_projects).to receive(:output).and_return('error') if fail
-      end
-
-      def expect_gitaly_call(fail, options = {})
+      def expect_call(fail, options = {})
         receive_fetch_remote =
           if fail
             receive(:fetch_remote).and_raise(GRPC::NotFound)
@@ -539,16 +519,6 @@ describe Gitlab::Shell do
         expect_any_instance_of(Gitlab::GitalyClient::RepositoryService).to receive_fetch_remote
       end
 
-      if gitaly_on
-        def expect_call(fail, options = {})
-          expect_gitaly_call(fail, options)
-        end
-      else
-        def expect_call(fail, options = {})
-          expect_gitlab_projects(fail, options)
-        end
-      end
-
       def build_ssh_auth(opts = {})
         defaults = {
           ssh_import?: true,
@@ -634,14 +604,6 @@ describe Gitlab::Shell do
           expect(fetch_remote(ssh_auth)).to be_truthy
         end
       end
-    end
-
-    describe '#fetch_remote local', :skip_gitaly_mock do
-      it_should_behave_like 'fetch_remote', false
-    end
-
-    describe '#fetch_remote gitaly' do
-      it_should_behave_like 'fetch_remote', true
 
       context 'gitaly call' do
         let(:remote_name) { 'remote-name' }
@@ -683,25 +645,6 @@ describe Gitlab::Shell do
           end.to raise_error(Gitlab::Shell::Error, "error")
         end
       end
-
-      context 'without gitaly', :disable_gitaly do
-        it 'returns true when the command succeeds' do
-          expect(gitlab_projects).to receive(:import_project).with(import_url, timeout) { true }
-
-          result = gitlab_shell.import_repository(project.repository_storage, project.disk_path, import_url)
-
-          expect(result).to be_truthy
-        end
-
-        it 'raises an exception when the command fails' do
-          allow(gitlab_projects).to receive(:output) { 'error' }
-          expect(gitlab_projects).to receive(:import_project) { false }
-
-          expect do
-            gitlab_shell.import_repository(project.repository_storage, project.disk_path, import_url)
-          end.to raise_error(Gitlab::Shell::Error, "error")
-        end
-      end
     end
   end
 

spec/lib/gitlab/workhorse_spec.rb

@@ -36,7 +36,6 @@ describe Gitlab::Workhorse do
       allow(described_class).to receive(:git_archive_cache_disabled?).and_return(cache_disabled)
     end
 
-    context 'when Gitaly workhorse_archive feature is enabled' do
     it 'sets the header correctly' do
       key, command, params = decode_workhorse_header(subject)
 
@@ -53,7 +52,6 @@ describe Gitlab::Workhorse do
         expect(params).to include({ 'DisableCache' => true })
       end
     end
-    end
 
     context "when the repository doesn't have an archive file path" do
       before do