Merge branch 'gitaly-mandatory-20180709-jv' into 'master'

Use Gitaly for fetches and creating bundles

Closes gitaly#387, gitaly#874, gitaly#750, and gitaly#665

See merge request gitlab-org/gitlab-ce!20490

lib/gitlab/git/repository.rb

@@ -904,12 +904,8 @@ module Gitlab
       end
 
       def fetch_source_branch!(source_repository, source_branch, local_ref)
-        Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled|
-          if is_enabled
-            gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref)
-          else
-            rugged_fetch_source_branch(source_repository, source_branch, local_ref)
-          end
+        wrapped_gitaly_errors do
+          gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref)
         end
       end
 
@@ -1064,12 +1060,8 @@ module Gitlab
       end
 
       def bundle_to_disk(save_path)
-        gitaly_migrate(:bundle_to_disk) do |is_enabled|
-          if is_enabled
-            gitaly_repository_client.create_bundle(save_path)
-          else
-            run_git!(%W(bundle create #{save_path} --all))
-          end
+        wrapped_gitaly_errors do
+          gitaly_repository_client.create_bundle(save_path)
         end
 
         true

lib/gitlab/shell.rb

@@ -92,21 +92,13 @@ module Gitlab
     # Ex.
     #   import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git")
     #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874
     def import_repository(storage, name, url)
       if url.start_with?('.', '/')
         raise Error.new("don't use disk paths with import_repository: #{url.inspect}")
       end
 
       relative_path = "#{name}.git"
-      cmd = gitaly_migrate(:import_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-        if is_enabled
-          GitalyGitlabProjects.new(storage, relative_path)
-        else
-          # The timeout ensures the subprocess won't hang forever
-          gitlab_projects(storage, relative_path)
-        end
-      end
+      cmd = GitalyGitlabProjects.new(storage, relative_path)
 
       success = cmd.import_project(url, git_timeout)
       raise Error, cmd.output unless success
@@ -126,12 +118,8 @@ module Gitlab
     #   fetch_remote(my_repo, "upstream")
     #
     def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true)
-      gitaly_migrate(:fetch_remote) do |is_enabled|
-        if is_enabled
-          repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune)
-        else
-          local_fetch_remote(repository.storage, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune)
-        end
+      wrapped_gitaly_errors do
+        repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune)
       end
     end
 
@@ -389,28 +377,6 @@ module Gitlab
       )
     end
 
-    def local_fetch_remote(storage_name, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true)
-      vars = { force: forced, tags: !no_tags, prune: prune }
-
-      if ssh_auth&.ssh_import?
-        if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
-          vars[:ssh_key] = ssh_auth.ssh_private_key
-        end
-
-        if ssh_auth.ssh_known_hosts.present?
-          vars[:known_hosts] = ssh_auth.ssh_known_hosts
-        end
-      end
-
-      cmd = gitlab_projects(storage_name, repository_relative_path)
-
-      success = cmd.fetch_remote(remote, git_timeout, vars)
-
-      raise Error, cmd.output unless success
-
-      success
-    end
-
     def gitlab_shell_fast_execute(cmd)
       output, status = gitlab_shell_fast_execute_helper(cmd)
 
@@ -440,10 +406,6 @@ module Gitlab
       Gitlab.config.gitlab_shell.git_timeout
     end
 
-    def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
-      wrapped_gitaly_errors { Gitlab::GitalyClient.migrate(method, status: status, &block) }
-    end
-
     def wrapped_gitaly_errors
       yield
     rescue GRPC::NotFound, GRPC::BadStatus => e

spec/lib/gitlab/git/repository_spec.rb

@@ -1716,59 +1716,51 @@ describe Gitlab::Git::Repository, seed_helper: true do
   end
 
   describe '#fetch_source_branch!' do
-    shared_examples '#fetch_source_branch!' do
-      let(:local_ref) { 'refs/merge-requests/1/head' }
-      let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') }
-      let(:source_repository) { Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') }
-
-      after do
-        ensure_seeds
-      end
+    let(:local_ref) { 'refs/merge-requests/1/head' }
+    let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') }
+    let(:source_repository) { Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') }
 
-      context 'when the branch exists' do
-        context 'when the commit does not exist locally' do
-          let(:source_branch) { 'new-branch-for-fetch-source-branch' }
-          let(:source_rugged) { Gitlab::GitalyClient::StorageSettings.allow_disk_access { source_repository.rugged } }
-          let(:new_oid) { new_commit_edit_old_file(source_rugged).oid }
+    after do
+      ensure_seeds
+    end
 
-          before do
-            source_rugged.branches.create(source_branch, new_oid)
-          end
+    context 'when the branch exists' do
+      context 'when the commit does not exist locally' do
+        let(:source_branch) { 'new-branch-for-fetch-source-branch' }
+        let(:source_rugged) { Gitlab::GitalyClient::StorageSettings.allow_disk_access { source_repository.rugged } }
+        let(:new_oid) { new_commit_edit_old_file(source_rugged).oid }
 
-          it 'writes the ref' do
-            expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true)
-            expect(repository.commit(local_ref).sha).to eq(new_oid)
-          end
+        before do
+          source_rugged.branches.create(source_branch, new_oid)
         end
 
-        context 'when the commit exists locally' do
-          let(:source_branch) { 'master' }
-          let(:expected_oid) { SeedRepo::LastCommit::ID }
-
-          it 'writes the ref' do
-            # Sanity check: the commit should already exist
-            expect(repository.commit(expected_oid)).not_to be_nil
-
-            expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true)
-            expect(repository.commit(local_ref).sha).to eq(expected_oid)
-          end
+        it 'writes the ref' do
+          expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true)
+          expect(repository.commit(local_ref).sha).to eq(new_oid)
         end
       end
 
-      context 'when the branch does not exist' do
-        let(:source_branch) { 'definitely-not-master' }
+      context 'when the commit exists locally' do
+        let(:source_branch) { 'master' }
+        let(:expected_oid) { SeedRepo::LastCommit::ID }
+
+        it 'writes the ref' do
+          # Sanity check: the commit should already exist
+          expect(repository.commit(expected_oid)).not_to be_nil
 
-        it 'does not write the ref' do
-          expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(false)
-          expect(repository.commit(local_ref)).to be_nil
+          expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true)
+          expect(repository.commit(local_ref).sha).to eq(expected_oid)
         end
       end
     end
 
-    it_behaves_like '#fetch_source_branch!'
+    context 'when the branch does not exist' do
+      let(:source_branch) { 'definitely-not-master' }
 
-    context 'without gitaly', :skip_gitaly_mock do
-      it_behaves_like '#fetch_source_branch!'
+      it 'does not write the ref' do
+        expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(false)
+        expect(repository.commit(local_ref)).to be_nil
+      end
     end
   end
 

spec/lib/gitlab/shell_spec.rb

@@ -403,46 +403,36 @@ describe Gitlab::Shell do
     end
 
     describe '#create_repository' do
-      shared_examples '#create_repository' do
-        let(:repository_storage) { 'default' }
-        let(:repository_storage_path) do
-          Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-            Gitlab.config.repositories.storages[repository_storage].legacy_disk_path
-          end
-        end
-        let(:repo_name) { 'project/path' }
-        let(:created_path) { File.join(repository_storage_path, repo_name + '.git') }
-
-        after do
-          FileUtils.rm_rf(created_path)
+      let(:repository_storage) { 'default' }
+      let(:repository_storage_path) do
+        Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          Gitlab.config.repositories.storages[repository_storage].legacy_disk_path
         end
+      end
+      let(:repo_name) { 'project/path' }
+      let(:created_path) { File.join(repository_storage_path, repo_name + '.git') }
 
-        it 'creates a repository' do
-          expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_truthy
-
-          expect(File.stat(created_path).mode & 0o777).to eq(0o770)
+      after do
+        FileUtils.rm_rf(created_path)
+      end
 
-          hooks_path = File.join(created_path, 'hooks')
-          expect(File.lstat(hooks_path)).to be_symlink
-          expect(File.realpath(hooks_path)).to eq(gitlab_shell_hooks_path)
-        end
+      it 'creates a repository' do
+        expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_truthy
 
-        it 'returns false when the command fails' do
-          FileUtils.mkdir_p(File.dirname(created_path))
-          # This file will block the creation of the repo's .git directory. That
-          # should cause #create_repository to fail.
-          FileUtils.touch(created_path)
+        expect(File.stat(created_path).mode & 0o777).to eq(0o770)
 
-          expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_falsy
-        end
+        hooks_path = File.join(created_path, 'hooks')
+        expect(File.lstat(hooks_path)).to be_symlink
+        expect(File.realpath(hooks_path)).to eq(gitlab_shell_hooks_path)
       end
 
-      context 'with gitaly' do
-        it_behaves_like '#create_repository'
-      end
+      it 'returns false when the command fails' do
+        FileUtils.mkdir_p(File.dirname(created_path))
+        # This file will block the creation of the repo's .git directory. That
+        # should cause #create_repository to fail.
+        FileUtils.touch(created_path)
 
-      context 'without gitaly', :skip_gitaly_mock do
-        it_behaves_like '#create_repository'
+        expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_falsy
       end
     end
 
@@ -513,22 +503,12 @@ describe Gitlab::Shell do
       end
     end
 
-    shared_examples 'fetch_remote' do |gitaly_on|
+    describe '#fetch_remote' do
       def fetch_remote(ssh_auth = nil, prune = true)
         gitlab_shell.fetch_remote(repository.raw_repository, 'remote-name', ssh_auth: ssh_auth, prune: prune)
       end
 
-      def expect_gitlab_projects(fail = false, options = {})
-        expect(gitlab_projects).to receive(:fetch_remote).with(
-          'remote-name',
-          timeout,
-          options
-        ).and_return(!fail)
-
-        allow(gitlab_projects).to receive(:output).and_return('error') if fail
-      end
-
-      def expect_gitaly_call(fail, options = {})
+      def expect_call(fail, options = {})
         receive_fetch_remote =
           if fail
             receive(:fetch_remote).and_raise(GRPC::NotFound)
@@ -539,16 +519,6 @@ describe Gitlab::Shell do
         expect_any_instance_of(Gitlab::GitalyClient::RepositoryService).to receive_fetch_remote
       end
 
-      if gitaly_on
-        def expect_call(fail, options = {})
-          expect_gitaly_call(fail, options)
-        end
-      else
-        def expect_call(fail, options = {})
-          expect_gitlab_projects(fail, options)
-        end
-      end
-
       def build_ssh_auth(opts = {})
         defaults = {
           ssh_import?: true,
@@ -634,14 +604,6 @@ describe Gitlab::Shell do
           expect(fetch_remote(ssh_auth)).to be_truthy
         end
       end
-    end
-
-    describe '#fetch_remote local', :skip_gitaly_mock do
-      it_should_behave_like 'fetch_remote', false
-    end
-
-    describe '#fetch_remote gitaly' do
-      it_should_behave_like 'fetch_remote', true
 
       context 'gitaly call' do
         let(:remote_name) { 'remote-name' }
@@ -683,25 +645,6 @@ describe Gitlab::Shell do
           end.to raise_error(Gitlab::Shell::Error, "error")
         end
       end
-
-      context 'without gitaly', :disable_gitaly do
-        it 'returns true when the command succeeds' do
-          expect(gitlab_projects).to receive(:import_project).with(import_url, timeout) { true }
-
-          result = gitlab_shell.import_repository(project.repository_storage, project.disk_path, import_url)
-
-          expect(result).to be_truthy
-        end
-
-        it 'raises an exception when the command fails' do
-          allow(gitlab_projects).to receive(:output) { 'error' }
-          expect(gitlab_projects).to receive(:import_project) { false }
-
-          expect do
-            gitlab_shell.import_repository(project.repository_storage, project.disk_path, import_url)
-          end.to raise_error(Gitlab::Shell::Error, "error")
-        end
-      end
     end
   end
 

spec/lib/gitlab/workhorse_spec.rb

@@ -36,22 +36,20 @@ describe Gitlab::Workhorse do
       allow(described_class).to receive(:git_archive_cache_disabled?).and_return(cache_disabled)
     end
 
-    context 'when Gitaly workhorse_archive feature is enabled' do
-      it 'sets the header correctly' do
-        key, command, params = decode_workhorse_header(subject)
+    it 'sets the header correctly' do
+      key, command, params = decode_workhorse_header(subject)
 
-        expect(key).to eq('Gitlab-Workhorse-Send-Data')
-        expect(command).to eq('git-archive')
-        expect(params).to include(gitaly_params)
-      end
+      expect(key).to eq('Gitlab-Workhorse-Send-Data')
+      expect(command).to eq('git-archive')
+      expect(params).to include(gitaly_params)
+    end
 
-      context 'when archive caching is disabled' do
-        let(:cache_disabled) { true }
+    context 'when archive caching is disabled' do
+      let(:cache_disabled) { true }
 
-        it 'tells workhorse not to use the cache' do
-          _, _, params = decode_workhorse_header(subject)
-          expect(params).to include({ 'DisableCache' => true })
-        end
+      it 'tells workhorse not to use the cache' do
+        _, _, params = decode_workhorse_header(subject)
+        expect(params).to include({ 'DisableCache' => true })
       end
     end