Merge branch 'feature/migrate-repo-restorer-to-gitaly' into 'master'

Migrate restoring repo from bundle to Gitaly

Closes gitaly#946

See merge request gitlab-org/gitlab-ce!16623

GITALY_SERVER_VERSION

-0.73.0
+0.74.0

Gemfile

@@ -406,7 +406,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly-proto', '~> 0.76.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 0.78.0', require: 'gitaly'
 
 gem 'toml-rb', '~> 0.3.15', require: false
 

Gemfile.lock

@@ -285,7 +285,7 @@ GEM
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
     gherkin-ruby (0.3.2)
-    gitaly-proto (0.76.0)
+    gitaly-proto (0.78.0)
       google-protobuf (~> 3.1)
       grpc (~> 1.0)
     github-linguist (4.7.6)
@@ -1056,7 +1056,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.2.0)
-  gitaly-proto (~> 0.76.0)
+  gitaly-proto (~> 0.78.0)
   github-linguist (~> 4.7.0)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-markup (~> 1.6.2)

app/models/repository.rb

@@ -20,7 +20,7 @@ class Repository
   attr_accessor :full_path, :disk_path, :project, :is_wiki
 
   delegate :ref_name_for_sha, to: :raw_repository
-  delegate :bundle_to_disk, to: :raw_repository
+  delegate :bundle_to_disk, :create_from_bundle, to: :raw_repository
 
   CreateTreeError = Class.new(StandardError)
 

lib/gitlab/git/repository.rb

@@ -1188,6 +1188,19 @@ module Gitlab
         end
       end
 
+      def create_from_bundle(bundle_path)
+        gitaly_migrate(:create_repo_from_bundle) do |is_enabled|
+          if is_enabled
+            gitaly_repository_client.create_from_bundle(bundle_path)
+          else
+            run_git!(%W(clone --bare -- #{bundle_path} #{path}), chdir: nil)
+            self.class.create_hooks(path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path))
+          end
+        end
+
+        true
+      end
+
       def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:)
         gitaly_migrate(:rebase) do |is_enabled|
           if is_enabled

lib/gitlab/gitaly_client/repository_service.rb

@@ -3,6 +3,8 @@ module Gitlab
     class RepositoryService
       include Gitlab::EncodingHelper
 
+      MAX_MSG_SIZE = 128.kilobytes.freeze
+
       def initialize(repository)
         @repository = repository
         @gitaly_repo = repository.gitaly_repository
@@ -178,6 +180,29 @@ module Gitlab
           end
         end
       end
+
+      def create_from_bundle(bundle_path)
+        request = Gitaly::CreateRepositoryFromBundleRequest.new(repository: @gitaly_repo)
+        enum = Enumerator.new do |y|
+          File.open(bundle_path, 'rb') do |f|
+            while data = f.read(MAX_MSG_SIZE)
+              request.data = data
+
+              y.yield request
+
+              request = Gitaly::CreateRepositoryFromBundleRequest.new
+            end
+          end
+        end
+
+        GitalyClient.call(
+          @storage,
+          :repository_service,
+          :create_repository_from_bundle,
+          enum,
+          timeout: GitalyClient.default_timeout
+        )
+      end
     end
   end
 end

lib/gitlab/import_export/command_line_util.rb

@@ -11,11 +11,6 @@ module Gitlab
         untar_with_options(archive: archive, dir: dir, options: 'zxf')
       end
 
-      def git_clone_bundle(repo_path:, bundle_path:)
-        execute(%W(#{git_bin_path} clone --bare -- #{bundle_path} #{repo_path}))
-        Gitlab::Git::Repository.create_hooks(repo_path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path))
-      end
-
       def mkdir_p(path)
         FileUtils.mkdir_p(path, mode: DEFAULT_MODE)
         FileUtils.chmod(DEFAULT_MODE, path)

lib/gitlab/import_export/repo_restorer.rb

@@ -13,7 +13,7 @@ module Gitlab
       def restore
         return true unless File.exist?(@path_to_bundle)
 
-        git_clone_bundle(repo_path: @project.repository.path_to_repo, bundle_path: @path_to_bundle)
+        @project.repository.create_from_bundle(@path_to_bundle)
       rescue => e
         @shared.error(e)
         false

spec/lib/gitlab/git/repository_spec.rb

@@ -1992,6 +1992,42 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
+  describe '#create_from_bundle' do
+    shared_examples 'creating repo from bundle' do
+      let(:bundle_path) { File.join(Dir.tmpdir, "repo-#{SecureRandom.hex}.bundle") }
+      let(:project) { create(:project) }
+      let(:imported_repo) { project.repository.raw }
+
+      before { expect(repository.bundle_to_disk(bundle_path)).to be true }
+      after { FileUtils.rm_rf(bundle_path) }
+
+      it 'creates a repo from a bundle file' do
+        expect(imported_repo).not_to exist
+
+        result = imported_repo.create_from_bundle(bundle_path)
+
+        expect(result).to be true
+        expect(imported_repo).to exist
+        expect { imported_repo.fsck }.not_to raise_exception
+      end
+
+      it 'creates a symlink to the global hooks dir' do
+        imported_repo.create_from_bundle(bundle_path)
+        hooks_path = File.join(imported_repo.path, 'hooks')
+
+        expect(File.readlink(hooks_path)).to eq(Gitlab.config.gitlab_shell.hooks_path)
+      end
+    end
+
+    context 'when Gitaly create_repo_from_bundle feature is enabled' do
+      it_behaves_like 'creating repo from bundle'
+    end
+
+    context 'when Gitaly create_repo_from_bundle feature is disabled', :disable_gitaly do
+      it_behaves_like 'creating repo from bundle'
+    end
+  end
+
   context 'gitlab_projects commands' do
     let(:gitlab_projects) { repository.gitlab_projects }
     let(:timeout) { Gitlab.config.gitlab_shell.git_timeout }