Add client methods for FetchIntoObjectPool RPC

Gitaly's FetchIntoObjectPool RPC will idempotently fetch objects into an
object pool. If the pool doesn't exist, it will create an empty pool
before attempting the fetch. This change adds client code as well as
specs to cover this behavior.

Gemfile

@@ -417,7 +417,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly-proto', '~> 1.22.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 1.26.0', require: 'gitaly'
 
 gem 'grpc', '~> 1.19.0'
 

Gemfile.lock

@@ -283,7 +283,7 @@ GEM
       gettext_i18n_rails (>= 0.7.1)
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
-    gitaly-proto (1.22.0)
+    gitaly-proto (1.26.0)
       grpc (~> 1.0)
     github-markup (1.7.0)
     gitlab-default_value_for (3.1.1)
@@ -1056,7 +1056,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.3)
-  gitaly-proto (~> 1.22.0)
+  gitaly-proto (~> 1.26.0)
   github-markup (~> 1.7.0)
   gitlab-default_value_for (~> 3.1.1)
   gitlab-labkit (~> 0.1.2)

lib/gitlab/git/object_pool.rb

@@ -40,6 +40,10 @@ module Gitlab
         @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY, gl_project_path)
       end
 
+      def fetch
+        object_pool_service.fetch(source_repository)
+      end
+
       private
 
       def object_pool_service

lib/gitlab/gitaly_client/object_pool_service.rb

@@ -33,6 +33,15 @@ module Gitlab
         GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool,
                           request, timeout: GitalyClient.fast_timeout)
       end
+
+      def fetch(repository)
+        request = Gitaly::FetchIntoObjectPoolRequest.new(
+          object_pool: object_pool,
+          origin: repository.gitaly_repository
+        )
+
+        GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, request)
+      end
     end
   end
 end

spec/lib/gitlab/git/object_pool_spec.rb

@@ -3,8 +3,12 @@
 require 'spec_helper'
 
 describe Gitlab::Git::ObjectPool do
+  include RepoHelpers
+
   let(:pool_repository) { create(:pool_repository) }
   let(:source_repository) { pool_repository.source_project.repository }
+  let(:source_repository_path) { File.join(TestEnv.repos_path, source_repository.relative_path) }
+  let(:source_repository_rugged) { Rugged::Repository.new(source_repository_path) }
 
   subject { pool_repository.object_pool }
 
@@ -76,4 +80,41 @@ describe Gitlab::Git::ObjectPool do
       end
     end
   end
+
+  describe '#fetch' do
+    let(:commit_count) { source_repository.commit_count }
+
+    context "when the object's pool repository exists" do
+      it 'does not raise an error' do
+        expect { subject.fetch }.not_to raise_error
+      end
+    end
+
+    context "when the object's pool repository does not exist" do
+      before do
+        subject.delete
+      end
+
+      it "re-creates the object pool's repository" do
+        subject.fetch
+
+        expect(subject.repository.exists?).to be(true)
+      end
+
+      it 'does not raise an error' do
+        expect { subject.fetch }.not_to raise_error
+      end
+
+      it 'fetches objects from the source repository' do
+        new_commit_id = new_commit_edit_old_file(source_repository_rugged).oid
+
+        expect(subject.repository.exists?).to be false
+
+        subject.fetch
+
+        expect(subject.repository.commit_count('refs/remotes/origin/master')).to eq(commit_count)
+        expect(subject.repository.commit(new_commit_id).id).to eq(new_commit_id)
+      end
+    end
+  end
 end

spec/lib/gitlab/git/repository_spec.rb

@@ -3,6 +3,7 @@ require "spec_helper"
 
 describe Gitlab::Git::Repository, :seed_helper do
   include Gitlab::EncodingHelper
+  include RepoHelpers
   using RSpec::Parameterized::TableSyntax
 
   shared_examples 'wrapping gRPC errors' do |gitaly_client_class, gitaly_client_method|
@@ -2209,83 +2210,6 @@ describe Gitlab::Git::Repository, :seed_helper do
     repository_rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", source_branch.dereferenced_target.sha)
   end
 
-  # Build the options hash that's passed to Rugged::Commit#create
-  def commit_options(repo, index, target, ref, message)
-    options = {}
-    options[:tree] = index.write_tree(repo)
-    options[:author] = {
-      email: "test@example.com",
-      name: "Test Author",
-      time: Time.gm(2014, "mar", 3, 20, 15, 1)
-    }
-    options[:committer] = {
-      email: "test@example.com",
-      name: "Test Author",
-      time: Time.gm(2014, "mar", 3, 20, 15, 1)
-    }
-    options[:message] ||= message
-    options[:parents] = repo.empty? ? [] : [target].compact
-    options[:update_ref] = ref
-
-    options
-  end
-
-  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
-  # contents of CHANGELOG with a single new line of text.
-  def new_commit_edit_old_file(repo)
-    oid = repo.write("I replaced the changelog with this text", :blob)
-    index = repo.index
-    index.read_tree(repo.head.target.tree)
-    index.add(path: "CHANGELOG", oid: oid, mode: 0100644)
-
-    options = commit_options(
-      repo,
-      index,
-      repo.head.target,
-      "HEAD",
-      "Edit CHANGELOG in its original location"
-    )
-
-    sha = Rugged::Commit.create(repo, options)
-    repo.lookup(sha)
-  end
-
-  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
-  # contents of the specified file_path with new text.
-  def new_commit_edit_new_file(repo, file_path, commit_message, text, branch = repo.head)
-    oid = repo.write(text, :blob)
-    index = repo.index
-    index.read_tree(branch.target.tree)
-    index.add(path: file_path, oid: oid, mode: 0100644)
-    options = commit_options(repo, index, branch.target, branch.canonical_name, commit_message)
-    sha = Rugged::Commit.create(repo, options)
-    repo.lookup(sha)
-  end
-
-  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
-  # contents of encoding/CHANGELOG with new text.
-  def new_commit_edit_new_file_on_branch(repo, file_path, branch_name, commit_message, text)
-    branch = repo.branches[branch_name]
-    new_commit_edit_new_file(repo, file_path, commit_message, text, branch)
-  end
-
-  # Writes a new commit to the repo and returns a Rugged::Commit.  Moves the
-  # CHANGELOG file to the encoding/ directory.
-  def new_commit_move_file(repo)
-    blob_oid = repo.head.target.tree.detect { |i| i[:name] == "CHANGELOG" }[:oid]
-    file_content = repo.lookup(blob_oid).content
-    oid = repo.write(file_content, :blob)
-    index = repo.index
-    index.read_tree(repo.head.target.tree)
-    index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644)
-    index.remove("CHANGELOG")
-
-    options = commit_options(repo, index, repo.head.target, "HEAD", "Move CHANGELOG to encoding/")
-
-    sha = Rugged::Commit.create(repo, options)
-    repo.lookup(sha)
-  end
-
   def refs(dir)
     IO.popen(%W[git -C #{dir} for-each-ref], &:read).split("\n").map do |line|
       line.split("\t").last

spec/lib/gitlab/gitaly_client/object_pool_service_spec.rb

@@ -43,4 +43,24 @@ describe Gitlab::GitalyClient::ObjectPoolService do
       end
     end
   end
+
+  describe '#fetch' do
+    before do
+      subject.delete
+    end
+
+    it 'restores the pool repository objects' do
+      subject.fetch(project.repository)
+
+      expect(object_pool.repository.exists?).to be(true)
+    end
+
+    context 'when called twice' do
+      it "doesn't raise an error" do
+        subject.delete
+
+        expect { subject.fetch(project.repository) }.not_to raise_error
+      end
+    end
+  end
 end

spec/support/helpers/repo_helpers.rb

@@ -11,6 +11,8 @@ module RepoHelpers
   #   blob.path # => 'files/js/commit.js.coffee'
   #   blob.data # => 'class Commit...'
   #
+  # Build the options hash that's passed to Rugged::Commit#create
+
   def sample_blob
     OpenStruct.new(
       oid: '5f53439ca4b009096571d3c8bc3d09d30e7431b3',
@@ -129,4 +131,80 @@ eos
       file_content: content
     ).execute
   end
+
+  def commit_options(repo, index, target, ref, message)
+    options = {}
+    options[:tree] = index.write_tree(repo)
+    options[:author] = {
+      email: "test@example.com",
+      name: "Test Author",
+      time: Time.gm(2014, "mar", 3, 20, 15, 1)
+    }
+    options[:committer] = {
+      email: "test@example.com",
+      name: "Test Author",
+      time: Time.gm(2014, "mar", 3, 20, 15, 1)
+    }
+    options[:message] ||= message
+    options[:parents] = repo.empty? ? [] : [target].compact
+    options[:update_ref] = ref
+
+    options
+  end
+
+  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
+  # contents of CHANGELOG with a single new line of text.
+  def new_commit_edit_old_file(repo)
+    oid = repo.write("I replaced the changelog with this text", :blob)
+    index = repo.index
+    index.read_tree(repo.head.target.tree)
+    index.add(path: "CHANGELOG", oid: oid, mode: 0100644)
+
+    options = commit_options(
+      repo,
+      index,
+      repo.head.target,
+      "HEAD",
+      "Edit CHANGELOG in its original location"
+    )
+
+    sha = Rugged::Commit.create(repo, options)
+    repo.lookup(sha)
+  end
+
+  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
+  # contents of the specified file_path with new text.
+  def new_commit_edit_new_file(repo, file_path, commit_message, text, branch = repo.head)
+    oid = repo.write(text, :blob)
+    index = repo.index
+    index.read_tree(branch.target.tree)
+    index.add(path: file_path, oid: oid, mode: 0100644)
+    options = commit_options(repo, index, branch.target, branch.canonical_name, commit_message)
+    sha = Rugged::Commit.create(repo, options)
+    repo.lookup(sha)
+  end
+
+  # Writes a new commit to the repo and returns a Rugged::Commit.  Replaces the
+  # contents of encoding/CHANGELOG with new text.
+  def new_commit_edit_new_file_on_branch(repo, file_path, branch_name, commit_message, text)
+    branch = repo.branches[branch_name]
+    new_commit_edit_new_file(repo, file_path, commit_message, text, branch)
+  end
+
+  # Writes a new commit to the repo and returns a Rugged::Commit.  Moves the
+  # CHANGELOG file to the encoding/ directory.
+  def new_commit_move_file(repo)
+    blob_oid = repo.head.target.tree.detect { |i| i[:name] == "CHANGELOG" }[:oid]
+    file_content = repo.lookup(blob_oid).content
+    oid = repo.write(file_content, :blob)
+    index = repo.index
+    index.read_tree(repo.head.target.tree)
+    index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644)
+    index.remove("CHANGELOG")
+
+    options = commit_options(repo, index, repo.head.target, "HEAD", "Move CHANGELOG to encoding/")
+
+    sha = Rugged::Commit.create(repo, options)
+    repo.lookup(sha)
+  end
 end