Commit 04b9de85 authored by John Cai's avatar John Cai

Modifying gitaly search files client to add chunking support

updates gitaly proto to 1.7.0, modifies the search files gitaly client
call to use the new chunked_response flag in the rpc request, and stitch
the responses together.

maintains backwards compatibility with older gitaly servers.
parent 684a1a17
---
title: Fix code search when text is larger than max gRPC message size
merge_request: 24111
author:
type: changed
...@@ -324,13 +324,40 @@ module Gitlab ...@@ -324,13 +324,40 @@ module Gitlab
GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files)
end end
def search_files_by_content(ref, query) def search_files_by_content(ref, query, chunked_response: true)
request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query) request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query, chunked_response: chunked_response)
GitalyClient.call(@storage, :repository_service, :search_files_by_content, request).flat_map(&:matches) response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request)
search_results_from_response(response)
end end
private private
def search_results_from_response(gitaly_response)
matches = []
current_match = +""
gitaly_response.each do |message|
next if message.nil?
# Old client will ignore :chunked_response flag
# and return messages with `matches` key.
# This code path will be removed post 12.0 release
if message.matches.any?
matches += message.matches
else
current_match << message.match_data
if message.end_of_match
matches << current_match
current_match = +""
end
end
end
matches
end
def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout) def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout)
request = request_class.new(repository: @gitaly_repo) request = request_class.new(repository: @gitaly_repo)
response = GitalyClient.call( response = GitalyClient.call(
......
...@@ -20,6 +20,8 @@ describe Gitlab::Git::Repository, :seed_helper do ...@@ -20,6 +20,8 @@ describe Gitlab::Git::Repository, :seed_helper do
end end
let(:mutable_repository) { Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '', 'group/project') } let(:mutable_repository) { Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '', 'group/project') }
let(:mutable_repository_path) { File.join(TestEnv.repos_path, mutable_repository.relative_path) }
let(:mutable_repository_rugged) { Rugged::Repository.new(mutable_repository_path) }
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '', 'group/project') } let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '', 'group/project') }
let(:repository_path) { File.join(TestEnv.repos_path, repository.relative_path) } let(:repository_path) { File.join(TestEnv.repos_path, repository.relative_path) }
let(:repository_rugged) { Rugged::Repository.new(repository_path) } let(:repository_rugged) { Rugged::Repository.new(repository_path) }
...@@ -497,6 +499,48 @@ describe Gitlab::Git::Repository, :seed_helper do ...@@ -497,6 +499,48 @@ describe Gitlab::Git::Repository, :seed_helper do
end end
end end
describe '#search_files_by_content' do
let(:repository) { mutable_repository }
let(:repository_rugged) { mutable_repository_rugged }
before do
repository.create_branch('search-files-by-content-branch', 'master')
new_commit_edit_new_file_on_branch(repository_rugged, 'encoding/CHANGELOG', 'search-files-by-content-branch', 'committing something', 'search-files-by-content change')
new_commit_edit_new_file_on_branch(repository_rugged, 'anotherfile', 'search-files-by-content-branch', 'committing something', 'search-files-by-content change')
end
after do
ensure_seeds
end
shared_examples 'search files by content' do
it 'should have 2 items' do
expect(search_results.size).to eq(2)
end
it 'should have the correct matching line' do
expect(search_results).to contain_exactly("search-files-by-content-branch:encoding/CHANGELOG\u00001\u0000search-files-by-content change\n",
"search-files-by-content-branch:anotherfile\u00001\u0000search-files-by-content change\n")
end
end
it_should_behave_like 'search files by content' do
let(:search_results) do
repository.search_files_by_content('search-files-by-content', 'search-files-by-content-branch')
end
end
it_should_behave_like 'search files by content' do
let(:search_results) do
repository.gitaly_repository_client.search_files_by_content(
'search-files-by-content-branch',
'search-files-by-content',
chunked_response: false
)
end
end
end
describe '#find_remote_root_ref' do describe '#find_remote_root_ref' do
it 'gets the remote root ref from GitalyClient' do it 'gets the remote root ref from GitalyClient' do
expect_any_instance_of(Gitlab::GitalyClient::RemoteService) expect_any_instance_of(Gitlab::GitalyClient::RemoteService)
...@@ -544,7 +588,7 @@ describe Gitlab::Git::Repository, :seed_helper do ...@@ -544,7 +588,7 @@ describe Gitlab::Git::Repository, :seed_helper do
# Add new commits so that there's a renamed file in the commit history # Add new commits so that there's a renamed file in the commit history
@commit_with_old_name_id = new_commit_edit_old_file(repository_rugged).oid @commit_with_old_name_id = new_commit_edit_old_file(repository_rugged).oid
@rename_commit_id = new_commit_move_file(repository_rugged).oid @rename_commit_id = new_commit_move_file(repository_rugged).oid
@commit_with_new_name_id = new_commit_edit_new_file(repository_rugged).oid @commit_with_new_name_id = new_commit_edit_new_file(repository_rugged, "encoding/CHANGELOG", "Edit encoding/CHANGELOG", "I'm a new changelog with different text").oid
end end
after do after do
...@@ -1964,7 +2008,7 @@ describe Gitlab::Git::Repository, :seed_helper do ...@@ -1964,7 +2008,7 @@ describe Gitlab::Git::Repository, :seed_helper do
end end
# Build the options hash that's passed to Rugged::Commit#create # Build the options hash that's passed to Rugged::Commit#create
def commit_options(repo, index, message) def commit_options(repo, index, target, ref, message)
options = {} options = {}
options[:tree] = index.write_tree(repo) options[:tree] = index.write_tree(repo)
options[:author] = { options[:author] = {
...@@ -1978,8 +2022,8 @@ describe Gitlab::Git::Repository, :seed_helper do ...@@ -1978,8 +2022,8 @@ describe Gitlab::Git::Repository, :seed_helper do
time: Time.gm(2014, "mar", 3, 20, 15, 1) time: Time.gm(2014, "mar", 3, 20, 15, 1)
} }
options[:message] ||= message options[:message] ||= message
options[:parents] = repo.empty? ? [] : [repo.head.target].compact options[:parents] = repo.empty? ? [] : [target].compact
options[:update_ref] = "HEAD" options[:update_ref] = ref
options options
end end
...@@ -1995,6 +2039,8 @@ describe Gitlab::Git::Repository, :seed_helper do ...@@ -1995,6 +2039,8 @@ describe Gitlab::Git::Repository, :seed_helper do
options = commit_options( options = commit_options(
repo, repo,
index, index,
repo.head.target,
"HEAD",
"Edit CHANGELOG in its original location" "Edit CHANGELOG in its original location"
) )
...@@ -2003,19 +2049,24 @@ describe Gitlab::Git::Repository, :seed_helper do ...@@ -2003,19 +2049,24 @@ describe Gitlab::Git::Repository, :seed_helper do
end end
# Writes a new commit to the repo and returns a Rugged::Commit. Replaces the # Writes a new commit to the repo and returns a Rugged::Commit. Replaces the
# contents of encoding/CHANGELOG with new text. # contents of the specified file_path with new text.
def new_commit_edit_new_file(repo) def new_commit_edit_new_file(repo, file_path, commit_message, text, branch = repo.head)
oid = repo.write("I'm a new changelog with different text", :blob) oid = repo.write(text, :blob)
index = repo.index index = repo.index
index.read_tree(repo.head.target.tree) index.read_tree(branch.target.tree)
index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644) index.add(path: file_path, oid: oid, mode: 0100644)
options = commit_options(repo, index, branch.target, branch.canonical_name, commit_message)
options = commit_options(repo, index, "Edit encoding/CHANGELOG")
sha = Rugged::Commit.create(repo, options) sha = Rugged::Commit.create(repo, options)
repo.lookup(sha) repo.lookup(sha)
end end
# Writes a new commit to the repo and returns a Rugged::Commit. Replaces the
# contents of encoding/CHANGELOG with new text.
def new_commit_edit_new_file_on_branch(repo, file_path, branch_name, commit_message, text)
branch = repo.branches[branch_name]
new_commit_edit_new_file(repo, file_path, commit_message, text, branch)
end
# Writes a new commit to the repo and returns a Rugged::Commit. Moves the # Writes a new commit to the repo and returns a Rugged::Commit. Moves the
# CHANGELOG file to the encoding/ directory. # CHANGELOG file to the encoding/ directory.
def new_commit_move_file(repo) def new_commit_move_file(repo)
...@@ -2027,7 +2078,7 @@ describe Gitlab::Git::Repository, :seed_helper do ...@@ -2027,7 +2078,7 @@ describe Gitlab::Git::Repository, :seed_helper do
index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644) index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644)
index.remove("CHANGELOG") index.remove("CHANGELOG")
options = commit_options(repo, index, "Move CHANGELOG to encoding/") options = commit_options(repo, index, repo.head.target, "HEAD", "Move CHANGELOG to encoding/")
sha = Rugged::Commit.create(repo, options) sha = Rugged::Commit.create(repo, options)
repo.lookup(sha) repo.lookup(sha)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment