Commit 201c1277 authored by Kerri Miller's avatar Kerri Miller

Merge branch '349099-use-reverse-filename-index-for-extension-filter' into 'master'

Advanced Search: Use reverse filename index

See merge request gitlab-org/gitlab!77226
parents 8bade920 30ae308b
---
name: elastic_file_name_reverse_optimization
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77226
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/349122
milestone: '14.7'
type: development
group: group::global search
default_enabled: false
......@@ -59,6 +59,10 @@ module Elastic
type: 'custom',
tokenizer: 'whitespace',
filter: %w(lowercase asciifolding)
},
whitespace_reverse: {
tokenizer: 'whitespace',
filter: %w(lowercase asciifolding reverse)
}
},
filter: {
......@@ -207,7 +211,8 @@ module Elastic
analyzer: :path_analyzer
indexes :file_name, type: :text,
analyzer: :code_analyzer,
search_analyzer: :code_search_analyzer
search_analyzer: :code_search_analyzer,
fields: { reverse: { type: :text, analyzer: :whitespace_reverse } }
indexes :content, type: :text,
index_options: 'positions',
analyzer: :code_analyzer,
......
......@@ -148,7 +148,13 @@ module Elastic
query = ::Gitlab::Search::Query.new(query) do
filter :filename, field: :file_name
filter :path, parser: ->(input) { "*#{input.downcase}*" }
filter :extension, field: :path, parser: ->(input) { '*.' + input.downcase }
if Feature.enabled?(:elastic_file_name_reverse_optimization, default_enabled: :yaml)
filter :extension, field: 'file_name.reverse', type: :prefix, parser: ->(input) { input.downcase.reverse + '.' }
else
filter :extension, field: :path, parser: ->(input) { '*.' + input.downcase }
end
filter :blob, field: :oid
end
......
......@@ -46,42 +46,59 @@ RSpec.describe Repository, :elastic do
'commit:match:search_terms')
end
it 'can filter blobs' do
project = create :project, :repository
index!(project)
context 'filtering' do
let(:project) { create :project, :repository }
before do
index!(project)
end
# Finds custom-highlighting/test.gitlab-custom
expect(project.repository.elastic_search('def | popen filename:test')[:blobs][:total_count]).to eq(1)
it 'can filter blobs' do
# Finds custom-highlighting/test.gitlab-custom
expect(project.repository.elastic_search('def | popen filename:test')[:blobs][:total_count]).to eq(1)
# Should not find anything, since filename doesn't match on path
expect(project.repository.elastic_search('def | popen filename:files')[:blobs][:total_count]).to eq(0)
# Should not find anything, since filename doesn't match on path
expect(project.repository.elastic_search('def | popen filename:files')[:blobs][:total_count]).to eq(0)
# Finds files/ruby/popen.rb, files/markdown/ruby-style-guide.md, files/ruby/regex.rb, files/ruby/version_info.rb
expect(project.repository.elastic_search('def | popen path:ruby')[:blobs][:total_count]).to eq(4)
# Finds files/ruby/popen.rb, files/markdown/ruby-style-guide.md, files/ruby/regex.rb, files/ruby/version_info.rb
expect(project.repository.elastic_search('def | popen path:ruby')[:blobs][:total_count]).to eq(4)
# Finds files/markdown/ruby-style-guide.md
expect(project.repository.elastic_search('def | popen extension:md')[:blobs][:total_count]).to eq(1)
# Finds files/markdown/ruby-style-guide.md
expect(project.repository.elastic_search('def | popen extension:md')[:blobs][:total_count]).to eq(1)
# Finds files/ruby/popen.rb
expect(project.repository.elastic_search('* blob:7e3e39ebb9b2bf433b4ad17313770fbe4051649c')[:blobs][:total_count]).to eq(1)
# Finds files/ruby/popen.rb
expect(project.repository.elastic_search('* blob:7e3e39ebb9b2bf433b4ad17313770fbe4051649c')[:blobs][:total_count]).to eq(1)
# filename filter without search term
count = project.repository.ls_files('master').count { |path| path.split('/')[-1].include?('test') }
expect(project.repository.elastic_search('filename:test')[:blobs][:total_count]).to eq(count)
expect(project.repository.elastic_search('filename:test')[:blobs][:total_count]).to be > 0
# filename filter without search term
count = project.repository.ls_files('master').count { |path| path.split('/')[-1].include?('test') }
expect(project.repository.elastic_search('filename:test')[:blobs][:total_count]).to eq(count)
expect(project.repository.elastic_search('filename:test')[:blobs][:total_count]).to be > 0
# extension filter without search term
count = project.repository.ls_files('master').count { |path| path.split('/')[-1].split('.')[-1].include?('md') }
expect(project.repository.elastic_search('extension:md')[:blobs][:total_count]).to eq(count)
expect(project.repository.elastic_search('extension:md')[:blobs][:total_count]).to be > 0
# extension filter without search term
count = project.repository.ls_files('master').count { |path| path.split('/')[-1].split('.')[-1].include?('md') }
expect(project.repository.elastic_search('extension:md')[:blobs][:total_count]).to eq(count)
expect(project.repository.elastic_search('extension:md')[:blobs][:total_count]).to be > 0
# path filter without search term
count = project.repository.ls_files('master').count { |path| path.include?('ruby') }
expect(project.repository.elastic_search('path:ruby')[:blobs][:total_count]).to eq(count)
expect(project.repository.elastic_search('path:ruby')[:blobs][:total_count]).to be > 0
# path filter without search term
count = project.repository.ls_files('master').count { |path| path.include?('ruby') }
expect(project.repository.elastic_search('path:ruby')[:blobs][:total_count]).to eq(count)
expect(project.repository.elastic_search('path:ruby')[:blobs][:total_count]).to be > 0
# blob filter without search term
expect(project.repository.elastic_search('blob:7e3e39ebb9b2bf433b4ad17313770fbe4051649c')[:blobs][:total_count]).to eq(1)
# blob filter without search term
expect(project.repository.elastic_search('blob:7e3e39ebb9b2bf433b4ad17313770fbe4051649c')[:blobs][:total_count]).to eq(1)
end
it 'filters by extension when optimization is disabled' do
stub_feature_flags(elastic_file_name_reverse_optimization: false)
# Finds files/markdown/ruby-style-guide.md
expect(project.repository.elastic_search('def | popen extension:md')[:blobs][:total_count]).to eq(1)
# extension filter without search term
count = project.repository.ls_files('master').count { |path| path.split('/')[-1].split('.')[-1].include?('md') }
expect(project.repository.elastic_search('extension:md')[:blobs][:total_count]).to eq(count)
expect(project.repository.elastic_search('extension:md')[:blobs][:total_count]).to be > 0
end
end
def search_and_check!(on, query, type:, per: 1000)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment