Merge branch '349099-use-reverse-filename-index-for-extension-filter' into 'master'

Advanced Search: Use reverse filename index

See merge request gitlab-org/gitlab!77226

ee/config/feature_flags/development/elastic_file_name_reverse_optimization.yml

+---
+name: elastic_file_name_reverse_optimization
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77226
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/349122
+milestone: '14.7'
+type: development
+group: group::global search
+default_enabled: false

ee/lib/elastic/latest/config.rb

@@ -59,6 +59,10 @@ module Elastic
                 type: 'custom',
                 tokenizer: 'whitespace',
                 filter: %w(lowercase asciifolding)
+              },
+              whitespace_reverse: {
+                tokenizer: 'whitespace',
+                filter: %w(lowercase asciifolding reverse)
               }
             },
             filter: {
@@ -207,7 +211,8 @@ module Elastic
             analyzer: :path_analyzer
           indexes :file_name, type: :text,
             analyzer: :code_analyzer,
-            search_analyzer: :code_search_analyzer
+            search_analyzer: :code_search_analyzer,
+            fields: { reverse: { type: :text, analyzer: :whitespace_reverse } }
           indexes :content, type: :text,
             index_options: 'positions',
             analyzer: :code_analyzer,

ee/lib/elastic/latest/git_class_proxy.rb

@@ -148,7 +148,13 @@ module Elastic
         query = ::Gitlab::Search::Query.new(query) do
           filter :filename, field: :file_name
           filter :path, parser: ->(input) { "*#{input.downcase}*" }
-          filter :extension, field: :path, parser: ->(input) { '*.' + input.downcase }
+
+          if Feature.enabled?(:elastic_file_name_reverse_optimization, default_enabled: :yaml)
+            filter :extension, field: 'file_name.reverse', type: :prefix, parser: ->(input) { input.downcase.reverse + '.' }
+          else
+            filter :extension, field: :path, parser: ->(input) { '*.' + input.downcase }
+          end
+
           filter :blob, field: :oid
         end
 

ee/spec/models/concerns/elastic/repository_spec.rb

@@ -46,42 +46,59 @@ RSpec.describe Repository, :elastic do
                          'commit:match:search_terms')
   end
 
-  it 'can filter blobs' do
-    project = create :project, :repository
-    index!(project)
+  context 'filtering' do
+    let(:project) { create :project, :repository }
+
+    before do
+      index!(project)
+    end
 
-    # Finds custom-highlighting/test.gitlab-custom
-    expect(project.repository.elastic_search('def | popen filename:test')[:blobs][:total_count]).to eq(1)
+    it 'can filter blobs' do
+      # Finds custom-highlighting/test.gitlab-custom
+      expect(project.repository.elastic_search('def | popen filename:test')[:blobs][:total_count]).to eq(1)
 
-    # Should not find anything, since filename doesn't match on path
-    expect(project.repository.elastic_search('def | popen filename:files')[:blobs][:total_count]).to eq(0)
+      # Should not find anything, since filename doesn't match on path
+      expect(project.repository.elastic_search('def | popen filename:files')[:blobs][:total_count]).to eq(0)
 
-    # Finds files/ruby/popen.rb, files/markdown/ruby-style-guide.md, files/ruby/regex.rb, files/ruby/version_info.rb
-    expect(project.repository.elastic_search('def | popen path:ruby')[:blobs][:total_count]).to eq(4)
+      # Finds files/ruby/popen.rb, files/markdown/ruby-style-guide.md, files/ruby/regex.rb, files/ruby/version_info.rb
+      expect(project.repository.elastic_search('def | popen path:ruby')[:blobs][:total_count]).to eq(4)
 
-    # Finds files/markdown/ruby-style-guide.md
-    expect(project.repository.elastic_search('def | popen extension:md')[:blobs][:total_count]).to eq(1)
+      # Finds files/markdown/ruby-style-guide.md
+      expect(project.repository.elastic_search('def | popen extension:md')[:blobs][:total_count]).to eq(1)
 
-    # Finds files/ruby/popen.rb
-    expect(project.repository.elastic_search('* blob:7e3e39ebb9b2bf433b4ad17313770fbe4051649c')[:blobs][:total_count]).to eq(1)
+      # Finds files/ruby/popen.rb
+      expect(project.repository.elastic_search('* blob:7e3e39ebb9b2bf433b4ad17313770fbe4051649c')[:blobs][:total_count]).to eq(1)
 
-    # filename filter without search term
-    count = project.repository.ls_files('master').count { |path| path.split('/')[-1].include?('test') }
-    expect(project.repository.elastic_search('filename:test')[:blobs][:total_count]).to eq(count)
-    expect(project.repository.elastic_search('filename:test')[:blobs][:total_count]).to be > 0
+      # filename filter without search term
+      count = project.repository.ls_files('master').count { |path| path.split('/')[-1].include?('test') }
+      expect(project.repository.elastic_search('filename:test')[:blobs][:total_count]).to eq(count)
+      expect(project.repository.elastic_search('filename:test')[:blobs][:total_count]).to be > 0
 
-    # extension filter without search term
-    count = project.repository.ls_files('master').count { |path| path.split('/')[-1].split('.')[-1].include?('md') }
-    expect(project.repository.elastic_search('extension:md')[:blobs][:total_count]).to eq(count)
-    expect(project.repository.elastic_search('extension:md')[:blobs][:total_count]).to be > 0
+      # extension filter without search term
+      count = project.repository.ls_files('master').count { |path| path.split('/')[-1].split('.')[-1].include?('md') }
+      expect(project.repository.elastic_search('extension:md')[:blobs][:total_count]).to eq(count)
+      expect(project.repository.elastic_search('extension:md')[:blobs][:total_count]).to be > 0
 
-    # path filter without search term
-    count = project.repository.ls_files('master').count { |path| path.include?('ruby') }
-    expect(project.repository.elastic_search('path:ruby')[:blobs][:total_count]).to eq(count)
-    expect(project.repository.elastic_search('path:ruby')[:blobs][:total_count]).to be > 0
+      # path filter without search term
+      count = project.repository.ls_files('master').count { |path| path.include?('ruby') }
+      expect(project.repository.elastic_search('path:ruby')[:blobs][:total_count]).to eq(count)
+      expect(project.repository.elastic_search('path:ruby')[:blobs][:total_count]).to be > 0
 
-    # blob filter without search term
-    expect(project.repository.elastic_search('blob:7e3e39ebb9b2bf433b4ad17313770fbe4051649c')[:blobs][:total_count]).to eq(1)
+      # blob filter without search term
+      expect(project.repository.elastic_search('blob:7e3e39ebb9b2bf433b4ad17313770fbe4051649c')[:blobs][:total_count]).to eq(1)
+    end
+
+    it 'filters by extension when optimization is disabled' do
+      stub_feature_flags(elastic_file_name_reverse_optimization: false)
+
+      # Finds files/markdown/ruby-style-guide.md
+      expect(project.repository.elastic_search('def | popen extension:md')[:blobs][:total_count]).to eq(1)
+
+      # extension filter without search term
+      count = project.repository.ls_files('master').count { |path| path.split('/')[-1].split('.')[-1].include?('md') }
+      expect(project.repository.elastic_search('extension:md')[:blobs][:total_count]).to eq(count)
+      expect(project.repository.elastic_search('extension:md')[:blobs][:total_count]).to be > 0
+    end
   end
 
   def search_and_check!(on, query, type:, per: 1000)