Commit f23cd251 authored by Micael Bergeron's avatar Micael Bergeron

Add `_name` to ES queries to improve readability

This MR adds a new helper class: `Elastic::Latest::QueryFactory` that
can be used to generate semantic names for queries.

In this MR, a hierarchical pattern has been used, to correlate
different part of the queries, such as:

- `issue:match:search_query`
- `issue:related:project`
- `doc:is_a:blob`

This MR sets a lot of names across the query parts, so it is expected
that some of these names might be sub-optimal.
parent ba0277c5
......@@ -99,6 +99,7 @@ gem 'graphiql-rails', '~> 1.4.10'
gem 'apollo_upload_server', '~> 2.0.2'
gem 'graphql-docs', '~> 1.6.0', group: [:development, :test]
gem 'hashie'
# Disable strong_params so that Mash does not respond to :permitted?
gem 'hashie-forbidden_attributes'
......
......@@ -1353,6 +1353,7 @@ DEPENDENCIES
haml_lint (~> 0.34.0)
hamlit (~> 2.11.0)
hangouts-chat (~> 0.0.5)
hashie
hashie-forbidden_attributes
health_check (~> 3.0)
hipchat (~> 1.5.0)
......
......@@ -55,13 +55,19 @@ module Elastic
bool: {
must: [{
simple_query_string: {
_name: QueryFactory.query_name(self.es_type, :match, :search_terms),
fields: fields,
query: query,
default_operator: default_operator
}
}],
filter: [{
term: { type: self.es_type }
term: {
type: {
_name: QueryFactory.query_name(:doc, :is_a, self.es_type),
value: self.es_type
}
}
}]
}
}
......@@ -87,8 +93,8 @@ module Elastic
query: {
bool: {
filter: [
{ term: { iid: iid } },
{ term: { type: self.es_type } }
{ term: { iid: { _name: QueryFactory.query_name(self.es_type, :related, :iid), value: iid } } },
{ term: { type: { _name: QueryFactory.query_name(:doc, :is_a, self.es_type), value: self.es_type } } }
]
}
}
......@@ -98,22 +104,25 @@ module Elastic
# Builds an elasticsearch query that will select child documents from a
# set of projects, taking user access rules into account.
def project_ids_filter(query_hash, options)
project_query = project_ids_query(
options[:current_user],
options[:project_ids],
options[:public_and_internal_projects],
options[:features]
)
query_hash[:query][:bool][:filter] ||= []
query_hash[:query][:bool][:filter] << {
has_parent: {
parent_type: "project",
query: {
bool: project_query
QueryFactory.query_context(:project) do |context|
project_query = project_ids_query(
options[:current_user],
options[:project_ids],
options[:public_and_internal_projects],
options[:features]
)
query_hash[:query][:bool][:filter] ||= []
query_hash[:query][:bool][:filter] << {
has_parent: {
_name: context.name,
parent_type: "project",
query: {
bool: project_query
}
}
}
}
end
query_hash
end
......@@ -153,17 +162,19 @@ module Elastic
conditions = pick_projects_by_membership(scoped_project_ids, user, features)
if public_and_internal_projects
# Skip internal projects for anonymous and external users.
# Others are given access to all internal projects.
#
# Admins & auditors get access to internal projects even
# if the feature is private.
conditions += pick_projects_by_visibility(Project::INTERNAL, user, features) if user && !user.external?
# All users, including anonymous, can access public projects.
# Admins & auditors get access to public projects where the feature is
# private.
conditions += pick_projects_by_visibility(Project::PUBLIC, user, features)
QueryFactory.query_context(:visibility) do
# Skip internal projects for anonymous and external users.
# Others are given access to all internal projects.
#
# Admins & auditors get access to internal projects even
# if the feature is private.
conditions += pick_projects_by_visibility(Project::INTERNAL, user, features) if user && !user.external?
# All users, including anonymous, can access public projects.
# Admins & auditors get access to public projects where the feature is
# private.
conditions += pick_projects_by_visibility(Project::PUBLIC, user, features)
end
end
{ should: conditions }
......@@ -179,24 +190,32 @@ module Elastic
def pick_projects_by_membership(project_ids, user, features = nil)
if features.nil?
if project_ids == :any
return [{ term: { visibility_level: Project::PRIVATE } }]
return [{ term: { visibility_level: { _name: QueryFactory.query_name(:any), value: Project::PRIVATE } } }]
else
return [{ terms: { id: project_ids } }]
return [{ terms: { _name: QueryFactory.query_name(:membership, :id), id: project_ids } }]
end
end
Array(features).map do |feature|
condition =
if project_ids == :any
{ term: { visibility_level: Project::PRIVATE } }
{ term: { visibility_level: { _name: QueryFactory.query_name(:any), value: Project::PRIVATE } } }
else
{ terms: { id: filter_ids_by_feature(project_ids, user, feature) } }
{ terms: { _name: QueryFactory.query_name(:membership, :id), id: filter_ids_by_feature(project_ids, user, feature) } }
end
limit =
{ terms: { "#{feature}_access_level" => [::ProjectFeature::ENABLED, ::ProjectFeature::PRIVATE] } }
limit = {
terms: {
_name: QueryFactory.query_name(feature, :enabled_or_private),
"#{feature}_access_level" => [::ProjectFeature::ENABLED, ::ProjectFeature::PRIVATE]
}
}
{ bool: { filter: [condition, limit] } }
{
bool: {
filter: [condition, limit]
}
}
end
end
......@@ -205,9 +224,11 @@ module Elastic
# If a project feature is specified, access is only granted if the feature
# is enabled or, for admins & auditors, private.
def pick_projects_by_visibility(visibility, user, features)
condition = { term: { visibility_level: visibility } }
QueryFactory.query_context(visibility) do |context|
condition = { term: { visibility_level: { _name: context.name, value: visibility } } }
limit_by_feature(condition, features, include_members_only: user&.can_read_all_resources?)
limit_by_feature(condition, features, include_members_only: user&.can_read_all_resources?)
end
end
# If a project feature(s) is specified, access is dependent on its visibility
......@@ -225,14 +246,33 @@ module Elastic
features = Array(features)
features.map do |feature|
limit =
if include_members_only
{ terms: { "#{feature}_access_level" => [::ProjectFeature::ENABLED, ::ProjectFeature::PRIVATE] } }
else
{ term: { "#{feature}_access_level" => ::ProjectFeature::ENABLED } }
end
QueryFactory.query_context(feature, :access_level) do |context|
limit =
if include_members_only
{
terms: {
_name: context.name(:enabled_or_private),
"#{feature}_access_level" => [::ProjectFeature::ENABLED, ::ProjectFeature::PRIVATE]
}
}
else
{
term: {
"#{feature}_access_level" => {
_name: context.name(:enabled),
value: ::ProjectFeature::ENABLED
}
}
}
end
{ bool: { filter: [condition, limit] } }
{
bool: {
_name: context.name,
filter: [condition, limit]
}
}
end
end
end
......
......@@ -43,8 +43,25 @@ module Elastic
languages = [options[:language]].flatten
filters = []
filters << { terms: { "#{type}.rid" => repository_ids } } if repository_ids.any?
filters << { terms: { "#{type}.language" => languages } } if languages.any?
if repository_ids.any?
filters << {
terms: {
_name: QueryFactory.query_name(type, :related, :repositories),
"#{type}.rid" => repository_ids
}
}
end
if languages.any?
filters << {
terms: {
_name: QueryFactory.query_name(type, :match, :languages),
"#{type}.language" => languages
}
}
end
filters << options[:additional_filter] if options[:additional_filter]
{ filter: filters }
......@@ -55,7 +72,8 @@ module Elastic
fields = %w(message^10 sha^5 author.name^2 author.email^2 committer.name committer.email).map {|i| "commit.#{i}"}
query_with_prefix = query.split(/\s+/).map { |s| s.gsub(SHA_REGEX) { |sha| "#{sha}*" } }.join(' ')
bool_expr = Gitlab::Elastic::BoolExpr.new
bool_expr = ::Gitlab::Elastic::BoolExpr.new
query_hash = {
query: { bool: bool_expr },
size: per,
......@@ -67,7 +85,7 @@ module Elastic
# we need to do a project visibility check.
#
# Note that `:current_user` might be `nil` for a anonymous user
query_hash = project_ids_filter(query_hash, options) if options.key?(:current_user)
query_hash = QueryFactory.query_context(:commit, :authorized) { project_ids_filter(query_hash, options) } if options.key?(:current_user)
if query.blank?
bool_expr[:must] = { match_all: {} }
......@@ -75,6 +93,7 @@ module Elastic
else
bool_expr[:must] = {
simple_query_string: {
_name: QueryFactory.query_name(:commit, :match, :search_terms),
fields: fields,
query: query_with_prefix,
default_operator: :and
......@@ -83,7 +102,14 @@ module Elastic
end
# add the document type filter
bool_expr[:filter] << { term: { type: 'commit' } }
bool_expr[:filter] << {
term: {
type: {
_name: QueryFactory.query_name(:doc, :is_a, :commit),
value: 'commit'
}
}
}
# add filters extracted from the options
options_filter_context = options_filter_context(:commit, options)
......@@ -131,6 +157,7 @@ module Elastic
# add the term matching
bool_expr[:must] = {
simple_query_string: {
_name: QueryFactory.query_name(:blob, :match, :search_terms),
query: query.term,
default_operator: :and,
fields: %w[blob.content blob.file_name]
......@@ -141,10 +168,17 @@ module Elastic
# we need to do a project visibility check.
#
# Note that `:current_user` might be `nil` for a anonymous user
query_hash = project_ids_filter(query_hash, options) if options.key?(:current_user)
query_hash = QueryFactory.query_context(:blob, :authorized) { project_ids_filter(query_hash, options) } if options.key?(:current_user)
# add the document type filter
bool_expr[:filter] << { term: { type: type } }
bool_expr[:filter] << {
term: {
type: {
_name: QueryFactory.query_name(:doc, :is_a, type),
value: type
}
}
}
# add filters extracted from the query
query_filter_context = query.elasticsearch_filter_context(:blob)
......
......@@ -21,9 +21,11 @@ module Elastic
end
options[:features] = 'issues'
query_hash = project_ids_filter(query_hash, options)
query_hash = confidentiality_filter(query_hash, options)
query_hash = state_filter(query_hash, options)
QueryFactory.query_context(:issue) do
query_hash = QueryFactory.query_context(:authorized) { project_ids_filter(query_hash, options) }
query_hash = QueryFactory.query_context(:confidentiality) { confidentiality_filter(query_hash, options) }
query_hash = QueryFactory.query_context(:match) { state_filter(query_hash, options) }
end
query_hash = apply_sort(query_hash, options)
search(query_hash, options)
......@@ -51,13 +53,14 @@ module Elastic
return query_hash if authorized_project_ids.to_set == scoped_project_ids.to_set
end
filter = { term: { confidential: false } }
context = QueryFactory.current_query_context
filter = { term: { confidential: { _name: context.name(:non_confidential), value: false } } }
if current_user
filter = {
bool: {
should: [
{ term: { confidential: false } },
{ term: { confidential: { _name: context.name(:non_confidential), value: false } } },
{
bool: {
must: [
......@@ -65,9 +68,9 @@ module Elastic
{
bool: {
should: [
{ term: { author_id: current_user.id } },
{ term: { assignee_id: current_user.id } },
{ terms: { project_id: authorized_project_ids } }
{ term: { author_id: { _name: context.name(:as_author), value: current_user.id } } },
{ term: { assignee_id: { _name: context.name(:as_assignee), value: current_user.id } } },
{ terms: { _name: context.name(:project, :membership, :id), project_id: authorized_project_ids } }
]
}
}
......
......@@ -21,8 +21,10 @@ module Elastic
end
options[:features] = 'merge_requests'
query_hash = project_ids_filter(query_hash, options)
query_hash = state_filter(query_hash, options)
QueryFactory.query_context(:merge_request) do
query_hash = QueryFactory.query_context(:authorized) { project_ids_filter(query_hash, options) }
query_hash = QueryFactory.query_context(:match) { state_filter(query_hash, options) }
end
query_hash = apply_sort(query_hash, options)
search(query_hash, options)
......
......@@ -6,9 +6,8 @@ module Elastic
def elastic_search(query, options: {})
options[:in] = %w(title^2 description)
query_hash = basic_query_hash(options[:in], query)
query_hash = project_ids_filter(query_hash, options)
query_hash = QueryFactory.query_context(:milestone, :match) { basic_query_hash(options[:in], query) }
query_hash = QueryFactory.query_context(:milestone, :related) { project_ids_filter(query_hash, options) }
search(query_hash, options)
end
......
......@@ -11,10 +11,12 @@ module Elastic
def elastic_search(query, options: {})
options[:in] = ['note']
query_hash = basic_query_hash(%w[note], query)
query_hash = project_ids_filter(query_hash, options)
query_hash = confidentiality_filter(query_hash, options)
QueryFactory.query_context(:note) do
query_hash = QueryFactory.query_context(:authorized) { project_ids_filter(query_hash, options) }
query_hash = QueryFactory.query_context(:confidentiality) { confidentiality_filter(query_hash, options) }
end
query_hash[:highlight] = highlight_options(options[:in])
......@@ -24,6 +26,7 @@ module Elastic
private
def confidentiality_filter(query_hash, options)
context = QueryFactory.current_query_context
current_user = options[:current_user]
return query_hash if current_user&.can_read_all_resources?
......@@ -35,18 +38,20 @@ module Elastic
must: [
{
bool: {
should: [
{ bool: { must_not: [{ exists: { field: :issue } }] } },
{ term: { "issue.confidential" => false } }
]
_name: context.name(:issue, :not_confidential),
should: [
{ bool: { must_not: [{ exists: { field: :issue } }] } },
{ term: { "issue.confidential" => false } }
]
}
},
{
bool: {
should: [
{ bool: { must_not: [{ exists: { field: :confidential } }] } },
{ term: { confidential: false } }
]
_name: context.name(:not_confidential),
should: [
{ bool: { must_not: [{ exists: { field: :confidential } }] } },
{ term: { confidential: false } }
]
}
}
]
......@@ -62,17 +67,17 @@ module Elastic
{
bool: {
should: [
{ term: { "issue.confidential" => true } },
{ term: { confidential: true } }
{ term: { "issue.confidential" => { _name: context.name(:issue, :confidential), value: true } } },
{ term: { confidential: { _name: context.name(:confidential), value: true } } }
]
}
},
{
bool: {
should: [
{ term: { "issue.author_id" => current_user.id } },
{ term: { "issue.assignee_id" => current_user.id } },
{ terms: { project_id: authorized_project_ids(current_user, options) } }
{ term: { "issue.author_id" => { _name: context.name(:as_author), value: current_user.id } } },
{ term: { "issue.assignee_id" => { _name: context.name(:as_assignee), value: current_user.id } } },
{ terms: { _name: context.name(:project, :membership, :id), project_id: authorized_project_ids(current_user, options) } }
]
}
}
......@@ -98,15 +103,18 @@ module Elastic
def project_ids_filter(query_hash, options)
query_hash[:query][:bool][:filter] ||= []
project_query = project_ids_query(
options[:current_user],
options[:project_ids],
options[:public_and_internal_projects],
options[:features]
)
project_query = QueryFactory.query_context(:project) do
project_ids_query(
options[:current_user],
options[:project_ids],
options[:public_and_internal_projects],
options[:features]
)
end
filters = {
bool: {
_name: QueryFactory.query_name,
should: []
}
}
......@@ -130,7 +138,7 @@ module Elastic
}
}
},
{ term: { noteable_type: noteable_type } }
{ term: { noteable_type: { _name: QueryFactory.query_name(:noteable, :is_a, noteable_type), value: noteable_type } } }
]
}
}
......@@ -146,17 +154,19 @@ module Elastic
override :pick_projects_by_membership
def pick_projects_by_membership(project_ids, user, _ = nil)
noteable_type_to_feature.map do |noteable_type, feature|
condition =
if project_ids == :any
{ term: { visibility_level: Project::PRIVATE } }
else
{ terms: { id: filter_ids_by_feature(project_ids, user, feature) } }
end
limit =
{ terms: { "#{feature}_access_level" => [::ProjectFeature::ENABLED, ::ProjectFeature::PRIVATE] } }
{ bool: { filter: [condition, limit] }, noteable_type: noteable_type }
QueryFactory.query_context(feature) do |context|
condition =
if project_ids == :any
{ term: { visibility_level: { _name: context.name(:any), value: Project::PRIVATE } } }
else
{ terms: { _name: context.name(:membership, :id), id: filter_ids_by_feature(project_ids, user, feature) } }
end
limit =
{ terms: { _name: context.name(:enabled_or_private), "#{feature}_access_level" => [::ProjectFeature::ENABLED, ::ProjectFeature::PRIVATE] } }
{ bool: { _name: context.name, filter: [condition, limit] }, noteable_type: noteable_type }
end
end
end
......@@ -166,14 +176,16 @@ module Elastic
override :limit_by_feature
def limit_by_feature(condition, _ = nil, include_members_only:)
noteable_type_to_feature.map do |noteable_type, feature|
limit =
if include_members_only
{ terms: { "#{feature}_access_level" => [::ProjectFeature::ENABLED, ::ProjectFeature::PRIVATE] } }
else
{ term: { "#{feature}_access_level" => ::ProjectFeature::ENABLED } }
end
{ bool: { filter: [condition, limit] }, noteable_type: noteable_type }
QueryFactory.query_context(feature) do |context|
limit =
if include_members_only
{ terms: { _name: context.name(:enabled_or_private), "#{feature}_access_level" => [::ProjectFeature::ENABLED, ::ProjectFeature::PRIVATE] } }
else
{ term: { "#{feature}_access_level" => { _name: context.name(:enabled), value: ::ProjectFeature::ENABLED } } }
end
{ bool: { _name: context.name, filter: [condition, limit] }, noteable_type: noteable_type }
end
end
end
end
......
......@@ -8,40 +8,45 @@ module Elastic
query_hash = basic_query_hash(options[:in], query)
filters = [{ terms: { type: [es_type] } }]
if options[:namespace_id]
filters << {
terms: {
namespace_id: [options[:namespace_id]].flatten
filters = [{ terms: { _name: QueryFactory.query_name(:doc, :is_a, es_type), type: [es_type] } }]
QueryFactory.query_context(:project) do |context|
if options[:namespace_id]
filters << {
terms: {
_name: context.name(:related, :namespaces),
namespace_id: [options[:namespace_id]].flatten
}
}
}
end
if options[:non_archived]
filters << {
terms: {
archived: [!options[:non_archived]].flatten
end
if options[:non_archived]
filters << {
terms: {
_name: context.name(:not_archived),
archived: [!options[:non_archived]].flatten
}
}
}
end
end
if options[:visibility_levels]
filters << {
terms: {
_name: context.name(:visibility_level),
visibility_level: [options[:visibility_levels]].flatten
}
}
end
if options[:visibility_levels]
filters << {
terms: {
visibility_level: [options[:visibility_levels]].flatten
if options[:project_ids]
filters << {
bool: project_ids_query(options[:current_user], options[:project_ids], options[:public_and_internal_projects])
}
}
end
end
if options[:project_ids]
filters << {
bool: project_ids_query(options[:current_user], options[:project_ids], options[:public_and_internal_projects])
}
query_hash[:query][:bool][:filter] = filters
end
query_hash[:query][:bool][:filter] = filters
search(query_hash, options)
end
end
......
# frozen_string_literal: true
module Elastic
module Latest
class QueryFactory
@stack = []
def self.query_context(*args, &block)
context = if current_query_context
::Gitlab::Elastic::ExprName.build(*current_query_context, *args)
else
::Gitlab::Elastic::ExprName.build(*args)
end
return context unless block_given?
begin
@stack.push(context)
yield context
ensure
@stack.pop
end
end
def self.query_name(*args)
return current_query_context.name(*args) if current_query_context
Gitlab::Elastic::ExprName.build(*args).name
end
def self.current_query_context
return if @stack.empty?
@stack.last
end
end
end
end
......@@ -5,7 +5,7 @@ module Elastic
class SnippetClassProxy < ApplicationClassProxy
def elastic_search(query, options: {})
query_hash = basic_query_hash(%w(title description), query)
query_hash = filter(query_hash, options)
query_hash = QueryFactory.query_context(:snippet, :authorized) { filter(query_hash, options) }
search(query_hash, options)
end
......@@ -27,6 +27,7 @@ module Elastic
# Match any of the filter conditions, in addition to the existing conditions
query_hash[:query][:bool][:filter] << {
bool: {
_name: QueryFactory.query_name,
should: filter_conditions
}
}
......@@ -40,6 +41,7 @@ module Elastic
# Include accessible personal snippets
filter_conditions << {
bool: {
_name: QueryFactory.query_name(:personal),
filter: [
{ terms: { visibility_level: Gitlab::VisibilityLevel.levels_for_user(user) } }
],
......@@ -51,8 +53,9 @@ module Elastic
if user
filter_conditions << {
bool: {
_name: QueryFactory.query_name(:authored),
filter: [
{ term: { author_id: user.id } }
{ term: { author_id: { _name: QueryFactory.query_name(:as_author), value: user.id } } }
],
must_not: { exists: { field: 'project_id' } }
}
......@@ -76,6 +79,7 @@ module Elastic
filter_conditions << {
bool: {
_name: QueryFactory.query_name(:membership, :id),
must: [
{ terms: { project_id: project_ids } }
]
......
......@@ -11,7 +11,7 @@ module Elastic
return query_hash if state.blank? || state == 'all'
return query_hash unless API::Helpers::SearchHelpers.search_states.include?(state)
filter = { match: { state: state } }
filter = { match: { state: { _name: QueryFactory.query_name(:state), query: state } } }
query_hash[:query][:bool][:filter] << filter
query_hash
......
......@@ -9,7 +9,7 @@ module Gitlab
# Takes a hash as returned by `ApplicationSetting#elasticsearch_config`,
# and configures itself based on those parameters
def self.build(config)
def self.build(config, &block)
base_config = {
urls: config[:url],
request_timeout: config[:client_request_timeout],
......@@ -23,9 +23,11 @@ module Gitlab
::Elasticsearch::Client.new(base_config) do |fmid|
fmid.request(:aws_sigv4, credentials_provider: creds, service: 'es', region: region)
yield fmid if block_given?
end
else
::Elasticsearch::Client.new(base_config)
::Elasticsearch::Client.new(base_config, &block)
end
end
......
# frozen_string_literal: true
module Gitlab
module Elastic
class ExprName < Array
def self.build(*context)
new(context)
end
def name(*context)
return to_s if context.empty?
ExprName.build(*self, *context).to_s
end
def to_s
map(&:to_s).join(":")
end
end
end
end
......@@ -2,13 +2,13 @@
require 'spec_helper'
RSpec.describe Elastic::Latest::GitClassProxy do
RSpec.describe Elastic::Latest::GitClassProxy, :elastic do
let_it_be(:project) { create(:project, :repository) }
let(:included_class) { Elastic::Latest::RepositoryClassProxy }
subject { included_class.new(project.repository) }
describe '#elastic_search_as_found_blob', :elastic do
describe '#elastic_search_as_found_blob' do
before do
stub_ee_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
......@@ -31,4 +31,13 @@ RSpec.describe Elastic::Latest::GitClassProxy do
expect(result.project).to eq(project)
end
end
it "names elasticsearch queries" do |example|
expect_named_queries(example) do |inspector|
subject.elastic_search_as_found_blob('*')
expect(inspector).to have_named_query('doc:is_a:blob')
expect(inspector).to have_named_query('blob:match:search_terms')
end
end
end
......@@ -2,12 +2,12 @@
require 'spec_helper'
RSpec.describe Elastic::Latest::ProjectWikiClassProxy do
RSpec.describe Elastic::Latest::ProjectWikiClassProxy, :elastic do
let_it_be(:project) { create(:project, :wiki_repo) }
subject { described_class.new(project.wiki.repository) }
describe '#elastic_search_as_wiki_page', :elastic do
describe '#elastic_search_as_wiki_page' do
let_it_be(:page) { create(:wiki_page, wiki: project.wiki) }
before do
......@@ -31,4 +31,13 @@ RSpec.describe Elastic::Latest::ProjectWikiClassProxy do
expect(result.project).to eq(project)
end
end
it 'names elasticsearch queries' do |example|
expect_named_queries(example) do |inspector|
subject.elastic_search_as_wiki_page('*')
expect(inspector).to have_named_query('doc:is_a:wiki_blob')
expect(inspector).to have_named_query('blob:match:search_terms')
end
end
end
......@@ -75,6 +75,16 @@ RSpec.describe Issue, :elastic do
expect(described_class.elastic_search('bla-bla', options: { project_ids: :any, public_and_internal_projects: true }).total_count).to eq(3)
end
it "names elasticsearch queries" do |example|
expect_named_queries(example) do |inspector|
described_class.elastic_search('*').total_count
expect(inspector).to have_named_query('doc:is_a:issue')
expect(inspector).to have_named_query('issue:match:search_terms')
expect(inspector).to have_named_query('issue:authorized:project')
end
end
it "searches by iid and scopes to type: issue only" do
issue = nil
......
......@@ -40,6 +40,16 @@ RSpec.describe MergeRequest, :elastic do
expect(described_class.elastic_search('term3', options: { project_ids: :any, public_and_internal_projects: true }).total_count).to eq(1)
end
it "names elasticsearch queries" do |example|
expect_named_queries(example) do |inspector|
described_class.elastic_search('*').total_count
expect(inspector).to have_named_query('doc:is_a:merge_request')
expect(inspector).to have_named_query('merge_request:match:search_terms')
expect(inspector).to have_named_query('merge_request:authorized:project')
end
end
it "searches by iid and scopes to type: merge_request only", :sidekiq_might_not_need_inline do
project = create :project, :public, :repository
merge_request = nil
......
......@@ -55,6 +55,16 @@ RSpec.describe Note, :elastic do
expect(described_class.elastic_search('bla-bla', options: { project_ids: :any }).records).to contain_exactly(outside_note)
end
it "names elasticsearch queries" do |example|
expect_named_queries(example) do |inspector|
described_class.elastic_search('*').total_count
expect(inspector).to have_named_query('doc:is_a:note')
expect(inspector).to have_named_query('note:match:search_terms')
expect(inspector).to have_named_query('note:authorized')
end
end
it "indexes && searches diff notes" do
notes = []
......
......@@ -198,6 +198,15 @@ RSpec.describe Project, :elastic do
expect(described_class.elastic_search('tesla', options: { project_ids: project_ids }).total_count).to eq(2)
end
it "names elasticsearch queries" do |example|
expect_named_queries(example) do |inspector|
described_class.elastic_search('*').total_count
expect(inspector).to have_named_query('doc:is_a:project')
expect(inspector).to have_named_query('project:match:search_terms')
end
end
it "returns json with all needed elements" do
project = create :project
......
......@@ -31,6 +31,21 @@ RSpec.describe Repository, :elastic do
expect(project.repository.elastic_search(partial_ref + '*')[:commits][:total_count]).to eq(1)
end
it "names elasticsearch queries" do |example|
project = create :project, :repository
expect_named_queries(example) do |inspector|
project.repository.elastic_search('*')
expect(inspector).to have_named_query('doc:is_a:blob')
expect(inspector).to have_named_query('doc:is_a:wiki_blob')
expect(inspector).to have_named_query('blob:match:search_terms')
expect(inspector).to have_named_query('doc:is_a:commit')
expect(inspector).to have_named_query('commit:match:search_terms')
end
end
it 'can filter blobs' do
project = create :project, :repository
index!(project)
......
......@@ -35,6 +35,16 @@ RSpec.describe Snippet, :elastic do
expect(described_class.elastic_search('test snippet', options: options).total_count).to eq(1)
end
it "names elasticsearch queries" do |example|
expect_named_queries(example) do |inspector|
described_class.elastic_search('*').total_count
expect(inspector).to have_named_query('doc:is_a:snippet')
expect(inspector).to have_named_query('snippet:match:search_terms')
expect(inspector).to have_named_query('snippet:authorized')
end
end
it 'returns json with all needed elements' do
snippet = create(:project_snippet)
......
# frozen_string_literal: true
RSpec.configure do |config|
config.before(:each, :elastic) do
config.before(:each, :elastic) do |example|
Elastic::ProcessBookkeepingService.clear_tracking!
Gitlab::Elastic::Helper.default.delete_index
Gitlab::Elastic::Helper.default.create_empty_index(options: { settings: { number_of_replicas: 0 } })
name_inspector = ElasticQueryNameInspector.new
es_config = Gitlab::CurrentSettings.elasticsearch_config
es_client = Gitlab::Elastic::Client.build(es_config) do |faraday|
faraday.use(ElasticQueryInspectorMiddleware, inspector: name_inspector)
end
example.metadata[:query_inspector] = name_inspector
# inject a client that records Elastic named queries
GemExtensions::Elasticsearch::Model::Client.cached_client = es_client
GemExtensions::Elasticsearch::Model::Client.cached_config = es_config
end
config.after(:each, :elastic) do
......
# frozen_string_literal: true
require 'hashie'
class ElasticQueryInspectorMiddleware < Faraday::Middleware
def initialize(app, options = {})
super(app)
@inspector = options.fetch(:inspector)
@env = nil
end
def call(env)
@env = env
return continue! unless is_search?
query = begin
payload = Gitlab::Json.parse(env[:request_body])
payload["query"]
rescue ::JSON::ParserError
nil
end
return continue! unless query.present?
query.extend(Hashie::Extensions::DeepFind)
@inspector.inspect(query)
continue!
end
def continue!
@app.call(@env)
end
def is_search?
@env.url.path.ends_with?("_search")
end
end
# frozen_string_literal: true
class ElasticQueryNameInspector
def initialize
@buckets = []
end
def inspect(query)
@buckets << query.deep_find_all("_name")
end
def reset!
@buckets = []
end
def names
@buckets.clone
end
def all_names
@buckets.flatten
end
def has_named_query?(*names)
names.all? { |name| all_names.include?(name) }
end
end
# frozen_string_literal: true
module ElasticsearchHelpers
def expect_named_queries(example, &block)
query_inspector = example.metadata[:query_inspector]
query_inspector.reset!
yield query_inspector
expect(query_inspector.names).not_to be_empty
expect(query_inspector.names).not_to include(be_empty)
end
def ensure_elasticsearch_index!
# Ensure that any enqueued updates are processed
Elastic::ProcessBookkeepingService.new.execute
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment