Merge branch '209854-cache-es-check' into 'master'

Cache ES enabled namespaces and projects

See merge request gitlab-org/gitlab!27348

changelogs/unreleased/209854-cache-es-check.yml

+---
+title: Cache ES enabled namespaces and projects
+merge_request: 27348
+author:
+type: performance

ee/app/models/concerns/elasticsearch_indexed_container.rb

@@ -3,9 +3,16 @@
 module ElasticsearchIndexedContainer
   extend ActiveSupport::Concern
 
+  CACHE_EXPIRES_IN = 10.minutes
+
   included do
     after_commit :index, on: :create
     after_commit :delete_from_index, on: :destroy
+    after_commit :drop_limited_ids_cache!, on: [:create, :destroy]
+  end
+
+  def drop_limited_ids_cache!
+    self.class.drop_limited_ids_cache!
   end
 
   class_methods do
@@ -13,6 +20,28 @@ module ElasticsearchIndexedContainer
       pluck(target_attr_name)
     end
 
+    def limited_ids
+      limited.pluck(:id)
+    end
+
+    def limited_ids_cache_key
+      [self.name.underscore.to_sym, :limited_ids]
+    end
+
+    def limited_ids_cached
+      Rails.cache.fetch(limited_ids_cache_key, expires_in: CACHE_EXPIRES_IN) do
+        limited_ids
+      end
+    end
+
+    def drop_limited_ids_cache!
+      Rails.cache.delete(limited_ids_cache_key)
+    end
+
+    def limited_include?(namespace_id)
+      limited_ids_cached.include?(namespace_id)
+    end
+
     def remove_all(except: [])
       self.where.not(target_attr_name => except).each_batch do |batch, _index|
         batch.destroy_all # #rubocop:disable Cop/DestroyAll

ee/app/models/ee/application_setting.rb

@@ -128,37 +128,26 @@ module EE
       ElasticsearchIndexedProject.target_ids
     end
 
+    def elasticsearch_limited_namespaces(ignore_descendants: false)
+      ElasticsearchIndexedNamespace.limited(ignore_descendants: ignore_descendants)
+    end
+
+    def elasticsearch_limited_projects(ignore_namespaces: false)
+      ElasticsearchIndexedProject.limited(ignore_namespaces: ignore_namespaces)
+    end
+
     def elasticsearch_indexes_project?(project)
       return false unless elasticsearch_indexing?
       return true unless elasticsearch_limit_indexing?
 
-      elasticsearch_limited_projects.exists?(project.id)
+      ElasticsearchIndexedProject.limited_include?(project.id)
     end
 
     def elasticsearch_indexes_namespace?(namespace)
       return false unless elasticsearch_indexing?
       return true unless elasticsearch_limit_indexing?
 
-      elasticsearch_limited_namespaces.exists?(namespace.id)
-    end
-
-    def elasticsearch_limited_projects(ignore_namespaces = false)
-      return ::Project.where(id: ElasticsearchIndexedProject.select(:project_id)) if ignore_namespaces
-
-      union = ::Gitlab::SQL::Union.new([
-                                         ::Project.where(namespace_id: elasticsearch_limited_namespaces.select(:id)),
-                                         ::Project.where(id: ElasticsearchIndexedProject.select(:project_id))
-                                       ]).to_sql
-
-      ::Project.from("(#{union}) projects")
-    end
-
-    def elasticsearch_limited_namespaces(ignore_descendants = false)
-      namespaces = ::Namespace.where(id: ElasticsearchIndexedNamespace.select(:namespace_id))
-
-      return namespaces if ignore_descendants
-
-      ::Gitlab::ObjectHierarchy.new(namespaces).base_and_descendants
+      ElasticsearchIndexedNamespace.limited_include?(namespace.id)
     end
 
     def pseudonymizer_available?

ee/app/models/elasticsearch_indexed_namespace.rb

@@ -18,6 +18,19 @@ class ElasticsearchIndexedNamespace < ApplicationRecord
     :namespace_id
   end
 
+  def self.limited(ignore_descendants: false)
+    namespaces = Namespace.where(id: target_ids)
+
+    return namespaces if ignore_descendants
+
+    Gitlab::ObjectHierarchy.new(namespaces).base_and_descendants
+  end
+
+  def self.drop_limited_ids_cache!
+    ElasticsearchIndexedProject.drop_limited_ids_cache!
+    super
+  end
+
   def self.index_first_n_namespaces_of_plan(plan, number_of_namespaces)
     indexed_namespaces = self.select(:namespace_id)
     now = Time.now

ee/app/models/elasticsearch_indexed_project.rb

@@ -14,6 +14,17 @@ class ElasticsearchIndexedProject < ApplicationRecord
     :project_id
   end
 
+  def self.limited(ignore_namespaces: false)
+    return Project.where(id: target_ids) if ignore_namespaces
+
+    Project.from_union(
+      [
+        Project.where(namespace_id: ElasticsearchIndexedNamespace.limited.select(:id)),
+        Project.where(id: target_ids)
+      ]
+    )
+  end
+
   private
 
   def index

ee/app/views/admin/application_settings/_elasticsearch_form.html.haml

@@ -92,14 +92,14 @@
             - if elasticsearch_too_many_namespaces?
               %p= _('Too many namespaces enabled. You will need to manage them via the console or the API.')
             - else
-              = f.text_field :elasticsearch_namespace_ids, class: 'js-elasticsearch-namespaces', value: elasticsearch_namespace_ids, data: { selected: elasticsearch_objects_options(@application_setting.elasticsearch_limited_namespaces(true)).to_json }
+              = f.text_field :elasticsearch_namespace_ids, class: 'js-elasticsearch-namespaces', value: elasticsearch_namespace_ids, data: { selected: elasticsearch_objects_options(@application_setting.elasticsearch_limited_namespaces(ignore_descendants: true)).to_json }
 
           .form-group.js-limit-projects{ class: ('hidden' unless @application_setting.elasticsearch_limit_indexing) }
             = f.label :elasticsearch_project_ids, _('Projects to index'), class: 'label-bold'
             - if elasticsearch_too_many_projects?
               %p= _('Too many projects enabled. You will need to manage them via the console or the API.')
             - else
-              = f.text_field :elasticsearch_project_ids, class: 'js-elasticsearch-projects', value: elasticsearch_project_ids, data: { selected: elasticsearch_objects_options(@application_setting.elasticsearch_limited_projects(true)).to_json }
+              = f.text_field :elasticsearch_project_ids, class: 'js-elasticsearch-projects', value: elasticsearch_project_ids, data: { selected: elasticsearch_objects_options(@application_setting.elasticsearch_limited_projects(ignore_namespaces: true)).to_json }
 
         .sub-section
           %h4= _('Elasticsearch AWS IAM credentials')

ee/spec/models/application_setting_spec.rb

@@ -259,7 +259,7 @@ describe ApplicationSetting do
 
           expect(setting.elasticsearch_limited_namespaces).to match_array(
             [namespaces.last, child_namespace, child_namespace_indexed_through_parent])
-          expect(setting.elasticsearch_limited_namespaces(true)).to match_array(
+          expect(setting.elasticsearch_limited_namespaces(ignore_descendants: true)).to match_array(
             [namespaces.last, child_namespace])
         end
       end

ee/spec/models/elasticsearch_indexed_namespace_spec.rb

@@ -28,6 +28,17 @@ describe ElasticsearchIndexedNamespace do
     end
   end
 
+  context 'caching' do
+    it 'invalidates indexed project cache' do
+      expect(ElasticsearchIndexedProject).to receive(:drop_limited_ids_cache!).and_call_original.twice
+      expect(ElasticsearchIndexedNamespace).to receive(:drop_limited_ids_cache!).and_call_original.twice
+
+      n = create(:elasticsearch_indexed_namespace)
+
+      n.destroy
+    end
+  end
+
   context 'with plans' do
     Plan::PAID_HOSTED_PLANS.each do |plan|
       plan_factory = "#{plan}_plan"

ee/spec/support/shared_examples/models/elasticsearch_indexed_container_shared_examples.rb

@@ -9,6 +9,25 @@ RSpec.shared_examples 'an elasticsearch indexed container' do
     end
   end
 
+  describe '.limited_ids_cached', :use_clean_rails_memory_store_caching do
+    subject { create container }
+
+    it 'returns correct values' do
+      initial_ids = subject.class.limited_ids
+
+      expect(initial_ids).not_to be_empty
+      expect(subject.class.limited_ids_cached).to match_array(initial_ids)
+
+      new_container = create container
+
+      expect(subject.class.limited_ids_cached).to match_array(initial_ids + [new_container.id])
+
+      new_container.destroy
+
+      expect(subject.class.limited_ids_cached).to match_array(initial_ids)
+    end
+  end
+
   describe 'callbacks' do
     subject { build container }
 
@@ -24,6 +43,12 @@ RSpec.shared_examples 'an elasticsearch indexed container' do
 
         subject.save!
       end
+
+      it 'invalidates limited_ids cache' do
+        is_expected.to receive(:drop_limited_ids_cache!)
+
+        subject.save!
+      end
     end
 
     describe 'on destroy' do
@@ -40,6 +65,12 @@ RSpec.shared_examples 'an elasticsearch indexed container' do
 
         subject.destroy!
       end
+
+      it 'invalidates limited_ids cache' do
+        is_expected.to receive(:drop_limited_ids_cache!)
+
+        subject.destroy!
+      end
     end
   end
 end