Implement zero-downtime reindexing admin UI

We implement UI for triggering and monitoring zero-downtime reindexing
in the admin UI

db/migrate/20200630110826_add_documents_count_target_to_elastic_reindexing_tasks.rb

+# frozen_string_literal: true
+
+class AddDocumentsCountTargetToElasticReindexingTasks < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    add_column :elastic_reindexing_tasks, :documents_count_target, :integer
+  end
+end

db/structure.sql

@@ -11104,6 +11104,7 @@ CREATE TABLE public.elastic_reindexing_tasks (
     index_name_to text,
     elastic_task text,
     error_message text,
+    documents_count_target integer,
     CONSTRAINT check_04151aca42 CHECK ((char_length(index_name_from) <= 255)),
     CONSTRAINT check_7f64acda8e CHECK ((char_length(error_message) <= 255)),
     CONSTRAINT check_85ebff7124 CHECK ((char_length(index_name_to) <= 255)),
@@ -23569,6 +23570,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200625190458
 20200626060151
 20200626130220
+20200630110826
 20200702123805
 20200703154822
 20200706005325

ee/app/controllers/admin/elasticsearch_controller.rb

@@ -16,4 +16,17 @@ class Admin::ElasticsearchController < Admin::ApplicationController
 
     redirect_to integrations_admin_application_settings_path(anchor: 'js-elasticsearch-settings')
   end
+
+  # POST
+  # Trigger reindexing task
+  def trigger_reindexing
+    if Elastic::ReindexingTask.running?
+      flash[:warning] = _('Elasticsearch reindexing is already in progress')
+    else
+      Elastic::ReindexingTask.create!
+      flash[:notice] = _('Elasticsearch reindexing triggered')
+    end
+
+    redirect_to integrations_admin_application_settings_path(anchor: 'js-elasticsearch-settings')
+  end
 end

ee/app/controllers/ee/admin/application_settings_controller.rb

@@ -4,9 +4,18 @@ module EE
   module Admin
     module ApplicationSettingsController
       extend ::Gitlab::Utils::Override
+      extend ActiveSupport::Concern
 
       include ::Admin::MergeRequestApprovalSettingsHelper
 
+      prepended do
+        before_action :elasticsearch_reindexing_task, only: [:integrations]
+
+        def elasticsearch_reindexing_task
+          @elasticsearch_reindexing_task = Elastic::ReindexingTask.last
+        end
+      end
+
       EE_VALID_SETTING_PANELS = %w(templates).freeze
 
       EE_VALID_SETTING_PANELS.each do |action|

ee/app/models/elastic/reindexing_task.rb

@@ -17,6 +17,10 @@ class Elastic::ReindexingTask < ApplicationRecord
     where(in_progress: true).last
   end
 
+  def self.running?
+    current.present?
+  end
+
   private
 
   def set_in_progress_flag

ee/app/services/elastic/cluster_reindexing_service.rb

@@ -2,8 +2,10 @@
 
 module Elastic
   class ClusterReindexingService
+    include Gitlab::Utils::StrongMemoize
+
     INITIAL_INDEX_OPTIONS = { # Optimized for writes
-        refresh_interval: -1, # Disable automatic refreshing
+        refresh_interval: '10s',
         number_of_replicas: 0,
         translog: { durability: 'async' }
     }.freeze
@@ -20,7 +22,9 @@ module Elastic
     end
 
     def current_task
-      Elastic::ReindexingTask.current
+      strong_memoize(:elastic_current_task) do
+        Elastic::ReindexingTask.current
+      end
     end
 
     private
@@ -74,41 +78,62 @@ module Elastic
       true
     end
 
-    def reindexing!
-      task = current_task
+    def save_documents_count!(refresh:)
+      elastic_helper.refresh_index(index_name: current_task.index_name_to) if refresh
+
+      new_documents_count = elastic_helper.index_size(index_name: current_task.index_name_to).dig('docs', 'count')
+      current_task.update!(documents_count_target: new_documents_count)
+    end
 
-      # Check if indexing is completed
-      task_status = elastic_helper.task_status(task_id: task.elastic_task)
+    def check_task_status
+      save_documents_count!(refresh: false)
+
+      task_status = elastic_helper.task_status(task_id: current_task.elastic_task)
       return false unless task_status['completed']
 
-      # Check if reindexing is failed
       reindexing_error = task_status.dig('error', 'type')
       if reindexing_error
-        abort_reindexing!("Task #{task.elastic_task} has failed with Elasticsearch error.", additional_logs: { elasticsearch_error_type: reindexing_error })
+        abort_reindexing!("Task #{current_task.elastic_task} has failed with Elasticsearch error.", additional_logs: { elasticsearch_error_type: reindexing_error })
         return false
       end
 
-      # Refresh a new index
-      elastic_helper.refresh_index(index_name: task.index_name_to)
+      true
+    end
 
-      # Compare documents count
-      old_documents_count = task.documents_count
-      new_documents_count = elastic_helper.index_size(index_name: task.index_name_to).dig('docs', 'count')
+    def compare_documents_count
+      save_documents_count!(refresh: true)
+
+      old_documents_count = current_task.documents_count
+      new_documents_count = current_task.documents_count_target
       if old_documents_count != new_documents_count
         abort_reindexing!("Documents count is different, Count from new index: #{new_documents_count} Count from original index: #{old_documents_count}. This likely means something went wrong during reindexing.")
         return false
       end
 
-      # Change index settings back
-      elastic_helper.update_settings(index_name: task.index_name_to, settings: default_index_options)
+      true
+    end
 
-      # Switch alias to a new index
-      elastic_helper.switch_alias(to: task.index_name_to)
+    def apply_default_index_options
+      elastic_helper.update_settings(index_name: current_task.index_name_to, settings: default_index_options)
+    end
 
-      # Unpause indexing
+    def switch_alias_to_new_index
+      elastic_helper.switch_alias(to: current_task.index_name_to)
+    end
+
+    def finalize_reindexing
       Gitlab::CurrentSettings.update!(elasticsearch_pause_indexing: false)
 
-      task.update!(state: :success)
+      current_task.update!(state: :success)
+    end
+
+    def reindexing!
+      return false unless check_task_status
+      return false unless compare_documents_count
+
+      apply_default_index_options
+      switch_alias_to_new_index
+      finalize_reindexing
 
       true
     end

ee/app/views/admin/application_settings/_elasticsearch_form.html.haml

@@ -88,6 +88,28 @@
               = _('Maximum concurrency of Elasticsearch bulk requests per indexing operation.')
               = _('This only applies to repository indexing operations.')
 
+        .sub-section
+          %h4= _('Elasticsearch zero-downtime reindexing')
+          = link_to _('Trigger cluster reindexing'), admin_elasticsearch_trigger_reindexing_path, class: 'btn btn-success', method: :post, disabled: @elasticsearch_reindexing_task&.in_progress?
+          .form-text.text-muted
+            = _('This feature should be used with an index that was created after 13.0')
+          - if @elasticsearch_reindexing_task
+            - expected_documents = @elasticsearch_reindexing_task.documents_count
+            - processed_documents = @elasticsearch_reindexing_task.documents_count_target
+            %h5= _('Reindexing status')
+            %p= _('State: %{last_reindexing_task_state}') % { last_reindexing_task_state: @elasticsearch_reindexing_task.state }
+            - if @elasticsearch_reindexing_task.elastic_task
+              %p= _('Task ID: %{elastic_task}') % { elastic_task: @elasticsearch_reindexing_task.elastic_task }
+            - if @elasticsearch_reindexing_task.error_message
+              %p= _('Error: %{error_message}') % { error_message: @elasticsearch_reindexing_task.error_message }
+            - if expected_documents
+              %p= _('Expected documents: %{expected_documents}') % { expected_documents: expected_documents }
+            - if processed_documents && expected_documents
+              - percentage = ((processed_documents / expected_documents.to_f) * 100).round(2)
+              %p= _('Documents reindexed: %{processed_documents} (%{percentage}%%)') % { processed_documents: processed_documents, percentage: percentage }
+              .progress
+                .progress-bar.progress-bar-striped.bg-success{ "aria-valuemax" => "100", "aria-valuemin" => "0", "aria-valuenow" => percentage, :role => "progressbar", :style => "width: #{percentage}%" }
+
         .sub-section
           %h4= _('Elasticsearch indexing restrictions')
           .form-group

ee/changelogs/unreleased/214418-cluster-reindexing-ui.yml

+---
+title: Admin UI change to trigger Elasticsearch in-cluster re-indexing
+merge_request: 35024
+author:
+type: added

ee/config/routes/admin.rb

@@ -70,5 +70,6 @@ namespace :admin do
 
   namespace :elasticsearch do
     post :enqueue_index
+    post :trigger_reindexing
   end
 end

ee/spec/controllers/admin/elasticsearch_controller_spec.rb

@@ -37,4 +37,29 @@ RSpec.describe Admin::ElasticsearchController do
       end
     end
   end
+
+  describe 'POST #trigger_reindexing' do
+    before do
+      sign_in(admin)
+    end
+
+    it 'creates a reindexing task' do
+      expect(Elastic::ReindexingTask).to receive(:create!)
+
+      post :trigger_reindexing
+
+      expect(controller).to set_flash[:notice].to include('reindexing triggered')
+      expect(response).to redirect_to integrations_admin_application_settings_path(anchor: 'js-elasticsearch-settings')
+    end
+
+    it 'does not create a reindexing task if there is another one' do
+      allow(Elastic::ReindexingTask).to receive(:current).and_return(build(:elastic_reindexing_task))
+      expect(Elastic::ReindexingTask).not_to receive(:create!)
+
+      post :trigger_reindexing
+
+      expect(controller).to set_flash[:warning].to include('already in progress')
+      expect(response).to redirect_to integrations_admin_application_settings_path(anchor: 'js-elasticsearch-settings')
+    end
+  end
 end

ee/spec/services/elastic/cluster_reindexing_service_spec.rb

@@ -55,27 +55,36 @@ RSpec.describe Elastic::ClusterReindexingService, :elastic do
       allow(Gitlab::Elastic::Helper.default).to receive(:refresh_index).and_return(true)
     end
 
-    it 'errors if documents count is different' do
-      expect(Gitlab::Elastic::Helper.default).to receive(:index_size).and_return('docs' => { 'count' => task.reload.documents_count * 2 })
-
-      expect { subject.execute }.to change { task.reload.state }.from('reindexing').to('failure')
-      expect(task.reload.error_message).to match(/count is different/)
+    context 'errors are raised' do
+      before do
+        allow(Gitlab::Elastic::Helper.default).to receive(:index_size).and_return('docs' => { 'count' => task.reload.documents_count * 2 })
+      end
+
+      it 'errors if documents count is different' do
+        expect { subject.execute }.to change { task.reload.state }.from('reindexing').to('failure')
+        expect(task.reload.error_message).to match(/count is different/)
+      end
+
+      it 'errors if reindexing is failed' do
+        allow(Gitlab::Elastic::Helper.default).to receive(:task_status).and_return({ 'completed' => true, 'error' => { 'type' => 'search_phase_execution_exception' } })
+
+        expect { subject.execute }.to change { task.reload.state }.from('reindexing').to('failure')
+        expect(task.reload.error_message).to match(/has failed with/)
+      end
     end
 
-    it 'errors if reindexing is failed' do
-      allow(Gitlab::Elastic::Helper.default).to receive(:task_status).and_return({ 'completed' => true, 'error' => { 'type' => 'search_phase_execution_exception' } })
-
-      expect { subject.execute }.to change { task.reload.state }.from('reindexing').to('failure')
-      expect(task.reload.error_message).to match(/has failed with/)
-    end
+    context 'task finishes correctly' do
+      before do
+        allow(Gitlab::Elastic::Helper.default).to receive(:index_size).and_return('docs' => { 'count' => task.reload.documents_count })
+      end
 
-    it 'launches all state steps' do
-      expect(Gitlab::Elastic::Helper.default).to receive(:index_size).and_return('docs' => { 'count' => task.reload.documents_count })
-      expect(Gitlab::Elastic::Helper.default).to receive(:update_settings).with(index_name: task.index_name_to, settings: expected_default_settings)
-      expect(Gitlab::Elastic::Helper.default).to receive(:switch_alias).with(to: task.index_name_to)
-      expect(Gitlab::CurrentSettings).to receive(:update!).with(elasticsearch_pause_indexing: false)
+      it 'launches all state steps' do
+        expect(Gitlab::Elastic::Helper.default).to receive(:update_settings).with(index_name: task.index_name_to, settings: expected_default_settings)
+        expect(Gitlab::Elastic::Helper.default).to receive(:switch_alias).with(to: task.index_name_to)
+        expect(Gitlab::CurrentSettings).to receive(:update!).with(elasticsearch_pause_indexing: false)
 
-      expect { subject.execute }.to change { task.reload.state }.from('reindexing').to('success')
+        expect { subject.execute }.to change { task.reload.state }.from('reindexing').to('success')
+      end
     end
   end
 end

locale/gitlab.pot

@@ -8128,6 +8128,9 @@ msgstr ""
 msgid "Documentation for popular identity providers"
 msgstr ""
 
+msgid "Documents reindexed: %{processed_documents} (%{percentage}%%)"
+msgstr ""
+
 msgid "Doing"
 msgstr ""
 
@@ -8368,9 +8371,18 @@ msgstr ""
 msgid "Elasticsearch integration. Elasticsearch AWS IAM."
 msgstr ""
 
+msgid "Elasticsearch reindexing is already in progress"
+msgstr ""
+
+msgid "Elasticsearch reindexing triggered"
+msgstr ""
+
 msgid "Elasticsearch returned status code: %{status_code}"
 msgstr ""
 
+msgid "Elasticsearch zero-downtime reindexing"
+msgstr ""
+
 msgid "Elastic|None. Select namespaces to index."
 msgstr ""
 
@@ -9220,6 +9232,9 @@ msgstr ""
 msgid "Error with Akismet. Please check the logs for more info."
 msgstr ""
 
+msgid "Error: %{error_message}"
+msgstr ""
+
 msgid "ErrorTracking|Active"
 msgstr ""
 
@@ -9415,6 +9430,9 @@ msgstr ""
 msgid "Expand up"
 msgstr ""
 
+msgid "Expected documents: %{expected_documents}"
+msgstr ""
+
 msgid "Experienced"
 msgstr ""
 
@@ -18878,6 +18896,9 @@ msgstr ""
 msgid "Regulate approvals by authors/committers, based on compliance frameworks. Can be changed only at the instance level."
 msgstr ""
 
+msgid "Reindexing status"
+msgstr ""
+
 msgid "Rejected (closed)"
 msgstr ""
 
@@ -21891,6 +21912,9 @@ msgstr ""
 msgid "State your message to activate"
 msgstr ""
 
+msgid "State: %{last_reindexing_task_state}"
+msgstr ""
+
 msgid "Static Application Security Testing (SAST)"
 msgstr ""
 
@@ -22518,6 +22542,9 @@ msgstr ""
 msgid "Target-Branch"
 msgstr ""
 
+msgid "Task ID: %{elastic_task}"
+msgstr ""
+
 msgid "Team"
 msgstr ""
 
@@ -23443,6 +23470,9 @@ msgstr ""
 msgid "This feature requires local storage to be enabled"
 msgstr ""
 
+msgid "This feature should be used with an index that was created after 13.0"
+msgstr ""
+
 msgid "This field is required."
 msgstr ""
 
@@ -24413,6 +24443,9 @@ msgstr ""
 msgid "Trigger"
 msgstr ""
 
+msgid "Trigger cluster reindexing"
+msgstr ""
+
 msgid "Trigger pipelines for mirror updates"
 msgstr ""