Copy issues into a separate ES index

Create an Elasticsearch background migration to move issues into their
own separate index

db/migrate/20201217132603_create_elastic_reindexing_subtasks.rb

+# frozen_string_literal: true
+
+class CreateElasticReindexingSubtasks < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class ReindexingTask < ActiveRecord::Base
+    self.table_name = 'elastic_reindexing_tasks'
+  end
+
+  class ReindexingSubtask < ActiveRecord::Base
+    self.table_name = 'elastic_reindexing_subtasks'
+  end
+
+  def up
+    unless table_exists?(:elastic_reindexing_subtasks)
+      create_table :elastic_reindexing_subtasks do |t|
+        t.references :elastic_reindexing_task, foreign_key: { on_delete: :cascade }, null: false
+        t.text :alias_name, null: false
+        t.text :index_name_from, null: false
+        t.text :index_name_to, null: false
+        t.text :elastic_task, null: false
+        t.integer :documents_count_target
+        t.integer :documents_count
+        t.timestamps_with_timezone null: false
+      end
+    end
+
+    add_text_limit :elastic_reindexing_subtasks, :index_name_from, 255
+    add_text_limit :elastic_reindexing_subtasks, :index_name_to, 255
+    add_text_limit :elastic_reindexing_subtasks, :elastic_task, 255
+    add_text_limit :elastic_reindexing_subtasks, :alias_name, 255
+
+    ReindexingTask.find_each do |task|
+      next if task.index_name_from.blank? || task.index_name_to.blank? || task.elastic_task.blank?
+      next if ReindexingSubtask.where(elastic_reindexing_task_id: task.id).exists?
+
+      ReindexingSubtask.create(
+        elastic_reindexing_task_id: task.id,
+        documents_count_target: task.documents_count_target,
+        documents_count: task.documents_count,
+        alias_name: 'gitlab-production',
+        index_name_from: task.index_name_from,
+        index_name_to: task.index_name_to,
+        elastic_task: task.elastic_task
+      )
+    end
+  end
+
+  def down
+    drop_table :elastic_reindexing_subtasks
+  end
+end

db/schema_migrations/20201217132603

+164bcc838beb7d51775f8b813b92d3ec7080d4c7937d6ad16cf973131b45359e
\ No newline at end of file

db/structure.sql

@@ -11947,6 +11947,32 @@ CREATE SEQUENCE draft_notes_id_seq
 
 ALTER SEQUENCE draft_notes_id_seq OWNED BY draft_notes.id;
 
+CREATE TABLE elastic_reindexing_subtasks (
+    id bigint NOT NULL,
+    elastic_reindexing_task_id bigint NOT NULL,
+    alias_name text NOT NULL,
+    index_name_from text NOT NULL,
+    index_name_to text NOT NULL,
+    elastic_task text NOT NULL,
+    documents_count_target integer,
+    documents_count integer,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    CONSTRAINT check_4910adc798 CHECK ((char_length(elastic_task) <= 255)),
+    CONSTRAINT check_88f56216a4 CHECK ((char_length(alias_name) <= 255)),
+    CONSTRAINT check_a1fbd9faa9 CHECK ((char_length(index_name_from) <= 255)),
+    CONSTRAINT check_f456494bd8 CHECK ((char_length(index_name_to) <= 255))
+);
+
+CREATE SEQUENCE elastic_reindexing_subtasks_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE elastic_reindexing_subtasks_id_seq OWNED BY elastic_reindexing_subtasks.id;
+
 CREATE TABLE elastic_reindexing_tasks (
     id bigint NOT NULL,
     created_at timestamp with time zone NOT NULL,
@@ -18387,6 +18413,8 @@ ALTER TABLE ONLY diff_note_positions ALTER COLUMN id SET DEFAULT nextval('diff_n
 
 ALTER TABLE ONLY draft_notes ALTER COLUMN id SET DEFAULT nextval('draft_notes_id_seq'::regclass);
 
+ALTER TABLE ONLY elastic_reindexing_subtasks ALTER COLUMN id SET DEFAULT nextval('elastic_reindexing_subtasks_id_seq'::regclass);
+
 ALTER TABLE ONLY elastic_reindexing_tasks ALTER COLUMN id SET DEFAULT nextval('elastic_reindexing_tasks_id_seq'::regclass);
 
 ALTER TABLE ONLY emails ALTER COLUMN id SET DEFAULT nextval('emails_id_seq'::regclass);
@@ -19548,6 +19576,9 @@ ALTER TABLE ONLY diff_note_positions
 ALTER TABLE ONLY draft_notes
     ADD CONSTRAINT draft_notes_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY elastic_reindexing_subtasks
+    ADD CONSTRAINT elastic_reindexing_subtasks_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY elastic_reindexing_tasks
     ADD CONSTRAINT elastic_reindexing_tasks_pkey PRIMARY KEY (id);
 
@@ -21393,6 +21424,8 @@ CREATE INDEX index_draft_notes_on_discussion_id ON draft_notes USING btree (disc
 
 CREATE INDEX index_draft_notes_on_merge_request_id ON draft_notes USING btree (merge_request_id);
 
+CREATE INDEX index_elastic_reindexing_subtasks_on_elastic_reindexing_task_id ON elastic_reindexing_subtasks USING btree (elastic_reindexing_task_id);
+
 CREATE UNIQUE INDEX index_elastic_reindexing_tasks_on_in_progress ON elastic_reindexing_tasks USING btree (in_progress) WHERE in_progress;
 
 CREATE INDEX index_elastic_reindexing_tasks_on_state ON elastic_reindexing_tasks USING btree (state);
@@ -25357,6 +25390,9 @@ ALTER TABLE ONLY requirements
 ALTER TABLE ONLY snippet_repositories
     ADD CONSTRAINT fk_rails_f21f899728 FOREIGN KEY (shard_id) REFERENCES shards(id) ON DELETE RESTRICT;
 
+ALTER TABLE ONLY elastic_reindexing_subtasks
+    ADD CONSTRAINT fk_rails_f2cc190164 FOREIGN KEY (elastic_reindexing_task_id) REFERENCES elastic_reindexing_tasks(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY ci_pipeline_chat_data
     ADD CONSTRAINT fk_rails_f300456b63 FOREIGN KEY (chat_name_id) REFERENCES chat_names(id) ON DELETE CASCADE;
 

doc/integration/elasticsearch.md

@@ -283,7 +283,7 @@ To disable the Elasticsearch integration:
 1. Expand the **Advanced Search** section and uncheck **Elasticsearch indexing**
    and **Search with Elasticsearch enabled**.
 1. Click **Save changes** for the changes to take effect.
-1. (Optional) Delete the existing index:
+1. (Optional) Delete the existing indexes:
 
    ```shell
    # Omnibus installations
@@ -347,7 +347,8 @@ To reclaim the `gitlab-production` index name, you need to first create a `secon
 
 To create a secondary index, run the following Rake task. The `SKIP_ALIAS`
 environment variable will disable the automatic creation of the Elasticsearch
-alias, which would conflict with the existing index under `$PRIMARY_INDEX`:
+alias, which would conflict with the existing index under `$PRIMARY_INDEX`, and will
+not create a separate Issue index:
 
 ```shell
 # Omnibus installation
@@ -523,8 +524,8 @@ The following are some available Rake tasks:
 | [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake)                   | Iterates over all projects and queues Sidekiq jobs to index them in the background.                                                                                                       |
 | [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake)            | Determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. |
 | [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake)               | Deletes all instances of IndexStatus for all projects. Note that this command will result in a complete wipe of the index, and it should be used with caution.                                                                                              |
-| [`sudo gitlab-rake gitlab:elastic:create_empty_index[<TARGET_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Generates an empty index and assigns an alias for it on the Elasticsearch side only if it doesn't already exist.                                                                                                      |
-| [`sudo gitlab-rake gitlab:elastic:delete_index[<TARGET_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake)       | Removes the GitLab index and alias (if exists) on the Elasticsearch instance.                                                                                                                                   |
+| [`sudo gitlab-rake gitlab:elastic:create_empty_index[<TARGET_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Generates empty indexes (the default index and a separate issues index) and assigns an alias for each on the Elasticsearch side only if it doesn't already exist.                                                                                                      |
+| [`sudo gitlab-rake gitlab:elastic:delete_index[<TARGET_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake)       | Removes the GitLab indexes and aliases (if they exist) on the Elasticsearch instance.                                                                                                                                   |
 | [`sudo gitlab-rake gitlab:elastic:recreate_index[<TARGET_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake)     | Wrapper task for `gitlab:elastic:delete_index[<TARGET_NAME>]` and `gitlab:elastic:create_empty_index[<TARGET_NAME>]`.                                                                       |
 | [`sudo gitlab-rake gitlab:elastic:index_snippets`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake)                   | Performs an Elasticsearch import that indexes the snippets data.                                                                                                                          |
 | [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake)             | Displays which projects are not indexed.                                                                                                                                                  |

ee/app/models/concerns/elastic/application_versioned_search.rb

@@ -6,9 +6,15 @@ module Elastic
     FORWARDABLE_INSTANCE_METHODS = [:es_id, :es_parent].freeze
     FORWARDABLE_CLASS_METHODS = [:elastic_search, :es_import, :es_type, :index_name, :document_type, :mapping, :mappings, :settings, :import].freeze
 
+    # rubocop:disable Gitlab/ModuleWithInstanceVariables
     def __elasticsearch__(&block)
-      @__elasticsearch__ ||= ::Elastic::MultiVersionInstanceProxy.new(self)
+      if self.class.use_separate_indices?
+        @__elasticsearch_separate__ ||= ::Elastic::MultiVersionInstanceProxy.new(self, use_separate_indices: true)
+      else
+        @__elasticsearch__ ||= ::Elastic::MultiVersionInstanceProxy.new(self)
+      end
     end
+    # rubocop:enable Gitlab/ModuleWithInstanceVariables
 
     # Should be overridden in the models where some records should be skipped
     def searchable?
@@ -77,7 +83,15 @@ module Elastic
 
     class_methods do
       def __elasticsearch__
-        @__elasticsearch__ ||= ::Elastic::MultiVersionClassProxy.new(self)
+        if use_separate_indices?
+          @__elasticsearch_separate__ ||= ::Elastic::MultiVersionClassProxy.new(self, use_separate_indices: true)
+        else
+          @__elasticsearch__ ||= ::Elastic::MultiVersionClassProxy.new(self)
+        end
+      end
+
+      def use_separate_indices?
+        Gitlab::Elastic::Helper::ES_SEPARATE_CLASSES.include?(self) && Elastic::DataMigrationService.migration_has_finished?(:migrate_issues_to_separate_index)
       end
 
       # Mark a dependant association as needing to be updated when a specific

ee/app/models/elastic/migration_record.rb

@@ -4,7 +4,7 @@ module Elastic
   class MigrationRecord
     attr_reader :version, :name, :filename
 
-    delegate :migrate, :skip_migration?, :completed?, :batched?, :throttle_delay, to: :migration
+    delegate :migrate, :skip_migration?, :completed?, :batched?, :throttle_delay, :pause_indexing?, to: :migration
 
     def initialize(version:, name:, filename:)
       @version = version
@@ -16,11 +16,17 @@ module Elastic
     def save!(completed:)
       raise 'Migrations index is not found' unless helper.index_exists?(index_name: index_name)
 
-      data = { completed: completed }.merge(timestamps(completed: completed))
+      data = { completed: completed, state: load_state }.merge(timestamps(completed: completed))
 
       client.index index: index_name, type: '_doc', id: version, body: data
     end
 
+    def save_state!(state)
+      completed = load_from_index&.dig('_source', 'completed')
+
+      client.index index: index_name, type: '_doc', id: version, body: { completed: completed, state: load_state.merge(state) }
+    end
+
     def persisted?
       load_from_index.present?
     end
@@ -31,6 +37,18 @@ module Elastic
       nil
     end
 
+    def load_state
+      load_from_index&.dig('_source', 'state')&.with_indifferent_access || {}
+    end
+
+    def halted?
+      !!load_state&.dig('halted')
+    end
+
+    def name_for_key
+      name.underscore
+    end
+
     def self.persisted_versions(completed:)
       helper = Gitlab::Elastic::Helper.default
       helper.client

ee/app/models/elastic/reindexing_subtask.rb

+# frozen_string_literal: true
+
+class Elastic::ReindexingSubtask < ApplicationRecord
+  self.table_name = 'elastic_reindexing_subtasks'
+
+  belongs_to :elastic_reindexing_task, class_name: 'Elastic::ReindexingTask'
+
+  validates :index_name_from, :index_name_to, :elastic_task, presence: true
+end

ee/app/models/elastic/reindexing_task.rb

@@ -3,6 +3,8 @@
 class Elastic::ReindexingTask < ApplicationRecord
   self.table_name = 'elastic_reindexing_tasks'
 
+  has_many :subtasks, class_name: 'Elastic::ReindexingSubtask', foreign_key: :elastic_reindexing_task_id
+
   enum state: {
     initial:                0,
     indexing_paused:        1,
@@ -27,8 +29,9 @@ class Elastic::ReindexingTask < ApplicationRecord
 
   def self.drop_old_indices!
     old_indices_to_be_deleted.find_each do |task|
-      next unless Gitlab::Elastic::Helper.default.delete_index(index_name: task.index_name_from)
-
+      task.subtasks.each do |subtask|
+        Gitlab::Elastic::Helper.default.delete_index(index_name: subtask.index_name_from)
+      end
       task.update!(state: :original_index_deleted)
     end
   end

ee/app/services/elastic/cluster_reindexing_service.rb

@@ -31,6 +31,10 @@ module Elastic
 
     private
 
+    def alias_names
+      [elastic_helper.target_name] + elastic_helper.standalone_indices_proxies.map(&:index_name)
+    end
+
     def default_index_options
       {
         refresh_interval: nil, # Change it back to the default
@@ -40,6 +44,12 @@ module Elastic
     end
 
     def initial!
+      if Elastic::DataMigrationService.pending_migrations?
+        # migrations may have paused indexing so we do not want to unpause when aborting the reindexing process
+        abort_reindexing!('You have unapplied advanced search migrations. Please wait until it is finished', unpause_indexing: false)
+        return false
+      end
+
       # Pause indexing
       Gitlab::CurrentSettings.update!(elasticsearch_pause_indexing: true)
 
@@ -48,7 +58,7 @@ module Elastic
         return false
       end
 
-      expected_free_size = elastic_helper.index_size_bytes * 2
+      expected_free_size = alias_names.sum {|name| elastic_helper.index_size_bytes(index_name: name) } * 2
       if elastic_helper.cluster_free_size_bytes < expected_free_size
         abort_reindexing!("You should have at least #{expected_free_size} bytes of storage available to perform reindexing. Please increase the storage in your Elasticsearch cluster before reindexing.")
         return false
@@ -60,43 +70,52 @@ module Elastic
     end
 
     def indexing_paused!
-      # Create an index with custom settings
-      index_name = elastic_helper.create_empty_index(with_alias: false, options: { settings: INITIAL_INDEX_OPTIONS })
-
-      # Record documents count
-      documents_count = elastic_helper.documents_count
-
-      # Trigger reindex
-      task_id = elastic_helper.reindex(to: index_name)
+      # Create indices with custom settings
+      main_index = elastic_helper.create_empty_index(with_alias: false, options: { settings: INITIAL_INDEX_OPTIONS })
+      standalone_indices = elastic_helper.create_standalone_indices(with_alias: false, options: { settings: INITIAL_INDEX_OPTIONS })
+
+      main_index.merge(standalone_indices).each do |new_index_name, alias_name|
+        old_index_name = elastic_helper.target_index_name(target: alias_name)
+        # Record documents count
+        documents_count = elastic_helper.documents_count(index_name: old_index_name)
+        # Trigger reindex
+        task_id = elastic_helper.reindex(from: old_index_name, to: new_index_name)
+
+        current_task.subtasks.create!(
+          alias_name: alias_name,
+          index_name_from: old_index_name,
+          index_name_to: new_index_name,
+          documents_count: documents_count,
+          elastic_task: task_id
+        )
+      end
 
-      current_task.update!(
-        index_name_from: elastic_helper.target_index_name,
-        index_name_to: index_name,
-        documents_count: documents_count,
-        elastic_task: task_id,
-        state: :reindexing
-      )
+      current_task.update!(state: :reindexing)
 
       true
     end
 
     def save_documents_count!(refresh:)
-      elastic_helper.refresh_index(index_name: current_task.index_name_to) if refresh
+      current_task.subtasks.each do |subtask|
+        elastic_helper.refresh_index(index_name: subtask.index_name_to) if refresh
 
-      new_documents_count = elastic_helper.documents_count(index_name: current_task.index_name_to)
-      current_task.update!(documents_count_target: new_documents_count)
+        new_documents_count = elastic_helper.documents_count(index_name: subtask.index_name_to)
+        subtask.update!(documents_count_target: new_documents_count)
+      end
     end
 
     def check_task_status
       save_documents_count!(refresh: false)
 
-      task_status = elastic_helper.task_status(task_id: current_task.elastic_task)
-      return false unless task_status['completed']
+      current_task.subtasks.each do |subtask|
+        task_status = elastic_helper.task_status(task_id: subtask.elastic_task)
+        return false unless task_status['completed']
 
-      reindexing_error = task_status.dig('error', 'type')
-      if reindexing_error
-        abort_reindexing!("Task #{current_task.elastic_task} has failed with Elasticsearch error.", additional_logs: { elasticsearch_error_type: reindexing_error })
-        return false
+        reindexing_error = task_status.dig('error', 'type')
+        if reindexing_error
+          abort_reindexing!("Task #{subtask.elastic_task} has failed with Elasticsearch error.", additional_logs: { elasticsearch_error_type: reindexing_error })
+          return false
+        end
       end
 
       true
@@ -109,22 +128,28 @@ module Elastic
     def compare_documents_count
       save_documents_count!(refresh: true)
 
-      old_documents_count = current_task.documents_count
-      new_documents_count = current_task.documents_count_target
-      if old_documents_count != new_documents_count
-        abort_reindexing!("Documents count is different, Count from new index: #{new_documents_count} Count from original index: #{old_documents_count}. This likely means something went wrong during reindexing.")
-        return false
+      current_task.subtasks.each do |subtask|
+        old_documents_count = subtask.documents_count
+        new_documents_count = subtask.documents_count_target
+        if old_documents_count != new_documents_count
+          abort_reindexing!("Documents count is different, Count from new index: #{new_documents_count} Count from original index: #{old_documents_count}. This likely means something went wrong during reindexing.")
+          return false
+        end
       end
 
       true
     end
 
     def apply_default_index_options
-      elastic_helper.update_settings(index_name: current_task.index_name_to, settings: default_index_options)
+      current_task.subtasks.each do |subtask|
+        elastic_helper.update_settings(index_name: subtask.index_name_to, settings: default_index_options)
+      end
     end
 
     def switch_alias_to_new_index
-      elastic_helper.switch_alias(to: current_task.index_name_to)
+      current_task.subtasks.each do |subtask|
+        elastic_helper.switch_alias(from: subtask.index_name_from, to: subtask.index_name_to, alias_name: subtask.alias_name)
+      end
     end
 
     def finalize_reindexing
@@ -144,8 +169,8 @@ module Elastic
       true
     end
 
-    def abort_reindexing!(reason, additional_logs: {})
-      error = { message: 'elasticsearch_reindex_error', error: reason, elasticsearch_task_id: current_task.elastic_task, gitlab_task_id: current_task.id, gitlab_task_state: current_task.state }
+    def abort_reindexing!(reason, additional_logs: {}, unpause_indexing: true)
+      error = { message: 'elasticsearch_reindex_error', error: reason, gitlab_task_id: current_task.id, gitlab_task_state: current_task.state }
       logger.error(error.merge(additional_logs))
 
       current_task.update!(
@@ -154,7 +179,7 @@ module Elastic
       )
 
       # Unpause indexing
-      Gitlab::CurrentSettings.update!(elasticsearch_pause_indexing: false)
+      Gitlab::CurrentSettings.update!(elasticsearch_pause_indexing: false) if unpause_indexing
     end
 
     def logger

ee/app/services/elastic/data_migration_service.rb

@@ -20,23 +20,32 @@ module Elastic
         migrations.sort_by(&:version)
       end
 
+      def [](version)
+        migrations.find { |m| m.version == version }
+      end
+
       def drop_migration_has_finished_cache!(migration)
-        name = migration.name.underscore
-        Rails.cache.delete cache_key(:migration_has_finished, name)
+        Rails.cache.delete cache_key(:migration_has_finished, migration.name_for_key)
       end
 
       def migration_has_finished?(name)
-        Rails.cache.fetch cache_key(:migration_has_finished, name), expires_in: 30.minutes do
+        Rails.cache.fetch cache_key(:migration_has_finished, name.to_s.underscore), expires_in: 30.minutes do
           migration_has_finished_uncached?(name)
         end
       end
 
       def migration_has_finished_uncached?(name)
-        migration = migrations.find { |migration| migration.name == name.to_s.camelize }
+        migration = migrations.find { |migration| migration.name_for_key == name.to_s.underscore }
 
         !!migration&.load_from_index&.dig('_source', 'completed')
       end
 
+      def pending_migrations?
+        migrations.reverse.any? do |migration|
+          !migration_has_finished?(migration.name_for_key)
+        end
+      end
+
       def mark_all_as_completed!
         migrations.each do |migration|
           migration.save!(completed: true)

ee/app/views/admin/application_settings/_elasticsearch_form.html.haml

@@ -23,7 +23,7 @@
             .form-check
               = f.check_box :elasticsearch_indexing, class: 'form-check-input', data: { qa_selector: 'indexing_checkbox' }
               = f.label :elasticsearch_indexing, class: 'form-check-label' do
-                Elasticsearch indexing
+                = _('Elasticsearch indexing')
               - unless Gitlab::CurrentSettings.elasticsearch_indexing?
                 .form-text.text-muted
                   = _('An empty index will be created if one does not already exist')
@@ -35,11 +35,16 @@
           .card-body
             .form-group
               .form-check
-                = f.check_box :elasticsearch_pause_indexing, class: 'form-check-input', data: { qa_selector: 'pause_checkbox' }, disabled: !Gitlab::CurrentSettings.elasticsearch_indexing?
+                - pending_migrations = Elastic::DataMigrationService.pending_migrations? && Gitlab::CurrentSettings.elasticsearch_pause_indexing? rescue false
+                - disable_checkbox = !Gitlab::CurrentSettings.elasticsearch_indexing? || pending_migrations
+                = f.check_box :elasticsearch_pause_indexing, class: 'form-check-input', data: { qa_selector: 'pause_checkbox' }, disabled: disable_checkbox
                 = f.label :elasticsearch_pause_indexing, class: 'form-check-label' do
-                  Pause Elasticsearch indexing
+                  = _('Pause Elasticsearch indexing')
                 .form-text.text-muted
                   = _('Changes are still tracked. Useful for cluster/index migrations.')
+                - if pending_migrations
+                  .form-text.text-warning
+                    = _('There are pending advanced search migrations. Indexing must remain paused until the migrations are completed.')
 
           .form-group
             .form-check
@@ -102,29 +107,31 @@
 
         .sub-section
           %h4= _('Elasticsearch zero-downtime reindexing')
-          = link_to _('Trigger cluster reindexing'), admin_elasticsearch_trigger_reindexing_path, class: 'gl-button btn btn-primary', data: { confirm: _('Are you sure you want to reindex?') }, method: :post, disabled: @elasticsearch_reindexing_task&.in_progress?
-          .form-text.text-muted
+          = link_to _('Trigger cluster reindexing'), admin_elasticsearch_trigger_reindexing_path, class: "gl-button btn btn-info", disabled: @elasticsearch_reindexing_task&.in_progress?, data: { confirm: _('Are you sure you want to reindex?') }, method: :post
+          .form-text.gl-text-gray-600
             = _('This feature should be used with an index that was created after 13.0')
           - Elastic::ReindexingTask.old_indices_scheduled_for_deletion.each do |task|
             .form-text.text-danger
-              = _("Unused, previous index '%{index_name}' will be deleted after %{time} automatically.") % { index_name: task.index_name_from, time: task.delete_original_index_at }
-              = link_to _('Cancel index deletion'), admin_elasticsearch_cancel_index_deletion_path(task_id: task.id), method: :post
+              = _("Unused, previous indices: %{index_names} will be deleted after %{time} automatically.") % { index_names: task.subtasks.map(&:index_name_from).join(', '), time: task.delete_original_index_at }
+              = link_to _('Cancel index deletion'), admin_elasticsearch_cancel_index_deletion_path(task_id: task.id), class: 'gl-mb-2', method: :post
           - if @elasticsearch_reindexing_task
-            - expected_documents = @elasticsearch_reindexing_task.documents_count
-            - processed_documents = @elasticsearch_reindexing_task.documents_count_target
-            %h5= _('Reindexing status')
-            %p= _('State: %{last_reindexing_task_state}') % { last_reindexing_task_state: @elasticsearch_reindexing_task.state }
-            - if @elasticsearch_reindexing_task.elastic_task
-              %p= _('Task ID: %{elastic_task}') % { elastic_task: @elasticsearch_reindexing_task.elastic_task }
+            %h5= _('Reindexing Status: %{status}') % { status: @elasticsearch_reindexing_task.state }
             - if @elasticsearch_reindexing_task.error_message
               %p= _('Error: %{error_message}') % { error_message: @elasticsearch_reindexing_task.error_message }
-            - if expected_documents
-              %p= _('Expected documents: %{expected_documents}') % { expected_documents: expected_documents }
-            - if processed_documents && expected_documents
-              - percentage = ((processed_documents / expected_documents.to_f) * 100).round(2)
-              %p= _('Documents reindexed: %{processed_documents} (%{percentage}%%)') % { processed_documents: processed_documents, percentage: percentage }
-              .progress
-                .progress-bar.progress-bar-striped.bg-primary{ "aria-valuemax" => "100", "aria-valuemin" => "0", "aria-valuenow" => percentage, :role => "progressbar", :style => "width: #{percentage}%" }
+            - @elasticsearch_reindexing_task.subtasks.each do |subtask|
+              .card-body.form-group
+                %h5= subtask.alias_name
+                - expected_documents = subtask.documents_count
+                - if subtask.elastic_task
+                  %p= _('Task ID: %{elastic_task}') % { elastic_task: subtask.elastic_task }
+                - if expected_documents
+                  - processed_documents = subtask.documents_count_target
+                  %p= _('Expected documents: %{expected_documents}') % { expected_documents: expected_documents }
+                  - if processed_documents && expected_documents
+                    - percentage = ((processed_documents / expected_documents.to_f) * 100).round(2)
+                    %p= _('Documents reindexed: %{processed_documents} (%{percentage}%%)') % { processed_documents: processed_documents, percentage: percentage }
+                    .progress
+                      .progress-bar{ "aria-valuemax" => "100", "aria-valuemin" => "0", "aria-valuenow" => percentage, :role => "progressbar", :style => "width: #{percentage}%" }
 
         .sub-section
           %h4= _('Elasticsearch indexing restrictions')

ee/app/workers/concerns/elastic/migration_options.rb

@@ -15,6 +15,10 @@ module Elastic
       self.class.get_throttle_delay
     end
 
+    def pause_indexing?
+      self.class.get_pause_indexing
+    end
+
     class_methods do
       def batched!
         class_attributes[:batched] = true
@@ -24,6 +28,14 @@ module Elastic
         class_attributes[:batched]
       end
 
+      def pause_indexing!
+        class_attributes[:pause_indexing] = true
+      end
+
+      def get_pause_indexing
+        class_attributes[:pause_indexing]
+      end
+
       def throttle_delay(value)
         class_attributes[:throttle_delay] = value
       end

ee/app/workers/concerns/elastic/migration_state.rb

+# frozen_string_literal: true
+
+module Elastic
+  module MigrationState
+    def migration_state
+      migration_record.load_state
+    end
+
+    def set_migration_state(state)
+      log "Setting migration_state to #{state.to_json}"
+
+      migration_record.save_state!(state)
+    end
+  end
+end

ee/app/workers/elastic/migration_worker.rb

@@ -29,12 +29,21 @@ module Elastic
           helper.create_migrations_index
         end
 
+        if migration.halted?
+          logger.info "MigrationWorker: migration[#{migration.name}] has been halted. All future migrations will be halted because of that. Exiting"
+          unpause_indexing!(migration)
+
+          break false
+        end
+
         execute_migration(migration)
 
         completed = migration.completed?
         logger.info "MigrationWorker: migration[#{migration.name}] updating with completed: #{completed}"
         migration.save!(completed: completed)
 
+        unpause_indexing!(migration) if completed
+
         Elastic::DataMigrationService.drop_migration_has_finished_cache!(migration)
       end
     end
@@ -45,6 +54,8 @@ module Elastic
       if migration.persisted? && !migration.batched?
         logger.info "MigrationWorker: migration[#{migration.name}] did not execute migrate method since it was already executed. Waiting for migration to complete"
       else
+        pause_indexing!(migration)
+
         logger.info "MigrationWorker: migration[#{migration.name}] executing migrate method"
         migration.migrate
 
@@ -61,6 +72,27 @@ module Elastic
       Elastic::DataMigrationService.migrations.find { |migration| !completed_migrations.include?(migration.version) }
     end
 
+    def pause_indexing!(migration)
+      return unless migration.pause_indexing?
+      return if migration.load_state[:pause_indexing].present?
+
+      pause_indexing = !Gitlab::CurrentSettings.elasticsearch_pause_indexing?
+      migration.save_state!(pause_indexing: pause_indexing)
+
+      if pause_indexing
+        logger.info 'MigrationWorker: Pausing indexing'
+        Gitlab::CurrentSettings.update!(elasticsearch_pause_indexing: true)
+      end
+    end
+
+    def unpause_indexing!(migration)
+      return unless migration.pause_indexing?
+      return unless migration.load_state[:pause_indexing]
+
+      logger.info 'MigrationWorker: unpausing indexing'
+      Gitlab::CurrentSettings.update!(elasticsearch_pause_indexing: false)
+    end
+
     def helper
       Gitlab::Elastic::Helper.default
     end

ee/app/workers/elastic_delete_project_worker.rb

@@ -16,9 +16,19 @@ class ElasticDeleteProjectWorker
 
   private
 
+  def indices
+    helper = Gitlab::Elastic::Helper.default
+
+    if Elastic::DataMigrationService.migration_has_finished?(:migrate_issues_to_separate_index)
+      [helper.target_name] + helper.standalone_indices_proxies.map(&:index_name)
+    else
+      [helper.target_name]
+    end
+  end
+
   def remove_project_and_children_documents(project_id, es_id)
     client.delete_by_query({
-      index: Project.__elasticsearch__.index_name,
+      index: indices,
       routing: es_id,
       body: {
         query: {

ee/changelogs/unreleased/273264-copy-issues-to-new-index.yml

+---
+title: 'Advanced Search: Copy issues to new index'
+merge_request: 48334
+author:
+type: changed

ee/elastic/migrate/20201123123400_migrate_issues_to_separate_index.rb

+# frozen_string_literal: true
+
+class MigrateIssuesToSeparateIndex < Elastic::Migration
+  pause_indexing!
+  batched!
+  throttle_delay 1.minute
+
+  MAX_ATTEMPTS = 30
+
+  FIELDS = %w(
+    type
+    id
+    iid
+    title
+    description
+    created_at
+    updated_at
+    state
+    project_id
+    author_id
+    confidential
+    assignee_id
+    visibility_level
+    issues_access_level
+  ).freeze
+
+  def migrate
+    # On initial batch we only create index
+    if migration_state[:slice].blank?
+      log "Create standalone issues index under #{issues_index_name}"
+      helper.create_standalone_indices unless helper.index_exists?(index_name: issues_index_name)
+
+      options = {
+        slice: 0,
+        retry_attempt: 0,
+        max_slices: get_number_of_shards
+      }
+      set_migration_state(options)
+
+      return
+    end
+
+    retry_attempt = migration_state[:retry_attempt].to_i
+    slice = migration_state[:slice]
+    max_slices = migration_state[:max_slices]
+
+    if retry_attempt >= MAX_ATTEMPTS
+      fail_migration_halt_error!(retry_attempt: retry_attempt)
+      return
+    end
+
+    if slice < max_slices
+      log "Launching reindexing for slice:#{slice} | max_slices:#{max_slices}"
+
+      response = reindex(slice: slice, max_slices: max_slices)
+      process_response(response)
+
+      log "Reindexing for slice:#{slice} | max_slices:#{max_slices} is completed with #{response.to_json}"
+
+      set_migration_state(
+        slice: slice + 1,
+        retry_attempt: retry_attempt,
+        max_slices: max_slices
+      )
+    end
+  rescue StandardError => e
+    log "migrate failed, increasing migration_state retry_attempt: #{retry_attempt} error:#{e.message}"
+
+    set_migration_state(
+      slice: slice,
+      retry_attempt: retry_attempt + 1,
+      max_slices: max_slices
+    )
+
+    raise e
+  end
+
+  def completed?
+    log "completed check: Refreshing #{issues_index_name}"
+    helper.refresh_index(index_name: issues_index_name)
+
+    original_count = original_issues_documents_count
+    new_count = new_issues_documents_count
+    log "Checking to see if migration is completed based on index counts: original_count:#{original_count}, new_count:#{new_count}"
+
+    original_count == new_count
+  end
+
+  private
+
+  def reindex(slice:, max_slices:)
+    body = query(slice: slice, max_slices: max_slices)
+
+    client.reindex(body: body, wait_for_completion: true)
+  end
+
+  def process_response(response)
+    if response['failures'].present?
+      log_raise "Reindexing failed with #{response['failures']}"
+    end
+
+    if response['total'] != (response['updated'] + response['created'] + response['deleted'])
+      log_raise "Slice reindexing seems to have failed, total is not equal to updated + created + deleted"
+    end
+  end
+
+  def query(slice:, max_slices:)
+    {
+      source: {
+        index: default_index_name,
+        _source: FIELDS,
+        query: {
+          match: {
+            type: 'issue'
+          }
+        },
+        slice: {
+          id: slice,
+          max: max_slices
+        }
+      },
+      dest: {
+        index: issues_index_name
+      }
+    }
+  end
+
+  def original_issues_documents_count
+    query = {
+      size: 0,
+      aggs: {
+        issues: {
+          filter: {
+            term: {
+              type: {
+                value: 'issue'
+              }
+            }
+          }
+        }
+      }
+    }
+
+    results = client.search(index: default_index_name, body: query)
+    results.dig('aggregations', 'issues', 'doc_count')
+  end
+
+  def new_issues_documents_count
+    helper.documents_count(index_name: issues_index_name)
+  end
+
+  def get_number_of_shards
+    helper.get_settings.dig('number_of_shards').to_i
+  end
+
+  def default_index_name
+    helper.target_name
+  end
+
+  def issues_index_name
+    "#{default_index_name}-issues"
+  end
+end

ee/lib/elastic/class_proxy_util.rb

@@ -6,15 +6,24 @@ module Elastic
   module ClassProxyUtil
     extend ActiveSupport::Concern
 
-    def initialize(target)
+    attr_reader :use_separate_indices
+
+    def initialize(target, use_separate_indices: false)
       super(target)
 
-      config = version_namespace.const_get('Config', false)
+      const_name = if use_separate_indices
+                     "#{target.name}Config"
+                   else
+                     'Config'
+                   end
+
+      config = version_namespace.const_get(const_name, false)
 
       @index_name = config.index_name
       @document_type = config.document_type
       @settings = config.settings
       @mapping = config.mapping
+      @use_separate_indices = use_separate_indices
     end
 
     ### Multi-version utils

ee/lib/elastic/instance_proxy_util.rb

@@ -6,10 +6,16 @@ module Elastic
   module InstanceProxyUtil
     extend ActiveSupport::Concern
 
-    def initialize(target)
+    def initialize(target, use_separate_indices: false)
       super(target)
 
-      config = version_namespace.const_get('Config', false)
+      const_name = if use_separate_indices
+                     "#{target.class.name}Config"
+                   else
+                     'Config'
+                   end
+
+      config = version_namespace.const_get(const_name, false)
 
       @index_name = config.index_name
       @document_type = config.document_type

ee/lib/elastic/latest/issue_config.rb

+# frozen_string_literal: true
+
+module Elastic
+  module Latest
+    module IssueConfig
+      # To obtain settings and mappings methods
+      extend Elasticsearch::Model::Indexing::ClassMethods
+      extend Elasticsearch::Model::Naming::ClassMethods
+
+      self.document_type = 'doc'
+      self.index_name = [Rails.application.class.module_parent_name.downcase, Rails.env, 'issues'].join('-')
+
+      settings Elastic::Latest::Config.settings.to_hash
+
+      mappings dynamic: 'strict' do
+        indexes :type, type: :keyword
+
+        indexes :id, type: :integer
+        indexes :iid, type: :integer
+
+        indexes :title, type: :text, index_options: 'positions'
+        indexes :description, type: :text, index_options: 'positions'
+        indexes :created_at, type: :date
+        indexes :updated_at, type: :date
+        indexes :state, type: :keyword
+        indexes :project_id, type: :integer
+        indexes :author_id, type: :integer
+        indexes :confidential, type: :boolean
+        indexes :assignee_id, type: :integer
+
+        indexes :visibility_level, type: :integer
+        indexes :issues_access_level, type: :integer
+      end
+    end
+  end
+end

ee/lib/elastic/latest/issue_instance_proxy.rb

@@ -26,6 +26,16 @@ module Elastic
 
         data.merge(generic_attributes)
       end
+
+      private
+
+      def generic_attributes
+        if Elastic::DataMigrationService.migration_has_finished?(:migrate_issues_to_separate_index)
+          super.except('join_field')
+        else
+          super
+        end
+      end
     end
   end
 end

ee/lib/elastic/migration.rb

@@ -3,6 +3,7 @@
 module Elastic
   class Migration
     include Elastic::MigrationOptions
+    include Elastic::MigrationState
 
     attr_reader :version
 
@@ -28,10 +29,26 @@ module Elastic
       helper.client
     end
 
+    def migration_record
+      Elastic::DataMigrationService[version]
+    end
+
+    def fail_migration_halt_error!(retry_attempt: 0)
+      set_migration_state(
+        retry_attempt: retry_attempt,
+        halted: true
+      )
+    end
+
     def log(message)
       logger.info "[Elastic::Migration: #{self.version}] #{message}"
     end
 
+    def log_raise(message)
+      logger.error "[Elastic::Migration: #{self.version}] #{message}"
+      raise message
+    end
+
     def logger
       @logger ||= ::Gitlab::Elasticsearch::Logger.build
     end

ee/lib/elastic/multi_version_class_proxy.rb

@@ -5,9 +5,10 @@ module Elastic
   class MultiVersionClassProxy
     include MultiVersionUtil
 
-    def initialize(data_target)
+    def initialize(data_target, use_separate_indices: false)
       @data_target = data_target
       @data_class = get_data_class(data_target)
+      @use_separate_indices = use_separate_indices
 
       generate_forwarding
     end

ee/lib/elastic/multi_version_instance_proxy.rb

@@ -5,9 +5,10 @@ module Elastic
   class MultiVersionInstanceProxy
     include MultiVersionUtil
 
-    def initialize(data_target)
+    def initialize(data_target, use_separate_indices: false)
       @data_target = data_target
       @data_class = get_data_class(data_target.class)
+      @use_separate_indices = use_separate_indices
 
       generate_forwarding
     end

ee/lib/elastic/multi_version_util.rb

@@ -5,7 +5,7 @@ module Elastic
     extend ActiveSupport::Concern
     include Gitlab::Utils::StrongMemoize
 
-    attr_reader :data_class, :data_target
+    attr_reader :data_class, :data_target, :use_separate_indices
 
     # TODO: remove once multi-version is functional https://gitlab.com/gitlab-org/gitlab/issues/10156
     TARGET_VERSION = 'V12p1'
@@ -13,7 +13,8 @@ module Elastic
     # @params version [String, Module] can be a string "V12p1" or module (Elastic::V12p1)
     def version(version)
       version = Elastic.const_get(version, false) if version.is_a?(String)
-      version.const_get(proxy_class_name, false).new(data_target)
+
+      version.const_get(proxy_class_name, false).new(data_target, use_separate_indices: use_separate_indices)
     end
 
     # TODO: load from db table https://gitlab.com/gitlab-org/gitlab/issues/12555

ee/lib/elastic/v12p1/config.rb

@@ -3,5 +3,6 @@
 module Elastic
   module V12p1
     Config = Elastic::Latest::Config
+    IssueConfig = Elastic::Latest::IssueConfig
   end
 end

ee/lib/gitlab/elastic/helper.rb

@@ -3,9 +3,8 @@
 module Gitlab
   module Elastic
     class Helper
-      ES_ENABLED_CLASSES = [
+      ES_MAPPINGS_CLASSES = [
           Project,
-          Issue,
           MergeRequest,
           Snippet,
           Note,
@@ -14,6 +13,10 @@ module Gitlab
           Repository
       ].freeze
 
+      ES_SEPARATE_CLASSES = [
+        Issue
+      ].freeze
+
       attr_reader :version, :client
       attr_accessor :target_name
 
@@ -40,13 +43,13 @@ module Gitlab
       end
 
       def default_settings
-        ES_ENABLED_CLASSES.inject({}) do |settings, klass|
+        ES_MAPPINGS_CLASSES.inject({}) do |settings, klass|
           settings.deep_merge(klass.__elasticsearch__.settings.to_hash)
         end
       end
 
       def default_mappings
-        mappings = ES_ENABLED_CLASSES.inject({}) do |m, klass|
+        mappings = ES_MAPPINGS_CLASSES.inject({}) do |m, klass|
           m.deep_merge(klass.__elasticsearch__.mappings.to_hash)
         end
         mappings.deep_merge(::Elastic::Latest::CustomLanguageAnalyzers.custom_analyzers_mappings)
@@ -64,6 +67,9 @@ module Gitlab
               completed: {
                 type: 'boolean'
               },
+              state: {
+                type: 'object'
+              },
               started_at: {
                 type: 'date'
               },
@@ -87,6 +93,54 @@ module Gitlab
         migrations_index_name
       end
 
+      def standalone_indices_proxies
+        ES_SEPARATE_CLASSES.map do |class_name|
+          ::Elastic::Latest::ApplicationClassProxy.new(class_name, use_separate_indices: true)
+        end
+      end
+
+      def create_standalone_indices(with_alias: true, options: {})
+        standalone_indices_proxies.each_with_object({}) do |proxy, indices|
+          alias_name = proxy.index_name
+          new_index_name = "#{alias_name}-#{Time.now.strftime("%Y%m%d-%H%M")}"
+
+          raise "Index under '#{new_index_name}' already exists" if index_exists?(index_name: new_index_name)
+
+          if with_alias
+            raise "Alias under '#{alias_name}' already exists" if alias_exists?(name: alias_name)
+          end
+
+          settings = proxy.settings.to_hash
+          settings = settings.merge(options[:settings]) if options[:settings]
+
+          mappings = proxy.mappings.to_hash
+          mappings = mappings.merge(options[:mappings]) if options[:mappings]
+
+          create_index_options = {
+            index: new_index_name,
+            body: {
+              settings: settings,
+              mappings: mappings
+            }
+          }.merge(additional_index_options)
+
+          client.indices.create create_index_options
+
+          client.indices.put_alias(name: alias_name, index: new_index_name) if with_alias
+
+          indices[new_index_name] = alias_name
+        end
+      end
+
+      def delete_standalone_indices
+        standalone_indices_proxies.map do |proxy|
+          index_name = target_index_name(target: proxy.index_name)
+          result = delete_index(index_name: index_name)
+
+          [index_name, proxy.index_name, result]
+        end
+      end
+
       def create_empty_index(with_alias: true, options: {})
         new_index_name = options[:index_name] || "#{target_name}-#{Time.now.strftime("%Y%m%d-%H%M")}"
 
@@ -111,11 +165,13 @@ module Gitlab
         client.indices.create create_index_options
         client.indices.put_alias(name: target_name, index: new_index_name) if with_alias
 
-        new_index_name
+        {
+          new_index_name => target_name
+        }
       end
 
       def delete_index(index_name: nil)
-        result = client.indices.delete(index: index_name || target_index_name)
+        result = client.indices.delete(index: target_index_name(target: index_name))
         result['acknowledged']
       rescue ::Elasticsearch::Transport::Transport::Errors::NotFound => e
         Gitlab::ErrorTracking.log_exception(e)
@@ -126,29 +182,40 @@ module Gitlab
         client.indices.exists?(index: index_name || target_name) # rubocop:disable CodeReuse/ActiveRecord
       end
 
-      def alias_exists?
-        client.indices.exists_alias(name: target_name)
+      def alias_exists?(name: nil)
+        client.indices.exists_alias(name: name || target_name)
       end
 
       # Calls Elasticsearch refresh API to ensure data is searchable
       # immediately.
       # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html
+      # By default refreshes main and standalone_indices
       def refresh_index(index_name: nil)
-        client.indices.refresh(index: index_name || target_name)
+        indices = if index_name.nil?
+                    [target_name] + standalone_indices_proxies.map(&:index_name)
+                  else
+                    [index_name]
+                  end
+
+        indices.each do |index|
+          client.indices.refresh(index: index)
+        end
       end
 
       def index_size(index_name: nil)
-        client.indices.stats['indices'][index_name || target_index_name]['total']
+        index = target_index_name(target: index_name || target_index_name)
+
+        client.indices.stats.dig('indices', index, 'total')
       end
 
       def documents_count(index_name: nil)
-        index = index_name || target_index_name
+        index = target_index_name(target: index_name || target_index_name)
 
         client.indices.stats.dig('indices', index, 'primaries', 'docs', 'count')
       end
 
-      def index_size_bytes
-        index_size['store']['size_in_bytes']
+      def index_size_bytes(index_name: nil)
+        index_size(index_name: index_name)['store']['size_in_bytes']
       end
 
       def cluster_free_size_bytes
@@ -184,13 +251,13 @@ module Gitlab
         client.indices.put_settings(index: index_name || target_index_name, body: settings)
       end
 
-      def switch_alias(from: target_index_name, to:)
+      def switch_alias(from: target_index_name, alias_name: target_name, to:)
         actions = [
           {
-            remove: { index: from, alias: target_name }
+            remove: { index: from, alias: alias_name }
           },
           {
-            add: { index: to, alias: target_name }
+            add: { index: to, alias: alias_name }
           }
         ]
 
@@ -199,11 +266,13 @@ module Gitlab
       end
 
       # This method is used when we need to get an actual index name (if it's used through an alias)
-      def target_index_name
-        if alias_exists?
-          client.indices.get_alias(name: target_name).each_key.first
+      def target_index_name(target: nil)
+        target ||= target_name
+
+        if alias_exists?(name: target)
+          client.indices.get_alias(name: target).each_key.first
         else
-          target_name
+          target
         end
       end
 

ee/lib/tasks/gitlab/elastic.rake

 namespace :gitlab do
   namespace :elastic do
-    desc "GitLab | Elasticsearch | Index eveything at once"
+    desc "GitLab | Elasticsearch | Index everything at once"
     task :index do
       # UPDATE_INDEX=true can cause some projects not to be indexed properly if someone were to push a commit to the
       # project before the rake task could get to it, so we set it to `nil` here to avoid that. It doesn't make sense
@@ -57,7 +57,7 @@ namespace :gitlab do
       logger.info("Indexing snippets... " + "done".color(:green))
     end
 
-    desc "GitLab | Elasticsearch | Create empty index and assign alias"
+    desc "GitLab | Elasticsearch | Create empty indexes and assigns an alias for each"
     task :create_empty_index, [:target_name] => [:environment] do |t, args|
       with_alias = ENV["SKIP_ALIAS"].nil?
       options = {}
@@ -68,14 +68,25 @@ namespace :gitlab do
       helper = Gitlab::Elastic::Helper.new(target_name: args[:target_name])
       index_name = helper.create_empty_index(with_alias: with_alias, options: options)
 
-      helper.create_migrations_index unless helper.index_exists?(index_name: helper.migrations_index_name)
-      ::Elastic::DataMigrationService.mark_all_as_completed!
+      # with_alias is used to support interacting with a specific index (such as when reclaiming the production index
+      # name when the index was created prior to 13.0). If the `SKIP_ALIAS` environment variable is set,
+      # do not create standalone indexes and do not create the migrations index
+      if with_alias
+        standalone_index_names = helper.create_standalone_indices(options: options)
+        standalone_index_names.each do |index_name, alias_name|
+          puts "Index '#{index_name}' has been created.".color(:green)
+          puts "Alias '#{alias_name}' -> '#{index_name}' has been created.".color(:green)
+        end
+
+        helper.create_migrations_index unless helper.index_exists?(index_name: helper.migrations_index_name)
+        ::Elastic::DataMigrationService.mark_all_as_completed!
+      end
 
       puts "Index '#{index_name}' has been created.".color(:green)
       puts "Alias '#{helper.target_name}' → '#{index_name}' has been created".color(:green) if with_alias
     end
 
-    desc "GitLab | Elasticsearch | Delete index"
+    desc "GitLab | Elasticsearch | Delete all indexes"
     task :delete_index, [:target_name] => [:environment] do |t, args|
       helper = Gitlab::Elastic::Helper.new(target_name: args[:target_name])
 
@@ -84,9 +95,18 @@ namespace :gitlab do
       else
         puts "Index/alias '#{helper.target_name}' was not found".color(:green)
       end
+
+      results = helper.delete_standalone_indices
+      results.each do |index_name, alias_name, result|
+        if result
+          puts "Index '#{index_name}' with alias '#{alias_name}' has been deleted".color(:green)
+        else
+          puts "Index '#{index_name}' with alias '#{alias_name}' was not found".color(:green)
+        end
+      end
     end
 
-    desc "GitLab | Elasticsearch | Recreate index"
+    desc "GitLab | Elasticsearch | Recreate indexes"
     task :recreate_index, [:target_name] => [:environment] do |t, args|
       Rake::Task["gitlab:elastic:delete_index"].invoke(*args)
       Rake::Task["gitlab:elastic:create_empty_index"].invoke(*args)

ee/lib/tasks/gitlab/elastic/test.rake

@@ -3,8 +3,14 @@ namespace :gitlab do
     namespace :test do
       desc 'GitLab | Elasticsearch | Test | Measure space taken by ES indices'
       task index_size: :environment do
-        puts "===== Size stats for index: #{Project.__elasticsearch__.index_name} ====="
-        pp Gitlab::Elastic::Helper.default.index_size.slice(*%w(docs store))
+        helper = Gitlab::Elastic::Helper.default
+
+        indices = [helper.target_name]
+        indices += helper.standalone_indices_proxies.map(&:index_name) if Elastic::DataMigrationService.migration_has_finished?(:migrate_issues_to_separate_index)
+        indices.each do |index_name|
+          puts "===== Size stats for index: #{index_name} ====="
+          pp helper.index_size(index_name: index_name).slice(*%w(docs store))
+        end
       end
 
       desc 'GitLab | Elasticsearch | Test | Measure space taken by ES indices, reindex, and measure space taken again'

ee/spec/controllers/admin/application_settings_controller_spec.rb

@@ -25,8 +25,7 @@ RSpec.describe Admin::ApplicationSettingsController do
         get :general
 
         expect(assigns(:elasticsearch_reindexing_task)).to eq(task)
-        expect(response.body).to include('Reindexing status')
-        expect(response.body).to include("State: #{task.state}")
+        expect(response.body).to include("Reindexing Status: #{task.state}")
       end
     end
   end

ee/spec/elastic/migrate/20201116142400_add_new_data_to_issues_documents_spec.rb

@@ -11,6 +11,10 @@ RSpec.describe AddNewDataToIssuesDocuments, :elastic, :sidekiq_inline do
   before do
     stub_ee_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
 
+    allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
+      .with(:migrate_issues_to_separate_index)
+      .and_return(false)
+
     # ensure issues are indexed
     issues
 

ee/spec/elastic/migrate/20201123123400_migrate_issues_to_separate_index_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require File.expand_path('ee/elastic/migrate/20201123123400_migrate_issues_to_separate_index.rb')
+
+RSpec.describe MigrateIssuesToSeparateIndex, :elastic, :sidekiq_inline do
+  let(:version) { 20201123123400 }
+  let(:migration) { described_class.new(version) }
+  let(:issues) { create_list(:issue, 3) }
+  let(:index_name) { "#{es_helper.target_name}-issues" }
+
+  before do
+    allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
+      .with(:migrate_issues_to_separate_index)
+      .and_return(false)
+
+    stub_ee_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
+
+    issues
+
+    ensure_elasticsearch_index!
+  end
+
+  describe 'migration_options' do
+    it 'has migration options set', :aggregate_failures do
+      expect(migration.batched?).to be_truthy
+      expect(migration.throttle_delay).to eq(1.minute)
+      expect(migration.pause_indexing?).to be_truthy
+    end
+  end
+
+  describe '.migrate', :clean_gitlab_redis_shared_state do
+    context 'initial launch' do
+      before do
+        es_helper.delete_index(index_name: es_helper.target_index_name(target: index_name))
+      end
+
+      it 'creates an index and sets migration_state' do
+        expect { migration.migrate }.to change { es_helper.alias_exists?(name: index_name) }.from(false).to(true)
+
+        expect(migration.migration_state).to include(slice: 0, max_slices: 5)
+      end
+    end
+
+    context 'batch run' do
+      it 'migrates all issues' do
+        total_shards = es_helper.get_settings.dig('number_of_shards').to_i
+        migration.set_migration_state(slice: 0, max_slices: total_shards)
+
+        total_shards.times do |i|
+          migration.migrate
+        end
+
+        expect(migration.completed?).to be_truthy
+        expect(es_helper.documents_count(index_name: "#{es_helper.target_name}-issues")).to eq(issues.count)
+      end
+    end
+
+    context 'failed run' do
+      let(:client) { double('Elasticsearch::Transport::Client') }
+
+      before do
+        allow(migration).to receive(:client).and_return(client)
+      end
+
+      context 'exception is raised' do
+        before do
+          allow(client).to receive(:reindex).and_raise(StandardError)
+        end
+
+        it 'increases retry_attempt' do
+          migration.set_migration_state(slice: 0, max_slices: 2, retry_attempt: 1)
+
+          expect { migration.migrate }.to raise_error(StandardError)
+          expect(migration.migration_state).to match(slice: 0, max_slices: 2, retry_attempt: 2)
+        end
+
+        it 'fails the migration after too many attempts' do
+          migration.set_migration_state(slice: 0, max_slices: 2, retry_attempt: 30)
+
+          migration.migrate
+
+          expect(migration.migration_state).to match(slice: 0, max_slices: 2, retry_attempt: 30, halted: true)
+          expect(migration).not_to receive(:process_response)
+        end
+      end
+
+      context 'elasticsearch failures' do
+        context 'total is not equal' do
+          before do
+            allow(client).to receive(:reindex).and_return({ "total" => 60, "updated" => 0, "created" => 45, "deleted" => 0, "failures" => [] })
+          end
+
+          it 'raises an error' do
+            migration.set_migration_state(slice: 0, max_slices: 2)
+
+            expect { migration.migrate }.to raise_error(/total is not equal/)
+          end
+        end
+
+        context 'reindexing failues' do
+          before do
+            allow(client).to receive(:reindex).and_return({ "total" => 60, "updated" => 0, "created" => 0, "deleted" => 0, "failures" => [{ "type": "es_rejected_execution_exception" }] })
+          end
+
+          it 'raises an error' do
+            migration.set_migration_state(slice: 0, max_slices: 2)
+
+            expect { migration.migrate }.to raise_error(/failed with/)
+          end
+        end
+      end
+    end
+  end
+
+  describe '.completed?' do
+    subject { migration.completed? }
+
+    before do
+      2.times do |slice|
+        migration.set_migration_state(slice: slice, max_slices: 2)
+        migration.migrate
+      end
+    end
+
+    context 'counts are equal' do
+      let(:issues_count) { issues.count }
+
+      it 'returns true' do
+        is_expected.to be_truthy
+      end
+    end
+  end
+end

ee/spec/factories/elastic/reindexing_subtasks.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :elastic_reindexing_subtask, class: 'Elastic::ReindexingSubtask' do
+    association :elastic_reindexing_task, in_progress: false, state: :success
+    sequence(:index_name_from) { |n| "old_index_name_#{n}" }
+    sequence(:index_name_to) { |n| "new_index_name_#{n}" }
+    sequence(:elastic_task) { |n| "elastic_task_#{n}" }
+    sequence(:alias_name) { |n| "alias_name_#{n}" }
+  end
+end

ee/spec/factories/elastic/reindexing_tasks.rb

@@ -6,5 +6,11 @@ FactoryBot.define do
     in_progress { true }
     index_name_from { 'old_index_name' }
     index_name_to { 'new_index_name' }
+
+    trait :with_subtask do
+      after(:create) do |task|
+        create :elastic_reindexing_subtask, elastic_reindexing_task: task
+      end
+    end
   end
 end

ee/spec/lib/ee/gitlab/elastic/helper_spec.rb

@@ -7,13 +7,13 @@ RSpec.describe Gitlab::Elastic::Helper do
 
   shared_context 'with a legacy index' do
     before do
-      @index_name = helper.create_empty_index(with_alias: false, options: { index_name: helper.target_name })
+      @index_name = helper.create_empty_index(with_alias: false, options: { index_name: helper.target_name }).each_key.first
     end
   end
 
   shared_context 'with an existing index and alias' do
     before do
-      @index_name = helper.create_empty_index(with_alias: true)
+      @index_name = helper.create_empty_index(with_alias: true).each_key.first
     end
   end
 
@@ -40,6 +40,10 @@ RSpec.describe Gitlab::Elastic::Helper do
   end
 
   describe '#default_mappings' do
+    it 'has only one type' do
+      expect(helper.default_mappings.keys).to match_array %i(doc)
+    end
+
     context 'custom analyzers' do
       let(:custom_analyzers_mappings) { { doc: { properties: { title: { fields: { custom: true } } } } } }
 
@@ -65,6 +69,44 @@ RSpec.describe Gitlab::Elastic::Helper do
     end
   end
 
+  describe '#create_standalone_indices' do
+    after do
+      @indices.each do |index_name, _|
+        helper.delete_index(index_name: index_name)
+      end
+    end
+
+    it 'creates standalone indices' do
+      @indices = helper.create_standalone_indices
+
+      @indices.each do |index|
+        expect(helper.index_exists?(index_name: index)).to be_truthy
+      end
+    end
+
+    it 'raises an exception when there is an existing alias' do
+      @indices = helper.create_standalone_indices
+
+      expect { helper.create_standalone_indices }.to raise_error(/already exists/)
+    end
+
+    it 'raises an exception when there is an existing index' do
+      @indices = helper.create_standalone_indices(with_alias: false)
+
+      expect { helper.create_standalone_indices(with_alias: false) }.to raise_error(/already exists/)
+    end
+  end
+
+  describe '#delete_standalone_indices' do
+    before do
+      helper.create_standalone_indices
+    end
+
+    subject { helper.delete_standalone_indices }
+
+    it_behaves_like 'deletes all standalone indices'
+  end
+
   describe '#create_empty_index' do
     context 'with an empty cluster' do
       context 'with alias and index' do
@@ -174,7 +216,7 @@ RSpec.describe Gitlab::Elastic::Helper do
     end
   end
 
-  describe '#cluster_free_size' do
+  describe '#cluster_free_size_bytes' do
     it 'returns valid cluster size' do
       expect(helper.cluster_free_size_bytes).to be_positive
     end
@@ -194,4 +236,51 @@ RSpec.describe Gitlab::Elastic::Helper do
       helper.delete_index(index_name: new_index_name)
     end
   end
+
+  describe '#index_size' do
+    subject { helper.index_size }
+
+    context 'when there is a legacy index' do
+      include_context 'with a legacy index'
+
+      it { is_expected.to have_key("docs") }
+      it { is_expected.to have_key("store") }
+    end
+
+    context 'when there is an alias', :aggregate_failures do
+      include_context 'with an existing index and alias'
+
+      it { is_expected.to have_key("docs") }
+      it { is_expected.to have_key("store") }
+
+      it 'supports providing the alias name' do
+        alias_name = helper.target_name
+
+        expect(helper.index_size(index_name: alias_name)).to have_key("docs")
+        expect(helper.index_size(index_name: alias_name)).to have_key("store")
+      end
+    end
+  end
+
+  describe '#documents_count' do
+    subject { helper.documents_count }
+
+    context 'when there is a legacy index' do
+      include_context 'with a legacy index'
+
+      it { is_expected.to eq(0) }
+    end
+
+    context 'when there is an alias' do
+      include_context 'with an existing index and alias'
+
+      it { is_expected.to eq(0) }
+
+      it 'supports providing the alias name' do
+        alias_name = helper.target_name
+
+        expect(helper.documents_count(index_name: alias_name)).to eq(0)
+      end
+    end
+  end
 end

ee/spec/lib/elastic/latest/issue_config_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Elastic::Latest::IssueConfig do
+  describe '.document_type' do
+    it 'returns config' do
+      expect(described_class.document_type).to eq('doc')
+    end
+  end
+
+  describe '.settings' do
+    it 'returns config' do
+      expect(described_class.settings).to be_a(Elasticsearch::Model::Indexing::Settings)
+    end
+  end
+
+  describe '.mappings' do
+    it 'returns config' do
+      expect(described_class.mapping).to be_a(Elasticsearch::Model::Indexing::Mappings)
+    end
+  end
+end

ee/spec/lib/gitlab/instrumentation/elasticsearch_transport_spec.rb

@@ -35,7 +35,7 @@ RSpec.describe ::Gitlab::Instrumentation::ElasticsearchTransport, :elastic, :req
 
       ::Gitlab::SafeRequestStore.clear!
 
-      create(:issue, title: "new issue")
+      create(:merge_request, title: "new MR")
       ensure_elasticsearch_index!
 
       request = ::Gitlab::Instrumentation::ElasticsearchTransport.detail_store.first

ee/spec/migrations/create_elastic_reindexing_subtasks_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe CreateElasticReindexingSubtasks do
+  let(:migration) { described_class.new }
+  let(:reindexing_tasks) { table(:elastic_reindexing_tasks) }
+  let(:reindexing_subtasks) { table(:elastic_reindexing_subtasks) }
+  let(:fields_to_migrate) { %w(documents_count documents_count_target index_name_from index_name_to elastic_task) }
+
+  describe "#up" do
+    it 'migrates old reindexing tasks' do
+      # these tasks should not be migrated
+      reindexing_tasks.create!(in_progress: false, state: 10)
+      reindexing_tasks.create!(in_progress: false, state: 10, index_name_from: 'index_name')
+      reindexing_tasks.create!(in_progress: false, state: 10, index_name_to: 'index_name')
+      reindexing_tasks.create!(in_progress: false, state: 10, elastic_task: 'TASK')
+      # these tasks should not be migrated
+
+      task1 = reindexing_tasks.create!(in_progress: false, documents_count: 100, state: 10, index_name_from: 'index1', index_name_to: 'index2', elastic_task: 'TASK_ID', documents_count_target: 100)
+      task2 = reindexing_tasks.create!(in_progress: false, documents_count: 50, state: 11, index_name_from: 'index3', index_name_to: 'index4', elastic_task: 'TASK_ID2', documents_count_target: 99)
+
+      migrate!
+
+      expect(reindexing_subtasks.count).to eq(2)
+
+      [task1, task2].each do |task|
+        subtask = reindexing_subtasks.find_by(elastic_reindexing_task_id: task.id)
+
+        expect(task.attributes.slice(*fields_to_migrate)).to match(subtask.attributes.slice(*fields_to_migrate))
+      end
+    end
+  end
+end

ee/spec/models/concerns/elastic/issue_spec.rb

@@ -122,11 +122,7 @@ RSpec.describe Issue, :elastic do
       'confidential'
     ).merge({
       'type' => issue.es_type,
-      'state' => issue.state,
-      'join_field' => {
-        'name' => issue.es_type,
-        'parent' => issue.es_parent
-      }
+      'state' => issue.state
     })
 
     expected_hash['assignee_id'] = [assignee.id]

ee/spec/models/elastic/reindexing_subtask_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Elastic::ReindexingSubtask, type: :model do
+  describe 'relations' do
+    it { is_expected.to belong_to(:elastic_reindexing_task) }
+  end
+
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:index_name_from) }
+    it { is_expected.to validate_presence_of(:index_name_to) }
+    it { is_expected.to validate_presence_of(:elastic_task) }
+  end
+end

ee/spec/models/elastic/reindexing_task_spec.rb

@@ -3,6 +3,10 @@
 require 'spec_helper'
 
 RSpec.describe Elastic::ReindexingTask, type: :model do
+  describe 'relations' do
+    it { is_expected.to have_many(:subtasks) }
+  end
+
   it 'only allows one running task at a time' do
     expect { create(:elastic_reindexing_task, state: :success) }.not_to raise_error
     expect { create(:elastic_reindexing_task) }.not_to raise_error
@@ -18,21 +22,21 @@ RSpec.describe Elastic::ReindexingTask, type: :model do
   end
 
   describe '.drop_old_indices!' do
-    let(:task_1) { create(:elastic_reindexing_task, index_name_from: 'original_index_1', state: :reindexing, delete_original_index_at: 1.day.ago) }
-    let(:task_2) { create(:elastic_reindexing_task, index_name_from: 'original_index_2', state: :success, delete_original_index_at: nil) }
-    let(:task_3) { create(:elastic_reindexing_task, index_name_from: 'original_index_3', state: :success, delete_original_index_at: 1.day.ago) }
-    let(:task_4) { create(:elastic_reindexing_task, index_name_from: 'original_index_4', state: :success, delete_original_index_at: 5.days.ago) }
-    let(:task_5) { create(:elastic_reindexing_task, index_name_from: 'original_index_5', state: :success, delete_original_index_at: 14.days.from_now) }
+    let(:task_1) { create(:elastic_reindexing_task, :with_subtask, state: :reindexing, delete_original_index_at: 1.day.ago) }
+    let(:task_2) { create(:elastic_reindexing_task, :with_subtask, state: :success, delete_original_index_at: nil) }
+    let(:task_3) { create(:elastic_reindexing_task, :with_subtask, state: :success, delete_original_index_at: 1.day.ago) }
+    let(:task_4) { create(:elastic_reindexing_task, :with_subtask, state: :success, delete_original_index_at: 5.days.ago) }
+    let(:task_5) { create(:elastic_reindexing_task, :with_subtask, state: :success, delete_original_index_at: 14.days.from_now) }
     let(:tasks_for_deletion) { [task_3, task_4] }
     let(:other_tasks) { [task_1, task_2, task_5] }
 
     it 'deletes the correct indices' do
       other_tasks.each do |task|
-        expect(Gitlab::Elastic::Helper.default).not_to receive(:delete_index).with(index_name: task.index_name_from)
+        expect(Gitlab::Elastic::Helper.default).not_to receive(:delete_index).with(index_name: task.subtasks.first.index_name_from)
       end
 
       tasks_for_deletion.each do |task|
-        expect(Gitlab::Elastic::Helper.default).to receive(:delete_index).with(index_name: task.index_name_from).and_return(true)
+        expect(Gitlab::Elastic::Helper.default).to receive(:delete_index).with(index_name: task.subtasks.first.index_name_from).and_return(true)
       end
 
       described_class.drop_old_indices!

ee/spec/services/elastic/cluster_reindexing_service_spec.rb

@@ -8,6 +8,20 @@ RSpec.describe Elastic::ClusterReindexingService, :elastic do
   context 'state: initial' do
     let(:task) { create(:elastic_reindexing_task, state: :initial) }
 
+    it 'aborts if the main index does not use aliases' do
+      allow(Gitlab::Elastic::Helper.default).to receive(:alias_exists?).and_return(false)
+
+      expect { subject.execute }.to change { task.reload.state }.from('initial').to('failure')
+      expect(task.reload.error_message).to match(/use aliases/)
+    end
+
+    it 'aborts if there are pending ES migrations' do
+      allow(Elastic::DataMigrationService).to receive(:pending_migrations?).and_return(true)
+
+      expect { subject.execute }.to change { task.reload.state }.from('initial').to('failure')
+      expect(task.reload.error_message).to match(/unapplied advanced search migrations/)
+    end
+
     it 'errors when there is not enough space' do
       allow(Gitlab::Elastic::Helper.default).to receive(:index_size_bytes).and_return(100.megabytes)
       allow(Gitlab::Elastic::Helper.default).to receive(:cluster_free_size_bytes).and_return(30.megabytes)
@@ -29,19 +43,26 @@ RSpec.describe Elastic::ClusterReindexingService, :elastic do
     it 'triggers reindexing' do
       task = create(:elastic_reindexing_task, state: :indexing_paused)
 
-      allow(Gitlab::Elastic::Helper.default).to receive(:create_empty_index).and_return('new_index_name')
-      allow(Gitlab::Elastic::Helper.default).to receive(:reindex).and_return('task_id')
+      allow(Gitlab::Elastic::Helper.default).to receive(:create_empty_index).and_return('new_index_name' => 'new_index')
+      allow(Gitlab::Elastic::Helper.default).to receive(:create_standalone_indices).and_return('new_issues_name' => 'new_issues')
+      allow(Gitlab::Elastic::Helper.default).to receive(:reindex).with(from: anything, to: 'new_index_name').and_return('task_id_1')
+      allow(Gitlab::Elastic::Helper.default).to receive(:reindex).with(from: anything, to: 'new_issues_name').and_return('task_id_2')
 
       expect { subject.execute }.to change { task.reload.state }.from('indexing_paused').to('reindexing')
 
-      task = task.reload
-      expect(task.index_name_to).to eq('new_index_name')
-      expect(task.elastic_task).to eq('task_id')
+      subtasks = task.subtasks
+      expect(subtasks.count).to eq(2)
+
+      expect(subtasks.first.index_name_to).to eq('new_index_name')
+      expect(subtasks.first.elastic_task).to eq('task_id_1')
+      expect(subtasks.last.index_name_to).to eq('new_issues_name')
+      expect(subtasks.last.elastic_task).to eq('task_id_2')
     end
   end
 
   context 'state: reindexing' do
-    let(:task) { create(:elastic_reindexing_task, state: :reindexing, documents_count: 10) }
+    let(:task) { create(:elastic_reindexing_task, state: :reindexing) }
+    let(:subtask) { create(:elastic_reindexing_subtask, elastic_reindexing_task: task, documents_count: 10)}
     let(:expected_default_settings) do
       {
         refresh_interval: nil,
@@ -57,7 +78,7 @@ RSpec.describe Elastic::ClusterReindexingService, :elastic do
 
     context 'errors are raised' do
       before do
-        allow(Gitlab::Elastic::Helper.default).to receive(:documents_count).with(index_name: task.index_name_to).and_return(task.reload.documents_count * 2)
+        allow(Gitlab::Elastic::Helper.default).to receive(:documents_count).with(index_name: subtask.index_name_to).and_return(subtask.reload.documents_count * 2)
       end
 
       it 'errors if documents count is different' do
@@ -82,12 +103,12 @@ RSpec.describe Elastic::ClusterReindexingService, :elastic do
 
     context 'task finishes correctly' do
       before do
-        allow(Gitlab::Elastic::Helper.default).to receive(:documents_count).with(index_name: task.index_name_to).and_return(task.reload.documents_count)
+        allow(Gitlab::Elastic::Helper.default).to receive(:documents_count).with(index_name: subtask.index_name_to).and_return(subtask.reload.documents_count)
       end
 
       it 'launches all state steps' do
-        expect(Gitlab::Elastic::Helper.default).to receive(:update_settings).with(index_name: task.index_name_to, settings: expected_default_settings)
-        expect(Gitlab::Elastic::Helper.default).to receive(:switch_alias).with(to: task.index_name_to)
+        expect(Gitlab::Elastic::Helper.default).to receive(:update_settings).with(index_name: subtask.index_name_to, settings: expected_default_settings)
+        expect(Gitlab::Elastic::Helper.default).to receive(:switch_alias).with(to: subtask.index_name_to, from: subtask.index_name_from, alias_name: subtask.alias_name)
         expect(Gitlab::CurrentSettings).to receive(:update!).with(elasticsearch_pause_indexing: false)
 
         expect { subject.execute }.to change { task.reload.state }.from('reindexing').to('success')

ee/spec/services/search/global_service_spec.rb

@@ -112,14 +112,21 @@ RSpec.describe Search::GlobalService do
       # finished we need a test to verify the old style searches work for
       # instances which haven't finished the migration yet
       context 'when add_new_data_to_issues_documents migration is not finished' do
-        let!(:issue) { create :issue, project: project }
-
         before do
+          allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
           allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
             .with(:add_new_data_to_issues_documents)
             .and_return(false)
+          allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
+            .with(:migrate_issues_to_separate_index)
+            .and_return(false)
         end
 
+        # issue cannot be defined prior to the migration mocks because it
+        # will cause the incorrect value to be passed to `use_separate_indices` when creating
+        # the proxy
+        let!(:issue) { create(:issue, project: project) }
+
         where(:project_level, :feature_access_level, :membership, :admin_mode, :expected_count) do
           permission_table_for_guest_feature_access
         end
@@ -158,6 +165,11 @@ RSpec.describe Search::GlobalService do
         let(:search_url) { Addressable::Template.new("#{es_host}/{index}/doc/_search{?params*}") }
 
         before do
+          allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_call_original
+          allow(Elastic::DataMigrationService).to receive(:migration_has_finished?)
+            .with(:migrate_issues_to_separate_index)
+            .and_return(false)
+
           ensure_elasticsearch_index!
         end
 

ee/spec/support/elastic.rb

@@ -6,20 +6,23 @@ RSpec.configure do |config|
 
     Elastic::ProcessBookkeepingService.clear_tracking!
 
-    # Delete the migration index and the main ES index
-    helper.delete_index(index_name: helper.migrations_index_name)
-    helper.delete_index
+    # Delete all test indices
+    indices = [helper.target_name, helper.migrations_index_name] + helper.standalone_indices_proxies.map(&:index_name)
+    indices.each do |index_name|
+      helper.delete_index(index_name: index_name)
+    end
 
     helper.create_empty_index(options: { settings: { number_of_replicas: 0 } })
     helper.create_migrations_index
     ::Elastic::DataMigrationService.mark_all_as_completed!
+    helper.create_standalone_indices
     refresh_index!
 
     example.run
 
-    helper.delete_index(index_name: helper.migrations_index_name)
-    helper.delete_index
-
+    indices.each do |index_name|
+      helper.delete_index(index_name: index_name)
+    end
     Elastic::ProcessBookkeepingService.clear_tracking!
   end
 

ee/spec/support/shared_examples/lib/gitlab/elastic/standalone_indices_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'deletes all standalone indices' do
+  Gitlab::Elastic::Helper::ES_SEPARATE_CLASSES.each do |class_name|
+    describe "#{class_name}" do
+      it 'removes a standalone index' do
+        proxy = ::Elastic::Latest::ApplicationClassProxy.new(class_name, use_separate_indices: true)
+
+        expect { subject }.to change { helper.index_exists?(index_name: proxy.index_name) }.from(true).to(false)
+      end
+    end
+  end
+end

ee/spec/tasks/gitlab/elastic_rake_spec.rb

@@ -13,12 +13,68 @@ RSpec.describe 'gitlab:elastic namespace rake tasks', :elastic do
     before do
       es_helper.delete_index
       es_helper.delete_index(index_name: es_helper.migrations_index_name)
+      es_helper.delete_standalone_indices
     end
 
-    it 'creates an index' do
+    it 'creates the default index' do
       expect { subject }.to change { es_helper.index_exists? }.from(false).to(true)
     end
 
+    context 'when SKIP_ALIAS environment variable is set' do
+      let(:secondary_index_name) { "gitlab-test-#{Time.now.strftime("%Y%m%d-%H%M")}"}
+
+      before do
+        stub_env('SKIP_ALIAS', '1')
+      end
+
+      after do
+        es_helper.delete_index(index_name: secondary_index_name)
+      end
+
+      subject { run_rake_task('gitlab:elastic:create_empty_index', secondary_index_name) }
+
+      it 'does not alias the new index' do
+        expect { subject }.not_to change { es_helper.alias_exists?(name: es_helper.target_name) }
+      end
+
+      it 'does not create the migrations index if it does not exist' do
+        migration_index_name = es_helper.migrations_index_name
+        es_helper.delete_index(index_name: migration_index_name)
+
+        expect { subject }.not_to change { es_helper.index_exists?(index_name: migration_index_name) }
+      end
+
+      it 'creates an index at the specified name' do
+        expect { subject }.to change { es_helper.index_exists?(index_name: secondary_index_name) }.from(false).to(true)
+      end
+
+      Gitlab::Elastic::Helper::ES_SEPARATE_CLASSES.each do |class_name|
+        describe "#{class_name}" do
+          it "does not create a standalone index" do
+            proxy = ::Elastic::Latest::ApplicationClassProxy.new(class_name, use_separate_indices: true)
+
+            expect { subject }.not_to change { es_helper.alias_exists?(name: proxy.index_name) }
+          end
+        end
+      end
+    end
+
+    it 'creates the migrations index if it does not exist' do
+      migration_index_name = es_helper.migrations_index_name
+      es_helper.delete_index(index_name: migration_index_name)
+
+      expect { subject }.to change { es_helper.index_exists?(index_name: migration_index_name) }.from(false).to(true)
+    end
+
+    Gitlab::Elastic::Helper::ES_SEPARATE_CLASSES.each do |class_name|
+      describe "#{class_name}" do
+        it "creates a standalone index" do
+          proxy = ::Elastic::Latest::ApplicationClassProxy.new(class_name, use_separate_indices: true)
+          expect { subject }.to change { es_helper.index_exists?(index_name: proxy.index_name) }.from(false).to(true)
+        end
+      end
+    end
+
     it 'marks all migrations as completed' do
       expect(Elastic::DataMigrationService).to receive(:mark_all_as_completed!).and_call_original
       expect(Elastic::MigrationRecord.persisted_versions(completed: true)).to eq([])
@@ -37,6 +93,10 @@ RSpec.describe 'gitlab:elastic namespace rake tasks', :elastic do
     it 'removes the index' do
       expect { subject }.to change { es_helper.index_exists? }.from(true).to(false)
     end
+
+    it_behaves_like 'deletes all standalone indices' do
+      let(:helper) { es_helper }
+    end
   end
 
   context "with elasticsearch_indexing enabled" do

ee/spec/views/admin/application_settings/_elasticsearch_form.html.haml_spec.rb

@@ -5,6 +5,8 @@ require 'spec_helper'
 RSpec.describe 'admin/application_settings/_elasticsearch_form' do
   let_it_be(:admin) { create(:admin) }
   let(:page) { Capybara::Node::Simple.new(rendered) }
+  let(:pause_indexing) { false }
+  let(:pending_migrations) { false }
 
   before do
     assign(:application_setting, application_setting)
@@ -18,7 +20,8 @@ RSpec.describe 'admin/application_settings/_elasticsearch_form' do
 
     before do
       allow(Gitlab::CurrentSettings).to(receive(:elasticsearch_indexing?)).and_return(es_indexing)
-      allow(Gitlab::CurrentSettings).to(receive(:elasticsearch_pause_indexing?)).and_return(true)
+      allow(Gitlab::CurrentSettings).to(receive(:elasticsearch_pause_indexing?)).and_return(pause_indexing)
+      allow(Elastic::DataMigrationService).to(receive(:pending_migrations?)).and_return(pending_migrations)
     end
 
     context 'indexing is enabled' do
@@ -36,6 +39,17 @@ RSpec.describe 'admin/application_settings/_elasticsearch_form' do
         expect(rendered).to have_css('input[id=application_setting_elasticsearch_pause_indexing]')
         expect(rendered).not_to have_css('input[id=application_setting_elasticsearch_pause_indexing][disabled="disabled"]')
       end
+
+      context 'pending migrations' do
+        let(:pending_migrations) { true }
+        let(:pause_indexing) { true }
+
+        it 'renders a disabled pause checkbox' do
+          render
+
+          expect(rendered).to have_css('input[id=application_setting_elasticsearch_pause_indexing][disabled="disabled"]')
+        end
+      end
     end
 
     context 'indexing is disabled' do
@@ -88,7 +102,7 @@ RSpec.describe 'admin/application_settings/_elasticsearch_form' do
       it 'renders the task' do
         render
 
-        expect(rendered).to include("State: #{task.state}")
+        expect(rendered).to include("Reindexing Status: #{task.state}")
         expect(rendered).not_to include("Task ID:")
         expect(rendered).not_to include("Error:")
         expect(rendered).not_to include("Expected documents:")
@@ -97,28 +111,30 @@ RSpec.describe 'admin/application_settings/_elasticsearch_form' do
     end
 
     context 'with extended details' do
-      let(:task) { build(:elastic_reindexing_task, state: :reindexing, elastic_task: 'elastic-task-id', error_message: 'error-message', documents_count_target: 5, documents_count: 10) }
+      let!(:task) { create(:elastic_reindexing_task, state: :reindexing, error_message: 'error-message') }
+      let!(:subtask) { create(:elastic_reindexing_subtask, elastic_reindexing_task: task, documents_count_target: 5, documents_count: 10) }
 
       it 'renders the task' do
         render
 
-        expect(rendered).to include("State: #{task.state}")
-        expect(rendered).to include("Task ID: #{task.elastic_task}")
+        expect(rendered).to include("Reindexing Status: #{task.state}")
+        expect(rendered).to include("Task ID: #{subtask.elastic_task}")
         expect(rendered).to include("Error: #{task.error_message}")
-        expect(rendered).to include("Expected documents: #{task.documents_count}")
-        expect(rendered).to include("Documents reindexed: #{task.documents_count_target} (50.0%)")
+        expect(rendered).to include("Expected documents: #{subtask.documents_count}")
+        expect(rendered).to include("Documents reindexed: #{subtask.documents_count_target} (50.0%)")
       end
     end
 
     context 'with extended details, but without documents_count_target' do
-      let(:task) { build(:elastic_reindexing_task, state: :reindexing, elastic_task: 'elastic-task-id', documents_count: 10) }
+      let!(:task) { create(:elastic_reindexing_task, state: :reindexing) }
+      let!(:subtask) { create(:elastic_reindexing_subtask, elastic_reindexing_task: task, documents_count: 10) }
 
       it 'renders the task' do
         render
 
-        expect(rendered).to include("State: #{task.state}")
-        expect(rendered).to include("Task ID: #{task.elastic_task}")
-        expect(rendered).to include("Expected documents: #{task.documents_count}")
+        expect(rendered).to include("Reindexing Status: #{task.state}")
+        expect(rendered).to include("Task ID: #{subtask.elastic_task}")
+        expect(rendered).to include("Expected documents: #{subtask.documents_count}")
         expect(rendered).not_to include("Error:")
         expect(rendered).not_to include("Documents reindexed:")
       end

ee/spec/workers/elastic/migration_worker_spec.rb

@@ -44,6 +44,45 @@ RSpec.describe Elastic::MigrationWorker, :elastic do
         end
       end
 
+      context 'migration is halted' do
+        before do
+          allow(Gitlab::CurrentSettings).to receive(:elasticsearch_pause_indexing?).and_return(true)
+          allow(subject).to receive(:current_migration).and_return(migration)
+          allow(migration).to receive(:pause_indexing?).and_return(true)
+          allow(migration).to receive(:halted?).and_return(true)
+        end
+
+        it 'skips execution' do
+          expect(migration).not_to receive(:migrate)
+
+          subject.perform
+        end
+
+        context 'pause indexing is not allowed' do
+          before do
+            migration.save_state!(pause_indexing: false)
+          end
+
+          it 'does not unpauses indexing' do
+            expect(Gitlab::CurrentSettings).not_to receive(:update!)
+
+            subject.perform
+          end
+        end
+
+        context 'pause indexing is allowed' do
+          before do
+            migration.save_state!(pause_indexing: true)
+          end
+
+          it 'unpauses indexing' do
+            expect(Gitlab::CurrentSettings).to receive(:update!).with(elasticsearch_pause_indexing: false)
+
+            subject.perform
+          end
+        end
+      end
+
       context 'migration process' do
         before do
           allow(migration).to receive(:persisted?).and_return(persisted)
@@ -91,6 +130,29 @@ RSpec.describe Elastic::MigrationWorker, :elastic do
             subject.perform
           end
         end
+
+        context 'indexing pause' do
+          before do
+            allow(migration).to receive(:pause_indexing?).and_return(true)
+          end
+
+          let(:batched) { true }
+
+          where(:persisted, :completed, :expected) do
+            false | false | false
+            true  | false | false
+            true  | true  | true
+          end
+
+          with_them do
+            it 'pauses and unpauses indexing' do
+              expect(Gitlab::CurrentSettings).to receive(:update!).with(elasticsearch_pause_indexing: true)
+              expect(Gitlab::CurrentSettings).to receive(:update!).with(elasticsearch_pause_indexing: false) if expected
+
+              subject.perform
+            end
+          end
+        end
       end
     end
   end

ee/spec/workers/elastic_delete_project_worker_spec.rb

@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-RSpec.describe ElasticDeleteProjectWorker, :elastic do
+RSpec.describe ElasticDeleteProjectWorker, :elastic, :sidekiq_inline do
   subject { described_class.new }
 
   # Create admin user and search globally to avoid dealing with permissions in
@@ -11,6 +11,7 @@ RSpec.describe ElasticDeleteProjectWorker, :elastic do
 
   before do
     stub_ee_application_setting(elasticsearch_indexing: true)
+    allow(Elastic::DataMigrationService).to receive(:migration_has_finished?).and_return(migration_has_finished)
   end
 
   # Extracted to a method as the `#elastic_search` methods using it below will
@@ -19,36 +20,51 @@ RSpec.describe ElasticDeleteProjectWorker, :elastic do
     { options: { current_user: user, project_ids: :any } }
   end
 
-  it 'deletes a project with all nested objects' do
-    project = create :project, :repository
-    issue = create :issue, project: project
-    milestone = create :milestone, project: project
-    note = create :note, project: project
-    merge_request = create :merge_request, target_project: project, source_project: project
-
-    ensure_elasticsearch_index!
-
-    ## All database objects + data from repository. The absolute value does not matter
-    expect(Project.elastic_search('*', **search_options).records).to include(project)
-    expect(Issue.elastic_search('*', **search_options).records).to include(issue)
-    expect(Milestone.elastic_search('*', **search_options).records).to include(milestone)
-    expect(Note.elastic_search('*', **search_options).records).to include(note)
-    expect(MergeRequest.elastic_search('*', **search_options).records).to include(merge_request)
-
-    subject.perform(project.id, project.es_id)
-    ensure_elasticsearch_index!
-
-    expect(Project.elastic_search('*', **search_options).total_count).to be(0)
-    expect(Issue.elastic_search('*', **search_options).total_count).to be(0)
-    expect(Milestone.elastic_search('*', **search_options).total_count).to be(0)
-    expect(Note.elastic_search('*', **search_options).total_count).to be(0)
-    expect(MergeRequest.elastic_search('*', **search_options).total_count).to be(0)
-
-    # verify that entire index is empty
-    # searches use joins on the parent record (project)
-    # and the previous queries will not find data left in the index
-    helper = Gitlab::Elastic::Helper.default
-
-    expect(helper.documents_count).to be(0)
+  shared_examples 'delete project and objects' do
+    it 'deletes a project with all nested objects' do
+      project = create :project, :repository
+      issue = create :issue, project: project
+      milestone = create :milestone, project: project
+      note = create :note, project: project
+      merge_request = create :merge_request, target_project: project, source_project: project
+
+      ensure_elasticsearch_index!
+
+      ## All database objects + data from repository. The absolute value does not matter
+      expect(Project.elastic_search('*', **search_options).records).to include(project)
+      expect(Issue.elastic_search('*', **search_options).records).to include(issue)
+      expect(Milestone.elastic_search('*', **search_options).records).to include(milestone)
+      expect(Note.elastic_search('*', **search_options).records).to include(note)
+      expect(MergeRequest.elastic_search('*', **search_options).records).to include(merge_request)
+
+      subject.perform(project.id, project.es_id)
+
+      ensure_elasticsearch_index!
+
+      expect(Project.elastic_search('*', **search_options).total_count).to be(0)
+      expect(Issue.elastic_search('*', **search_options).total_count).to be(0)
+      expect(Milestone.elastic_search('*', **search_options).total_count).to be(0)
+      expect(Note.elastic_search('*', **search_options).total_count).to be(0)
+      expect(MergeRequest.elastic_search('*', **search_options).total_count).to be(0)
+
+      # verify that entire index is empty
+      # searches use joins on the parent record (project)
+      # and the previous queries will not find data left in the index
+      helper = Gitlab::Elastic::Helper.default
+
+      expect(helper.documents_count).to be(0)
+    end
+  end
+
+  context 'migration has finished' do
+    let(:migration_has_finished) { true }
+
+    include_examples 'delete project and objects'
+  end
+
+  context 'migration has not finished' do
+    let(:migration_has_finished) { false }
+
+    include_examples 'delete project and objects'
   end
 end

locale/gitlab.pot

@@ -10398,6 +10398,9 @@ msgstr ""
 msgid "Elasticsearch HTTP client timeout value in seconds."
 msgstr ""
 
+msgid "Elasticsearch indexing"
+msgstr ""
+
 msgid "Elasticsearch indexing restrictions"
 msgstr ""
 
@@ -20371,6 +20374,9 @@ msgstr ""
 msgid "Pause"
 msgstr ""
 
+msgid "Pause Elasticsearch indexing"
+msgstr ""
+
 msgid "Pause replication"
 msgstr ""
 
@@ -23196,7 +23202,7 @@ msgstr ""
 msgid "Regulate approvals by authors/committers. Affects all projects."
 msgstr ""
 
-msgid "Reindexing status"
+msgid "Reindexing Status: %{status}"
 msgstr ""
 
 msgid "Rejected (closed)"
@@ -26687,9 +26693,6 @@ msgstr ""
 msgid "State your message to activate"
 msgstr ""
 
-msgid "State: %{last_reindexing_task_state}"
-msgstr ""
-
 msgid "Static Application Security Testing (SAST)"
 msgstr ""
 
@@ -28212,6 +28215,9 @@ msgstr ""
 msgid "There are no variables yet."
 msgstr ""
 
+msgid "There are pending advanced search migrations. Indexing must remain paused until the migrations are completed."
+msgstr ""
+
 msgid "There are running deployments on the environment. Please retry later."
 msgstr ""
 
@@ -30138,7 +30144,7 @@ msgstr ""
 msgid "Until"
 msgstr ""
 
-msgid "Unused, previous index '%{index_name}' will be deleted after %{time} automatically."
+msgid "Unused, previous indices: %{index_names} will be deleted after %{time} automatically."
 msgstr ""
 
 msgid "Unverified"