Merge branch '2087-make-num-shards-configurable' into 'master'

Make the number of ES shards and replicas configurable

Closes #2087

See merge request gitlab-org/gitlab-ee!12713

db/migrate/20190515125613_add_application_settings_elasticsearch_shards.rb

+# frozen_string_literal: true
+
+class AddApplicationSettingsElasticsearchShards < ActiveRecord::Migration[5.1]
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings, :elasticsearch_shards, :integer, null: false, default: 5
+    add_column :application_settings, :elasticsearch_replicas, :integer, null: false, default: 1
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20190506135400) do
+ActiveRecord::Schema.define(version: 20190515125613) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -221,6 +221,8 @@ ActiveRecord::Schema.define(version: 20190506135400) do
     t.string "lets_encrypt_notification_email"
     t.boolean "lets_encrypt_terms_of_service_accepted", default: false, null: false
     t.string "geo_node_allowed_ips", default: "0.0.0.0/0, ::/0"
+    t.integer "elasticsearch_shards", default: 5, null: false
+    t.integer "elasticsearch_replicas", default: 1, null: false
     t.index ["custom_project_templates_group_id"], name: "index_application_settings_on_custom_project_templates_group_id", using: :btree
     t.index ["file_template_project_id"], name: "index_application_settings_on_file_template_project_id", using: :btree
     t.index ["usage_stats_set_by_user_id"], name: "index_application_settings_on_usage_stats_set_by_user_id", using: :btree

doc/integration/elasticsearch.md

@@ -131,6 +131,8 @@ The following Elasticsearch settings are available:
 | `Use the new repository indexer (beta)` | Perform repository indexing using [GitLab Elasticsearch Indexer](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). |
 | `Search with Elasticsearch enabled` | Enables/disables using Elasticsearch in search. |
 | `URL`                              | The URL to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://host1, https://host2:9200"). If your Elasticsearch instance is password protected, pass the `username:password` in the URL (e.g., `http://<username>:<password>@<elastic_host>:9200/`). |
+| `Number of Elasticsearch shards` | Elasticsearch indexes are split into multiple shards for performance reasons. In general, larger indexes need to have more shards. Changes to this value do not take effect until the index is recreated. You can read more about tradeoffs in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#create-index-settings) |
+| `Number of Elasticsearch replicas` | Each Elasticsearch shard can have a number of replicas. These are a complete copy of the shard, and can provide increased query performance or resilience against hardware failure. Increasing this value will greatly increase total disk space required by the index. |
 | `Limit namespaces and projects that can be indexed` | Enabling this will allow you to select namespaces and projects to index. All other namespaces and projects will use database search instead. Please note that if you enable this option but do not select any namespaces or projects, none will be indexed. [Read more below](#limiting-namespaces-and-projects).
 | `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) or [AWS EC2 Instance Profile Credentials](http://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli). The policies must be configured to allow `es:*` actions. |
 | `AWS Region` | The AWS region your Elasticsearch service is located in. |

ee/app/helpers/ee/application_settings_helper.rb

@@ -27,7 +27,9 @@ module EE
         :elasticsearch_aws_secret_access_key,
         :elasticsearch_experimental_indexer,
         :elasticsearch_indexing,
+        :elasticsearch_replicas,
         :elasticsearch_search,
+        :elasticsearch_shards,
         :elasticsearch_url,
         :elasticsearch_limit_indexing,
         :elasticsearch_namespace_ids,

ee/app/models/concerns/elastic/application_search.rb

@@ -3,6 +3,21 @@ module Elastic
   module ApplicationSearch
     extend ActiveSupport::Concern
 
+    # Defer evaluation from class-definition time to index-creation time
+    class AsJSON
+      def initialize(&blk)
+        @blk = blk
+      end
+
+      def call
+        @blk.call
+      end
+
+      def as_json(*args, &blk)
+        call
+      end
+    end
+
     included do
       include Elasticsearch::Model
 
@@ -13,6 +28,8 @@ module Elastic
 
       settings \
         index: {
+          number_of_shards: AsJSON.new { Gitlab::CurrentSettings.elasticsearch_shards },
+          number_of_replicas: AsJSON.new { Gitlab::CurrentSettings.elasticsearch_replicas },
           codec: 'best_compression',
           analysis: {
             analyzer: {

ee/app/models/ee/application_setting.rb

@@ -39,6 +39,14 @@ module EE
                 presence: true,
                 numericality: { only_integer: true, greater_than_or_equal_to: 0 }
 
+      validates :elasticsearch_shards,
+                presence: true,
+                numericality: { only_integer: true, greater_than: 0 }
+
+      validates :elasticsearch_replicas,
+                presence: true,
+                numericality: { only_integer: true, greater_than: 0 }
+
       validates :elasticsearch_url,
                 presence: { message: "can't be blank when indexing is enabled" },
                 if: ->(setting) { setting.elasticsearch_indexing? }
@@ -69,6 +77,8 @@ module EE
           allow_group_owners_to_manage_ldap: true,
           elasticsearch_aws: false,
           elasticsearch_aws_region: ENV['ELASTIC_REGION'] || 'us-east-1',
+          elasticsearch_replicas: 1,
+          elasticsearch_shards: 5,
           elasticsearch_url: ENV['ELASTIC_URL'] || 'http://localhost:9200',
           email_additional_text: nil,
           mirror_capacity_threshold: Settings.gitlab['mirror_capacity_threshold'],

ee/app/views/admin/application_settings/_elasticsearch_form.html.haml

 - return unless License.feature_available?(:elastic_search)
 
+- recreate_index_url = help_page_url('integration/elasticsearch.md')
+- recreate_index_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: recreate_index_url }
+- recreate_index_text = s_("Changes won't take place until the index is %{link_start}recreated%{link_end}.").html_safe % { link_start: recreate_index_link_start, link_end: '</a>'.html_safe }
+
 %section.settings.as-elasticsearch.no-animate#js-elasticsearch-settings{ class: ('expanded' if expanded) }
   .settings-header
     %h4
@@ -42,6 +46,22 @@
             .form-text.text-muted
               The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201").
 
+          .form-group
+            = f.label :elasticsearch_shards, _('Number of Elasticsearch shards'), class: 'label-bold'
+            = f.number_field :elasticsearch_shards, value: @application_setting.elasticsearch_shards, class: 'form-control'
+            .form-text.text-muted
+              = _('How many shards to split the Elasticsearch index over.')
+              = recreate_index_text
+
+          .form-group
+            = f.label :elasticsearch_replicas, _('Number of Elasticsearch replicas'), class: 'label-bold'
+            = f.number_field :elasticsearch_replicas, value: @application_setting.elasticsearch_replicas, class: 'form-control'
+            .form-text.text-muted
+              = _('How many replicas each Elasticsearch shard has.')
+              = recreate_index_text
+
+        .sub-section
+          %h4= _('Elasticsearch indexing restrictions')
           .form-group
             .form-check
               = f.check_box :elasticsearch_limit_indexing, class: 'form-check-input js-limit-checkbox'

ee/changelogs/unreleased/2087-make-num-shards-configurable.yml

+---
+title: Make the number of Elasticsearch shards and replicas configurable
+merge_request: 12713
+author:
+type: added

ee/spec/features/admin/admin_settings_spec.rb

@@ -52,14 +52,23 @@ describe 'Admin updates EE-only settings' do
       visit integrations_admin_application_settings_path
     end
 
-    it 'Enable elastic search indexing' do
+    it 'changes elasticsearch settings' do
       page.within('.as-elasticsearch') do
         check 'Elasticsearch indexing'
+        check 'Search with Elasticsearch enabled'
+        fill_in 'Number of Elasticsearch shards', with: '120'
+        fill_in 'Number of Elasticsearch replicas', with: '2'
+
         click_button 'Save changes'
       end
 
-      expect(Gitlab::CurrentSettings.elasticsearch_indexing).to be_truthy
-      expect(page).to have_content "Application settings saved successfully"
+      aggregate_failures do
+        expect(Gitlab::CurrentSettings.elasticsearch_indexing).to be_truthy
+        expect(Gitlab::CurrentSettings.elasticsearch_search).to be_truthy
+        expect(Gitlab::CurrentSettings.elasticsearch_shards).to eq(120)
+        expect(Gitlab::CurrentSettings.elasticsearch_replicas).to eq(2)
+        expect(page).to have_content "Application settings saved successfully"
+      end
     end
 
     it 'Allows limiting projects and namespaces to index', :js do

ee/spec/models/application_setting_spec.rb

@@ -27,6 +27,18 @@ describe ApplicationSetting do
     it { is_expected.not_to allow_value(subject.mirror_max_capacity + 1).for(:mirror_capacity_threshold) }
     it { is_expected.to allow_value(nil).for(:custom_project_templates_group_id) }
 
+    it { is_expected.to allow_value(10).for(:elasticsearch_shards) }
+    it { is_expected.not_to allow_value(nil).for(:elasticsearch_shards) }
+    it { is_expected.not_to allow_value(0).for(:elasticsearch_shards) }
+    it { is_expected.not_to allow_value(1.1).for(:elasticsearch_shards) }
+    it { is_expected.not_to allow_value(-1).for(:elasticsearch_shards) }
+
+    it { is_expected.to allow_value(10).for(:elasticsearch_replicas) }
+    it { is_expected.not_to allow_value(nil).for(:elasticsearch_replicas) }
+    it { is_expected.not_to allow_value(0).for(:elasticsearch_replicas) }
+    it { is_expected.not_to allow_value(1.1).for(:elasticsearch_replicas) }
+    it { is_expected.not_to allow_value(-1).for(:elasticsearch_replicas) }
+
     describe 'when additional email text is enabled' do
       before do
         stub_licensed_features(email_additional_text: true)

locale/gitlab.pot

@@ -2254,6 +2254,9 @@ msgstr ""
 msgid "Changes the title to \"%{title_param}\"."
 msgstr ""
 
+msgid "Changes won't take place until the index is %{link_start}recreated%{link_end}."
+msgstr ""
+
 msgid "Charts"
 msgstr ""
 
@@ -4364,6 +4367,9 @@ msgstr ""
 msgid "Elasticsearch"
 msgstr ""
 
+msgid "Elasticsearch indexing restrictions"
+msgstr ""
+
 msgid "Elasticsearch indexing started"
 msgstr ""
 
@@ -6492,6 +6498,12 @@ msgstr ""
 msgid "Housekeeping, export, path, transfer, remove, archive."
 msgstr ""
 
+msgid "How many replicas each Elasticsearch shard has."
+msgstr ""
+
+msgid "How many shards to split the Elasticsearch index over."
+msgstr ""
+
 msgid "However, you are already a member of this %{member_source}. Sign in using a different account to accept the invitation."
 msgstr ""
 
@@ -8570,6 +8582,12 @@ msgstr ""
 msgid "November"
 msgstr ""
 
+msgid "Number of Elasticsearch replicas"
+msgstr ""
+
+msgid "Number of Elasticsearch shards"
+msgstr ""
+
 msgid "OK"
 msgstr ""