Merge branch 'ak/official-helm-chart' into 'master'

Move to supported Elastic helm charts

See merge request gitlab-org/gitlab!30528

app/assets/javascripts/clusters/components/applications.vue

@@ -662,7 +662,7 @@ Crossplane runs inside your Kubernetes cluster and supports secure connectivity
         :uninstall-successful="applications.elastic_stack.uninstallSuccessful"
         :uninstall-failed="applications.elastic_stack.uninstallFailed"
         :disabled="!helmInstalled"
-        title-link="https://github.com/helm/charts/tree/master/stable/elastic-stack"
+        title-link="https://gitlab.com/gitlab-org/charts/elastic-stack"
       >
         <div slot="description">
           <p>

app/models/clusters/applications/elastic_stack.rb

@@ -3,7 +3,7 @@
 module Clusters
   module Applications
     class ElasticStack < ApplicationRecord
-      VERSION = '2.0.0'
+      VERSION = '3.0.0'
 
       ELASTICSEARCH_PORT = 9200
 
@@ -18,7 +18,11 @@ module Clusters
       default_value_for :version, VERSION
 
       def chart
-        'stable/elastic-stack'
+        'elastic-stack/elastic-stack'
+      end
+
+      def repository
+        'https://charts.gitlab.io'
       end
 
       def install_command
@@ -27,8 +31,9 @@ module Clusters
           version: VERSION,
           rbac: cluster.platform_kubernetes_rbac?,
           chart: chart,
+          repository: repository,
           files: files,
-          preinstall: migrate_to_2_script,
+          preinstall: migrate_to_3_script,
           postinstall: post_install_script
         )
       end
@@ -50,7 +55,7 @@ module Clusters
         strong_memoize(:elasticsearch_client) do
           next unless kube_client
 
-          proxy_url = kube_client.proxy_url('service', 'elastic-stack-elasticsearch-client', ::Clusters::Applications::ElasticStack::ELASTICSEARCH_PORT, Gitlab::Kubernetes::Helm::NAMESPACE)
+          proxy_url = kube_client.proxy_url('service', service_name, ::Clusters::Applications::ElasticStack::ELASTICSEARCH_PORT, Gitlab::Kubernetes::Helm::NAMESPACE)
 
           Elasticsearch::Client.new(url: proxy_url) do |faraday|
             # ensures headers containing auth data are appended to original client options
@@ -70,21 +75,33 @@ module Clusters
         end
       end
 
-      def filebeat7?
+      def chart_above_v2?
         Gem::Version.new(version) >= Gem::Version.new('2.0.0')
       end
 
+      def chart_above_v3?
+        Gem::Version.new(version) >= Gem::Version.new('3.0.0')
+      end
+
       private
 
+      def service_name
+        chart_above_v3? ? 'elastic-stack-elasticsearch-master' : 'elastic-stack-elasticsearch-client'
+      end
+
+      def pvc_selector
+        chart_above_v3? ? "app=elastic-stack-elasticsearch-master" : "release=elastic-stack"
+      end
+
       def post_install_script
         [
-          "timeout -t60 sh /data/helm/elastic-stack/config/wait-for-elasticsearch.sh http://elastic-stack-elasticsearch-client:9200"
+          "timeout -t60 sh /data/helm/elastic-stack/config/wait-for-elasticsearch.sh http://elastic-stack-elasticsearch-master:9200"
         ]
       end
 
       def post_delete_script
         [
-          Gitlab::Kubernetes::KubectlCmd.delete("pvc", "--selector", "release=elastic-stack")
+          Gitlab::Kubernetes::KubectlCmd.delete("pvc", "--selector", pvc_selector, "--namespace", Gitlab::Kubernetes::Helm::NAMESPACE)
         ]
       end
 
@@ -92,25 +109,19 @@ module Clusters
         cluster&.kubeclient&.core_client
       end
 
-      def migrate_to_2_script
-        # Updating the chart to 2.0.0 includes an update of the filebeat chart from 1.7.0 to 3.1.1 https://github.com/helm/charts/pull/21640
-        # This includes the following commit that changes labels on the filebeat deployment https://github.com/helm/charts/commit/9b009170686c6f4b202c36ceb1da4bb9ba15ddd0
-        # Unfortunately those fields are immutable, and we can't use `helm upgrade` to change them. We first have to delete the associated filebeat resources
-        # The following pre-install command runs before updating to 2.0.0 and sets filebeat.enable=false so the filebeat deployment is deleted.
-        # Then the main install command re-creates them properly
-        if updating? && !filebeat7?
-          [
-            Gitlab::Kubernetes::Helm::InstallCommand.new(
-              name: 'elastic-stack',
-              version: version,
-              rbac: cluster.platform_kubernetes_rbac?,
-              chart: chart,
-              files: files
-            ).install_command + ' --set filebeat.enabled\\=false'
-          ]
-        else
-          []
-        end
+      def migrate_to_3_script
+        return [] if !updating? || chart_above_v3?
+
+        # Chart version 3.0.0 moves to our own chart at https://gitlab.com/gitlab-org/charts/elastic-stack
+        # and is not compatible with pre-existing resources. We first remove them.
+        [
+          Gitlab::Kubernetes::Helm::DeleteCommand.new(
+            name: 'elastic-stack',
+            rbac: cluster.platform_kubernetes_rbac?,
+            files: files
+          ).delete_command,
+          Gitlab::Kubernetes::KubectlCmd.delete("pvc", "--selector", "release=elastic-stack", "--namespace", Gitlab::Kubernetes::Helm::NAMESPACE)
+        ]
       end
     end
   end

app/services/pod_logs/elasticsearch_service.rb

@@ -70,7 +70,7 @@ module PodLogs
       client = cluster&.application_elastic_stack&.elasticsearch_client
       return error(_('Unable to connect to Elasticsearch')) unless client
 
-      filebeat7 = cluster.application_elastic_stack.filebeat7?
+      chart_above_v2 = cluster.application_elastic_stack.chart_above_v2?
 
       response = ::Gitlab::Elasticsearch::Logs::Lines.new(client).pod_logs(
         namespace,
@@ -80,7 +80,7 @@ module PodLogs
         start_time: result[:start_time],
         end_time: result[:end_time],
         cursor: result[:cursor],
-        filebeat7: filebeat7
+        chart_above_v2: chart_above_v2
       )
 
       result.merge!(response)

changelogs/unreleased/ak-official-helm-chart.yml

+---
+title: Move to supported Elastic helm charts
+merge_request: 30528
+author:
+type: changed

doc/user/clusters/applications.md

@@ -505,20 +505,23 @@ Log data is automatically deleted after 30 days using [Curator](https://www.elas
 
 To enable log shipping:
 
+1. Ensure your cluster contains at least 3 nodes of instance types larger than
+   `f1-micro`, `g1-small`, or `n1-standard-1`.
 1. Navigate to **{cloud-gear}** **Operations > Kubernetes**.
 1. In **Kubernetes Cluster**, select a cluster.
 1. In the **Applications** section, find **Elastic Stack** and click **Install**.
 
 NOTE: **Note:**
-The [`stable/elastic-stack`](https://github.com/helm/charts/tree/master/stable/elastic-stack)
+The [`gitlab/elastic-stack`](https://gitlab.com/gitlab-org/charts/elastic-stack)
 chart is used to install this application with a
 [`values.yaml`](https://gitlab.com/gitlab-org/gitlab/blob/master/vendor/elastic_stack/values.yaml)
 file.
 
 NOTE: **Note:**
-The chart will deploy 5 Elasticsearch nodes: 2 masters, 2 data and 1 client node,
-with resource requests totalling 0.125 CPU and 4.5GB RAM. Each data node requests 1.5GB of memory,
-which makes it incompatible with clusters of `f1-micro` and `g1-small` instance types.
+The chart deploys 3 identical Elasticsearch pods which can't be colocated, and each
+require 1 CPU and 2 GB of RAM, making them incompatible with clusters containing
+fewer than 3 nodes or consisting of `f1-micro`, `g1-small`, `n1-standard-1`, or
+`*-highcpu-2` instance types.
 
 NOTE: **Note:**
 The Elastic Stack cluster application is intended as a log aggregation solution and is not related to our
@@ -542,20 +545,23 @@ logstash:
 
 kibana:
   enabled: true
-  env:
-    ELASTICSEARCH_HOSTS: http://elastic-stack-elasticsearch-client.gitlab-managed-apps.svc.cluster.local:9200
+  elasticsearchHosts: http://elastic-stack-elasticsearch-master.gitlab-managed-apps.svc.cluster.local:9200
+
+elasticseach-curator:
+  enabled: false
 ```
 
 Then install it on your cluster:
 
 ```shell
-helm install --name kibana stable/elastic-stack --values kibana.yml
+helm repo add gitlab https://charts.gitlab.io
+helm install --name kibana gitlab/elastic-stack --values kibana.yml
 ```
 
 To access Kibana, forward the port to your local machine:
 
 ```shell
-kubectl port-forward svc/kibana 5601:443
+kubectl port-forward svc/kibana 5601:5601
 ```
 
 Then, you can visit Kibana at `http://localhost:5601`.
@@ -1069,7 +1075,7 @@ You can check the default [`values.yaml`](https://gitlab.com/gitlab-org/gitlab/-
 You can customize the installation of Elastic Stack by defining
 `.gitlab/managed-apps/elastic-stack/values.yaml` file in your cluster
 management project. Refer to the
-[chart](https://github.com/helm/charts/blob/master/stable/elastic-stack/values.yaml) for the
+[chart](https://gitlab.com/gitlab-org/charts/elastic-stack) for the
 available configuration options.
 
 NOTE: **Note:**

ee/app/services/security/waf_anomaly_summary_service.rb

@@ -21,8 +21,13 @@ module Security
       aggregate_results = elasticsearch_client.msearch(body: body)
       nginx_results, modsec_results = aggregate_results['responses']
 
-      nginx_total_requests = nginx_results.dig('hits', 'total').to_f
-      modsec_total_requests = modsec_results.dig('hits', 'total').to_f
+      if chart_above_v3?
+        nginx_total_requests = nginx_results.dig('hits', 'total', 'value').to_f
+        modsec_total_requests = modsec_results.dig('hits', 'total', 'value').to_f
+      else
+        nginx_total_requests = nginx_results.dig('hits', 'total').to_f
+        modsec_total_requests = modsec_results.dig('hits', 'total').to_f
+      end
 
       anomalous_traffic_count = nginx_total_requests.zero? ? 0 : (modsec_total_requests / nginx_total_requests).round(2)
 
@@ -41,11 +46,19 @@ module Security
     end
 
     def elasticsearch_client
-      @client ||= @environment.deployment_platform&.cluster&.application_elastic_stack&.elasticsearch_client
+      @elasticsearch_client ||= application_elastic_stack&.elasticsearch_client
     end
 
     private
 
+    def application_elastic_stack
+      @application_elastic_stack ||= @environment.deployment_platform&.cluster&.application_elastic_stack
+    end
+
+    def chart_above_v3?
+      application_elastic_stack.chart_above_v3?
+    end
+
     def body
       [
         { index: indices },
@@ -68,7 +81,7 @@ module Security
     # indices
     def indices
       (@from.to_date..@to.to_date).map do |day|
-        "filebeat-*-#{day.strftime('%Y.%m.%d')}"
+        chart_above_v3? ? "filebeat-*-#{day.strftime('%Y.%m.%d')}-*" : "filebeat-*-#{day.strftime('%Y.%m.%d')}"
       end
     end
 

ee/spec/controllers/projects/security/waf_anomalies_controller_spec.rb

@@ -24,6 +24,7 @@ describe Projects::Security::WafAnomaliesController do
 
       allow_next_instance_of(::Security::WafAnomalySummaryService) do |instance|
         allow(instance).to receive(:elasticsearch_client).at_most(3).times { es_client }
+        allow(instance).to receive(:chart_above_v3?) { true }
       end
     end
 

ee/spec/services/security/waf_anomaly_summary_service_spec.rb

@@ -9,13 +9,14 @@ describe Security::WafAnomalySummaryService do
   end
 
   let(:es_client) { double(Elasticsearch::Client) }
+  let(:chart_above_v3) { true }
 
   let(:empty_response) do
     {
       'took' => 40,
       'timed_out' => false,
       '_shards' => { 'total' => 11, 'successful' => 11, 'skipped' => 0, 'failed' => 0 },
-      'hits' => { 'total' => 0, 'max_score' => 0.0, 'hits' => [] },
+      'hits' => { 'total' => { 'value' => 0, 'relation' => 'gte' }, 'max_score' => 0.0, 'hits' => [] },
       'aggregations' => {
         'counts' => {
           'buckets' => []
@@ -27,7 +28,7 @@ describe Security::WafAnomalySummaryService do
 
   let(:nginx_response) do
     empty_response.deep_merge(
-      'hits' => { 'total' => 3 },
+      'hits' => { 'total' => { 'value' => 3 } },
       'aggregations' => {
         'counts' => {
           'buckets' => [
@@ -42,6 +43,38 @@ describe Security::WafAnomalySummaryService do
   end
 
   let(:modsec_response) do
+    empty_response.deep_merge(
+      'hits' => { 'total' => { 'value' => 1 } },
+      'aggregations' => {
+        'counts' => {
+          'buckets' => [
+            { 'key_as_string' => '2019-12-04T23:00:00.000Z', 'key' => 1575500400000, 'doc_count' => 0 },
+            { 'key_as_string' => '2019-12-05T00:00:00.000Z', 'key' => 1575504000000, 'doc_count' => 0 },
+            { 'key_as_string' => '2019-12-05T01:00:00.000Z', 'key' => 1575507600000, 'doc_count' => 0 },
+            { 'key_as_string' => '2019-12-05T08:00:00.000Z', 'key' => 1575532800000, 'doc_count' => 1 }
+          ]
+        }
+      }
+    )
+  end
+
+  let(:nginx_response_es6) do
+    empty_response.deep_merge(
+      'hits' => { 'total' => 3 },
+      'aggregations' => {
+        'counts' => {
+          'buckets' => [
+            { 'key_as_string' => '2020-02-14T23:00:00.000Z', 'key' => 1575500400000, 'doc_count' => 1 },
+            { 'key_as_string' => '2020-02-15T00:00:00.000Z', 'key' => 1575504000000, 'doc_count' => 0 },
+            { 'key_as_string' => '2020-02-15T01:00:00.000Z', 'key' => 1575507600000, 'doc_count' => 0 },
+            { 'key_as_string' => '2020-02-15T08:00:00.000Z', 'key' => 1575532800000, 'doc_count' => 2 }
+          ]
+        }
+      }
+    )
+  end
+
+  let(:modsec_response_es6) do
     empty_response.deep_merge(
       'hits' => { 'total' => 1 },
       'aggregations' => {
@@ -99,6 +132,9 @@ describe Security::WafAnomalySummaryService do
         allow(environment.deployment_platform.cluster).to receive_message_chain(
           :application_elastic_stack, :elasticsearch_client
         ) { es_client }
+        allow(environment.deployment_platform.cluster).to receive_message_chain(
+          :application_elastic_stack, :chart_above_v3?
+        ) { chart_above_v3 }
       end
 
       context 'no requests' do
@@ -142,11 +178,28 @@ describe Security::WafAnomalySummaryService do
           expect(results.fetch(:anomalous_traffic)).to eq 0.33
         end
       end
+
+      context 'with legacy es6 cluster' do
+        let(:chart_above_v3) { false }
+
+        let(:nginx_results) { nginx_response_es6 }
+        let(:modsec_results) { modsec_response_es6 }
+
+        it 'returns results', :aggregate_failures do
+          results = subject.execute
+
+          expect(results.fetch(:status)).to eq :success
+          expect(results.fetch(:interval)).to eq 'day'
+          expect(results.fetch(:total_traffic)).to eq 3
+          expect(results.fetch(:anomalous_traffic)).to eq 0.33
+        end
+      end
     end
 
     context 'with review app' do
       it 'resolves transaction_id from external_url' do
         allow(subject).to receive(:elasticsearch_client) { es_client }
+        allow(subject).to receive(:chart_above_v3?) { chart_above_v3 }
 
         expect(es_client).to receive(:msearch).with(
           body: array_including(
@@ -182,6 +235,7 @@ describe Security::WafAnomalySummaryService do
         )
 
         allow(subject).to receive(:elasticsearch_client) { es_client }
+        allow(subject).to receive(:chart_above_v3?) { chart_above_v3 }
 
         expect(es_client).to receive(:msearch).with(
           body: array_including(
@@ -218,6 +272,7 @@ describe Security::WafAnomalySummaryService do
         )
 
         allow(subject).to receive(:elasticsearch_client) { es_client }
+        allow(subject).to receive(:chart_above_v3?) { chart_above_v3 }
 
         expect(es_client).to receive(:msearch).with(
           body: array_including(

lib/gitlab/elasticsearch/logs/lines.rb

@@ -13,7 +13,7 @@ module Gitlab
           @client = client
         end
 
-        def pod_logs(namespace, pod_name: nil, container_name: nil, search: nil, start_time: nil, end_time: nil, cursor: nil, filebeat7: true)
+        def pod_logs(namespace, pod_name: nil, container_name: nil, search: nil, start_time: nil, end_time: nil, cursor: nil, chart_above_v2: true)
           query = { bool: { must: [] } }.tap do |q|
             filter_pod_name(q, pod_name)
             filter_namespace(q, namespace)
@@ -22,7 +22,7 @@ module Gitlab
             filter_times(q, start_time, end_time)
           end
 
-          body = build_body(query, cursor, filebeat7)
+          body = build_body(query, cursor, chart_above_v2)
           response = @client.search body: body
 
           format_response(response)
@@ -30,8 +30,8 @@ module Gitlab
 
         private
 
-        def build_body(query, cursor = nil, filebeat7 = true)
-          offset_field = filebeat7 ? "log.offset" : "offset"
+        def build_body(query, cursor = nil, chart_above_v2 = true)
+          offset_field = chart_above_v2 ? "log.offset" : "offset"
           body = {
             query: query,
             # reverse order so we can query N-most recent records

lib/gitlab/kubernetes/helm/delete_command.rb

@@ -36,8 +36,6 @@ module Gitlab
           @rbac
         end
 
-        private
-
         def delete_command
           command = ['helm', 'delete', '--purge', name] + tls_flags_if_remote_tiller
 

lib/gitlab/kubernetes/helm/install_command.rb

@@ -37,6 +37,8 @@ module Gitlab
           @rbac
         end
 
+        private
+
         # Uses `helm upgrade --install` which means we can use this for both
         # installation and uprade of applications
         def install_command
@@ -53,8 +55,6 @@ module Gitlab
           command.shelljoin
         end
 
-        private
-
         def install_flag
           ['--install']
         end

spec/lib/gitlab/elasticsearch/logs/lines_spec.rb

@@ -90,7 +90,7 @@ describe Gitlab::Elasticsearch::Logs::Lines do
     it 'can search on filebeat 6' do
       expect(client).to receive(:search).with(body: a_hash_equal_to_json(body_with_filebeat_6)).and_return(es_response)
 
-      result = subject.pod_logs(namespace, pod_name: pod_name, filebeat7: false)
+      result = subject.pod_logs(namespace, pod_name: pod_name, chart_above_v2: false)
       expect(result).to eq(logs: [es_message_4, es_message_3, es_message_2, es_message_1], cursor: cursor)
     end
   end

spec/lib/gitlab/kubernetes/helm/delete_command_spec.rb

@@ -75,4 +75,10 @@ describe Gitlab::Kubernetes::Helm::DeleteCommand do
   it_behaves_like 'helm command' do
     let(:command) { delete_command }
   end
+
+  describe '#delete_command' do
+    it 'deletes the release' do
+      expect(subject.delete_command).to eq('helm delete --purge app-name')
+    end
+  end
 end

spec/models/clusters/applications/elastic_stack_spec.rb

@@ -19,8 +19,9 @@ describe Clusters::Applications::ElasticStack do
 
     it 'is initialized with elastic stack arguments' do
       expect(subject.name).to eq('elastic-stack')
-      expect(subject.chart).to eq('stable/elastic-stack')
-      expect(subject.version).to eq('2.0.0')
+      expect(subject.chart).to eq('elastic-stack/elastic-stack')
+      expect(subject.version).to eq('3.0.0')
+      expect(subject.repository).to eq('https://charts.gitlab.io')
       expect(subject).to be_rbac
       expect(subject.files).to eq(elastic_stack.files)
       expect(subject.preinstall).to be_empty
@@ -42,7 +43,19 @@ describe Clusters::Applications::ElasticStack do
 
       it 'includes a preinstall script' do
         expect(subject.preinstall).not_to be_empty
-        expect(subject.preinstall.first).to include("filebeat.enable")
+        expect(subject.preinstall.first).to include("delete")
+      end
+    end
+
+    context 'on versions older than 3' do
+      before do
+        elastic_stack.status = elastic_stack.status_states[:updating]
+        elastic_stack.version = "2.9.0"
+      end
+
+      it 'includes a preinstall script' do
+        expect(subject.preinstall).not_to be_empty
+        expect(subject.preinstall.first).to include("delete")
       end
     end
 
@@ -50,11 +63,47 @@ describe Clusters::Applications::ElasticStack do
       let(:elastic_stack) { create(:clusters_applications_elastic_stack, :errored, version: '0.0.1') }
 
       it 'is initialized with the locked version' do
-        expect(subject.version).to eq('2.0.0')
+        expect(subject.version).to eq('3.0.0')
       end
     end
   end
 
+  describe '#chart_above_v2?' do
+    let(:elastic_stack) { create(:clusters_applications_elastic_stack, version: version) }
+
+    subject { elastic_stack.chart_above_v2? }
+
+    context 'on v1.9.0' do
+      let(:version) { '1.9.0' }
+
+      it { is_expected.to be_falsy }
+    end
+
+    context 'on v2.0.0' do
+      let(:version) { '2.0.0' }
+
+      it { is_expected.to be_truthy }
+    end
+  end
+
+  describe '#chart_above_v3?' do
+    let(:elastic_stack) { create(:clusters_applications_elastic_stack, version: version) }
+
+    subject { elastic_stack.chart_above_v3? }
+
+    context 'on v1.9.0' do
+      let(:version) { '1.9.0' }
+
+      it { is_expected.to be_falsy }
+    end
+
+    context 'on v3.0.0' do
+      let(:version) { '3.0.0' }
+
+      it { is_expected.to be_truthy }
+    end
+  end
+
   describe '#uninstall_command' do
     let!(:elastic_stack) { create(:clusters_applications_elastic_stack) }
 
@@ -70,7 +119,7 @@ describe Clusters::Applications::ElasticStack do
 
     it 'specifies a post delete command to remove custom resource definitions' do
       expect(subject.postdelete).to eq([
-        'kubectl delete pvc --selector release\\=elastic-stack'
+        'kubectl delete pvc --selector app\\=elastic-stack-elasticsearch-master --namespace gitlab-managed-apps'
       ])
     end
   end

spec/services/pod_logs/elasticsearch_service_spec.rb

@@ -255,7 +255,7 @@ describe ::PodLogs::ElasticsearchService do
         .and_return(Elasticsearch::Transport::Client.new)
       allow_any_instance_of(::Gitlab::Elasticsearch::Logs::Lines)
         .to receive(:pod_logs)
-        .with(namespace, pod_name: pod_name, container_name: container_name, search: search, start_time: start_time, end_time: end_time, cursor: cursor, filebeat7: true)
+        .with(namespace, pod_name: pod_name, container_name: container_name, search: search, start_time: start_time, end_time: end_time, cursor: cursor, chart_above_v2: true)
         .and_return({ logs: expected_logs, cursor: expected_cursor })
 
       result = subject.send(:pod_logs, result_arg)

vendor/elastic_stack/values.yaml

+# Default values for elastic-stack.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
 elasticsearch:
   enabled: true
-  cluster:
-    env:
-      MINIMUM_MASTER_NODES: "1"
-  master:
-    replicas: 2
-  client:
-    replicas: 1
-  data:
-    replicas: 2
-
-kibana:
-  enabled: false
-
-logstash:
-  enabled: false
+  # prefix elasticsearch resources with the name of the releases
+  # looks like we can't use {{ .Release.Name }}-elasticsearch
+  # https://github.com/helm/helm/issues/2133
+  clusterName: "elastic-stack-elasticsearch"
 
 filebeat:
   enabled: true
-  config:
-    output.file.enabled: false
-    output.elasticsearch:
-      enabled: true
-      hosts: ["http://elastic-stack-elasticsearch-client:9200"]
-    filebeat.inputs:
-    - type: log
-      enabled: true
-      paths:
-        - /var/log/*.log
-        - /var/log/messages
-        - /var/log/syslog
-    - type: docker
-      containers.ids:
-      - "*"
-      json.keys_under_root: true
-      json.ignore_decoding_error: true
-      processors:
-        - add_kubernetes_metadata:
-        - drop_event:
-            when:
-              equals:
-                kubernetes.container.name: "filebeat"
-        - decode_json_fields:
-            fields: ["message"]
-            when:
-              equals:
-                kubernetes.container.name: "modsecurity-log"
-
-fluentd:
-  enabled: false
-
-fluent-bit:
-  enabled: false
-
-nginx-ldapauth-proxy:
+  filebeatConfig:
+    filebeat.yml: |
+      output.file.enabled: false
+      output.elasticsearch:
+        hosts: ["http://elastic-stack-elasticsearch-master:9200"]
+      filebeat.inputs:
+      - type: container
+        paths:
+          - '/var/lib/docker/containers/*/*.log'
+        json.keys_under_root: true
+        json.ignore_decoding_error: true
+        processors:
+          - add_id:
+              target_field: tie_breaker_id
+          - add_cloud_metadata: ~
+          - add_kubernetes_metadata: ~
+          - decode_json_fields:
+              fields: ["message"]
+              when:
+                equals:
+                  kubernetes.container.namespace: "gitlab-managed-apps"
+                  kubernetes.container.name: "modsecurity-log"
+kibana:
   enabled: false
+  elasticsearchHosts: "http://elastic-stack-elasticsearch-master:9200"
 
 elasticsearch-curator:
   enabled: true
@@ -63,7 +44,7 @@ elasticsearch-curator:
       ---
       client:
         hosts:
-          - elastic-stack-elasticsearch-client
+          - elastic-stack-elasticsearch-master
         port: 9200
     action_file_yml: |-
       ---
@@ -76,6 +57,7 @@ elasticsearch-curator:
             actionable list of indices (ignore_empty_list) and exit cleanly.
           options:
             ignore_empty_list: True
+            allow_ilm_indices: True
           filters:
           - filtertype: pattern
             kind: prefix
@@ -86,17 +68,3 @@ elasticsearch-curator:
             timestring: '%Y.%m.%d'
             unit: days
             unit_count: 30
-        2:
-          action: delete_indices
-          description: >-
-            Indices created by filebeat 6.7.0 are incompatible with filebeat 7,
-            so they will be deleted.
-          options:
-            ignore_empty_list: True
-          filters:
-          - filtertype: pattern
-            kind: prefix
-            value: filebeat-6.7.0-
-
-elasticsearch-exporter:
-  enabled: false