Merge branch 'add_fluentd_model_as_cluster_app' into 'master'

Add Fluentd model for cluster apps

See merge request gitlab-org/gitlab!28846

app/assets/javascripts/clusters/services/application_state_machine.js

@@ -191,7 +191,8 @@ const applicationStateMachine = {
  * @param {*} event
  */
 const transitionApplicationState = (application, event) => {
-  const newState = applicationStateMachine[application.status].on[event];
+  const stateMachine = applicationStateMachine[application.status];
+  const newState = stateMachine !== undefined ? stateMachine.on[event] : false;
 
   return newState
     ? {

app/controllers/clusters/applications_controller.rb

@@ -47,7 +47,7 @@ class Clusters::ApplicationsController < Clusters::BaseController
   end
 
   def cluster_application_params
-    params.permit(:application, :hostname, :pages_domain_id, :email, :stack, :modsecurity_enabled, :modsecurity_mode)
+    params.permit(:application, :hostname, :pages_domain_id, :email, :stack, :modsecurity_enabled, :modsecurity_mode, :host, :port, :protocol)
   end
 
   def cluster_application_destroy_params

app/models/clusters/applications/fluentd.rb

+# frozen_string_literal: true
+
+module Clusters
+  module Applications
+    class Fluentd < ApplicationRecord
+      VERSION = '2.4.0'
+
+      self.table_name = 'clusters_applications_fluentd'
+
+      include ::Clusters::Concerns::ApplicationCore
+      include ::Clusters::Concerns::ApplicationStatus
+      include ::Clusters::Concerns::ApplicationVersion
+      include ::Clusters::Concerns::ApplicationData
+
+      default_value_for :version, VERSION
+      default_value_for :port, 514
+      default_value_for :protocol, :tcp
+
+      enum protocol: { tcp: 0, udp: 1 }
+
+      def chart
+        'stable/fluentd'
+      end
+
+      def install_command
+        Gitlab::Kubernetes::Helm::InstallCommand.new(
+          name: 'fluentd',
+          repository: repository,
+          version: VERSION,
+          rbac: cluster.platform_kubernetes_rbac?,
+          chart: chart,
+          files: files
+        )
+      end
+
+      def values
+        content_values.to_yaml
+      end
+
+      private
+
+      def content_values
+        YAML.load_file(chart_values_file).deep_merge!(specification)
+      end
+
+      def specification
+        {
+          "configMaps" => {
+            "output.conf" => output_configuration_content,
+            "general.conf" => general_configuration_content
+          }
+        }
+      end
+
+      def output_configuration_content
+        <<~EOF
+        <match kubernetes.**>
+          @type remote_syslog
+          @id out_kube_remote_syslog
+          host #{host}
+          port #{port}
+          program fluentd
+          hostname ${kubernetes_host}
+          protocol #{protocol}
+          packet_size 65535
+          <buffer kubernetes_host>
+          </buffer>
+          <format>
+            @type ltsv
+          </format>
+        </match>
+        EOF
+      end
+
+      def general_configuration_content
+        <<~EOF
+        <match fluent.**>
+          @type null
+        </match>
+        <source>
+          @type http
+          port 9880
+          bind 0.0.0.0
+        </source>
+        <source>
+          @type tail
+          @id in_tail_container_logs
+          path /var/log/containers/*#{Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log
+          pos_file /var/log/fluentd-containers.log.pos
+          tag kubernetes.*
+          read_from_head true
+          <parse>
+            @type json
+            time_format %Y-%m-%dT%H:%M:%S.%NZ
+          </parse>
+        </source>
+        EOF
+      end
+    end
+  end
+end

app/models/clusters/cluster.rb

@@ -19,7 +19,8 @@ module Clusters
       Clusters::Applications::Runner.application_name => Clusters::Applications::Runner,
       Clusters::Applications::Jupyter.application_name => Clusters::Applications::Jupyter,
       Clusters::Applications::Knative.application_name => Clusters::Applications::Knative,
-      Clusters::Applications::ElasticStack.application_name => Clusters::Applications::ElasticStack
+      Clusters::Applications::ElasticStack.application_name => Clusters::Applications::ElasticStack,
+      Clusters::Applications::Fluentd.application_name => Clusters::Applications::Fluentd
     }.freeze
     DEFAULT_ENVIRONMENT = '*'
     KUBE_INGRESS_BASE_DOMAIN = 'KUBE_INGRESS_BASE_DOMAIN'
@@ -57,6 +58,7 @@ module Clusters
     has_one_cluster_application :jupyter
     has_one_cluster_application :knative
     has_one_cluster_application :elastic_stack
+    has_one_cluster_application :fluentd
 
     has_many :kubernetes_namespaces
     has_many :metrics_dashboard_annotations, class_name: 'Metrics::Dashboard::Annotation', inverse_of: :cluster

app/serializers/cluster_application_entity.rb

@@ -16,4 +16,7 @@ class ClusterApplicationEntity < Grape::Entity
   expose :available_domains, using: Serverless::DomainEntity, if: -> (e, _) { e.respond_to?(:available_domains) }
   expose :pages_domain, using: Serverless::DomainEntity, if: -> (e, _) { e.respond_to?(:pages_domain) }
   expose :modsecurity_mode, if: -> (e, _) { e.respond_to?(:modsecurity_mode) }
+  expose :host, if: -> (e, _) { e.respond_to?(:host) }
+  expose :port, if: -> (e, _) { e.respond_to?(:port) }
+  expose :protocol, if: -> (e, _) { e.respond_to?(:protocol) }
 end

app/services/clusters/applications/base_service.rb

@@ -35,6 +35,18 @@ module Clusters
             application.modsecurity_mode = params[:modsecurity_mode] || 0
           end
 
+          if application.has_attribute?(:host)
+            application.host = params[:host]
+          end
+
+          if application.has_attribute?(:protocol)
+            application.protocol = params[:protocol]
+          end
+
+          if application.has_attribute?(:port)
+            application.port = params[:port]
+          end
+
           if application.respond_to?(:oauth_application)
             application.oauth_application = create_oauth_application(application, request)
           end

changelogs/unreleased/add_fluentd_model_as_cluster_app.yml

+---
+title: Add Fluentd model for cluster apps
+merge_request: 28846
+author:
+type: changed

spec/factories/clusters/applications/helm.rb

@@ -139,5 +139,14 @@ FactoryBot.define do
         cluster factory: %i(cluster provided_by_gcp)
       end
     end
+
+    factory :clusters_applications_fluentd, class: 'Clusters::Applications::Fluentd' do
+      host { 'example.com' }
+      cluster factory: %i(cluster with_installed_helm provided_by_gcp)
+
+      trait :no_helm_installed do
+        cluster factory: %i(cluster provided_by_gcp)
+      end
+    end
   end
 end

spec/fixtures/api/schemas/cluster_status.json

@@ -39,6 +39,9 @@
         "stack": { "type": ["string", "null"] },
         "modsecurity_enabled": { "type": ["boolean", "null"] },
         "modsecurity_mode": {"type": ["integer", "0"]},
+        "host": {"type": ["string", "null"]},
+        "port": {"type": ["integer", "514"]},
+        "protocol": {"type": ["integer", "0"]},
         "update_available": { "type": ["boolean", "null"] },
         "can_uninstall": { "type": "boolean" },
         "available_domains": {

spec/frontend/clusters/services/application_state_machine_spec.js

@@ -161,4 +161,20 @@ describe('applicationStateMachine', () => {
       });
     });
   });
+
+  describe('current state is undefined', () => {
+    it('returns the current state without having any effects', () => {
+      const currentAppState = {};
+      expect(transitionApplicationState(currentAppState, INSTALLABLE)).toEqual(currentAppState);
+    });
+  });
+
+  describe('with event is undefined', () => {
+    it('returns the current state without having any effects', () => {
+      const currentAppState = {
+        status: NO_STATUS,
+      };
+      expect(transitionApplicationState(currentAppState, undefined)).toEqual(currentAppState);
+    });
+  });
 });

spec/models/clusters/applications/fluentd_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Clusters::Applications::Fluentd do
+  let(:fluentd) { create(:clusters_applications_fluentd) }
+
+  include_examples 'cluster application core specs', :clusters_applications_fluentd
+  include_examples 'cluster application status specs', :clusters_applications_fluentd
+  include_examples 'cluster application version specs', :clusters_applications_fluentd
+  include_examples 'cluster application initial status specs'
+
+  describe '#can_uninstall?' do
+    subject { fluentd.can_uninstall? }
+
+    it { is_expected.to be true }
+  end
+
+  describe '#install_command' do
+    subject { fluentd.install_command }
+
+    it { is_expected.to be_an_instance_of(Gitlab::Kubernetes::Helm::InstallCommand) }
+
+    it 'is initialized with fluentd arguments' do
+      expect(subject.name).to eq('fluentd')
+      expect(subject.chart).to eq('stable/fluentd')
+      expect(subject.version).to eq('2.4.0')
+      expect(subject).to be_rbac
+    end
+
+    context 'application failed to install previously' do
+      let(:fluentd) { create(:clusters_applications_fluentd, :errored, version: '0.0.1') }
+
+      it 'is initialized with the locked version' do
+        expect(subject.version).to eq('2.4.0')
+      end
+    end
+  end
+
+  describe '#files' do
+    let(:application) { fluentd }
+    let(:values) { subject[:'values.yaml'] }
+
+    subject { application.files }
+
+    it 'includes fluentd specific keys in the values.yaml file' do
+      expect(values).to include('output.conf', 'general.conf')
+    end
+  end
+end

spec/models/clusters/cluster_spec.rb

@@ -582,9 +582,10 @@ describe Clusters::Cluster, :use_clean_rails_memory_store_caching do
       let!(:jupyter) { create(:clusters_applications_jupyter, cluster: cluster) }
       let!(:knative) { create(:clusters_applications_knative, cluster: cluster) }
       let!(:elastic_stack) { create(:clusters_applications_elastic_stack, cluster: cluster) }
+      let!(:fluentd) { create(:clusters_applications_fluentd, cluster: cluster) }
 
       it 'returns a list of created applications' do
-        is_expected.to contain_exactly(helm, ingress, cert_manager, crossplane, prometheus, runner, jupyter, knative, elastic_stack)
+        is_expected.to contain_exactly(helm, ingress, cert_manager, crossplane, prometheus, runner, jupyter, knative, elastic_stack, fluentd)
       end
     end
   end

vendor/fluentd/values.yaml

+plugins:
+  enabled: true
+  pluginsList: ["fluent-plugin-remote_syslog"]
+
+extraVolumes:
+  - name: varlog
+    hostPath:
+      path: /var/log
+  - name: varlibdockercontainers
+    hostPath:
+      path: /var/lib/docker/containers
+
+extraVolumeMounts:
+  - name: varlog
+    mountPath: /var/log
+  - name: varlibdockercontainers
+    mountPath: /var/lib/docker/containers
+    readOnly: true