Add custom metric importer

Adds a importer and "synchronizer" for ingesting custom metrics into the
 database.

app/models/concerns/enums/prometheus_metric.rb

@@ -20,7 +20,8 @@ module Enums
       {
         business: 0,
         response: 1,
-        system: 2
+        system: 2,
+        custom: 3
       }.freeze
     end
 
@@ -79,7 +80,11 @@ module Enums
         system: {
           group_title: _('System metrics (Custom)'),
           priority: -10
-        }.freeze
+        }.freeze,
+        custom: {
+          group_title: _('Custom metrics'),
+          priority: 0
+        }
       }.freeze
     end
   end

app/models/prometheus_metric.rb

@@ -16,11 +16,13 @@ class PrometheusMetric < ApplicationRecord
   validates :project, presence: true, unless: :common?
   validates :project, absence: true, if: :common?
 
+  scope :for_dashboard_path, -> (dashboard_path) { where(dashboard_path: dashboard_path) }
   scope :for_project, -> (project) { where(project: project) }
   scope :for_group, -> (group) { where(group: group) }
   scope :for_title, -> (title) { where(title: title) }
   scope :for_y_label, -> (y_label) { where(y_label: y_label) }
   scope :for_identifier, -> (identifier) { where(identifier: identifier) }
+  scope :not_identifier, -> (identifier) { where.not(identifier: identifier) }
   scope :common, -> { where(common: true) }
   scope :ordered, -> { reorder(created_at: :asc) }
 

lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb

@@ -18,6 +18,7 @@ module Gitlab
             business: 0,
             response: 1,
             system: 2,
+            custom: 3,
 
             cluster_health: -100
           }
@@ -34,7 +35,8 @@ module Gitlab
             aws_elb: _('Response metrics (AWS ELB)'),
             nginx: _('Response metrics (NGINX)'),
             kubernetes: _('System metrics (Kubernetes)'),
-            cluster_health: _('Cluster Health')
+            cluster_health: _('Cluster Health'),
+            custom: _('Custom metrics')
           }
         end
       end

lib/gitlab/metrics/dashboard/importer.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      class Importer
+        def initialize(dashboard_path, project)
+          @dashboard_path = dashboard_path.to_s
+          @project = project
+        end
+
+        def execute
+          return false unless Dashboard::Validator.validate(dashboard_hash, project: project, dashboard_path: dashboard_path)
+
+          Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute
+        end
+
+        def execute!
+          Dashboard::Validator.validate!(dashboard_hash, project: project, dashboard_path: dashboard_path)
+
+          Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute!
+        end
+
+        private
+
+        attr_accessor :dashboard_path, :project
+
+        def dashboard_hash
+          @dashboard_hash ||= begin
+            raw_dashboard = Dashboard::RepoDashboardFinder.read_dashboard(project, dashboard_path)
+            return unless raw_dashboard.present?
+
+            ::Gitlab::Config::Loader::Yaml.new(raw_dashboard).load_raw!
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Importers
+        class PrometheusMetrics
+          ALLOWED_ATTRIBUTES = %i(title query y_label unit legend group dashboard_path).freeze
+
+          # Takes a JSON schema validated dashboard hash and
+          # imports metrics to database
+          def initialize(dashboard_hash, project:, dashboard_path:)
+            @dashboard_hash = dashboard_hash
+            @project = project
+            @dashboard_path = dashboard_path
+          end
+
+          def execute
+            import
+          rescue ActiveRecord::RecordInvalid, ::Gitlab::Metrics::Dashboard::Transformers::TransformerError
+            false
+          end
+
+          def execute!
+            import
+          end
+
+          private
+
+          attr_reader :dashboard_hash, :project, :dashboard_path
+
+          def import
+            delete_stale_metrics
+            create_or_update_metrics
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def create_or_update_metrics
+            # TODO: use upsert and worker for callbacks?
+            prometheus_metrics_attributes&.each do |attributes|
+              prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:identifier, :project))
+              prometheus_metric.update!(attributes.slice(*ALLOWED_ATTRIBUTES))
+            end
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
+          def delete_stale_metrics
+            identifiers = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] }
+
+            stale_metrics = PrometheusMetric.for_project(project)
+              .for_dashboard_path(dashboard_path)
+              .for_group(Enums::PrometheusMetric.groups[:custom])
+              .not_identifier(identifiers)
+
+            # TODO: use destroy_all and worker for callbacks?
+            stale_metrics.each(&:destroy)
+          end
+
+          def prometheus_metrics_attributes
+            @prometheus_metrics_attributes ||= begin
+              Dashboard::Transformers::Yml::V1::PrometheusMetrics.new(
+                dashboard_hash,
+                project: project,
+                dashboard_path: dashboard_path
+              ).execute
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/metrics/dashboard/transformers/errors.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Transformers
+        TransformerError = Class.new(StandardError)
+
+        module Errors
+          class MissingAttribute < TransformerError
+            def initialize(attribute_name)
+              super("Missing attribute: '#{attribute_name}'")
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Transformers
+        module Yml
+          module V1
+            # Takes a JSON schema validated dashboard hash and
+            # maps it to PrometheusMetric model attributes
+            class PrometheusMetrics
+              def initialize(dashboard_hash, project: nil, dashboard_path: nil)
+                @dashboard_hash = dashboard_hash.with_indifferent_access
+                @project = project
+                @dashboard_path = dashboard_path
+
+                @dashboard_hash.default_proc = -> (h, k) { raise Transformers::Errors::MissingAttribute, k.to_s }
+              end
+
+              def execute
+                prometheus_metrics = []
+
+                dashboard_hash[:panel_groups].each do |panel_group|
+                  panel_group[:panels].each do |panel|
+                    panel[:metrics].each do |metric|
+                      prometheus_metrics << {
+                        project:        project,
+                        title:          panel[:title],
+                        y_label:        panel[:y_label],
+                        query:          metric[:query_range] || metric[:query],
+                        unit:           metric[:unit],
+                        legend:         metric[:label],
+                        identifier:     metric[:id],
+                        group:          Enums::PrometheusMetric.groups[:custom],
+                        common:         false,
+                        dashboard_path: dashboard_path
+                      }.compact
+                    end
+                  end
+                end
+
+                prometheus_metrics
+              end
+
+              private
+
+              attr_reader :dashboard_hash, :project, :dashboard_path
+            end
+          end
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/metrics/dashboard/importer_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Metrics::Dashboard::Importer do
+  include MetricsDashboardHelpers
+
+  describe '.execute' do
+    let_it_be(:dashboard_path) { '.gitlab/dashboards/sample_dashboard.yml' }
+    let_it_be(:dashboard_hash) { load_sample_dashboard }
+    let_it_be(:project) { create(:project) }
+
+    subject { described_class.new(dashboard_path, project) }
+
+    before do
+      allow(subject).to receive(:dashboard_hash).and_return(dashboard_hash)
+    end
+
+    it 'imports metrics to database' do
+      expect { subject.execute }
+        .to change { PrometheusMetric.count }.from(0).to(3)
+    end
+  end
+
+  describe '.execute!' do
+    let_it_be(:dashboard_path) { '.gitlab/dashboards/sample_dashboard.yml' }
+    let_it_be(:project) { create(:project) }
+
+    subject { described_class.new(dashboard_path, project) }
+
+    before do
+      allow(subject).to receive(:dashboard_hash).and_return(dashboard_hash)
+    end
+
+    context 'invalid dashboard hash' do
+      let(:dashboard_hash) { {} }
+
+      it 'raises error' do
+        expect { subject.execute! }.to raise_error(Gitlab::Metrics::Dashboard::Validator::Errors::SchemaValidationError,
+          'root is missing required keys: dashboard, panel_groups')
+      end
+    end
+  end
+end

spec/lib/gitlab/metrics/dashboard/importers/prometheus_metrics_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Metrics::Dashboard::Importers::PrometheusMetrics do
+  include MetricsDashboardHelpers
+
+  describe '#execute' do
+    let(:project) { create(:project) }
+    let(:dashboard_path) { 'path/to/dashboard.yml' }
+
+    subject { described_class.new(dashboard_hash, project: project, dashboard_path: dashboard_path) }
+
+    context 'valid dashboard' do
+      let(:dashboard_hash) { load_sample_dashboard }
+
+      context 'with all new metrics' do
+        it 'creates PrometheusMetrics' do
+          expect { subject.execute }.to change { PrometheusMetric.count }.by(3)
+        end
+      end
+
+      context 'with existing metrics' do
+        let!(:existing_metric) do
+          create(:prometheus_metric, {
+            project:    project,
+            identifier: 'metric_b',
+            title:      'overwrite',
+            y_label:    'overwrite',
+            query:      'overwrite',
+            unit:       'overwrite',
+            legend:     'overwrite'
+          })
+        end
+
+        it 'updates existing PrometheusMetrics' do
+          described_class.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute
+
+          expect(existing_metric.reload.attributes.with_indifferent_access).to include({
+            title:   'Super Chart B',
+            y_label: 'y_label',
+            query:   'query',
+            unit:    'unit',
+            legend:  'Legend Label'
+          })
+        end
+
+        it 'creates new PrometheusMetrics' do
+          expect { subject.execute }.to change { PrometheusMetric.count }.by(2)
+        end
+
+        context 'with stale metrics' do
+          let!(:stale_metric) do
+            create(:prometheus_metric,
+              project: project,
+              identifier: 'stale_metric',
+              dashboard_path: dashboard_path,
+              group: 3
+            )
+          end
+
+          it 'deletes stale metrics' do
+            subject.execute
+
+            expect { stale_metric.reload }.to raise_error(ActiveRecord::RecordNotFound)
+          end
+        end
+      end
+    end
+
+    context 'invalid dashboard' do
+      let(:dashboard_hash) { {} }
+
+      it 'returns false' do
+        expect(subject.execute).to eq(false)
+      end
+    end
+  end
+end

spec/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Metrics::Dashboard::Transformers::Yml::V1::PrometheusMetrics do
+  include MetricsDashboardHelpers
+
+  describe '#execute' do
+    subject { described_class.new(dashboard_hash) }
+
+    context 'valid dashboard' do
+      let_it_be(:dashboard_hash) do
+        {
+          panel_groups: [{
+            panels: [
+              {
+                title: 'Panel 1 title',
+                y_label: 'Panel 1 y_label',
+                metrics: [
+                  {
+                    query_range: 'Panel 1 metric 1 query_range',
+                    unit: 'Panel 1 metric 1 unit',
+                    label: 'Panel 1 metric 1 label',
+                    id: 'Panel 1 metric 1 id'
+                  },
+                  {
+                    query: 'Panel 1 metric 2 query',
+                    unit: 'Panel 1 metric 2 unit',
+                    label: 'Panel 1 metric 2 label',
+                    id: 'Panel 1 metric 2 id'
+                  }
+                ]
+              },
+              {
+                title: 'Panel 2 title',
+                y_label: 'Panel 2 y_label',
+                metrics: [{
+                  query_range: 'Panel 2 metric 1 query_range',
+                  unit: 'Panel 2 metric 1 unit',
+                  label: 'Panel 2 metric 1 label',
+                  id: 'Panel 2 metric 1 id'
+                }]
+              }
+            ]
+          }]
+        }
+      end
+
+      let(:expected_metrics) do
+        [
+          {
+            title: 'Panel 1 title',
+            y_label: 'Panel 1 y_label',
+            query: "Panel 1 metric 1 query_range",
+            unit: 'Panel 1 metric 1 unit',
+            legend: 'Panel 1 metric 1 label',
+            identifier: 'Panel 1 metric 1 id',
+            group: 3,
+            common: false
+          },
+          {
+            title: 'Panel 1 title',
+            y_label: 'Panel 1 y_label',
+            query: 'Panel 1 metric 2 query',
+            unit: 'Panel 1 metric 2 unit',
+            legend: 'Panel 1 metric 2 label',
+            identifier: 'Panel 1 metric 2 id',
+            group: 3,
+            common: false
+          },
+          {
+            title: 'Panel 2 title',
+            y_label: 'Panel 2 y_label',
+            query: 'Panel 2 metric 1 query_range',
+            unit: 'Panel 2 metric 1 unit',
+            legend: 'Panel 2 metric 1 label',
+            identifier: 'Panel 2 metric 1 id',
+            group: 3,
+            common: false
+          }
+        ]
+      end
+
+      it 'returns collection of metrics with correct attributes' do
+        expect(subject.execute).to match_array(expected_metrics)
+      end
+    end
+
+    context 'invalid dashboard' do
+      let(:dashboard_hash) { {} }
+
+      it 'raises missing attribute error' do
+        expect { subject.execute }.to raise_error(
+          ::Gitlab::Metrics::Dashboard::Transformers::Errors::MissingAttribute, "Missing attribute: 'panel_groups'"
+        )
+      end
+    end
+  end
+end