Add Sidekiq dead job metrics

This adds a counter, sidekiq_jobs_dead_total, that increments when a job
is added to the dead jobs queue. It has the same basic labels (worker,
queue, feature category, etc.) as the other Sidekiq metrics.

changelogs/unreleased/add-sidekiq-dead-job-metrics.yml

+---
+title: Add metric for dead Sidekiq jobs
+merge_request: 48361
+author:
+type: changed

config/initializers/sidekiq.rb

@@ -46,6 +46,8 @@ Sidekiq.configure_server do |config|
 
   config.client_middleware(&Gitlab::SidekiqMiddleware.client_configurator)
 
+  config.death_handlers << Gitlab::SidekiqDeathHandler.method(:handler)
+
   config.on :startup do
     # Clear any connections that might have been obtained before starting
     # Sidekiq (e.g. in an initializer).

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -142,6 +142,7 @@ configuration option in `gitlab.yml`. These metrics are served from the
 | `sidekiq_jobs_queue_duration_seconds`          | Histogram | 12.5 | Duration in seconds that a Sidekiq job was queued before being executed                             | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` |
 | `sidekiq_jobs_failed_total`                    | Counter   | 12.2 | Sidekiq jobs failed                                                                                 | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` |
 | `sidekiq_jobs_retried_total`                   | Counter   | 12.2 | Sidekiq jobs retried                                                                                | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` |
+| `sidekiq_jobs_dead_total`                      | Counter   | 13.7 | Sidekiq dead jobs (jobs that have run out of retries)                                               | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` |
 | `sidekiq_redis_requests_total`                 | Counter   | 13.1 | Redis requests during a Sidekiq job execution                                                       | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` |
 | `sidekiq_elasticsearch_requests_total`         | Counter   | 13.1 | Elasticsearch requests during a Sidekiq job execution                                                       | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` |
 | `sidekiq_running_jobs`                         | Gauge     | 12.2 | Number of Sidekiq jobs running                                                                      | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` |

lib/gitlab/sidekiq_death_handler.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqDeathHandler
+    class << self
+      include ::Gitlab::SidekiqMiddleware::MetricsHelper
+
+      def handler(job, _exception)
+        labels = create_labels(job['class'].constantize, job['queue'])
+
+        counter.increment(labels)
+      end
+
+      def counter
+        @counter ||= ::Gitlab::Metrics.counter(:sidekiq_jobs_dead_total, 'Sidekiq dead jobs')
+      end
+    end
+  end
+end

lib/gitlab/sidekiq_middleware/client_metrics.rb

@@ -2,7 +2,9 @@
 
 module Gitlab
   module SidekiqMiddleware
-    class ClientMetrics < SidekiqMiddleware::Metrics
+    class ClientMetrics
+      include ::Gitlab::SidekiqMiddleware::MetricsHelper
+
       ENQUEUED = :sidekiq_enqueued_jobs_total
 
       def initialize

lib/gitlab/sidekiq_middleware/metrics.rb → lib/gitlab/sidekiq_middleware/metrics_helper.rb

@@ -2,7 +2,7 @@
 
 module Gitlab
   module SidekiqMiddleware
-    class Metrics
+    module MetricsHelper
       TRUE_LABEL = "yes"
       FALSE_LABEL = "no"
 

lib/gitlab/sidekiq_middleware/server_metrics.rb

@@ -2,7 +2,9 @@
 
 module Gitlab
   module SidekiqMiddleware
-    class ServerMetrics < SidekiqMiddleware::Metrics
+    class ServerMetrics
+      include ::Gitlab::SidekiqMiddleware::MetricsHelper
+
       # SIDEKIQ_LATENCY_BUCKETS are latency histogram buckets better suited to Sidekiq
       # timeframes than the DEFAULT_BUCKET definition. Defined in seconds.
       SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze

spec/lib/gitlab/sidekiq_death_handler_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::SidekiqDeathHandler, :clean_gitlab_redis_queues do
+  describe '.handler' do
+    context 'when the job class has worker attributes' do
+      let(:test_worker) do
+        Class.new do
+          include WorkerAttributes
+
+          urgency :low
+          worker_has_external_dependencies!
+          worker_resource_boundary :cpu
+          feature_category :users
+        end
+      end
+
+      before do
+        stub_const('TestWorker', test_worker)
+      end
+
+      it 'uses the attributes from the worker' do
+        expect(described_class.counter)
+          .to receive(:increment)
+                .with(queue: 'test_queue', worker: 'TestWorker',
+                      urgency: 'low', external_dependencies: 'yes',
+                      feature_category: 'users', boundary: 'cpu')
+
+        described_class.handler({ 'class' => 'TestWorker', 'queue' => 'test_queue' }, nil)
+      end
+    end
+
+    context 'when the job class does not have worker attributes' do
+      before do
+        stub_const('TestWorker', Class.new)
+      end
+
+      it 'uses blank attributes' do
+        expect(described_class.counter)
+          .to receive(:increment)
+                .with(queue: 'test_queue', worker: 'TestWorker',
+                      urgency: '', external_dependencies: 'no',
+                      feature_category: '', boundary: '')
+
+        described_class.handler({ 'class' => 'TestWorker', 'queue' => 'test_queue' }, nil)
+      end
+    end
+  end
+end