Collect thread pool metrics for ActionCable

Since AC runs its own dedicated thread pool separate from the
Puma worker pool, we need to monitor this separately too.

changelogs/unreleased/217314-more-actioncable-prometheus-metrics.yml

+---
+title: New ActionCable Prometheus metrics added
+merge_request: 41771
+author:
+type: added

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -107,6 +107,12 @@ The following metrics are available:
 | `auto_devops_pipelines_completed_total`                        | Counter   |                   12.7 | Counter of completed Auto DevOps pipelines, labeled by status                                       |                                                     |
 | `gitlab_metrics_dashboard_processing_time_ms`                  | Summary   |                  12.10 | Metrics dashboard processing time in milliseconds | service, stages |
 | `action_cable_active_connections`                              | Gauge     |                   13.4 | Number of ActionCable WS clients currently connected | `server_mode` |
+| `action_cable_pool_min_size`                                   | Gauge     |                   13.4 | Minimum number of worker threads in ActionCable thread pool | `server_mode` |
+| `action_cable_pool_max_size`                                   | Gauge     |                   13.4 | Maximum number of worker threads in ActionCable thread pool | `server_mode` |
+| `action_cable_pool_current_size`                               | Gauge     |                   13.4 | Current number of worker threads in ActionCable thread pool | `server_mode` |
+| `action_cable_pool_largest_size`                               | Gauge     |                   13.4 | Largest number of worker threads observed so far in ActionCable thread pool | `server_mode` |
+| `action_cable_pool_pending_tasks`                              | Gauge     |                   13.4 | Number of tasks waiting to be executed in ActionCable thread pool | `server_mode` |
+| `action_cable_pool_tasks_total`                                | Gauge     |                   13.4 | Total number of tasks executed in ActionCable thread pool | `server_mode` |
 
 ## Metrics controlled by a feature flag
 

lib/gitlab/metrics/samplers/action_cable_sampler.rb

@@ -6,31 +6,54 @@ module Gitlab
       class ActionCableSampler < BaseSampler
         SAMPLING_INTERVAL_SECONDS = 5
 
+        def initialize(interval = SAMPLING_INTERVAL_SECONDS, action_cable: ::ActionCable.server)
+          super(interval)
+          @action_cable = action_cable
+        end
+
         def metrics
           @metrics ||= {
             active_connections: ::Gitlab::Metrics.gauge(
               :action_cable_active_connections, 'Number of ActionCable WS clients currently connected'
+            ),
+            pool_min_size: ::Gitlab::Metrics.gauge(
+              :action_cable_pool_min_size, 'Minimum number of worker threads in ActionCable thread pool'
+            ),
+            pool_max_size: ::Gitlab::Metrics.gauge(
+              :action_cable_pool_max_size, 'Maximum number of worker threads in ActionCable thread pool'
+            ),
+            pool_current_size: ::Gitlab::Metrics.gauge(
+              :action_cable_pool_current_size, 'Current number of worker threads in ActionCable thread pool'
+            ),
+            pool_largest_size: ::Gitlab::Metrics.gauge(
+              :action_cable_pool_largest_size, 'Largest number of worker threads observed so far in ActionCable thread pool'
+            ),
+            pool_completed_tasks: ::Gitlab::Metrics.gauge(
+              :action_cable_pool_tasks_total, 'Total number of tasks executed in ActionCable thread pool'
+            ),
+            pool_pending_tasks: ::Gitlab::Metrics.gauge(
+              :action_cable_pool_pending_tasks, 'Number of tasks waiting to be executed in ActionCable thread pool'
             )
           }
         end
 
         def sample
-          stats = sample_stats
+          pool = @action_cable.worker_pool.executor
           labels = {
             server_mode: server_mode
           }
 
-          metrics[:active_connections].set(labels, stats[:active_connections])
+          metrics[:active_connections].set(labels, @action_cable.connections.size)
+          metrics[:pool_min_size].set(labels, pool.min_length)
+          metrics[:pool_max_size].set(labels, pool.max_length)
+          metrics[:pool_current_size].set(labels, pool.length)
+          metrics[:pool_largest_size].set(labels, pool.largest_length)
+          metrics[:pool_completed_tasks].set(labels, pool.completed_task_count)
+          metrics[:pool_pending_tasks].set(labels, pool.queue_length)
         end
 
         private
 
-        def sample_stats
-          {
-            active_connections: ::ActionCable.server.connections.size
-          }
-        end
-
         def server_mode
           Gitlab::ActionCable::Config.in_app? ? 'in-app' : 'standalone'
         end

spec/lib/gitlab/metrics/samplers/action_cable_sampler_spec.rb

@@ -3,7 +3,9 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Metrics::Samplers::ActionCableSampler do
-  subject { described_class.new }
+  let(:action_cable) { instance_double(ActionCable::Server::Base) }
+
+  subject { described_class.new(action_cable: action_cable) }
 
   describe '#interval' do
     it 'samples every five seconds by default' do
@@ -16,28 +18,77 @@ RSpec.describe Gitlab::Metrics::Samplers::ActionCableSampler do
   end
 
   describe '#sample' do
+    let(:pool) { instance_double(Concurrent::ThreadPoolExecutor) }
+
     before do
-      expect(::ActionCable.server.connections).to receive(:size).and_return(42)
+      allow(action_cable).to receive_message_chain(:worker_pool, :executor).and_return(pool)
+      allow(action_cable).to receive(:connections).and_return([])
+      allow(pool).to receive(:min_length).and_return(1)
+      allow(pool).to receive(:max_length).and_return(2)
+      allow(pool).to receive(:length).and_return(3)
+      allow(pool).to receive(:largest_length).and_return(4)
+      allow(pool).to receive(:completed_task_count).and_return(5)
+      allow(pool).to receive(:queue_length).and_return(6)
     end
 
-    context 'for in-app mode' do
-      it 'samples statistic with correct labels attached' do
-        expect(Gitlab::ActionCable::Config).to receive(:in_app?).and_return(true)
+    shared_examples 'collects metrics' do |expected_labels|
+      it 'includes active connections' do
+        expect(subject.metrics[:active_connections]).to receive(:set).with(expected_labels, 0)
+
+        subject.sample
+      end
 
-        expect(subject.metrics[:active_connections]).to receive(:set).with({ server_mode: 'in-app' }, 42)
+      it 'includes minimum worker pool size' do
+        expect(subject.metrics[:pool_min_size]).to receive(:set).with(expected_labels, 1)
 
         subject.sample
       end
-    end
 
-    context 'for standalone mode' do
-      it 'samples statistic with correct labels attached' do
-        expect(Gitlab::ActionCable::Config).to receive(:in_app?).and_return(false)
+      it 'includes maximum worker pool size' do
+        expect(subject.metrics[:pool_max_size]).to receive(:set).with(expected_labels, 2)
 
-        expect(subject.metrics[:active_connections]).to receive(:set).with({ server_mode: 'standalone' }, 42)
+        subject.sample
+      end
+
+      it 'includes current worker pool size' do
+        expect(subject.metrics[:pool_current_size]).to receive(:set).with(expected_labels, 3)
 
         subject.sample
       end
+
+      it 'includes largest worker pool size' do
+        expect(subject.metrics[:pool_largest_size]).to receive(:set).with(expected_labels, 4)
+
+        subject.sample
+      end
+
+      it 'includes worker pool completed task count' do
+        expect(subject.metrics[:pool_completed_tasks]).to receive(:set).with(expected_labels, 5)
+
+        subject.sample
+      end
+
+      it 'includes worker pool pending task count' do
+        expect(subject.metrics[:pool_pending_tasks]).to receive(:set).with(expected_labels, 6)
+
+        subject.sample
+      end
+    end
+
+    context 'for in-app mode' do
+      before do
+        expect(Gitlab::ActionCable::Config).to receive(:in_app?).and_return(true)
+      end
+
+      it_behaves_like 'collects metrics', server_mode: 'in-app'
+    end
+
+    context 'for standalone mode' do
+      before do
+        expect(Gitlab::ActionCable::Config).to receive(:in_app?).and_return(false)
+      end
+
+      it_behaves_like 'collects metrics', server_mode: 'standalone'
     end
   end
 end