server_metrics_spec.rb 10.6 KB
Newer Older
1 2
# frozen_string_literal: true

3
require 'spec_helper'
4

5
# rubocop: disable RSpec/MultipleMemoizedHelpers
6
RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do
7 8 9
  shared_examples "a metrics middleware" do
    context "with mocked prometheus" do
      include_context 'server metrics with mocked prometheus'
10

11
      describe '.initialize_process_metrics' do
12 13
        it 'sets concurrency metrics' do
          expect(concurrency_metric).to receive(:set).with({}, Sidekiq.options[:concurrency].to_i)
14

15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
          described_class.initialize_process_metrics
        end

        it 'initializes sidekiq_jobs_completion_seconds for the workers in the current Sidekiq process' do
          allow(Gitlab::SidekiqConfig)
            .to receive(:current_worker_queue_mappings)
                  .and_return('MergeWorker' => 'merge', 'BuildFinishedWorker' => 'default')

          expect(completion_seconds_metric)
            .to receive(:get).with(queue: 'merge',
                                   worker: 'MergeWorker',
                                   urgency: 'high',
                                   external_dependencies: 'no',
                                   feature_category: 'source_code_management',
                                   boundary: '',
                                   job_status: 'done')

          expect(completion_seconds_metric)
            .to receive(:get).with(queue: 'merge',
                                   worker: 'MergeWorker',
                                   urgency: 'high',
                                   external_dependencies: 'no',
                                   feature_category: 'source_code_management',
                                   boundary: '',
                                   job_status: 'fail')

          expect(completion_seconds_metric)
            .to receive(:get).with(queue: 'default',
                                   worker: 'BuildFinishedWorker',
                                   urgency: 'high',
                                   external_dependencies: 'no',
                                   feature_category: 'continuous_integration',
                                   boundary: 'cpu',
                                   job_status: 'done')

          expect(completion_seconds_metric)
            .to receive(:get).with(queue: 'default',
                                   worker: 'BuildFinishedWorker',
                                   urgency: 'high',
                                   external_dependencies: 'no',
                                   feature_category: 'continuous_integration',
                                   boundary: 'cpu',
                                   job_status: 'fail')

          described_class.initialize_process_metrics
60
        end
61
      end
62

63 64
      describe '#call' do
        include_context 'server metrics call'
65

66 67
        it 'yields block' do
          expect { |b| subject.call(worker, job, :test, &b) }.to yield_control.once
68 69
        end

70 71 72
        it 'calls BackgroundTransaction' do
          expect_next_instance_of(Gitlab::Metrics::BackgroundTransaction) do |instance|
            expect(instance).to receive(:run)
73 74
          end

75 76
          subject.call(worker, job, :test) {}
        end
77

78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
        it 'sets queue specific metrics' do
          expect(running_jobs_metric).to receive(:increment).with(labels, -1)
          expect(running_jobs_metric).to receive(:increment).with(labels, 1)
          expect(queue_duration_seconds).to receive(:observe).with(labels, queue_duration_for_job) if queue_duration_for_job
          expect(user_execution_seconds_metric).to receive(:observe).with(labels_with_job_status, thread_cputime_duration)
          expect(db_seconds_metric).to receive(:observe).with(labels_with_job_status, db_duration)
          expect(gitaly_seconds_metric).to receive(:observe).with(labels_with_job_status, gitaly_duration)
          expect(completion_seconds_metric).to receive(:observe).with(labels_with_job_status, monotonic_time_duration)
          expect(redis_seconds_metric).to receive(:observe).with(labels_with_job_status, redis_duration)
          expect(elasticsearch_seconds_metric).to receive(:observe).with(labels_with_job_status, elasticsearch_duration)
          expect(redis_requests_total).to receive(:increment).with(labels_with_job_status, redis_calls)
          expect(elasticsearch_requests_total).to receive(:increment).with(labels_with_job_status, elasticsearch_calls)

          subject.call(worker, job, :test) { nil }
        end
93

94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
        it 'sets sidekiq_jobs_completion_seconds values that are compatible with those from .initialize_process_metrics' do
          label_validator = Prometheus::Client::LabelSetValidator.new([:le])

          allow(Gitlab::SidekiqConfig)
            .to receive(:current_worker_queue_mappings)
                  .and_return('MergeWorker' => 'merge', 'BuildFinishedWorker' => 'default')

          allow(completion_seconds_metric).to receive(:get) do |labels|
            expect { label_validator.validate(labels) }.not_to raise_error
          end

          allow(completion_seconds_metric).to receive(:observe) do |labels, _duration|
            expect { label_validator.validate(labels) }.not_to raise_error
          end

          described_class.initialize_process_metrics

          subject.call(worker, job, :test) { nil }
        end

114 115 116
        it 'sets the thread name if it was nil' do
          allow(Thread.current).to receive(:name).and_return(nil)
          expect(Thread.current).to receive(:name=).with(Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME)
117

118 119
          subject.call(worker, job, :test) { nil }
        end
120

121 122
        context 'when job_duration is not available' do
          let(:queue_duration_for_job) { nil }
123

124 125
          it 'does not set the queue_duration_seconds histogram' do
            expect(queue_duration_seconds).not_to receive(:observe)
126 127 128

            subject.call(worker, job, :test) { nil }
          end
129
        end
130

131 132
        context 'when error is raised' do
          let(:job_status) { :fail }
133

134 135
          it 'sets sidekiq_jobs_failed_total and reraises' do
            expect(failed_total_metric).to receive(:increment).with(labels, 1)
136

137
            expect { subject.call(worker, job, :test) { raise StandardError, "Failed" } }.to raise_error(StandardError, "Failed")
138 139
          end
        end
140

141 142
        context 'when job is retried' do
          let(:job) { { 'retry_count' => 1 } }
143

144 145
          it 'sets sidekiq_jobs_retried_total metric' do
            expect(retried_total_metric).to receive(:increment)
146

147
            subject.call(worker, job, :test) { nil }
148 149 150 151
          end
        end
      end
    end
152

153 154 155 156
    context "with prometheus integrated" do
      describe '#call' do
        it 'yields block' do
          expect { |b| subject.call(worker, job, :test, &b) }.to yield_control.once
157 158
        end

159 160
        context 'when error is raised' do
          let(:job_status) { :fail }
161

162 163 164
          it 'sets sidekiq_jobs_failed_total and reraises' do
            expect { subject.call(worker, job, :test) { raise StandardError, "Failed" } }.to raise_error(StandardError, "Failed")
          end
165 166
        end
      end
167
    end
168
  end
169

170 171 172
  it_behaves_like 'metrics middleware with worker attribution' do
    let(:job_status) { :done }
    let(:labels_with_job_status) { labels.merge(job_status: job_status.to_s) }
173
  end
174 175 176 177 178 179

  context 'DB load balancing' do
    subject { described_class.new }

    let(:queue) { :test }
    let(:worker_class) { worker.class }
180 181 182 183 184 185 186 187 188 189
    let(:worker) { TestWorker.new }
    let(:client_middleware) { Gitlab::Database::LoadBalancing::SidekiqClientMiddleware.new }
    let(:load_balancer) { double.as_null_object }
    let(:load_balancing_metric) { double('load balancing metric') }
    let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e" } }

    def process_job
      client_middleware.call(worker_class, job, queue, double) do
        worker_class.process_job(job)
      end
190 191 192 193 194 195 196
    end

    before do
      stub_const('TestWorker', Class.new)
      TestWorker.class_eval do
        include Sidekiq::Worker
        include WorkerAttributes
197 198 199

        def perform(*args)
        end
200
      end
201 202 203 204

      allow(::Gitlab::Database::LoadBalancing).to receive_message_chain(:proxy, :load_balancer).and_return(load_balancer)
      allow(load_balancing_metric).to receive(:increment)
      allow(Gitlab::Metrics).to receive(:counter).with(:sidekiq_load_balancing_count, anything).and_return(load_balancing_metric)
205 206
    end

207 208 209 210 211 212 213
    around do |example|
      with_sidekiq_server_middleware do |chain|
        chain.add Gitlab::Database::LoadBalancing::SidekiqServerMiddleware
        chain.add described_class
        Sidekiq::Testing.inline! { example.run }
      end
    end
214 215

    include_context 'server metrics with mocked prometheus'
216 217
    include_context 'server metrics call'
    include_context 'clear DB Load Balancing configuration'
218

219 220
    shared_context 'worker declaring data consistency' do
      let(:worker_class) { LBTestWorker }
221 222

      before do
223 224 225
        stub_const('LBTestWorker', Class.new(TestWorker))
        LBTestWorker.class_eval do
          include ApplicationWorker
226

227
          data_consistency :delayed
228 229
        end
      end
230
    end
231

232 233 234 235
    context 'when load_balancing is enabled' do
      before do
        allow(::Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(true)
      end
236

237 238 239
      describe '#call' do
        context 'when worker declares data consistency' do
          include_context 'worker declaring data consistency'
240

241 242
          it 'increments load balancing counter' do
            process_job
243

244 245 246 247 248
            expect(load_balancing_metric).to have_received(:increment).with(
              a_hash_including(
                data_consistency: :delayed,
                load_balancing_strategy: 'replica'
              ), 1)
249 250 251
          end
        end

252 253 254
        context 'when worker does not declare data consistency' do
          it 'does not increment load balancing counter' do
            process_job
255

256
            expect(load_balancing_metric).not_to have_received(:increment)
257 258 259 260 261 262
          end
        end
      end
    end

    context 'when load_balancing is disabled' do
263
      include_context 'worker declaring data consistency'
264 265 266 267 268 269

      before do
        allow(::Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(false)
      end

      describe '#initialize' do
270
        it 'does not set load_balancing metrics' do
271 272 273 274 275
          expect(Gitlab::Metrics).not_to receive(:counter).with(:sidekiq_load_balancing_count, anything)

          subject
        end
      end
276 277 278 279 280 281 282 283

      describe '#call' do
        it 'does not increment load balancing counter' do
          process_job

          expect(load_balancing_metric).not_to have_received(:increment)
        end
      end
284 285
    end
  end
286
end
287
# rubocop: enable RSpec/MultipleMemoizedHelpers