Commit fad4ab7d authored by Douwe Maan's avatar Douwe Maan

Merge branch 'pawel/update_prometheus_gem_to_well_tested_version' into 'master'

Update Prometheus Gem version and disable Prometheus method call instrumentation by default.

Closes gitlab-ee#4139 and #40457

See merge request gitlab-org/gitlab-ce!15558
parents cf631ddb 46cd2d93
...@@ -283,7 +283,7 @@ group :metrics do ...@@ -283,7 +283,7 @@ group :metrics do
gem 'influxdb', '~> 0.2', require: false gem 'influxdb', '~> 0.2', require: false
# Prometheus # Prometheus
gem 'prometheus-client-mmap', '~>0.7.0.beta18' gem 'prometheus-client-mmap', '~> 0.7.0.beta36'
gem 'raindrops', '~> 0.18' gem 'raindrops', '~> 0.18'
end end
......
...@@ -488,7 +488,7 @@ GEM ...@@ -488,7 +488,7 @@ GEM
mini_mime (0.1.4) mini_mime (0.1.4)
mini_portile2 (2.3.0) mini_portile2 (2.3.0)
minitest (5.7.0) minitest (5.7.0)
mmap2 (2.2.7) mmap2 (2.2.9)
mousetrap-rails (1.4.6) mousetrap-rails (1.4.6)
multi_json (1.12.2) multi_json (1.12.2)
multi_xml (0.6.0) multi_xml (0.6.0)
...@@ -625,8 +625,8 @@ GEM ...@@ -625,8 +625,8 @@ GEM
parser parser
unparser unparser
procto (0.0.3) procto (0.0.3)
prometheus-client-mmap (0.7.0.beta18) prometheus-client-mmap (0.7.0.beta36)
mmap2 (~> 2.2, >= 2.2.7) mmap2 (~> 2.2, >= 2.2.9)
pry (0.10.4) pry (0.10.4)
coderay (~> 1.1.0) coderay (~> 1.1.0)
method_source (~> 0.8.1) method_source (~> 0.8.1)
...@@ -1111,7 +1111,7 @@ DEPENDENCIES ...@@ -1111,7 +1111,7 @@ DEPENDENCIES
peek-sidekiq (~> 1.0.3) peek-sidekiq (~> 1.0.3)
pg (~> 0.18.2) pg (~> 0.18.2)
premailer-rails (~> 1.9.7) premailer-rails (~> 1.9.7)
prometheus-client-mmap (~> 0.7.0.beta18) prometheus-client-mmap (~> 0.7.0.beta36)
pry-byebug (~> 3.4.1) pry-byebug (~> 3.4.1)
pry-rails (~> 0.3.4) pry-rails (~> 0.3.4)
rack-attack (~> 4.4.1) rack-attack (~> 4.4.1)
......
---
title: Reenable Prometheus metrics, add more control over Prometheus method instrumentation
merge_request: 15558
author:
type: fixed
...@@ -12,16 +12,20 @@ Prometheus::Client.configure do |config| ...@@ -12,16 +12,20 @@ Prometheus::Client.configure do |config|
end end
config.pid_provider = -> do config.pid_provider = -> do
wid = Prometheus::Client::Support::Unicorn.worker_id worker_id = Prometheus::Client::Support::Unicorn.worker_id
wid = Process.pid if wid.nil? if worker_id.nil?
if wid.nil?
"process_pid_#{Process.pid}" "process_pid_#{Process.pid}"
else else
"worker_id_#{wid}" "worker_id_#{worker_id}"
end end
end end
end end
Gitlab::Application.configure do |config|
# 0 should be Sentry to catch errors in this middleware
config.middleware.insert(1, Gitlab::Metrics::RequestsRackMiddleware)
end
Sidekiq.configure_server do |config| Sidekiq.configure_server do |config|
config.on(:startup) do config.on(:startup) do
Gitlab::Metrics::SidekiqMetricsExporter.instance.start Gitlab::Metrics::SidekiqMetricsExporter.instance.start
......
...@@ -118,11 +118,6 @@ def instrument_classes(instrumentation) ...@@ -118,11 +118,6 @@ def instrument_classes(instrumentation)
end end
# rubocop:enable Metrics/AbcSize # rubocop:enable Metrics/AbcSize
Gitlab::Application.configure do |config|
# 0 should be Sentry to catch errors in this middleware
config.middleware.insert(1, Gitlab::Metrics::RequestsRackMiddleware)
end
if Gitlab::Metrics.enabled? if Gitlab::Metrics.enabled?
require 'pathname' require 'pathname'
require 'influxdb' require 'influxdb'
......
...@@ -6,29 +6,15 @@ module Gitlab ...@@ -6,29 +6,15 @@ module Gitlab
BASE_LABELS = { module: nil, method: nil }.freeze BASE_LABELS = { module: nil, method: nil }.freeze
attr_reader :real_time, :cpu_time, :call_count, :labels attr_reader :real_time, :cpu_time, :call_count, :labels
def self.call_real_duration_histogram def self.call_duration_histogram
return @call_real_duration_histogram if @call_real_duration_histogram return @call_duration_histogram if @call_duration_histogram
MUTEX.synchronize do
@call_real_duration_histogram ||= Gitlab::Metrics.histogram(
:gitlab_method_call_real_duration_seconds,
'Method calls real duration',
Transaction::BASE_LABELS.merge(BASE_LABELS),
[0.1, 0.2, 0.5, 1, 2, 5, 10]
)
end
end
def self.call_cpu_duration_histogram
return @call_cpu_duration_histogram if @call_cpu_duration_histogram
MUTEX.synchronize do MUTEX.synchronize do
@call_duration_histogram ||= Gitlab::Metrics.histogram( @call_duration_histogram ||= Gitlab::Metrics.histogram(
:gitlab_method_call_cpu_duration_seconds, :gitlab_method_call_duration_seconds,
'Method calls cpu duration', 'Method calls real duration',
Transaction::BASE_LABELS.merge(BASE_LABELS), Transaction::BASE_LABELS.merge(BASE_LABELS),
[0.1, 0.2, 0.5, 1, 2, 5, 10] [0.01, 0.05, 0.1, 0.5, 1])
)
end end
end end
...@@ -59,8 +45,9 @@ module Gitlab ...@@ -59,8 +45,9 @@ module Gitlab
@cpu_time += cpu_time @cpu_time += cpu_time
@call_count += 1 @call_count += 1
self.class.call_real_duration_histogram.observe(@transaction.labels.merge(labels), real_time / 1000.0) if call_measurement_enabled? && above_threshold?
self.class.call_cpu_duration_histogram.observe(@transaction.labels.merge(labels), cpu_time / 1000.0) self.class.call_duration_histogram.observe(@transaction.labels.merge(labels), real_time / 1000.0)
end
retval retval
end end
...@@ -83,6 +70,10 @@ module Gitlab ...@@ -83,6 +70,10 @@ module Gitlab
def above_threshold? def above_threshold?
real_time >= Metrics.method_call_threshold real_time >= Metrics.method_call_threshold
end end
def call_measurement_enabled?
Feature.get(:prometheus_metrics_method_instrumentation).enabled?
end
end end
end end
end end
...@@ -17,9 +17,9 @@ module Gitlab ...@@ -17,9 +17,9 @@ module Gitlab
end end
def prometheus_metrics_enabled? def prometheus_metrics_enabled?
# force disable prometheus_metrics until return @prometheus_metrics_enabled if defined?(@prometheus_metrics_enabled)
# https://gitlab.com/gitlab-org/prometheus-client-mmap/merge_requests/11 is ready
false @prometheus_metrics_enabled = prometheus_metrics_enabled_unmemoized
end end
def registry def registry
......
...@@ -13,18 +13,54 @@ describe Gitlab::Metrics::MethodCall do ...@@ -13,18 +13,54 @@ describe Gitlab::Metrics::MethodCall do
expect(method_call.call_count).to eq(1) expect(method_call.call_count).to eq(1)
end end
context 'when measurement is above threshold' do
before do
allow(method_call).to receive(:above_threshold?).and_return(true)
end
context 'prometheus instrumentation is enabled' do
before do
Feature.get(:prometheus_metrics_method_instrumentation).enable
end
it 'observes the performance of the supplied block' do it 'observes the performance of the supplied block' do
expect(described_class.call_real_duration_histogram) expect(described_class.call_duration_histogram)
.to receive(:observe) .to receive(:observe)
.with({ module: :Foo, method: '#bar' }, be_a_kind_of(Numeric)) .with({ module: :Foo, method: '#bar' }, be_a_kind_of(Numeric))
expect(described_class.call_cpu_duration_histogram) method_call.measure { 'foo' }
.to receive(:observe) end
.with({ module: :Foo, method: '#bar' }, be_a_kind_of(Numeric)) end
context 'prometheus instrumentation is disabled' do
before do
Feature.get(:prometheus_metrics_method_instrumentation).disable
end
it 'does not observe the performance' do
expect(described_class.call_duration_histogram)
.not_to receive(:observe)
method_call.measure { 'foo' } method_call.measure { 'foo' }
end end
end end
end
context 'when measurement is below threshold' do
before do
allow(method_call).to receive(:above_threshold?).and_return(false)
Feature.get(:prometheus_metrics_method_instrumentation).enable
end
it 'does not observe the performance' do
expect(described_class.call_duration_histogram)
.not_to receive(:observe)
method_call.measure { 'foo' }
end
end
end
describe '#to_metric' do describe '#to_metric' do
it 'returns a Metric instance' do it 'returns a Metric instance' do
...@@ -43,7 +79,13 @@ describe Gitlab::Metrics::MethodCall do ...@@ -43,7 +79,13 @@ describe Gitlab::Metrics::MethodCall do
end end
describe '#above_threshold?' do describe '#above_threshold?' do
before do
allow(Gitlab::Metrics).to receive(:method_call_threshold).and_return(100)
end
it 'returns false when the total call time is not above the threshold' do it 'returns false when the total call time is not above the threshold' do
expect(method_call).to receive(:real_time).and_return(9)
expect(method_call.above_threshold?).to eq(false) expect(method_call.above_threshold?).to eq(false)
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment