Commit 552d3886 authored by Peter Leitzen's avatar Peter Leitzen Committed by James Lopez

Automatically set Prometheus step interval

By computing the step interval passed to the query_range Prometheus API
call we improve the performance on the Prometheus server and GitLab by
reducing the amount of data points sent back and prevent Prometheus
from sending errors when requesting longer intervals.
parent c7f918aa
---
title: Automatically set Prometheus step interval
merge_request: 26441
author:
type: changed
...@@ -6,6 +6,14 @@ module Gitlab ...@@ -6,6 +6,14 @@ module Gitlab
Error = Class.new(StandardError) Error = Class.new(StandardError)
QueryError = Class.new(Gitlab::PrometheusClient::Error) QueryError = Class.new(Gitlab::PrometheusClient::Error)
# Target number of data points for `query_range`.
# Please don't exceed the limit of 11000 data points
# See https://github.com/prometheus/prometheus/blob/91306bdf24f5395e2601773316945a478b4b263d/web/api/v1/api.go#L347
QUERY_RANGE_DATA_POINTS = 600
# Minimal value of the `step` parameter for `query_range` in seconds.
QUERY_RANGE_MIN_STEP = 60
attr_reader :rest_client, :headers attr_reader :rest_client, :headers
def initialize(rest_client) def initialize(rest_client)
...@@ -23,12 +31,18 @@ module Gitlab ...@@ -23,12 +31,18 @@ module Gitlab
end end
def query_range(query, start: 8.hours.ago, stop: Time.now) def query_range(query, start: 8.hours.ago, stop: Time.now)
start = start.to_f
stop = stop.to_f
step = self.class.compute_step(start, stop)
get_result('matrix') do get_result('matrix') do
json_api_get('query_range', json_api_get(
query: query, 'query_range',
start: start.to_f, query: query,
end: stop.to_f, start: start,
step: 1.minute.to_i) end: stop,
step: step
)
end end
end end
...@@ -40,6 +54,14 @@ module Gitlab ...@@ -40,6 +54,14 @@ module Gitlab
json_api_get('series', 'match': matches, start: start.to_f, end: stop.to_f) json_api_get('series', 'match': matches, start: start.to_f, end: stop.to_f)
end end
def self.compute_step(start, stop)
diff = stop - start
step = (diff / QUERY_RANGE_DATA_POINTS).ceil
[QUERY_RANGE_MIN_STEP, step].max
end
private private
def json_api_get(type, args = {}) def json_api_get(type, args = {})
......
...@@ -230,4 +230,32 @@ describe Gitlab::PrometheusClient do ...@@ -230,4 +230,32 @@ describe Gitlab::PrometheusClient do
let(:execute_query) { subject.query_range(prometheus_query) } let(:execute_query) { subject.query_range(prometheus_query) }
end end
end end
describe '.compute_step' do
using RSpec::Parameterized::TableSyntax
let(:now) { Time.now.utc }
subject { described_class.compute_step(start, stop) }
where(:time_interval_in_seconds, :step) do
0 | 60
10.hours | 60
10.hours + 1 | 61
# frontend options
30.minutes | 60
3.hours | 60
8.hours | 60
1.day | 144
3.days | 432
1.week | 1008
end
with_them do
let(:start) { now - time_interval_in_seconds }
let(:stop) { now }
it { is_expected.to eq(step) }
end
end
end end
...@@ -25,12 +25,16 @@ module PrometheusHelpers ...@@ -25,12 +25,16 @@ module PrometheusHelpers
"https://prometheus.example.com/api/v1/query?#{query}" "https://prometheus.example.com/api/v1/query?#{query}"
end end
def prometheus_query_range_url(prometheus_query, start: 8.hours.ago, stop: Time.now.to_f) def prometheus_query_range_url(prometheus_query, start: 8.hours.ago, stop: Time.now, step: nil)
start = start.to_f
stop = stop.to_f
step ||= Gitlab::PrometheusClient.compute_step(start, stop)
query = { query = {
query: prometheus_query, query: prometheus_query,
start: start.to_f, start: start,
end: stop, end: stop,
step: 1.minute.to_i step: step
}.to_query }.to_query
"https://prometheus.example.com/api/v1/query_range?#{query}" "https://prometheus.example.com/api/v1/query_range?#{query}"
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment