Automatically set Prometheus step interval

By computing the step interval passed to the query_range Prometheus API
call we improve the performance on the Prometheus server and GitLab by
reducing the amount of data points sent back and prevent Prometheus
from sending errors when requesting longer intervals.

changelogs/unreleased/58839-automatically-set-prometheus-step-interval.yml

+---
+title: Automatically set Prometheus step interval
+merge_request: 26441
+author:
+type: changed

lib/gitlab/prometheus_client.rb

@@ -6,6 +6,14 @@ module Gitlab
     Error = Class.new(StandardError)
     QueryError = Class.new(Gitlab::PrometheusClient::Error)
 
+    # Target number of data points for `query_range`.
+    # Please don't exceed the limit of 11000 data points
+    # See https://github.com/prometheus/prometheus/blob/91306bdf24f5395e2601773316945a478b4b263d/web/api/v1/api.go#L347
+    QUERY_RANGE_DATA_POINTS = 600
+
+    # Minimal value of the `step` parameter for `query_range` in seconds.
+    QUERY_RANGE_MIN_STEP = 60
+
     attr_reader :rest_client, :headers
 
     def initialize(rest_client)
@@ -23,12 +31,18 @@ module Gitlab
     end
 
     def query_range(query, start: 8.hours.ago, stop: Time.now)
+      start = start.to_f
+      stop = stop.to_f
+      step = self.class.compute_step(start, stop)
+
       get_result('matrix') do
-        json_api_get('query_range',
-                     query: query,
-                     start: start.to_f,
-                     end: stop.to_f,
-                     step: 1.minute.to_i)
+        json_api_get(
+          'query_range',
+          query: query,
+          start: start,
+          end: stop,
+          step: step
+        )
       end
     end
 
@@ -40,6 +54,14 @@ module Gitlab
       json_api_get('series', 'match': matches, start: start.to_f, end: stop.to_f)
     end
 
+    def self.compute_step(start, stop)
+      diff = stop - start
+
+      step = (diff / QUERY_RANGE_DATA_POINTS).ceil
+
+      [QUERY_RANGE_MIN_STEP, step].max
+    end
+
     private
 
     def json_api_get(type, args = {})

spec/lib/gitlab/prometheus_client_spec.rb

@@ -230,4 +230,32 @@ describe Gitlab::PrometheusClient do
       let(:execute_query) { subject.query_range(prometheus_query) }
     end
   end
+
+  describe '.compute_step' do
+    using RSpec::Parameterized::TableSyntax
+
+    let(:now) { Time.now.utc }
+
+    subject { described_class.compute_step(start, stop) }
+
+    where(:time_interval_in_seconds, :step) do
+      0               | 60
+      10.hours        | 60
+      10.hours + 1    | 61
+      # frontend options
+      30.minutes      | 60
+      3.hours         | 60
+      8.hours         | 60
+      1.day           | 144
+      3.days          | 432
+      1.week          | 1008
+    end
+
+    with_them do
+      let(:start) { now - time_interval_in_seconds }
+      let(:stop) { now }
+
+      it { is_expected.to eq(step) }
+    end
+  end
 end

spec/support/helpers/prometheus_helpers.rb

@@ -25,12 +25,16 @@ module PrometheusHelpers
     "https://prometheus.example.com/api/v1/query?#{query}"
   end
 
-  def prometheus_query_range_url(prometheus_query, start: 8.hours.ago, stop: Time.now.to_f)
+  def prometheus_query_range_url(prometheus_query, start: 8.hours.ago, stop: Time.now, step: nil)
+    start = start.to_f
+    stop = stop.to_f
+    step ||= Gitlab::PrometheusClient.compute_step(start, stop)
+
     query = {
       query: prometheus_query,
-      start: start.to_f,
+      start: start,
       end: stop,
-      step: 1.minute.to_i
+      step: step
     }.to_query
 
     "https://prometheus.example.com/api/v1/query_range?#{query}"