Commit 3857ab87 authored by Sarah Yasonik's avatar Sarah Yasonik Committed by Dmitriy Zaporozhets

Add service to fetch and format a grafana embed

Adds a new Metrics Dashboard Service which fetches a grafana
dashboard and datasource based on an input url, reformatting
the output to be compatible with our existing dashboard
rendering behavior on the FE. Ultimately, this will be used
as a part of embedding prometheus metrics in GFM via grafana.

Note that this service introduces usage of ReactiveCaching in
the service as a result of external API calls. This will mean
that usage of this new service will need to be polled.
parent 5ef3c06a
# frozen_string_literal: true
# Responsible for returning a gitlab-compatible dashboard
# containing info based on a grafana dashboard and datasource.
#
# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards.
module Metrics
module Dashboard
class GrafanaMetricEmbedService < ::Metrics::Dashboard::BaseService
include ReactiveCaching
SEQUENCE = [
::Gitlab::Metrics::Dashboard::Stages::GrafanaFormatter
].freeze
self.reactive_cache_key = ->(service) { service.cache_key }
self.reactive_cache_lease_timeout = 30.seconds
self.reactive_cache_refresh_interval = 30.minutes
self.reactive_cache_lifetime = 30.minutes
self.reactive_cache_worker_finder = ->(_id, *args) { from_cache(*args) }
class << self
# Determines whether the provided params are sufficient
# to uniquely identify a grafana dashboard.
def valid_params?(params)
[
params[:embedded],
params[:grafana_url]
].all?
end
def from_cache(project_id, user_id, grafana_url)
project = Project.find(project_id)
user = User.find(user_id)
new(project, user, grafana_url: grafana_url)
end
end
def get_dashboard
with_reactive_cache(*cache_key) { |result| result }
end
# Inherits the primary logic from the parent class and
# maintains the service's API while including ReactiveCache
def calculate_reactive_cache(*)
::Metrics::Dashboard::BaseService
.instance_method(:get_dashboard)
.bind(self)
.call() # rubocop:disable Style/MethodCallWithoutArgsParentheses
end
def cache_key(*args)
[project.id, current_user.id, grafana_url]
end
# Required for ReactiveCaching; Usage overridden by
# self.reactive_cache_worker_finder
def id
nil
end
private
def get_raw_dashboard
raise MissingIntegrationError unless client
grafana_dashboard = fetch_dashboard
datasource = fetch_datasource(grafana_dashboard)
params.merge!(grafana_dashboard: grafana_dashboard, datasource: datasource)
{}
end
def fetch_dashboard
uid = GrafanaUidParser.new(grafana_url, project).parse
raise DashboardProcessingError.new('Dashboard uid not found') unless uid
response = client.get_dashboard(uid: uid)
parse_json(response.body)
end
def fetch_datasource(dashboard)
name = DatasourceNameParser.new(grafana_url, dashboard).parse
raise DashboardProcessingError.new('Datasource name not found') unless name
response = client.get_datasource(name: name)
parse_json(response.body)
end
def grafana_url
params[:grafana_url]
end
def client
project.grafana_integration&.client
end
def allowed?
Ability.allowed?(current_user, :read_project, project)
end
def sequence
SEQUENCE
end
def parse_json(json)
JSON.parse(json, symbolize_names: true)
rescue JSON::ParserError
raise DashboardProcessingError.new('Grafana response contains invalid json')
end
end
# Identifies the uid of the dashboard based on url format
class GrafanaUidParser
def initialize(grafana_url, project)
@grafana_url, @project = grafana_url, project
end
def parse
@grafana_url.match(uid_regex) { |m| m.named_captures['uid'] }
end
private
# URLs are expected to look like https://domain.com/d/:uid/other/stuff
def uid_regex
base_url = @project.grafana_integration.grafana_url.chomp('/')
%r{(#{Regexp.escape(base_url)}\/d\/(?<uid>\w+)\/)}x
end
end
# Identifies the name of the datasource for a dashboard
# based on the panelId query parameter found in the url
class DatasourceNameParser
def initialize(grafana_url, grafana_dashboard)
@grafana_url, @grafana_dashboard = grafana_url, grafana_dashboard
end
def parse
@grafana_dashboard[:dashboard][:panels]
.find { |panel| panel[:id].to_s == query_params[:panelId] }
.try(:[], :datasource)
end
private
def query_params
Gitlab::Metrics::Dashboard::Url.parse_query(@grafana_url)
end
end
end
end
......@@ -9,6 +9,7 @@ module Gitlab
module Errors
DashboardProcessingError = Class.new(StandardError)
PanelNotFoundError = Class.new(StandardError)
MissingIntegrationError = Class.new(StandardError)
LayoutError = Class.new(DashboardProcessingError)
MissingQueryError = Class.new(DashboardProcessingError)
......@@ -22,6 +23,10 @@ module Gitlab
error("#{dashboard_path} could not be found.", :not_found)
when PanelNotFoundError
error(error.message, :not_found)
when ::Grafana::Client::Error
error(error.message, :service_unavailable)
when MissingIntegrationError
error('Proxy support for this API is not available currently', :bad_request)
else
raise error
end
......
......@@ -17,7 +17,10 @@ module Gitlab
# Returns a new dashboard hash with the results of
# running transforms on the dashboard.
# @return [Hash, nil]
def process
return unless @dashboard
@dashboard.deep_symbolize_keys.tap do |dashboard|
@sequence.each do |stage|
stage.new(@project, dashboard, @params).transform!
......
# frozen_string_literal: true
module Gitlab
module Metrics
module Dashboard
module Stages
class GrafanaFormatter < BaseStage
include Gitlab::Utils::StrongMemoize
CHART_TYPE = 'area-chart'
PROXY_PATH = 'api/v1/query_range'
# Reformats the specified panel in the Gitlab
# dashboard-yml format
def transform!
InputFormatValidator.new(
grafana_dashboard,
datasource,
panel,
query_params
).validate!
new_dashboard = formatted_dashboard
dashboard.clear
dashboard.merge!(new_dashboard)
end
private
def formatted_dashboard
{ panel_groups: [{ panels: [formatted_panel] }] }
end
def formatted_panel
{
title: panel[:title],
type: CHART_TYPE,
y_label: '', # Grafana panels do not include a Y-Axis label
metrics: panel[:targets].map.with_index do |target, idx|
formatted_metric(target, idx)
end
}
end
def formatted_metric(metric, idx)
{
id: "#{metric[:legendFormat]}_#{idx}",
query_range: format_query(metric),
label: replace_variables(metric[:legendFormat]),
prometheus_endpoint_path: prometheus_endpoint_for_metric(metric)
}.compact
end
# Panel specified by the url from the Grafana dashboard
def panel
strong_memoize(:panel) do
grafana_dashboard[:dashboard][:panels].find do |panel|
panel[:id].to_s == query_params[:panelId]
end
end
end
# Grafana url query parameters. Includes information
# on which panel to select and time range.
def query_params
strong_memoize(:query_params) do
Gitlab::Metrics::Dashboard::Url.parse_query(grafana_url)
end
end
# Endpoint which will return prometheus metric data
# for the metric
def prometheus_endpoint_for_metric(metric)
Gitlab::Routing.url_helpers.project_grafana_api_path(
project,
datasource_id: datasource[:id],
proxy_path: PROXY_PATH,
query: format_query(metric)
)
end
# Reformats query for compatibility with prometheus api.
def format_query(metric)
expression = remove_new_lines(metric[:expr])
expression = replace_variables(expression)
expression = replace_global_variables(expression, metric)
expression
end
# Accomodates instance-defined Grafana variables.
# These are variables defined by users, and values
# must be provided in the query parameters.
def replace_variables(expression)
return expression unless grafana_dashboard[:dashboard][:templating]
grafana_dashboard[:dashboard][:templating][:list]
.sort_by { |variable| variable[:name].length }
.each do |variable|
variable_value = query_params[:"var-#{variable[:name]}"]
expression = expression.gsub("$#{variable[:name]}", variable_value)
expression = expression.gsub("[[#{variable[:name]}]]", variable_value)
expression = expression.gsub("{{#{variable[:name]}}}", variable_value)
end
expression
end
# Replaces Grafana global built-in variables with values.
# Only $__interval and $__from and $__to are supported.
#
# See https://grafana.com/docs/reference/templating/#global-built-in-variables
def replace_global_variables(expression, metric)
expression = expression.gsub('$__interval', metric[:interval]) if metric[:interval]
expression = expression.gsub('$__from', query_params[:from])
expression = expression.gsub('$__to', query_params[:to])
expression
end
# Removes new lines from expression.
def remove_new_lines(expression)
expression.gsub(/\R+/, '')
end
# Grafana datasource object corresponding to the
# specified dashboard
def datasource
params[:datasource]
end
# The specified Grafana dashboard
def grafana_dashboard
params[:grafana_dashboard]
end
# The URL specifying which Grafana panel to embed
def grafana_url
params[:grafana_url]
end
end
class InputFormatValidator
include ::Gitlab::Metrics::Dashboard::Errors
attr_reader :grafana_dashboard, :datasource, :panel, :query_params
UNSUPPORTED_GRAFANA_GLOBAL_VARS = %w(
$__interval_ms
$__timeFilter
$__name
$timeFilter
$interval
).freeze
def initialize(grafana_dashboard, datasource, panel, query_params)
@grafana_dashboard = grafana_dashboard
@datasource = datasource
@panel = panel
@query_params = query_params
end
def validate!
validate_query_params!
validate_datasource!
validate_panel_type!
validate_variable_definitions!
validate_global_variables!
end
private
def validate_datasource!
return if datasource[:access] == 'proxy' && datasource[:type] == 'prometheus'
raise_error 'Only Prometheus datasources with proxy access in Grafana are supported.'
end
def validate_query_params!
return if [:panelId, :from, :to].all? { |param| query_params.include?(param) }
raise_error 'Grafana query parameters must include panelId, from, and to.'
end
def validate_panel_type!
return if panel[:type] == 'graph' && panel[:lines]
raise_error 'Panel type must be a line graph.'
end
def validate_variable_definitions!
return unless grafana_dashboard[:dashboard][:templating]
return if grafana_dashboard[:dashboard][:templating][:list].all? do |variable|
query_params[:"var-#{variable[:name]}"].present?
end
raise_error 'All Grafana variables must be defined in the query parameters.'
end
def validate_global_variables!
return unless panel_contains_unsupported_vars?
raise_error 'Prometheus must not include'
end
def panel_contains_unsupported_vars?
panel[:targets].any? do |target|
UNSUPPORTED_GRAFANA_GLOBAL_VARS.any? do |variable|
target[:expr].include?(variable)
end
end
end
def raise_error(message)
raise DashboardProcessingError.new(message)
end
end
end
end
end
end
......@@ -11,6 +11,18 @@ module Grafana
@token = token
end
# @param uid [String] Unique identifier for a Grafana dashboard
def get_dashboard(uid:)
http_get("#{@api_url}/api/dashboards/uid/#{uid}")
end
# @param name [String] Unique identifier for a Grafana datasource
def get_datasource(name:)
# CGI#escape formats strings such that the Grafana endpoint
# will not recognize the dashboard name. Preferring URI#escape.
http_get("#{@api_url}/api/datasources/name/#{URI.escape(name)}") # rubocop:disable Lint/UriEscapeUnescape
end
# @param datasource_id [String] Grafana ID for the datasource
# @param proxy_path [String] Path to proxy - ex) 'api/v1/query_range'
def proxy_datasource(datasource_id:, proxy_path:, query: {})
......@@ -57,7 +69,7 @@ module Grafana
def handle_response(response)
return response if response.code == 200
raise_error "Grafana response status code: #{response.code}"
raise_error "Grafana response status code: #{response.code}, Message: #{response.body}"
end
def raise_error(message)
......
......@@ -3,7 +3,7 @@
FactoryBot.define do
factory :grafana_integration, class: GrafanaIntegration do
project
grafana_url { 'https://grafana.com' }
grafana_url { 'https://grafana.example.com' }
token { SecureRandom.hex(10) }
end
end
This diff is collapsed.
{
"id": 1,
"orgId": 1,
"name": "GitLab Omnibus",
"type": "prometheus",
"typeLogoUrl": "",
"access": "proxy",
"url": "http://localhost:9090",
"password": "",
"user": "",
"database": "",
"basicAuth": false,
"basicAuthUser": "",
"basicAuthPassword": "",
"withCredentials": false,
"isDefault": true,
"jsonData": {},
"secureJsonFields": {},
"version": 1,
"readOnly": true
}
{
"panel_groups": [
{
"panels": [
{
"title": "Network I/O",
"type": "area-chart",
"y_label": "",
"metrics": [
{
"id": "In_0",
"query_range": "sum( rate(redis_net_input_bytes_total{instance=~\"localhost:9121\"}[1m]))",
"label": "In",
"prometheus_endpoint_path": "/foo/bar/-/grafana/proxy/1/api/v1/query_range?query=sum%28++rate%28redis_net_input_bytes_total%7Binstance%3D~%22localhost%3A9121%22%7D%5B1m%5D%29%29"
},
{
"id": "Out_1",
"query_range": "sum( rate(redis_net_output_bytes_total{instance=~\"localhost:9121\"}[1m]))",
"label": "Out",
"prometheus_endpoint_path": "/foo/bar/-/grafana/proxy/1/api/v1/query_range?query=sum%28++rate%28redis_net_output_bytes_total%7Binstance%3D~%22localhost%3A9121%22%7D%5B1m%5D%29%29"
}
]
}
]
}
]
}
{
"dashboard": {
"panels": [
{
"datasource": "GitLab Omnibus",
"id": 8,
"lines": true,
"targets": [
{
"expr": "sum(\n rate(redis_net_input_bytes_total{instance=~\"$instance\"}[$__interval])\n)",
"format": "time_series",
"interval": "1m",
"legendFormat": "In",
"refId": "A"
},
{
"expr": "sum(\n rate(redis_net_output_bytes_total{instance=~\"[[instance]]\"}[$__interval])\n)",
"format": "time_series",
"interval": "1m",
"legendFormat": "Out",
"refId": "B"
}
],
"title": "Network I/O",
"type": "graph",
"yaxes": [{ "format": "Bps" }, { "format": "short" }]
}
],
"templating": {
"list": [
{
"current": {
"value": "localhost:9121"
},
"name": "instance"
}
]
}
}
}
{
"type": "object",
"required": [
"unit",
"label",
"prometheus_endpoint_path"
],
......
......@@ -3,7 +3,6 @@
"required": [
"title",
"y_label",
"weight",
"metrics"
],
"properties": {
......
......@@ -25,6 +25,14 @@ describe Gitlab::Metrics::Dashboard::Processor do
end
end
context 'when the dashboard is not present' do
let(:dashboard_yml) { nil }
it 'returns nil' do
expect(dashboard).to be_nil
end
end
context 'when dashboard config corresponds to common metrics' do
let!(:common_metric) { create(:prometheus_metric, :common, identifier: 'metric_a1') }
......
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Metrics::Dashboard::Stages::GrafanaFormatter do
include GrafanaApiHelpers
let_it_be(:namespace) { create(:namespace, name: 'foo') }
let_it_be(:project) { create(:project, namespace: namespace, name: 'bar') }
describe '#transform!' do
let(:grafana_dashboard) { JSON.parse(fixture_file('grafana/simplified_dashboard_response.json'), symbolize_names: true) }
let(:datasource) { JSON.parse(fixture_file('grafana/datasource_response.json'), symbolize_names: true) }
let(:dashboard) { described_class.new(project, {}, params).transform! }
let(:params) do
{
grafana_dashboard: grafana_dashboard,
datasource: datasource,
grafana_url: valid_grafana_dashboard_link('https://grafana.example.com')
}
end
context 'when the query and resources are configured correctly' do
let(:expected_dashboard) { JSON.parse(fixture_file('grafana/expected_grafana_embed.json'), symbolize_names: true) }
it 'generates a gitlab-yml formatted dashboard' do
expect(dashboard).to eq(expected_dashboard)
end
end
context 'when the inputs are invalid' do
shared_examples_for 'processing error' do
it 'raises a processing error' do
expect { dashboard }
.to raise_error(Gitlab::Metrics::Dashboard::Stages::InputFormatValidator::DashboardProcessingError)
end
end
context 'when the datasource is not proxyable' do
before do
params[:datasource][:access] = 'not-proxy'
end
it_behaves_like 'processing error'
end
context 'when query param "panelId" is not specified' do
before do
params[:grafana_url].gsub!('panelId=8', '')
end
it_behaves_like 'processing error'
end
context 'when query param "from" is not specified' do
before do
params[:grafana_url].gsub!('from=1570397739557', '')
end
it_behaves_like 'processing error'
end
context 'when query param "to" is not specified' do
before do
params[:grafana_url].gsub!('to=1570484139557', '')
end
it_behaves_like 'processing error'
end
context 'when the panel is not a graph' do
before do
params[:grafana_dashboard][:dashboard][:panels][0][:type] = 'singlestat'
end
it_behaves_like 'processing error'
end
context 'when the panel is not a line graph' do
before do
params[:grafana_dashboard][:dashboard][:panels][0][:lines] = false
end
it_behaves_like 'processing error'
end
context 'when the query dashboard includes undefined variables' do
before do
params[:grafana_url].gsub!('&var-instance=localhost:9121', '')
end
it_behaves_like 'processing error'
end
context 'when the expression contains unsupported global variables' do
before do
params[:grafana_dashboard][:dashboard][:panels][0][:targets][0][:expr] = 'sum(important_metric[$__interval_ms])'
end
it_behaves_like 'processing error'
end
end
end
end
......@@ -35,7 +35,7 @@ describe Grafana::Client do
it 'does not follow redirects' do
expect { subject }.to raise_exception(
Grafana::Client::Error,
'Grafana response status code: 302'
'Grafana response status code: 302, Message: {}'
)
expect(redirect_req_stub).to have_been_requested
......@@ -67,6 +67,30 @@ describe Grafana::Client do
end
end
describe '#get_dashboard' do
let(:grafana_api_url) { 'https://grafanatest.com/-/grafana-project/api/dashboards/uid/FndfgnX' }
subject do
client.get_dashboard(uid: 'FndfgnX')
end
it_behaves_like 'calls grafana api'
it_behaves_like 'no redirects'
it_behaves_like 'handles exceptions'
end
describe '#get_datasource' do
let(:grafana_api_url) { 'https://grafanatest.com/-/grafana-project/api/datasources/name/Test%20Name' }
subject do
client.get_datasource(name: 'Test Name')
end
it_behaves_like 'calls grafana api'
it_behaves_like 'no redirects'
it_behaves_like 'handles exceptions'
end
describe '#proxy_datasource' do
let(:grafana_api_url) do
'https://grafanatest.com/-/grafana-project/' \
......
# frozen_string_literal: true
require 'spec_helper'
describe Metrics::Dashboard::GrafanaMetricEmbedService do
include MetricsDashboardHelpers
include ReactiveCachingHelpers
include GrafanaApiHelpers
let_it_be(:project) { build(:project) }
let_it_be(:user) { create(:user) }
let_it_be(:grafana_integration) { create(:grafana_integration, project: project) }
let(:grafana_url) do
valid_grafana_dashboard_link(grafana_integration.grafana_url)
end
before do
project.add_maintainer(user)
end
describe '.valid_params?' do
let(:valid_params) { { embedded: true, grafana_url: grafana_url } }
subject { described_class.valid_params?(params) }
let(:params) { valid_params }
it { is_expected.to be_truthy }
context 'not embedded' do
let(:params) { valid_params.except(:embedded) }
it { is_expected.to be_falsey }
end
context 'undefined grafana_url' do
let(:params) { valid_params.except(:grafana_url) }
it { is_expected.to be_falsey }
end
end
describe '.from_cache' do
let(:params) { [project.id, user.id, grafana_url] }
subject { described_class.from_cache(*params) }
it 'initializes an instance of GrafanaMetricEmbedService' do
expect(subject).to be_an_instance_of(described_class)
expect(subject.project).to eq(project)
expect(subject.current_user).to eq(user)
expect(subject.params[:grafana_url]).to eq(grafana_url)
end
end
describe '#get_dashboard', :use_clean_rails_memory_store_caching do
let(:service_params) do
[
project,
user,
{
embedded: true,
grafana_url: grafana_url
}
]
end
let(:service) { described_class.new(*service_params) }
let(:service_call) { service.get_dashboard }
context 'without caching' do
before do
synchronous_reactive_cache(service)
end
it_behaves_like 'raises error for users with insufficient permissions'
context 'without a grafana integration' do
before do
allow(project).to receive(:grafana_integration).and_return(nil)
end
it_behaves_like 'misconfigured dashboard service response', :bad_request
end
context 'when grafana cannot be reached' do
before do
allow(grafana_integration.client).to receive(:get_dashboard).and_raise(::Grafana::Client::Error)
end
it_behaves_like 'misconfigured dashboard service response', :service_unavailable
end
context 'when panelId is missing' do
let(:grafana_url) do
grafana_integration.grafana_url +
'/d/XDaNK6amz/gitlab-omnibus-redis' \
'?from=1570397739557&to=1570484139557'
end
before do
stub_dashboard_request(grafana_integration.grafana_url)
end
it_behaves_like 'misconfigured dashboard service response', :unprocessable_entity
end
context 'when uid is missing' do
let(:grafana_url) { grafana_integration.grafana_url + '/d/' }
before do
stub_dashboard_request(grafana_integration.grafana_url)
end
it_behaves_like 'misconfigured dashboard service response', :unprocessable_entity
end
context 'when the dashboard response contains misconfigured json' do
before do
stub_dashboard_request(grafana_integration.grafana_url, body: '')
end
it_behaves_like 'misconfigured dashboard service response', :unprocessable_entity
end
context 'when the datasource response contains misconfigured json' do
before do
stub_dashboard_request(grafana_integration.grafana_url)
stub_datasource_request(grafana_integration.grafana_url, body: '')
end
it_behaves_like 'misconfigured dashboard service response', :unprocessable_entity
end
context 'when the embed was created successfully' do
before do
stub_dashboard_request(grafana_integration.grafana_url)
stub_datasource_request(grafana_integration.grafana_url)
end
it_behaves_like 'valid embedded dashboard service response'
end
end
context 'with caching', :use_clean_rails_memory_store_caching do
let(:cache_params) { [project.id, user.id, grafana_url] }
context 'when value not present in cache' do
it 'returns nil' do
expect(ReactiveCachingWorker)
.to receive(:perform_async)
.with(service.class, service.id, *cache_params)
expect(service_call).to eq(nil)
end
end
context 'when value present in cache' do
let(:return_value) { { 'http_status' => :ok, 'dashboard' => '{}' } }
before do
stub_reactive_cache(service, return_value, cache_params)
end
it 'returns cached value' do
expect(ReactiveCachingWorker)
.not_to receive(:perform_async)
.with(service.class, service.id, *cache_params)
expect(service_call[:http_status]).to eq(return_value[:http_status])
expect(service_call[:dashboard]).to eq(return_value[:dashboard])
end
end
end
end
end
# frozen_string_literal: true
module GrafanaApiHelpers
def valid_grafana_dashboard_link(base_url)
base_url +
'/d/XDaNK6amz/gitlab-omnibus-redis' \
'?from=1570397739557&to=1570484139557' \
'&var-instance=localhost:9121&panelId=8'
end
def stub_dashboard_request(base_url, path: '/api/dashboards/uid/XDaNK6amz', body: nil)
body ||= fixture_file('grafana/dashboard_response.json')
stub_request(:get, "#{base_url}#{path}")
.to_return(
status: 200,
body: body,
headers: { 'Content-Type' => 'application/json' }
)
end
def stub_datasource_request(base_url, path: '/api/datasources/name/GitLab%20Omnibus', body: nil)
body ||= fixture_file('grafana/datasource_response.json')
stub_request(:get, "#{base_url}#{path}")
.to_return(
status: 200,
body: body,
headers: { 'Content-Type' => 'application/json' }
)
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment