Merge branch 'generate-test-prometheus-data' into 'master'

Generate sample Prometheus data

See merge request gitlab-org/gitlab!19987

app/controllers/projects/environments/sample_metrics_controller.rb

+# frozen_string_literal: true
+
+class Projects::Environments::SampleMetricsController < Projects::ApplicationController
+  def query
+    result = Metrics::SampleMetricsService.new(params[:identifier]).query
+
+    if result
+      render json: { "status": "success", "data": { "resultType": "matrix", "result": result } }
+    else
+      render_404
+    end
+  end
+end

app/services/metrics/sample_metrics_service.rb

+# frozen_string_literal: true
+
+module Metrics
+  class SampleMetricsService
+    DIRECTORY = "sample_metrics"
+
+    attr_reader :identifier
+
+    def initialize(identifier)
+      @identifier = identifier
+    end
+
+    def query
+      return unless identifier && File.exist?(file_location)
+
+      YAML.load_file(File.expand_path(file_location, __dir__))
+    end
+
+    private
+
+    def file_location
+      sanitized_string = identifier.gsub(/[^0-9A-Za-z_]/, '')
+      File.join(Rails.root, DIRECTORY, "#{sanitized_string}.yml")
+    end
+  end
+end

changelogs/unreleased/generate-test-prometheus-data.yml

+---
+title: Genereate a set of sample prometheus metrics and route to the sample metrics
+  when enabled
+merge_request: 19987
+author:
+type: added

config/routes/project.rb

@@ -232,6 +232,8 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
             get '/terminal.ws/authorize', to: 'environments#terminal_websocket_authorize', format: false
 
             get '/prometheus/api/v1/*proxy_path', to: 'environments/prometheus_api#proxy', as: :prometheus_api
+
+            get '/sample_metrics', to: 'environments/sample_metrics#query' if ENV['USE_SAMPLE_METRICS']
           end
 
           collection do

doc/raketasks/generate_sample_prometheus_data.md

+# Generate Sample Prometheus Data
+
+This command will run Prometheus queries for each of the metrics of a specific environment
+for a default time interval of 7 days ago to now. The results of each of query are stored
+under a `sample_metrics` directory as a yaml file named by the metric's `identifier`.
+When the environmental variable `USE_SAMPLE_METRICS` is set, the Prometheus API query is
+re-routed to `Projects::Environments::SampleMetricsController` which loads the appropriate
+data set if it is present within the `sample_metrics` directory.
+
+- This command requires an id from an Environment with an available Prometheus installation.
+
+**Example:**
+
+```
+bundle exec rake gitlab:generate_sample_prometheus_data[21]
+```

lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb

@@ -16,12 +16,20 @@ module Gitlab
           private
 
           def endpoint_for_metric(metric)
-            Gitlab::Routing.url_helpers.prometheus_api_project_environment_path(
-              project,
-              params[:environment],
-              proxy_path: query_type(metric),
-              query: query_for_metric(metric)
-            )
+            if ENV['USE_SAMPLE_METRICS']
+              Gitlab::Routing.url_helpers.sample_metrics_project_environment_path(
+                project,
+                params[:environment],
+                identifier: metric[:id]
+              )
+            else
+              Gitlab::Routing.url_helpers.prometheus_api_project_environment_path(
+                project,
+                params[:environment],
+                proxy_path: query_type(metric),
+                query: query_for_metric(metric)
+              )
+            end
           end
 
           def query_type(metric)

lib/tasks/gitlab/generate_sample_prometheus_data.rake

+namespace :gitlab do
+  desc "GitLab | Generate Sample Prometheus Data"
+  task :generate_sample_prometheus_data, [:environment_id] => :gitlab_environment do |_, args|
+    environment = Environment.find(args[:environment_id])
+    metrics = PrometheusMetric.where(project_id: [environment.project.id, nil])
+    query_variables = Gitlab::Prometheus::QueryVariables.call(environment)
+
+    sample_metrics_directory_name = Metrics::SampleMetricsService::DIRECTORY
+    FileUtils.mkdir_p(sample_metrics_directory_name)
+
+    metrics.each do |metric|
+      query = metric.query % query_variables
+      result = environment.prometheus_adapter.prometheus_client.query_range(query, start: 7.days.ago)
+
+      next unless metric.identifier
+
+      File.write("#{sample_metrics_directory_name}/#{metric.identifier}.yml", result.to_yaml)
+    end
+  end
+end

spec/controllers/projects/environments/sample_metrics_controller_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Projects::Environments::SampleMetricsController do
+  include StubENV
+
+  let_it_be(:project) { create(:project) }
+  let_it_be(:environment) { create(:environment, project: project) }
+  let_it_be(:user) { create(:user) }
+
+  before(:context) do
+    RSpec::Mocks.with_temporary_scope do
+      stub_env('USE_SAMPLE_METRICS', 'true')
+      Rails.application.reload_routes!
+    end
+  end
+
+  after(:context) do
+    Rails.application.reload_routes!
+  end
+
+  before do
+    project.add_reporter(user)
+    sign_in(user)
+  end
+
+  describe 'GET #query' do
+    context 'when the file is not found' do
+      before do
+        get :query, params: environment_params
+      end
+
+      it 'returns a 404' do
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+
+    context 'when the sample data is found' do
+      before do
+        allow_next_instance_of(Metrics::SampleMetricsService) do |service|
+          allow(service).to receive(:query).and_return([])
+        end
+        get :query, params: environment_params
+      end
+
+      it 'returns JSON with a message and a 200 status code' do
+        expect(json_response.keys).to contain_exactly('status', 'data')
+        expect(response).to have_gitlab_http_status(:ok)
+      end
+    end
+  end
+
+  private
+
+  def environment_params(params = {})
+    {
+      id: environment.id.to_s,
+      namespace_id: project.namespace.full_path,
+      project_id: project.name,
+      identifier: 'sample_metric_query_result'
+    }.merge(params)
+  end
+end

spec/fixtures/gitlab/sample_metrics/sample_metric_query_result.yml

+---
+- metric: {}
+  values:
+  - - 1573560714.209
+    - '0.02361297607421875'
+  - - 1573560774.209
+    - '0.02361297607421875'
+  - - 1573560834.209
+    - '0.02362823486328125'
+  - - 1573560894.209
+    - '0.02361297607421875'
+  - - 1573560954.209
+    - '0.02385711669921875'
+  - - 1573561014.209
+    - '0.02361297607421875'
+  - - 1573561074.209
+    - '0.02361297607421875'
+  - - 1573561134.209
+    - '0.02362060546875'
+  - - 1573561194.209
+    - '0.02362060546875'
+  - - 1573561254.209
+    - '0.02362060546875'
+  - - 1573561314.209
+    - '0.02362060546875'
+  - - 1573561374.209
+    - '0.023624420166015625'
+  - - 1573561434.209
+    - '0.023651123046875'
+  - - 1573561494.209
+    - '0.02362060546875'
+  - - 1573561554.209
+    - '0.0236358642578125'
+  - - 1573561614.209
+    - '0.02362060546875'
+  - - 1573561674.209
+    - '0.02362060546875'
+  - - 1573561734.209
+    - '0.02362060546875'
+  - - 1573561794.209
+    - '0.02362060546875'
+  - - 1573561854.209
+    - '0.02362060546875'
+  - - 1573561914.209
+    - '0.023651123046875'
+  - - 1573561974.209
+    - '0.02362060546875'
+  - - 1573562034.209
+    - '0.02362060546875'
+  - - 1573562094.209
+    - '0.02362060546875'
+  - - 1573562154.209
+    - '0.02362060546875'
+  - - 1573562214.209
+    - '0.023624420166015625'
+  - - 1573562274.209
+    - '0.02362060546875'
+  - - 1573562334.209
+    - '0.023868560791015625'
+  - - 1573562394.209
+    - '0.02374267578125'
+  - - 1573562454.209
+    - '0.02362060546875'
+  - - 1573562514.209
+    - '0.02362060546875'
+  - - 1573562574.209
+    - '0.02362060546875'
+  - - 1573562634.209
+    - '0.02362060546875'
+  - - 1573562694.209
+    - '0.023639678955078125'
+  - - 1573562754.209
+    - '0.0236358642578125'
+  - - 1573562814.209
+    - '0.02362060546875'
+  - - 1573562874.209
+    - '0.0236358642578125'
+  - - 1573562934.209
+    - '0.023651123046875'
+  - - 1573562994.209
+    - '0.02362060546875'
+  - - 1573563054.209
+    - '0.023624420166015625'
+  - - 1573563114.209
+    - '0.02362060546875'
+  - - 1573563174.209
+    - '0.02362060546875'
+  - - 1573563234.209
+    - '0.02362060546875'
+  - - 1573563294.209
+    - '0.02362060546875'
+  - - 1573563354.209
+    - '0.02362060546875'
+  - - 1573563414.209
+    - '0.023651123046875'
+  - - 1573563474.209
+    - '0.023651123046875'
+  - - 1573563534.209
+    - '0.023651123046875'
+  - - 1573563594.209
+    - '0.023773193359375'
+  - - 1573563654.209
+    - '0.023681640625'
+  - - 1573563714.209
+    - '0.023895263671875'
+  - - 1573563774.209
+    - '0.023651123046875'
+  - - 1573563834.209
+    - '0.023651123046875'
+  - - 1573563894.209
+    - '0.023651123046875'
+  - - 1573563954.209
+    - '0.0236663818359375'
+  - - 1573564014.209
+    - '0.023651123046875'
+  - - 1573564074.209
+    - '0.023681640625'
+  - - 1573564134.209
+    - '0.0236663818359375'
+  - - 1573564194.209
+    - '0.0236663818359375'
+  - - 1573564254.209
+    - '0.023651123046875'
+  - - 1573564314.209
+    - '0.023651123046875'
+  - - 1573564374.209
+    - '0.023651123046875'
+  - - 1573564434.209
+    - '0.023773193359375'
+  - - 1573564494.209
+    - '0.023651123046875'
+  - - 1573564554.209
+    - '0.023681640625'
+  - - 1573564614.209
+    - '0.023773193359375'
+  - - 1573564674.209
+    - '0.023651123046875'
+  - - 1573564734.209
+    - '0.023651123046875'
+  - - 1573564794.209
+    - '0.023651123046875'
+  - - 1573564854.209
+    - '0.023651123046875'
+  - - 1573564914.209
+    - '0.023651123046875'
+  - - 1573564974.209
+    - '0.023651123046875'
+  - - 1573565034.209
+    - '0.023651123046875'
+  - - 1573565094.209
+    - '0.023895263671875'
\ No newline at end of file

spec/services/metrics/sample_metrics_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Metrics::SampleMetricsService do
+  describe 'query' do
+    subject { described_class.new(identifier).query }
+
+    context 'when the file is not found' do
+      let(:identifier) { nil }
+
+      it { is_expected.to be_nil }
+    end
+
+    context 'when the file is found' do
+      let(:identifier) { 'sample_metric_query_result' }
+      let(:source) { File.join(Rails.root, 'spec/fixtures/gitlab/sample_metrics', "#{identifier}.yml") }
+      let(:destination) { File.join(Rails.root, Metrics::SampleMetricsService::DIRECTORY, "#{identifier}.yml") }
+
+      around do |example|
+        FileUtils.mkdir_p(Metrics::SampleMetricsService::DIRECTORY)
+        FileUtils.cp(source, destination)
+
+        example.run
+      ensure
+        FileUtils.rm(destination)
+      end
+
+      subject { described_class.new(identifier).query }
+
+      it 'loads data from the sample file correctly' do
+        expect(subject).to eq(YAML.load_file(source))
+      end
+    end
+
+    context 'when the identifier is for a path outside of sample_metrics' do
+      let(:identifier) { '../config/secrets' }
+
+      it { is_expected.to be_nil }
+    end
+  end
+end

spec/tasks/gitlab/generate_sample_prometheus_data_spec.rb

+# frozen_string_literal: true
+
+require 'rake_helper'
+
+describe 'gitlab:generate_sample_prometheus_data rake task' do
+  let(:cluster) { create(:cluster, :provided_by_user, :project) }
+  let(:environment) { create(:environment, project: cluster.project) }
+  let(:sample_query_file) { File.join(Rails.root, Metrics::SampleMetricsService::DIRECTORY, 'test_query_result.yml') }
+  let!(:metric) { create(:prometheus_metric, project: cluster.project, identifier: 'test_query_result') }
+
+  around do |example|
+    example.run
+  ensure
+    FileUtils.rm(sample_query_file)
+  end
+
+  it 'creates the file correctly' do
+    Rake.application.rake_require 'tasks/gitlab/generate_sample_prometheus_data'
+    allow(Environment).to receive(:find).and_return(environment)
+    allow(environment).to receive_message_chain(:prometheus_adapter, :prometheus_client, :query_range) { sample_query_result }
+    run_rake_task('gitlab:generate_sample_prometheus_data', [environment.id])
+
+    expect(File.exist?(sample_query_file)).to be true
+
+    query_file_content = YAML.load_file(sample_query_file)
+
+    expect(query_file_content).to eq(sample_query_result)
+  end
+end
+
+def sample_query_result
+  file = File.join(Rails.root, 'spec/fixtures/gitlab/sample_metrics', 'sample_metric_query_result.yml')
+  YAML.load_file(File.expand_path(file, __dir__))
+end