Commit b6d93e3c authored by Roy Zwambag's avatar Roy Zwambag

Revert "Merge branch '350152-purge-request-profiler' into 'master'"

This reverts merge request !78911
parent 0f17f745
......@@ -160,6 +160,7 @@ Lint/MixedRegexpCaptureTypes:
- 'lib/gitlab/diff/suggestions_parser.rb'
- 'lib/gitlab/github_import/representation/note.rb'
- 'lib/gitlab/metrics/system.rb'
- 'lib/gitlab/request_profiler/profile.rb'
- 'lib/gitlab/slash_commands/issue_move.rb'
- 'lib/gitlab/slash_commands/issue_new.rb'
- 'lib/gitlab/slash_commands/run.rb'
......
......@@ -875,6 +875,7 @@ Gitlab/NamespacedClass:
- app/workers/repository_import_worker.rb
- app/workers/repository_remove_remote_worker.rb
- app/workers/repository_update_remote_mirror_worker.rb
- app/workers/requests_profiles_worker.rb
- app/workers/run_pipeline_schedule_worker.rb
- app/workers/schedule_merge_request_cleanup_refs_worker.rb
- app/workers/schedule_migrate_external_diffs_worker.rb
......
# frozen_string_literal: true
class Admin::RequestsProfilesController < Admin::ApplicationController
feature_category :not_owned
def index
@profile_token = Gitlab::RequestProfiler.profile_token
@profiles = Gitlab::RequestProfiler.all.group_by(&:request_path)
end
def show
clean_name = Rack::Utils.clean_path_info(params[:name])
profile = Gitlab::RequestProfiler.find(clean_name)
unless profile && profile.content_type
return redirect_to admin_requests_profiles_path, alert: 'Profile not found'
end
send_file profile.file_path, type: "#{profile.content_type}; charset=utf-8", disposition: 'inline'
end
end
......@@ -56,7 +56,7 @@ module NavHelper
end
def admin_monitoring_nav_links
%w(system_info background_migrations background_jobs health_check)
%w(system_info background_migrations background_jobs health_check requests_profiles)
end
def admin_analytics_nav_links
......
- page_title _('Requests Profiles')
%h3.page-title
= page_title
.bs-callout.clearfix
= html_escape(_('Pass the header %{codeOpen} X-Profile-Token: %{profile_token} %{codeClose} to profile the request')) % { profile_token: @profile_token, codeOpen: '<code>'.html_safe, codeClose: '</code>'.html_safe }
- if @profiles.present?
.gl-mt-3
- @profiles.each do |path, profiles|
.card
.card-header
%code= path
%ul.content-list
- profiles.each do |profile|
%li
= link_to profile.time.to_s(:long) + ' ' + profile.profile_mode.capitalize,
admin_requests_profile_path(profile)
- else
%p
= _('No profiles found')
......@@ -103,6 +103,10 @@
= link_to admin_health_check_path, title: _('Health Check') do
%span
= _('Health Check')
= nav_link(controller: :requests_profiles) do
= link_to admin_requests_profiles_path, title: _('Requests Profiles') do
%span
= _('Requests Profiles')
- if Gitlab::CurrentSettings.current_application_settings.grafana_enabled?
= nav_link do
= link_to Gitlab::CurrentSettings.current_application_settings.grafana_url, target: '_blank', title: _('Metrics Dashboard'), rel: 'noopener noreferrer' do
......
......@@ -597,6 +597,15 @@
:weight: 1
:idempotent:
:tags: []
- :name: cronjob:requests_profiles
:worker_name: RequestsProfilesWorker
:feature_category: :source_code_management
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent:
:tags: []
- :name: cronjob:schedule_merge_request_cleanup_refs
:worker_name: ScheduleMergeRequestCleanupRefsWorker
:feature_category: :code_review
......
# frozen_string_literal: true
class RequestsProfilesWorker # rubocop:disable Scalability/IdempotentWorker
include ApplicationWorker
data_consistency :always
# rubocop:disable Scalability/CronWorkerContext
# This worker does not perform work scoped to a context
include CronjobQueue
# rubocop:enable Scalability/CronWorkerContext
feature_category :source_code_management
def perform
Gitlab::RequestProfiler.remove_all_profiles
end
end
......@@ -479,6 +479,9 @@ Settings.cron_jobs['import_export_project_cleanup_worker']['job_class'] = 'Impor
Settings.cron_jobs['ci_archive_traces_cron_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['ci_archive_traces_cron_worker']['cron'] ||= '17 * * * *'
Settings.cron_jobs['ci_archive_traces_cron_worker']['job_class'] = 'Ci::ArchiveTracesCronWorker'
Settings.cron_jobs['requests_profiles_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['requests_profiles_worker']['cron'] ||= '0 0 * * *'
Settings.cron_jobs['requests_profiles_worker']['job_class'] = 'RequestsProfilesWorker'
Settings.cron_jobs['remove_expired_members_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['remove_expired_members_worker']['cron'] ||= '10 0 * * *'
Settings.cron_jobs['remove_expired_members_worker']['job_class'] = 'RemoveExpiredMembersWorker'
......
# frozen_string_literal: true
Rails.application.configure do |config|
config.middleware.use(Gitlab::RequestProfiler::Middleware)
config.middleware.use(Gitlab::Middleware::Speedscope)
end
......@@ -100,6 +100,7 @@ namespace :admin do
resource :background_jobs, controller: 'background_jobs', only: [:show]
resource :system_info, controller: 'system_info', only: [:show]
resources :requests_profiles, only: [:index, :show], param: :name, constraints: { name: /.+\.(html|txt)/ }
resources :projects, only: [:index]
......
......@@ -17,6 +17,7 @@ module Gitlab
lib/gitlab/profiler.rb
lib/gitlab/query_limiting/
lib/gitlab/request_context.rb
lib/gitlab/request_profiler/
lib/gitlab/sidekiq_logging/
lib/gitlab/sidekiq_middleware/
lib/gitlab/sidekiq_status/
......
......@@ -16,6 +16,7 @@ module Gitlab
lib/gitlab/middleware/
ee/lib/gitlab/middleware/
lib/gitlab/performance_bar/
lib/gitlab/request_profiler/
lib/gitlab/query_limiting/
lib/gitlab/tracing/
lib/gitlab/profiler.rb
......
# frozen_string_literal: true
require 'fileutils'
module Gitlab
module RequestProfiler
PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles"
def all
Dir["#{PROFILES_DIR}/*.{html,txt}"].map do |path|
Profile.new(File.basename(path))
end.select(&:valid?)
end
module_function :all # rubocop: disable Style/AccessModifierDeclarations
def find(name)
file_path = File.join(PROFILES_DIR, name)
return unless File.exist?(file_path)
Profile.new(name)
end
module_function :find # rubocop: disable Style/AccessModifierDeclarations
def profile_token
Rails.cache.fetch('profile-token') do
Devise.friendly_token
end
end
module_function :profile_token # rubocop: disable Style/AccessModifierDeclarations
def remove_all_profiles
FileUtils.rm_rf(PROFILES_DIR)
end
module_function :remove_all_profiles # rubocop: disable Style/AccessModifierDeclarations
end
end
# frozen_string_literal: true
require 'ruby-prof'
require 'memory_profiler'
module Gitlab
module RequestProfiler
class Middleware
def initialize(app)
@app = app
end
def call(env)
if profile?(env)
call_with_profiling(env)
else
@app.call(env)
end
end
def profile?(env)
header_token = env['HTTP_X_PROFILE_TOKEN']
return unless header_token.present?
profile_token = Gitlab::RequestProfiler.profile_token
return unless profile_token.present?
header_token == profile_token
end
def call_with_profiling(env)
case env['HTTP_X_PROFILE_MODE']
when 'execution', nil
call_with_call_stack_profiling(env)
when 'memory'
call_with_memory_profiling(env)
else
raise ActionController::BadRequest, invalid_profile_mode(env)
end
end
def invalid_profile_mode(env)
<<~HEREDOC
Invalid X-Profile-Mode: #{env['HTTP_X_PROFILE_MODE']}.
Supported profile mode request header:
- X-Profile-Mode: execution
- X-Profile-Mode: memory
HEREDOC
end
def call_with_call_stack_profiling(env)
ret = nil
report = RubyProf::Profile.profile do
ret = catch(:warden) do # rubocop:disable Cop/BanCatchThrow
@app.call(env)
end
end
generate_report(env, 'execution', 'html') do |file|
printer = RubyProf::CallStackPrinter.new(report)
printer.print(file)
end
handle_request_ret(ret)
end
def call_with_memory_profiling(env)
ret = nil
report = MemoryProfiler.report do
ret = catch(:warden) do # rubocop:disable Cop/BanCatchThrow
@app.call(env)
end
end
generate_report(env, 'memory', 'txt') do |file|
report.pretty_print(to_file: file)
end
handle_request_ret(ret)
end
def generate_report(env, report_type, extension)
file_name = "#{env['PATH_INFO'].tr('/', '|')}_#{Time.current.to_i}"\
"_#{report_type}.#{extension}"
file_path = "#{PROFILES_DIR}/#{file_name}"
FileUtils.mkdir_p(PROFILES_DIR)
begin
File.open(file_path, 'wb') do |file|
yield(file)
end
rescue StandardError
FileUtils.rm(file_path)
end
end
def handle_request_ret(ret)
if ret.is_a?(Array)
ret
else
throw(:warden, ret) # rubocop:disable Cop/BanCatchThrow
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module RequestProfiler
class Profile
attr_reader :name, :time, :file_path, :request_path, :profile_mode, :type
alias_method :to_param, :name
def initialize(name)
@name = name
@file_path = File.join(PROFILES_DIR, name)
set_attributes
end
def valid?
@request_path.present?
end
def content_type
case type
when 'html'
'text/html'
when 'txt'
'text/plain'
end
end
private
def set_attributes
matches = name.match(/^(?<path>.*)_(?<timestamp>\d+)(_(?<profile_mode>\w+))?\.(?<type>html|txt)$/)
return unless matches
@request_path = matches[:path].tr('|', '/')
@time = Time.at(matches[:timestamp].to_i).utc
@profile_mode = matches[:profile_mode] || 'unknown'
@type = matches[:type]
end
end
end
end
......@@ -24195,6 +24195,9 @@ msgstr ""
msgid "No prioritized labels with such name or description"
msgstr ""
msgid "No profiles found"
msgstr ""
msgid "No project subscribes to the pipelines in this project."
msgstr ""
......@@ -25742,6 +25745,9 @@ msgstr ""
msgid "Pass job variables"
msgstr ""
msgid "Pass the header %{codeOpen} X-Profile-Token: %{profile_token} %{codeClose} to profile the request"
msgstr ""
msgid "Passed"
msgstr ""
......@@ -30340,6 +30346,9 @@ msgstr ""
msgid "Requests"
msgstr ""
msgid "Requests Profiles"
msgstr ""
msgid "Requests for pages at %{code_start}%{help_text_url}%{code_end} redirect to the URL. The destination must meet certain requirements. %{docs_link_start}Learn more.%{docs_link_end}"
msgstr ""
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Admin::RequestsProfilesController do
let_it_be(:admin) { create(:admin) }
before do
sign_in(admin)
end
describe '#show' do
let(:tmpdir) { Dir.mktmpdir('profiler-test') }
let(:test_file) { File.join(tmpdir, basename) }
subject do
get :show, params: { name: basename }
end
before do
stub_const('Gitlab::RequestProfiler::PROFILES_DIR', tmpdir)
File.write(test_file, sample_data)
end
after do
FileUtils.rm_rf(tmpdir)
end
context 'when loading HTML profile' do
let(:basename) { "profile_#{Time.current.to_i}_execution.html" }
let(:sample_data) do
'<html> <body> <h1>Heading</h1> <p>paragraph.</p> </body> </html>'
end
it 'renders the data' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.body).to eq(sample_data)
end
end
context 'when loading TXT profile' do
let(:basename) { "profile_#{Time.current.to_i}_memory.txt" }
let(:sample_data) do
<<~TXT
Total allocated: 112096396 bytes (1080431 objects)
Total retained: 10312598 bytes (53567 objects)
TXT
end
it 'renders the data' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.body).to eq(sample_data)
end
end
context 'when loading PDF profile' do
let(:basename) { "profile_#{Time.current.to_i}_anything.pdf" }
let(:sample_data) { 'mocked pdf content' }
it 'fails to render the data' do
expect { subject }.to raise_error(ActionController::UrlGenerationError, /No route matches.*unmatched constraints:/)
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe 'Admin::RequestsProfilesController' do
let(:tmpdir) { Dir.mktmpdir('profiler-test') }
before do
stub_const('Gitlab::RequestProfiler::PROFILES_DIR', tmpdir)
admin = create(:admin)
sign_in(admin)
gitlab_enable_admin_mode_sign_in(admin)
end
after do
FileUtils.rm_rf(tmpdir)
end
describe 'GET /admin/requests_profiles' do
it 'shows the current profile token' do
allow(Rails).to receive(:cache).and_return(ActiveSupport::Cache::MemoryStore.new)
visit admin_requests_profiles_path
expect(page).to have_content("X-Profile-Token: #{Gitlab::RequestProfiler.profile_token}")
end
context 'when having multiple profiles' do
let(:time1) { 1.hour.ago }
let(:time2) { 2.hours.ago }
let(:profiles) do
[
{
request_path: '/gitlab-org/gitlab-foss',
name: "|gitlab-org|gitlab-foss_#{time1.to_i}_execution.html",
created: time1,
profile_mode: 'Execution'
},
{
request_path: '/gitlab-org/gitlab-foss',
name: "|gitlab-org|gitlab-foss_#{time2.to_i}_execution.html",
created: time2,
profile_mode: 'Execution'
},
{
request_path: '/gitlab-org/gitlab-foss',
name: "|gitlab-org|gitlab-foss_#{time1.to_i}_memory.html",
created: time1,
profile_mode: 'Memory'
},
{
request_path: '/gitlab-org/gitlab-foss',
name: "|gitlab-org|gitlab-foss_#{time2.to_i}_memory.html",
created: time2,
profile_mode: 'Memory'
},
{
request_path: '/gitlab-org/infrastructure',
name: "|gitlab-org|infrastructure_#{time1.to_i}_execution.html",
created: time1,
profile_mode: 'Execution'
},
{
request_path: '/gitlab-org/infrastructure',
name: "|gitlab-org|infrastructure_#{time2.to_i}_memory.html",
created: time2,
profile_mode: 'Memory'
},
{
request_path: '/gitlab-org/infrastructure',
name: "|gitlab-org|infrastructure_#{time2.to_i}.html",
created: time2,
profile_mode: 'Unknown'
}
]
end
before do
profiles.each do |profile|
FileUtils.touch(File.join(Gitlab::RequestProfiler::PROFILES_DIR, profile[:name]))
end
end
it 'lists all available profiles' do
visit admin_requests_profiles_path
profiles.each do |profile|
within('.card', text: profile[:request_path]) do
expect(page).to have_selector(
"a[href='#{admin_requests_profile_path(profile[:name])}']",
text: "#{profile[:created].to_s(:long)} #{profile[:profile_mode]}")
end
end
end
end
end
describe 'GET /admin/requests_profiles/:profile' do
context 'when a profile exists' do
before do
File.write("#{Gitlab::RequestProfiler::PROFILES_DIR}/#{profile}", content)
end
context 'when is valid call stack profile' do
let(:content) { 'This is a call stack request profile' }
let(:profile) { "|gitlab-org|gitlab-ce_#{Time.now.to_i}_execution.html" }
it 'displays the content' do
visit admin_requests_profile_path(profile)
expect(page).to have_content(content)
end
end
context 'when is valid memory profile' do
let(:content) { 'This is a memory request profile' }
let(:profile) { "|gitlab-org|gitlab-ce_#{Time.now.to_i}_memory.txt" }
it 'displays the content' do
visit admin_requests_profile_path(profile)
expect(page).to have_content(content)
end
end
end
context 'when a profile does not exist' do
it 'shows an error message' do
visit admin_requests_profile_path('|non|existent_12345.html')
expect(page).to have_content('Profile not found')
end
end
end
end
......@@ -25,6 +25,7 @@ RSpec.describe Gitlab::BacktraceCleaner do
"app/models/repository.rb:113:in `commit'",
"lib/gitlab/i18n.rb:50:in `with_locale'",
"lib/gitlab/middleware/multipart.rb:95:in `call'",
"lib/gitlab/request_profiler/middleware.rb:14:in `call'",
"ee/lib/gitlab/database/load_balancing/rack_middleware.rb:37:in `call'",
"ee/lib/gitlab/jira/middleware.rb:15:in `call'"
]
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::RequestProfiler::Profile do
let(:profile) { described_class.new(filename) }
describe '.new' do
context 'using old filename' do
let(:filename) { '|api|v4|version.txt_1562854738.html' }
it 'returns valid data' do
expect(profile).to be_valid
expect(profile.request_path).to eq('/api/v4/version.txt')
expect(profile.time).to eq(Time.at(1562854738).utc)
expect(profile.type).to eq('html')
end
end
context 'using new filename' do
let(:filename) { '|api|v4|version.txt_1563547949_execution.html' }
it 'returns valid data' do
expect(profile).to be_valid
expect(profile.request_path).to eq('/api/v4/version.txt')
expect(profile.profile_mode).to eq('execution')
expect(profile.time).to eq(Time.at(1563547949).utc)
expect(profile.type).to eq('html')
end
end
end
describe '#content_type' do
context 'when using html file' do
let(:filename) { '|api|v4|version.txt_1562854738_memory.html' }
it 'returns valid data' do
expect(profile).to be_valid
expect(profile.content_type).to eq('text/html')
end
end
context 'when using text file' do
let(:filename) { '|api|v4|version.txt_1562854738_memory.txt' }
it 'returns valid data' do
expect(profile).to be_valid
expect(profile.content_type).to eq('text/plain')
end
end
context 'when file is unknown' do
let(:filename) { '|api|v4|version.txt_1562854738_memory.xxx' }
it 'returns valid data' do
expect(profile).not_to be_valid
expect(profile.content_type).to be_nil
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::RequestProfiler do
describe '.profile_token' do
it 'returns a token' do
expect(described_class.profile_token).to be_present
end
it 'caches the token' do
expect(Rails.cache).to receive(:fetch).with('profile-token')
described_class.profile_token
end
end
context 'with temporary PROFILES_DIR' do
let(:tmpdir) { Dir.mktmpdir('profiler-test') }
let(:profile_name) { '|api|v4|version.txt_1562854738_memory.html' }
let(:profile_path) { File.join(tmpdir, profile_name) }
before do
stub_const('Gitlab::RequestProfiler::PROFILES_DIR', tmpdir)
FileUtils.touch(profile_path)
end
after do
FileUtils.rm_rf(tmpdir)
end
describe '.remove_all_profiles' do
it 'removes Gitlab::RequestProfiler::PROFILES_DIR directory' do
described_class.remove_all_profiles
expect(Dir.exist?(tmpdir)).to be false
end
end
describe '.all' do
subject { described_class.all }
it 'returns all profiles' do
expect(subject.map(&:name)).to contain_exactly(profile_name)
end
end
describe '.find' do
subject { described_class.find(profile_name) }
it 'returns all profiles' do
expect(subject.name).to eq(profile_name)
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe 'Request Profiler' do
let(:user) { create(:user) }
shared_examples 'profiling a request' do |profile_type, extension|
before do
allow(Rails).to receive(:cache).and_return(ActiveSupport::Cache::MemoryStore.new)
allow(RubyProf::Profile).to receive(:profile) do |&blk|
blk.call
RubyProf::Profile.new
end
allow(MemoryProfiler).to receive(:report) do |&blk|
blk.call
MemoryProfiler.start
MemoryProfiler.stop
end
end
it 'creates a profile of the request' do
project = create(:project, namespace: user.namespace)
time = Time.now
path = "/#{project.full_path}"
travel_to(time) do
get path, params: {}, headers: { 'X-Profile-Token' => Gitlab::RequestProfiler.profile_token, 'X-Profile-Mode' => profile_type }
end
profile_type = 'execution' if profile_type.nil?
profile_path = "#{Gitlab.config.shared.path}/tmp/requests_profiles/#{path.tr('/', '|')}_#{time.to_i}_#{profile_type}.#{extension}"
expect(File.exist?(profile_path)).to be true
end
after do
Gitlab::RequestProfiler.remove_all_profiles
end
end
context "when user is logged-in" do
before do
login_as(user)
end
include_examples 'profiling a request', 'execution', 'html'
include_examples 'profiling a request', nil, 'html'
include_examples 'profiling a request', 'memory', 'txt'
end
context "when user is not logged-in" do
include_examples 'profiling a request', 'execution', 'html'
include_examples 'profiling a request', nil, 'html'
include_examples 'profiling a request', 'memory', 'txt'
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment