Commit e3fa9d12 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch '62722-add-gem-size-data-to-metrics-report' into 'master'

Add gem size information to metrics.txt in memory CI artifacts

See merge request gitlab-org/gitlab-ce!29827
parents b08b036d 940fcc9f
......@@ -5,15 +5,15 @@ memory-static:
# Loads each of gems in the Gemfile and checks how much memory they consume when they are required.
# 'derailed_benchmarks' internally uses 'get_process_mem'
- scripts/memory-static 'tmp/memory_static_full_report.txt' 'tmp/memory_static_metrics.txt'
- bundle exec derailed bundle:mem > tmp/memory_bundle_mem.txt
- scripts/generate-gems-size-metrics-static tmp/memory_bundle_mem.txt >> 'tmp/memory_metrics.txt'
# Outputs detailed information about objects created while gems are loaded.
# 'derailed_benchmarks' internally uses 'memory_profiler'
- scripts/memory-static-objects 'tmp/memory_static_objects_full_report.txt' 'tmp/memory_static_metrics.txt'
- bundle exec derailed bundle:objects > tmp/memory_bundle_objects.txt
- scripts/generate-gems-memory-metrics-static tmp/memory_bundle_objects.txt >> 'tmp/memory_metrics.txt'
artifacts:
paths:
- tmp/memory_static_full_report.txt
- tmp/memory_static_objects_full_report.txt
- tmp/memory_static_metrics.txt
- tmp/memory_*.txt
reports:
metrics: tmp/memory_static_metrics.txt
metrics: tmp/memory_metrics.txt
#!/usr/bin/env ruby
abort "usage: #{__FILE__} <memory_bundle_objects_file_name>" unless ARGV.length == 1
memory_bundle_objects_file_name = ARGV.first
full_report = File.readlines(memory_bundle_objects_file_name)
allocated_str = full_report[1]
retained_str = full_report[2]
allocated_stats = /Total allocated: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(allocated_str)
retained_stats = /Total retained: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(retained_str)
abort 'failed to process the benchmark output' unless allocated_stats && retained_stats
puts "memory_static_objects_allocated_mb #{(allocated_stats[:bytes].to_f / (1024 * 1024)).round(1)}"
puts "memory_static_objects_retained_mb #{(retained_stats[:bytes].to_f / (1024 * 1024)).round(1)}"
puts "memory_static_objects_allocated_items #{allocated_stats[:objects]}"
puts "memory_static_objects_retained_items #{retained_stats[:objects]}"
#!/usr/bin/env ruby
abort "usage: #{__FILE__} <memory_bundle_mem_file_name>" unless ARGV.length == 1
memory_bundle_mem_file_name = ARGV.first
full_report = File.readlines(memory_bundle_mem_file_name)
def total_size(memory_bundle_mem_report)
stats = /TOP: (?<total_mibs_str>.*) MiB/.match(memory_bundle_mem_report.first)
abort 'failed to process the benchmark output' unless stats
"gem_total_size_mb #{stats[:total_mibs_str].to_f.round(1)}"
end
TOP_LEVEL_GEM_LOG_FORMAT = /^ (?<gem_name>\S.*):\s*(?<gem_size>\d[.\d]*)\s*MiB/.freeze
def all_gems(memory_bundle_mem_report)
memory_bundle_mem_report.map do |line|
TOP_LEVEL_GEM_LOG_FORMAT.match(line)
end.compact
end
def gems_as_metrics(gems_match_data)
gems_match_data.map do |gem|
gem_name = gem[:gem_name]
gem_size_mb = gem[:gem_size].to_f.round(1)
"gem_size_mb{name=\"#{gem_name}\"} #{gem_size_mb}"
end
end
puts total_size(full_report)
puts gems_as_metrics(all_gems(full_report)).sort(&:casecmp)
#!/usr/bin/env ruby
require_relative '../lib/gitlab/popen'
full_report_filename, metrics_filename = ARGV
abort 'usage: memory-static <full_report_filename> <metrics_filename>' unless full_report_filename && metrics_filename
full_report, status = Gitlab::Popen.popen(%w(bundle exec derailed bundle:mem))
abort 'failed to execute the benchmark' unless status.zero?
File.open(full_report_filename, 'w') do |f|
f.write(full_report)
end
stats = /TOP: (?<total_mibs_str>.*) MiB/.match(full_report.lines.first)
abort 'failed to process the benchmark output' unless stats
File.open(metrics_filename, 'a') do |f|
f.puts "memory_static_total_mb #{stats[:total_mibs_str].to_f.round(1)}"
end
#!/usr/bin/env ruby
require_relative '../lib/gitlab/popen'
full_report_filename, metrics_filename = ARGV
abort 'usage: memory-static-objects <full_report_filename> <metrics_filename>' unless full_report_filename && metrics_filename
full_report, status = Gitlab::Popen.popen(%w(bundle exec derailed bundle:objects))
abort 'failed to execute the benchmark' unless status.zero?
File.open(full_report_filename, 'w') do |f|
f.write(full_report)
end
allocated_str = full_report.lines[1]
retained_str = full_report.lines[2]
allocated_stats = /Total allocated: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(allocated_str)
retained_stats = /Total retained: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(retained_str)
abort 'failed to process the benchmark output' unless allocated_stats && retained_stats
File.open(metrics_filename, 'a') do |f|
f.puts "memory_static_objects_allocated_mb #{(allocated_stats[:bytes].to_f / (1024 * 1024)).round(1)}"
f.puts "memory_static_objects_retained_mb #{(retained_stats[:bytes].to_f / (104 * 1024)).round(1)}"
f.puts "memory_static_objects_allocated_items #{allocated_stats[:objects]}"
f.puts "memory_static_objects_retained_items #{retained_stats[:objects]}"
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment