Skip to content

M
misc
Commit 501d7a4f authored by Elvis Pranskevichus's avatar Elvis Pranskevichus
Browse files
Options

Add support for HTML report generation

parent b81b9367
Hide whitespace changes
Inline Side-by-side
Showing with 593 additions and 1 deletion
report/report.html 0 → 100644
View file @ 501d7a4f
<!DOCTYPE html>
<html>
<head>
<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/normalize/4.1.1/normalize.min.css" />
<script src="https://d3js.org/d3.v3.min.js" charset="utf-8"></script>
<style>
body {
padding: 20px;
}
.axis path, .axis line {
fill: none;
stroke: #999;
shape-rendering: crispEdges;
}
.axis.x2 path, .axis.x2 line {
display: none;
}
.axis text {
font: 12px sans-serif;
fill: #333;
}
.axis.x path {
display: none;
}
.axis text {
fill: #555;
}
.axis line, .axis path {
stroke: #888;
}
.focus line {
stroke: #900;
}
.focus text {
fill: #900;
font: 12px sans-serif;
}
table.results {
width: 100%;
border-collapse: collapse;
}
table.results td, table.results th {
border: 1px solid #999;
padding: 5px;
}
table.results tr.benchmark td {
font-weight: bold;
border-bottom: none;
text-align: center;
}
table.results tr.metric td {
border-top: none;
border-bottom: none;
text-align: right;
}
table.results tr.metric:last-child td {
border-bottom: 1px solid #999;
}
</style>
</head>
<body>
<p><em>${__BENCHMARK_DATE__}</em></p>
<h1>Server Performance Benchmark Report</h1>
Below are the results of testing network server implementations. Each server
is constrained to run in a single process.
Test environment: ${__BENCHMARK_PLATFORM__}.
<h2>Results</h2>
<svg id="bars"></svg>
<svg id="lats"></svg>
<h2>Detailed Benchmark Data</h2>
${__BENCHMARK_DATA_TABLE__}
<script>
function drawBars(elSelector, data, options) {'use strict';
options = options || {};
// geometry
var margin = {top: 10, right: 0, bottom: 40, left: 65},
width = (options.width || 1000) - margin.left - margin.right,
height = (options.height || 370) - margin.top - margin.bottom;
// data reshape
var maxRps = 0;
data.benchmarks.forEach(function(bench) {
bench.variations.forEach(function(v) {
if (v.rps > maxRps) {
maxRps = v.rps
}
})
});
var sizes = data.payload_size_levels;
var names = data.benchmarks.map(function(d) { return d.name });
// charting
var color = d3.scale.ordinal()
.range(["#98abc5", "#6b486b", "#ff8c00"]);
var x0 = d3.scale.ordinal()
.rangeRoundBands([0, width], .2)
.domain(names);
var x1 = d3.scale.ordinal();
x1.domain(sizes).rangeRoundBands([0, x0.rangeBand()], .2);
var y = d3.scale.linear()
.range([height, 0])
.domain([0, maxRps]);
var xAxis = d3.svg.axis()
.scale(x0)
.orient("bottom")
.tickFormat(function(d) { return d.split('-')[1] });
var xAxis2 = d3.svg.axis()
.scale(x0)
.orient("bottom")
.tickFormat(function(d) { return d.split('-')[2] });
var yAxis = d3.svg.axis()
.scale(y)
.orient("left");
var chart = d3.select(elSelector)
.attr('width', width + margin.left + margin.right)
.attr('height', height + margin.top + margin.bottom)
.append("g")
.attr("transform",
"translate(" + margin.left + "," + margin.top + ")");
chart.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis);
chart.append("g")
.attr("class", "x axis x2")
.attr("transform", "translate(0," + (height + 14) + ")")
.call(xAxis2);
chart.append("g")
.attr("class", "y axis")
.call(yAxis)
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 6)
.attr("dy", ".71em")
.style("text-anchor", "end")
.text("Requests / sec");
var bench = chart.selectAll(".bench")
.data(data.benchmarks)
.enter().append("g")
.attr("class", "bench")
.attr("transform", function(d) {
return "translate(" + x0(d.name) + ",0)";
});
bench.selectAll("rect")
.data(function(d) { return d.variations; })
.enter().append("rect")
.attr("width", x1.rangeBand())
.attr("x", function(d, i) { return x1(sizes[i]); })
.attr("y", function(d) { return y(d.rps); })
.attr("height", function(d) { return height - y(d.rps); })
.style("fill", function(d, i) { return color(sizes[i]); })
.on("mouseover", function(d, i) {
focusRect
.attr('y', y(d.rps) - 9);
focusLine
.attr('y1', y(d.rps))
.attr('y2', y(d.rps));
focusText
.attr('y', y(d.rps))
.text(d3.format("0,000")(Math.round(d.rps)));
focus.style("display", null);
})
.on("mouseout", function() { focus.style("display", 'none'); });
var focus = chart.append('g')
.attr('class', 'focus')
.style('display', 'none');
var focusRect = focus.append('rect')
.attr('x', -margin.left)
.attr('width', margin.left - 6)
.attr('y', 0)
.attr('height', 18)
.attr('fill', 'rgba(255, 255, 255, 0.9)');
var focusLine = focus.append('line')
.attr('x1', -6)
.attr('x2', width - 20)
.attr('y1', 0)
.attr('y2', 0)
.style("stroke-dasharray", "2,2");
var focusText = focus.append('text')
.attr('y', 0)
.attr('x', -9)
.attr('text-anchor', 'end')
.attr('alignment-baseline', 'middle');
};
function drawLats(elSelector, data) {'use strict';
options = options || {};
// geometry
var margin = {top: 10, right: 0, bottom: 40, left: 65},
width = (options.width || 1000) - margin.left - margin.right,
height = (options.height || 370) - margin.top - margin.bottom;
// data reshape
var maxLat = 0;
data.benchmarks.forEach(function(bench) {
bench.variations.forEach(function(v) {
if (v.latency_percentiles[4][1] > maxLat) {
maxLat = v.latency_percentiles[4][1];
}
})
});
var sizes = data.payload_size_levels;
var names = data.benchmarks.map(function(d) { return d.name });
// charting
var color = d3.scale.ordinal()
.range(["#98abc5", "#6b486b", "#ff8c00"]);
var x0 = d3.scale.ordinal()
.rangeRoundBands([0, width], .2)
.domain(names);
var x1 = d3.scale.ordinal();
x1.domain(sizes).rangeRoundBands([0, x0.rangeBand()], 0.3);
var y = d3.scale.linear()
.range([height, 0])
.domain([0, maxLat]);
var xAxis = d3.svg.axis()
.scale(x0)
.orient("bottom")
.tickFormat(function(d) { return d.split('-')[1] });
var xAxis2 = d3.svg.axis()
.scale(x0)
.orient("bottom")
.tickFormat(function(d) { return d.split('-')[2] });
var yAxis = d3.svg.axis()
.scale(y)
.orient("left");
var chart = d3.select(elSelector)
.attr('width', width + margin.left + margin.right)
.attr('height', height + margin.top + margin.bottom)
.append("g")
.attr("transform",
"translate(" + margin.left + "," + margin.top + ")");
chart.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis);
chart.append("g")
.attr("class", "x axis x2")
.attr("transform", "translate(0," + (height + 14) + ")")
.call(xAxis2);
chart.append("g")
.attr("class", "y axis")
.call(yAxis)
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 6)
.attr("dy", ".71em")
.style("text-anchor", "end")
.text("Latency (msec)");
var bench = chart.selectAll(".bench")
.data(data.benchmarks)
.enter().append("g")
.attr("class", "bench")
.attr("transform", function(d) {
return "translate(" + x0(d.name) + ",0)";
});
var g = bench.selectAll("rect")
.data(function(d) { return d.variations; })
.enter().append("g")
.attr('class', 'sub');
g.append('line')
.attr('y1', function(d) { return y(d.latency_percentiles[4][1]); })
.attr('y2', function(d) { return y(d.latency_percentiles[4][1]); })
.attr('x1', function(d, i) { return x1(sizes[i]); })
.attr('x2', function(d, i) { return x1(sizes[i]) + x1.rangeBand(); })
.style("stroke", function(d, i) { return color(sizes[i]); });
g.append('line')
.attr('y1', function(d) { return y(d.latency_min); })
.attr('y2', function(d) { return y(d.latency_min); })
.attr('x1', function(d, i) { return x1(sizes[i]); })
.attr('x2', function(d, i) { return x1(sizes[i]) + x1.rangeBand(); })
.style("stroke", function(d, i) { return color(sizes[i]); });
g.append('line')
.attr('y1', function(d) { return y(d.latency_percentiles[1][1]); })
.attr('y2', function(d) { return y(d.latency_percentiles[1][1]); })
.attr('x1', function(d, i) { return x1(sizes[i]); })
.attr('x2', function(d, i) { return x1(sizes[i]) + x1.rangeBand(); })
.style("stroke", function(d, i) { return color(sizes[i]); });
g.append('line')
.attr('y1', function(d) { return y(d.latency_min); })
.attr('y2', function(d) { return y(d.latency_percentiles[0][1]) })
.attr('x1', function(d, i) { return x1(sizes[i]) + x1.rangeBand() / 2; })
.attr('x2', function(d, i) { return x1(sizes[i]) + x1.rangeBand() / 2; })
.style("stroke", function(d, i) { return color(sizes[i]); })
.style("stroke-dasharray", "2,2");
g.append('line')
.attr('y1', function(d) { return y(d.latency_percentiles[4][1]); })
.attr('y2', function(d) { return y(d.latency_percentiles[2][1]) })
.attr('x1', function(d, i) { return x1(sizes[i]) + x1.rangeBand() / 2; })
.attr('x2', function(d, i) { return x1(sizes[i]) + x1.rangeBand() / 2; })
.style("stroke", function(d, i) { return color(sizes[i]); })
.style("stroke-dasharray", "2,2");
g.append('rect')
.attr('y', function(d) { return y(d.latency_percentiles[2][1]); })
.attr('x', function(d, i) { return x1(sizes[i]); })
.attr("width", x1.rangeBand())
.attr('height', function(d) { return Math.abs(y(d.latency_percentiles[2][1]) - y(d.latency_percentiles[0][1])) })
.style("stroke", function(d, i) { return color(sizes[i]); })
.style("fill", 'rgba(0, 0, 0, 0)');
g.append('rect')
.attr('y', 0)
.attr('height', height)
.attr('x', function(d, i) { return x1(sizes[i]); })
.attr('width', function(d) { return x1.rangeBand(); })
.style('fill', 'rgba(0, 0, 0, 0)')
.on("mouseout", function(d, i) {
d3.select(this).style('fill', 'rgba(0, 0, 0, 0)');
focus.style('display', 'none');
})
.on("mouseover", function(d, i) {
d3.select(this).style('fill', 'rgba(0, 0, 0, 0.04)');
var yMedian = y(d.latency_percentiles[1][1]);
focus
.style('display', null);
focusLine
.attr('y1', yMedian)
.attr('y2', yMedian);
focusRect
.attr('y', yMedian - 9);
focusLine
.attr('y1', yMedian)
.attr('y2', yMedian);
focusText
.attr('y', yMedian)
.text(d3.format(".2f")(d.latency_percentiles[1][1]));
});
var focus = chart.append('g')
.attr('class', 'focus')
.style('display', 'none');
var focusRect = focus.append('rect')
.attr('x', -margin.left)
.attr('width', margin.left - 6)
.attr('y', 0)
.attr('height', 18)
.attr('fill', 'rgba(255, 255, 255, 0.9)');
var focusLine = focus.append('line')
.attr('x1', -6)
.attr('x2', width - 20)
.attr('y1', 0)
.attr('y2', 0)
.style("stroke-dasharray", "2,2");
var focusText = focus.append('text')
.attr('y', 0)
.attr('x', -9)
.attr('text-anchor', 'end')
.attr('alignment-baseline', 'middle');
};
var data = ${__BENCHMARK_DATA_JSON__};
var tcpData = {
payload_size_levels: data.payload_size_levels,
benchmarks: data.benchmarks
}
function _total_rps(entry) {
var total = 0;
for (var i = 0; i < entry.variations.length; i+=1) {
total += entry.variations[i].rps;
}
return total;
}
tcpData.benchmarks.sort(function(e1, e2) {
return d3.ascending(_total_rps(e1), _total_rps(e2));
})
options = {
width: 800,
height: 300
}
drawBars('#bars', tcpData, options);
drawLats('#lats', tcpData, options);
</script>
</body>
</html>
run_benchmarks
View file @ 501d7a4f
...@@ -2,11 +2,14 @@ ...@@ -2,11 +2,14 @@
import argparse import argparse
import collections
import datetime
import json import json
import os import os
import os.path import os.path
import re import re
import socket import socket
import string
import subprocess import subprocess
import sys import sys
import textwrap import textwrap
...@@ -275,6 +278,140 @@ def kill_server(): ...@@ -275,6 +278,140 @@ def kill_server():
subprocess.check_output(['docker', 'rm', 'magicbench']) subprocess.check_output(['docker', 'rm', 'magicbench'])
def format_report(data, target_file):
tpl_path = os.path.join(os.path.dirname(__file__), 'report', 'report.html')
with open(tpl_path, 'r') as f:
tpl = string.Template(f.read())
now = datetime.datetime.now()
date = now.strftime('%c')
platform = '{system} ({dist}, {arch}) on {cpu}'.format(
system=data['platform']['system'],
dist=data['platform']['distribution'],
arch=data['platform']['arch'],
cpu=data['platform']['cpu'],
)
i = 0
entries = collections.OrderedDict()
btypes = []
for benchmark in data['benchmarks']:
entry = {}
bench = benchmark['name'].split('-')
btype = bench[0]
if btype not in btypes:
btypes.append(btype)
bname = ' '.join(bench[1:])
try:
entry = entries[bname]
except KeyError:
entry = entries[bname] = {
'name': bname,
'benchmarks': collections.OrderedDict()
}
try:
brecords = entry['benchmarks'][btype]
except KeyError:
brecords = entry['benchmarks'][btype] = collections.OrderedDict((
('Requests/sec', []),
('Transfer/sec', []),
('Min latency', []),
('Mean latency', []),
('Max latency', []),
('Latency variation', []),
))
variations = benchmark['variations']
i = 0
for concurrency in data['concurrency_levels']:
for msgsize in data['payload_size_levels']:
variation = variations[i]
i += 1
brecords['Requests/sec'].append(
variation['rps'])
brecords['Transfer/sec'].append(
'{}MiB'.format(variation['transfer']))
brecords['Min latency'].append(
'{}ms'.format(variation['latency_min']))
brecords['Mean latency'].append(
'{}ms'.format(variation['latency_mean']))
brecords['Max latency'].append(
'{}ms'.format(variation['latency_max']))
brecords['Latency variation'].append('{}ms ({}%)'.format(
variation['latency_std'], variation['latency_cv']))
vc = len(data['concurrency_levels']) * len(data['payload_size_levels'])
btypes_html = '\n'.join(['<th colspan="{span}">{btype}</th>'.format(
span=vc, btype=bt) for bt in btypes])
variations_th = []
for bt in btypes:
for concurrency in data['concurrency_levels']:
for msgsize in data['payload_size_levels']:
variations_th.append(
'<th>{}</th>'.format(
'{}KiB, c {}'.format(msgsize / 1024, concurrency)
)
)
record_trs = []
for bname, entry in entries.items():
record_trs.append(
'''<tr class="benchmark">
<td>{name}</td>
{empty_tds}
</tr>'''.format(name=bname, empty_tds='<td></td>' * vc)
)
for bt in btypes:
for metric, metric_data in entry['benchmarks'][bt].items():
record_trs.append(
'<tr class="metric"><td>{metric}</td>{data}</tr>'.format(
metric=metric,
data='\n'.join('<td>{}</td>'.format(v)
for v in metric_data)
)
)
table = '''
<table class="results">
<thead>
<tr>
<th rowspan="2"></th>
{btypes}
</tr>
<tr>
{variations_header}
</tr>
</thead>
<tbody>
{records}
</tbody>
</table>
'''.format(btypes=btypes_html, variations_header='\n'.join(variations_th),
records='\n'.join(record_trs))
output = tpl.safe_substitute(
__BENCHMARK_DATE__=date,
__BENCHMARK_PLATFORM__=platform,
__BENCHMARK_DATA_TABLE__=table,
__BENCHMARK_DATA_JSON__=json.dumps(data)
)
with open(target_file, 'wt') as f:
f.write(output)
def main(): def main():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('--duration', '-D', default=30, type=int, parser.add_argument('--duration', '-D', default=30, type=int,
...@@ -291,6 +428,8 @@ def main(): ...@@ -291,6 +428,8 @@ def main():
'to use (in bytes)') 'to use (in bytes)')
parser.add_argument('--save-json', '-J', type=str, parser.add_argument('--save-json', '-J', type=str,
help='path to save benchmark results in JSON format') help='path to save benchmark results in JSON format')
parser.add_argument('--save-html', '-H', type=str,
help='path to save benchmark results in HTML format')
args = parser.parse_args() args = parser.parse_args()
if not os.path.exists(_socket): if not os.path.exists(_socket):
...@@ -387,22 +526,28 @@ def main(): ...@@ -387,22 +526,28 @@ def main():
print() print()
if args.save_json: if args.save_json or args.save_html:
info_cmd = server_base + python + ['/usr/src/servers/platinfo.py'] info_cmd = server_base + python + ['/usr/src/servers/platinfo.py']
print(' ' + ' '.join(info_cmd)) print(' ' + ' '.join(info_cmd))
output = subprocess.check_output(info_cmd, universal_newlines=True) output = subprocess.check_output(info_cmd, universal_newlines=True)
platform_info = json.loads(output) platform_info = json.loads(output)
benchmarks_data = { benchmarks_data = {
'date': '%Y-%m-%dT%H:%M:%S%z',
'duration': args.duration,
'platform': platform_info, 'platform': platform_info,
'concurrency_levels': args.concurrency_levels, 'concurrency_levels': args.concurrency_levels,
'payload_size_levels': args.payload_size_levels, 'payload_size_levels': args.payload_size_levels,
'benchmarks': benchmarks_data, 'benchmarks': benchmarks_data,
} }
if args.save_json:
with open(args.save_json, 'w') as f: with open(args.save_json, 'w') as f:
json.dump(benchmarks_data, f) json.dump(benchmarks_data, f)
if args.save_html:
format_report(benchmarks_data, args.save_html)
if __name__ == '__main__': if __name__ == '__main__':
main() main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7