Commit ceb95e59 authored by Sebastien Robin's avatar Sebastien Robin

distribution graph and use plotly instead of flotcharts

Apdex give good measure to know if rendering time is globally good
or not. By adding a distribution graph, we can even see more quickly
how much we need to improve.
parent fc6baf0c
......@@ -34,6 +34,7 @@ from datetime import datetime, timedelta, date, tzinfo
from functools import partial
from operator import itemgetter
from urllib import splittype, splithost
from copy import copy
import argparse
import bz2
import calendar
......@@ -57,6 +58,7 @@ try:
except ImportError:
pytz = None
global current_line
def getResource(name, encoding='utf-8'):
return pkgutil.get_data(__name__, name).decode(encoding)
......@@ -160,6 +162,7 @@ def getDataPoints(apdex_dict, status_period_dict={}):
apdex.getApdex() * 100,
apdex.hit,
period_error_dict.get(value_date, 0),
apdex.distribution_dict,
) for value_date, apdex in sorted(apdex_dict.iteritems(), key=ITEMGETTER0)
]
......@@ -168,26 +171,27 @@ def prepareDataForGraph(daily_data, date_format, placeholder_delta,
current_date = datetime.strptime(x_min or daily_data[0][0], date_format)
new_daily_data = []
append = new_daily_data.append
for (measure_date_string, apdex, hit, error_hit) in daily_data:
for (measure_date_string, apdex, hit, error_hit, distribution_dict) in daily_data:
measure_date = datetime.strptime(measure_date_string, date_format)
if current_date < measure_date:
append((current_date.strftime(date_format), 100, 0, 0))
append((current_date.strftime(date_format), 100, 0, 0, {}))
placeholder_end_date = measure_date - placeholder_delta
if placeholder_end_date > current_date:
append((placeholder_end_date.strftime(date_format), 100, 0, 0))
append((placeholder_end_date.strftime(date_format), 100, 0, 0, {}))
coef = coefficient_callback(measure_date)
append((measure_date_string, apdex, hit * coef, error_hit * coef))
append((measure_date_string, apdex, hit * coef, error_hit * coef, distribution_dict))
current_date = measure_date + placeholder_delta
if x_max is not None and current_date < datetime.strptime(x_max,
date_format):
append((current_date.strftime(date_format), 100, 0, 0))
append((x_max, 100, 0, 0))
append((current_date.strftime(date_format), 100, 0, 0, {}))
append((x_max, 100, 0, 0, {}))
return new_daily_data
def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
hit_y_min=None, hit_y_max=None, apdex_y_scale=None, hit_y_scale=None):
date_list = [int(calendar.timegm(time.strptime(x[0], date_format)) * 1000)
for x in daily_data]
string_date_list = [x[0].replace("/","-") for x in daily_data]
timeformat = '%Y/<br/>%m/%d<br/> %H:%M'
# There is room for about 10 labels on the X axis.
minTickSize = (max(1,
......@@ -195,73 +199,86 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
# Guesstimation: 6px per digit. If only em were allowed...
yLabelWidth = max(int(math.log10(max(x[2] for x in daily_data))) + 1,
3) * 6
return graph('apdex',
[zip(date_list, (round(x[1], 2) for x in daily_data))],
graph_list = []
graph_list.append(graph('apdex',
[{
"x": string_date_list,
"y": [round(x[1], 2) for x in daily_data]
}],
{
'xaxis': {
'mode': 'time',
'timeformat': timeformat,
'minTickSize': minTickSize,
},
'yaxis': {
'min': apdex_y_min,
'max': 100,
'axisLabel': 'apdex (%)',
'labelWidth': yLabelWidth,
'transform': apdex_y_scale,
},
'lines': {'show': True},
'grid': {
'hoverable': True,
},
"margin": { "t": 0 },
"yaxis": {"title": "apdex (%)"},
}
))
graph_list.append(graph('Hits (per %s)' % graph_period,
[{
"x": string_date_list,
"y": [x[2] for x in daily_data],
"name": "Hits"
},{
"x": string_date_list,
"y": [x[3] for x in daily_data],
"name": "Errors"
},
) + graph('Hits (per %s)' % graph_period,
[
{
'label': 'Errors',
'data': zip(date_list, (x[3] for x in daily_data)),
'color': 'red',
},
{
'label': 'Hits',
'data': zip(date_list, (x[2] for x in daily_data)),
},
],
{
'xaxis': {
'mode': 'time',
'timeformat': timeformat,
'minTickSize': minTickSize,
},
'yaxis': {
'min': hit_y_min,
'max': hit_y_max,
'axisLabel': 'Hits',
'labelWidth': yLabelWidth,
'tickDecimals': 0,
'transform': hit_y_scale,
},
'lines': {'show': True},
'grid': {
'hoverable': True,
},
'legend': {
'backgroundOpacity': 0.25,
"margin": { "t": 0 },
"yaxis": {"title": "Hits"},
}
))
x_list = []
y_list = []
y_list_append = y_list.append
size_list = []
text_list = []
text_list_append = text_list.append
size_list_append = size_list.append
total_hit = 0
log10 = math.log10
for date, distribution_dict in zip(string_date_list, (x[4] for x in daily_data)):
x_list.extend([date] * len(distribution_dict))
for rendering_time, hit in distribution_dict.iteritems():
y_list_append(rendering_time)
size_list_append(log10(hit+1))
text_list_append("Hits: " + str(hit))
total_hit += hit
graph_list.append(graph("Distribution", [{
"mode": 'markers',
"marker": {
"sizemode": "area",
"sizeref": 0.01,
"size" : size_list
},
},
"x": x_list,
"y": y_list,
"text": text_list,
}],
{
"margin": { "t": 0 },
"hovermode": "closest",
"yaxis": {"title": "Time (seconds)"},
}, height=450
)
)
return "\n".join(graph_list)
def graph(title, data, options={}):
global count_graph
count_graph = 0
def graph(title, data, options={}, height=300):
result = []
global count_graph
count_graph += 1
div_id = "graph_%i" % count_graph
append = result.append
append('<h2>%s</h2><div class="graph" '
'style="width:600px;height:300px" data-points="' % title)
append('<h2>%s</h2><div id="%s" class="graph" '
'style="width:600px;height:%ipx" ' % (title, div_id, height))
append(' data-points="')
append(escape(json.dumps(data), quote=True))
append('" data-options="')
append(escape(json.dumps(options), quote=True))
append('"></div><div class="tooltip">'
'<span class="x"></span><br/>'
'<span class="y"></span></div>')
append('"></div>"')
return ''.join(result)
class APDEXStats(object):
......@@ -275,9 +292,11 @@ class APDEXStats(object):
self.duration_total = 0
self.duration_max = 0
self.getDuration = getDuration
self.distribution_dict = defaultdict(lambda: 0)
def accumulate(self, match):
duration = self.getDuration(match)
self.distribution_dict[int(duration/1000000)] += 1
self.duration_total += duration
self.duration_max = max(self.duration_max, duration)
if not statusIsError(match.group('status')):
......@@ -292,6 +311,8 @@ class APDEXStats(object):
setattr(self, attribute,
getattr(self, attribute) + getattr(other, attribute))
self.duration_max = max(self.duration_max, other.duration_max)
for key,value in other.distribution_dict.iteritems():
self.distribution_dict[key] += value
def getApdex(self):
if self.hit:
......@@ -1105,8 +1126,8 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
else:
out.write('<link rel="stylesheet" type="text/css" '
'href="%s/apachedex.css"/>' % js_path)
for script in ('jquery.js', 'jquery.flot.js', 'jquery.flot.time.js',
'jquery.flot.axislabels.js', 'jquery-ui.js', 'apachedex.js'):
for script in (
'jquery.js', 'jquery-ui.js', 'plotly.min.js', 'apachedex.js'):
if js_embed:
out.write('<script type="text/javascript">//<![CDATA[\n')
out.write(getResource(script))
......@@ -1146,7 +1167,7 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
if apdex_data_list:
x_min = min(x_min, apdex_data_list[0][0])
x_max = max(x_max, apdex_data_list[-1][0])
for hit_date, _, hit, _ in apdex_data_list:
for hit_date, _, hit, _, _ in apdex_data_list:
hit_per_day[decimator(hit_date)] += hit
if x_min == LARGER_THAN_INTEGER_STR:
x_min = None
......@@ -1528,6 +1549,8 @@ def main():
for lineno, line in enumerate(logfile, 1):
if show_progress and lineno % 5000 == 0:
print(lineno, end='\r', file=sys.stderr)
global current_line
current_line = line
match = matchline(line)
if match is None:
match = expensive_matchline(line)
......
......@@ -49,21 +49,9 @@ function updateAxisTransform(axis) {
function renderGraph(container) {
var container = $(container);
var previousIndex = null;
var tooltip = container.next(".tooltip");
var options = $.parseJSON(container.attr("data-options"));
updateAxisTransform(options.xaxis);
updateAxisTransform(options.yaxis);
var plot = $.plot(
container,
$.parseJSON(container.attr("data-points")),
options
);
tooltip.detach();
container.append(tooltip);
container.bind("plothover", function (event, pos, item) {
previousIndex = updateGraphTooltip(event, pos, item, previousIndex,
tooltip, plot);
});
var data = $.parseJSON(container.attr("data-points"));
Plotly.plot(container[0], data, options, {modeBarButtonsToRemove: ["sendDataToCloud"]});
}
function toggleGraph(node) {
var container = $(node).parent().find(".container");
......@@ -79,5 +67,5 @@ function hideGraph(node) {
}
$(function() {
$(".graph:visible").each(function (i){renderGraph(this)});
$(".hidden_graph .container").draggable();
$(".hidden_graph .container").draggable({cancel: "div.graph"});
});
......@@ -10,24 +10,14 @@ if sys.version_info >= (3, ):
else:
from urllib import urlretrieve
FLOT_SHA = 'aefe4e729b2d14efe6e8c0db359cb0e9aa6aae52'
FLOT_AXISLABELS_SHA = '80453cd7fb8a9cad084cf6b581034ada3339dbf8'
PLOTLY_VERSION = '1.52.1'
JQUERY_VERSION = '1.9.1'
JQUERY_UI_VERSION = '1.10.2'
DEPS = {
'jquery.flot.js': (
'http://raw.github.com/flot/flot/%s/jquery.flot.js' % FLOT_SHA,
'7b599c575f19c33bf0d93a6bbac3af02',
),
'jquery.flot.time.js': (
'http://raw.github.com/flot/flot/%s/jquery.flot.time.js' % FLOT_SHA,
'c0aec1608bf2fbb79f24d1905673e2c3',
),
'jquery.flot.axislabels.js': (
'http://raw.github.com/markrcote/flot-axislabels/%s/'
'jquery.flot.axislabels.js' % FLOT_AXISLABELS_SHA,
'a8526e0c1ed3b5cbc1a6b3ebb22bf334',
'plotly.min.js': (
'https://github.com/plotly/plotly.js/raw/v%s/dist/plotly.min.js' % PLOTLY_VERSION,
'02c8285a64ba86691d6c05eba366438e',
),
'jquery.js': (
'http://code.jquery.com/jquery-%s.min.js' % JQUERY_VERSION,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment