Commit ceb95e59 authored by Sebastien Robin's avatar Sebastien Robin

distribution graph and use plotly instead of flotcharts

Apdex give good measure to know if rendering time is globally good
or not. By adding a distribution graph, we can even see more quickly
how much we need to improve.
parent fc6baf0c
...@@ -34,6 +34,7 @@ from datetime import datetime, timedelta, date, tzinfo ...@@ -34,6 +34,7 @@ from datetime import datetime, timedelta, date, tzinfo
from functools import partial from functools import partial
from operator import itemgetter from operator import itemgetter
from urllib import splittype, splithost from urllib import splittype, splithost
from copy import copy
import argparse import argparse
import bz2 import bz2
import calendar import calendar
...@@ -57,6 +58,7 @@ try: ...@@ -57,6 +58,7 @@ try:
except ImportError: except ImportError:
pytz = None pytz = None
global current_line
def getResource(name, encoding='utf-8'): def getResource(name, encoding='utf-8'):
return pkgutil.get_data(__name__, name).decode(encoding) return pkgutil.get_data(__name__, name).decode(encoding)
...@@ -160,6 +162,7 @@ def getDataPoints(apdex_dict, status_period_dict={}): ...@@ -160,6 +162,7 @@ def getDataPoints(apdex_dict, status_period_dict={}):
apdex.getApdex() * 100, apdex.getApdex() * 100,
apdex.hit, apdex.hit,
period_error_dict.get(value_date, 0), period_error_dict.get(value_date, 0),
apdex.distribution_dict,
) for value_date, apdex in sorted(apdex_dict.iteritems(), key=ITEMGETTER0) ) for value_date, apdex in sorted(apdex_dict.iteritems(), key=ITEMGETTER0)
] ]
...@@ -168,26 +171,27 @@ def prepareDataForGraph(daily_data, date_format, placeholder_delta, ...@@ -168,26 +171,27 @@ def prepareDataForGraph(daily_data, date_format, placeholder_delta,
current_date = datetime.strptime(x_min or daily_data[0][0], date_format) current_date = datetime.strptime(x_min or daily_data[0][0], date_format)
new_daily_data = [] new_daily_data = []
append = new_daily_data.append append = new_daily_data.append
for (measure_date_string, apdex, hit, error_hit) in daily_data: for (measure_date_string, apdex, hit, error_hit, distribution_dict) in daily_data:
measure_date = datetime.strptime(measure_date_string, date_format) measure_date = datetime.strptime(measure_date_string, date_format)
if current_date < measure_date: if current_date < measure_date:
append((current_date.strftime(date_format), 100, 0, 0)) append((current_date.strftime(date_format), 100, 0, 0, {}))
placeholder_end_date = measure_date - placeholder_delta placeholder_end_date = measure_date - placeholder_delta
if placeholder_end_date > current_date: if placeholder_end_date > current_date:
append((placeholder_end_date.strftime(date_format), 100, 0, 0)) append((placeholder_end_date.strftime(date_format), 100, 0, 0, {}))
coef = coefficient_callback(measure_date) coef = coefficient_callback(measure_date)
append((measure_date_string, apdex, hit * coef, error_hit * coef)) append((measure_date_string, apdex, hit * coef, error_hit * coef, distribution_dict))
current_date = measure_date + placeholder_delta current_date = measure_date + placeholder_delta
if x_max is not None and current_date < datetime.strptime(x_max, if x_max is not None and current_date < datetime.strptime(x_max,
date_format): date_format):
append((current_date.strftime(date_format), 100, 0, 0)) append((current_date.strftime(date_format), 100, 0, 0, {}))
append((x_max, 100, 0, 0)) append((x_max, 100, 0, 0, {}))
return new_daily_data return new_daily_data
def graphPair(daily_data, date_format, graph_period, apdex_y_min=None, def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
hit_y_min=None, hit_y_max=None, apdex_y_scale=None, hit_y_scale=None): hit_y_min=None, hit_y_max=None, apdex_y_scale=None, hit_y_scale=None):
date_list = [int(calendar.timegm(time.strptime(x[0], date_format)) * 1000) date_list = [int(calendar.timegm(time.strptime(x[0], date_format)) * 1000)
for x in daily_data] for x in daily_data]
string_date_list = [x[0].replace("/","-") for x in daily_data]
timeformat = '%Y/<br/>%m/%d<br/> %H:%M' timeformat = '%Y/<br/>%m/%d<br/> %H:%M'
# There is room for about 10 labels on the X axis. # There is room for about 10 labels on the X axis.
minTickSize = (max(1, minTickSize = (max(1,
...@@ -195,73 +199,86 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None, ...@@ -195,73 +199,86 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
# Guesstimation: 6px per digit. If only em were allowed... # Guesstimation: 6px per digit. If only em were allowed...
yLabelWidth = max(int(math.log10(max(x[2] for x in daily_data))) + 1, yLabelWidth = max(int(math.log10(max(x[2] for x in daily_data))) + 1,
3) * 6 3) * 6
return graph('apdex', graph_list = []
[zip(date_list, (round(x[1], 2) for x in daily_data))], graph_list.append(graph('apdex',
[{
"x": string_date_list,
"y": [round(x[1], 2) for x in daily_data]
}],
{ {
'xaxis': { "margin": { "t": 0 },
'mode': 'time', "yaxis": {"title": "apdex (%)"},
'timeformat': timeformat, }
'minTickSize': minTickSize, ))
}, graph_list.append(graph('Hits (per %s)' % graph_period,
'yaxis': { [{
'min': apdex_y_min, "x": string_date_list,
'max': 100, "y": [x[2] for x in daily_data],
'axisLabel': 'apdex (%)', "name": "Hits"
'labelWidth': yLabelWidth, },{
'transform': apdex_y_scale, "x": string_date_list,
}, "y": [x[3] for x in daily_data],
'lines': {'show': True}, "name": "Errors"
'grid': {
'hoverable': True,
},
},
) + graph('Hits (per %s)' % graph_period,
[
{
'label': 'Errors',
'data': zip(date_list, (x[3] for x in daily_data)),
'color': 'red',
},
{
'label': 'Hits',
'data': zip(date_list, (x[2] for x in daily_data)),
}, },
], ],
{ {
'xaxis': { "margin": { "t": 0 },
'mode': 'time', "yaxis": {"title": "Hits"},
'timeformat': timeformat, }
'minTickSize': minTickSize, ))
}, x_list = []
'yaxis': { y_list = []
'min': hit_y_min, y_list_append = y_list.append
'max': hit_y_max, size_list = []
'axisLabel': 'Hits', text_list = []
'labelWidth': yLabelWidth, text_list_append = text_list.append
'tickDecimals': 0, size_list_append = size_list.append
'transform': hit_y_scale, total_hit = 0
}, log10 = math.log10
'lines': {'show': True}, for date, distribution_dict in zip(string_date_list, (x[4] for x in daily_data)):
'grid': { x_list.extend([date] * len(distribution_dict))
'hoverable': True, for rendering_time, hit in distribution_dict.iteritems():
}, y_list_append(rendering_time)
'legend': { size_list_append(log10(hit+1))
'backgroundOpacity': 0.25, text_list_append("Hits: " + str(hit))
}, total_hit += hit
graph_list.append(graph("Distribution", [{
"mode": 'markers',
"marker": {
"sizemode": "area",
"sizeref": 0.01,
"size" : size_list
}, },
"x": x_list,
"y": y_list,
"text": text_list,
}],
{
"margin": { "t": 0 },
"hovermode": "closest",
"yaxis": {"title": "Time (seconds)"},
}, height=450
) )
)
return "\n".join(graph_list)
def graph(title, data, options={}): global count_graph
count_graph = 0
def graph(title, data, options={}, height=300):
result = [] result = []
global count_graph
count_graph += 1
div_id = "graph_%i" % count_graph
append = result.append append = result.append
append('<h2>%s</h2><div class="graph" ' append('<h2>%s</h2><div id="%s" class="graph" '
'style="width:600px;height:300px" data-points="' % title) 'style="width:600px;height:%ipx" ' % (title, div_id, height))
append(' data-points="')
append(escape(json.dumps(data), quote=True)) append(escape(json.dumps(data), quote=True))
append('" data-options="') append('" data-options="')
append(escape(json.dumps(options), quote=True)) append(escape(json.dumps(options), quote=True))
append('"></div><div class="tooltip">' append('"></div>"')
'<span class="x"></span><br/>'
'<span class="y"></span></div>')
return ''.join(result) return ''.join(result)
class APDEXStats(object): class APDEXStats(object):
...@@ -275,9 +292,11 @@ class APDEXStats(object): ...@@ -275,9 +292,11 @@ class APDEXStats(object):
self.duration_total = 0 self.duration_total = 0
self.duration_max = 0 self.duration_max = 0
self.getDuration = getDuration self.getDuration = getDuration
self.distribution_dict = defaultdict(lambda: 0)
def accumulate(self, match): def accumulate(self, match):
duration = self.getDuration(match) duration = self.getDuration(match)
self.distribution_dict[int(duration/1000000)] += 1
self.duration_total += duration self.duration_total += duration
self.duration_max = max(self.duration_max, duration) self.duration_max = max(self.duration_max, duration)
if not statusIsError(match.group('status')): if not statusIsError(match.group('status')):
...@@ -292,6 +311,8 @@ class APDEXStats(object): ...@@ -292,6 +311,8 @@ class APDEXStats(object):
setattr(self, attribute, setattr(self, attribute,
getattr(self, attribute) + getattr(other, attribute)) getattr(self, attribute) + getattr(other, attribute))
self.duration_max = max(self.duration_max, other.duration_max) self.duration_max = max(self.duration_max, other.duration_max)
for key,value in other.distribution_dict.iteritems():
self.distribution_dict[key] += value
def getApdex(self): def getApdex(self):
if self.hit: if self.hit:
...@@ -1105,8 +1126,8 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict, ...@@ -1105,8 +1126,8 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
else: else:
out.write('<link rel="stylesheet" type="text/css" ' out.write('<link rel="stylesheet" type="text/css" '
'href="%s/apachedex.css"/>' % js_path) 'href="%s/apachedex.css"/>' % js_path)
for script in ('jquery.js', 'jquery.flot.js', 'jquery.flot.time.js', for script in (
'jquery.flot.axislabels.js', 'jquery-ui.js', 'apachedex.js'): 'jquery.js', 'jquery-ui.js', 'plotly.min.js', 'apachedex.js'):
if js_embed: if js_embed:
out.write('<script type="text/javascript">//<![CDATA[\n') out.write('<script type="text/javascript">//<![CDATA[\n')
out.write(getResource(script)) out.write(getResource(script))
...@@ -1146,7 +1167,7 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict, ...@@ -1146,7 +1167,7 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
if apdex_data_list: if apdex_data_list:
x_min = min(x_min, apdex_data_list[0][0]) x_min = min(x_min, apdex_data_list[0][0])
x_max = max(x_max, apdex_data_list[-1][0]) x_max = max(x_max, apdex_data_list[-1][0])
for hit_date, _, hit, _ in apdex_data_list: for hit_date, _, hit, _, _ in apdex_data_list:
hit_per_day[decimator(hit_date)] += hit hit_per_day[decimator(hit_date)] += hit
if x_min == LARGER_THAN_INTEGER_STR: if x_min == LARGER_THAN_INTEGER_STR:
x_min = None x_min = None
...@@ -1528,6 +1549,8 @@ def main(): ...@@ -1528,6 +1549,8 @@ def main():
for lineno, line in enumerate(logfile, 1): for lineno, line in enumerate(logfile, 1):
if show_progress and lineno % 5000 == 0: if show_progress and lineno % 5000 == 0:
print(lineno, end='\r', file=sys.stderr) print(lineno, end='\r', file=sys.stderr)
global current_line
current_line = line
match = matchline(line) match = matchline(line)
if match is None: if match is None:
match = expensive_matchline(line) match = expensive_matchline(line)
......
...@@ -49,21 +49,9 @@ function updateAxisTransform(axis) { ...@@ -49,21 +49,9 @@ function updateAxisTransform(axis) {
function renderGraph(container) { function renderGraph(container) {
var container = $(container); var container = $(container);
var previousIndex = null; var previousIndex = null;
var tooltip = container.next(".tooltip");
var options = $.parseJSON(container.attr("data-options")); var options = $.parseJSON(container.attr("data-options"));
updateAxisTransform(options.xaxis); var data = $.parseJSON(container.attr("data-points"));
updateAxisTransform(options.yaxis); Plotly.plot(container[0], data, options, {modeBarButtonsToRemove: ["sendDataToCloud"]});
var plot = $.plot(
container,
$.parseJSON(container.attr("data-points")),
options
);
tooltip.detach();
container.append(tooltip);
container.bind("plothover", function (event, pos, item) {
previousIndex = updateGraphTooltip(event, pos, item, previousIndex,
tooltip, plot);
});
} }
function toggleGraph(node) { function toggleGraph(node) {
var container = $(node).parent().find(".container"); var container = $(node).parent().find(".container");
...@@ -79,5 +67,5 @@ function hideGraph(node) { ...@@ -79,5 +67,5 @@ function hideGraph(node) {
} }
$(function() { $(function() {
$(".graph:visible").each(function (i){renderGraph(this)}); $(".graph:visible").each(function (i){renderGraph(this)});
$(".hidden_graph .container").draggable(); $(".hidden_graph .container").draggable({cancel: "div.graph"});
}); });
...@@ -10,24 +10,14 @@ if sys.version_info >= (3, ): ...@@ -10,24 +10,14 @@ if sys.version_info >= (3, ):
else: else:
from urllib import urlretrieve from urllib import urlretrieve
FLOT_SHA = 'aefe4e729b2d14efe6e8c0db359cb0e9aa6aae52' PLOTLY_VERSION = '1.52.1'
FLOT_AXISLABELS_SHA = '80453cd7fb8a9cad084cf6b581034ada3339dbf8'
JQUERY_VERSION = '1.9.1' JQUERY_VERSION = '1.9.1'
JQUERY_UI_VERSION = '1.10.2' JQUERY_UI_VERSION = '1.10.2'
DEPS = { DEPS = {
'jquery.flot.js': ( 'plotly.min.js': (
'http://raw.github.com/flot/flot/%s/jquery.flot.js' % FLOT_SHA, 'https://github.com/plotly/plotly.js/raw/v%s/dist/plotly.min.js' % PLOTLY_VERSION,
'7b599c575f19c33bf0d93a6bbac3af02', '02c8285a64ba86691d6c05eba366438e',
),
'jquery.flot.time.js': (
'http://raw.github.com/flot/flot/%s/jquery.flot.time.js' % FLOT_SHA,
'c0aec1608bf2fbb79f24d1905673e2c3',
),
'jquery.flot.axislabels.js': (
'http://raw.github.com/markrcote/flot-axislabels/%s/'
'jquery.flot.axislabels.js' % FLOT_AXISLABELS_SHA,
'a8526e0c1ed3b5cbc1a6b3ebb22bf334',
), ),
'jquery.js': ( 'jquery.js': (
'http://code.jquery.com/jquery-%s.min.js' % JQUERY_VERSION, 'http://code.jquery.com/jquery-%s.min.js' % JQUERY_VERSION,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment