Commit f7452cce authored by Sebastien Robin's avatar Sebastien Robin

distribution graph and using plotly instead of flotcharts

parent fc6baf0c
......@@ -34,6 +34,7 @@ from datetime import datetime, timedelta, date, tzinfo
from functools import partial
from operator import itemgetter
from urllib import splittype, splithost
from copy import copy
import argparse
import bz2
import calendar
......@@ -57,6 +58,7 @@ try:
except ImportError:
pytz = None
global current_line
def getResource(name, encoding='utf-8'):
return pkgutil.get_data(__name__, name).decode(encoding)
......@@ -160,6 +162,7 @@ def getDataPoints(apdex_dict, status_period_dict={}):
apdex.getApdex() * 100,
apdex.hit,
period_error_dict.get(value_date, 0),
apdex.distribution_dict,
) for value_date, apdex in sorted(apdex_dict.iteritems(), key=ITEMGETTER0)
]
......@@ -168,26 +171,27 @@ def prepareDataForGraph(daily_data, date_format, placeholder_delta,
current_date = datetime.strptime(x_min or daily_data[0][0], date_format)
new_daily_data = []
append = new_daily_data.append
for (measure_date_string, apdex, hit, error_hit) in daily_data:
for (measure_date_string, apdex, hit, error_hit, distribution_dict) in daily_data:
measure_date = datetime.strptime(measure_date_string, date_format)
if current_date < measure_date:
append((current_date.strftime(date_format), 100, 0, 0))
append((current_date.strftime(date_format), 100, 0, 0, {}))
placeholder_end_date = measure_date - placeholder_delta
if placeholder_end_date > current_date:
append((placeholder_end_date.strftime(date_format), 100, 0, 0))
append((placeholder_end_date.strftime(date_format), 100, 0, 0, {}))
coef = coefficient_callback(measure_date)
append((measure_date_string, apdex, hit * coef, error_hit * coef))
append((measure_date_string, apdex, hit * coef, error_hit * coef, distribution_dict))
current_date = measure_date + placeholder_delta
if x_max is not None and current_date < datetime.strptime(x_max,
date_format):
append((current_date.strftime(date_format), 100, 0, 0))
append((x_max, 100, 0, 0))
append((current_date.strftime(date_format), 100, 0, 0, {}))
append((x_max, 100, 0, 0, {}))
return new_daily_data
def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
hit_y_min=None, hit_y_max=None, apdex_y_scale=None, hit_y_scale=None):
date_list = [int(calendar.timegm(time.strptime(x[0], date_format)) * 1000)
for x in daily_data]
string_date_list = [x[0].replace("/","-") for x in daily_data]
timeformat = '%Y/<br/>%m/%d<br/> %H:%M'
# There is room for about 10 labels on the X axis.
minTickSize = (max(1,
......@@ -195,73 +199,90 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
# Guesstimation: 6px per digit. If only em were allowed...
yLabelWidth = max(int(math.log10(max(x[2] for x in daily_data))) + 1,
3) * 6
return graph('apdex',
[zip(date_list, (round(x[1], 2) for x in daily_data))],
graph_list = []
graph_list.append(graph('apdex',
[{
"x": string_date_list,
"y": [round(x[1], 2) for x in daily_data]
}],
{
'xaxis': {
'mode': 'time',
'timeformat': timeformat,
'minTickSize': minTickSize,
},
'yaxis': {
'min': apdex_y_min,
'max': 100,
'axisLabel': 'apdex (%)',
'labelWidth': yLabelWidth,
'transform': apdex_y_scale,
},
'lines': {'show': True},
'grid': {
'hoverable': True,
},
},
) + graph('Hits (per %s)' % graph_period,
[
{
'label': 'Errors',
'data': zip(date_list, (x[3] for x in daily_data)),
'color': 'red',
},
{
'label': 'Hits',
'data': zip(date_list, (x[2] for x in daily_data)),
"margin": { "t": 0 },
"yaxis": {"title": "apdex (%)"},
}
))
graph_list.append(graph('Hits (per %s)' % graph_period,
[{
"x": string_date_list,
"y": [x[2] for x in daily_data],
"name": "Hits"
},{
"x": string_date_list,
"y": [x[3] for x in daily_data],
"name": "Errors"
},
],
{
'xaxis': {
'mode': 'time',
'timeformat': timeformat,
'minTickSize': minTickSize,
},
'yaxis': {
'min': hit_y_min,
'max': hit_y_max,
'axisLabel': 'Hits',
'labelWidth': yLabelWidth,
'tickDecimals': 0,
'transform': hit_y_scale,
},
'lines': {'show': True},
'grid': {
'hoverable': True,
},
'legend': {
'backgroundOpacity': 0.25,
},
"margin": { "t": 0 },
"yaxis": {"title": "Hits"},
}
))
x_list = []
y_list = []
y_list_append = y_list.append
size_list = []
text_list = []
text_list_append = text_list.append
size_list_append = size_list.append
total_hit = 0
log10 = math.log10
for date, distribution_dict in zip(string_date_list, (x[4] for x in daily_data)):
x_list.extend([date] * len(distribution_dict))
for rendering_time, hit in distribution_dict.iteritems():
y_list_append(rendering_time)
size_list_append(log10(hit+1))
text_list_append("Hits: " + str(hit))
total_hit += hit
graph_list.append(graph("Distribution", [{
"mode": 'markers',
"marker": {
"sizemode": "area",
"sizeref": 0.01,
"size" : size_list
},
"x": x_list,
"y": y_list,
"text": text_list,
}],
{
"margin": { "t": 0 },
"hovermode": "closest",
"yaxis": {"title": "Time (seconds)"},
}, height=450
)
)
return "\n".join(graph_list)
def graph(title, data, options={}):
global count_graph
count_graph = 0
def graph(title, data, options={}, height=300):
result = []
global count_graph
count_graph += 1
div_id = "graph_%i" % count_graph
append = result.append
append('<h2>%s</h2><div class="graph" '
'style="width:600px;height:300px" data-points="' % title)
append(escape(json.dumps(data), quote=True))
append('" data-options="')
append(escape(json.dumps(options), quote=True))
append('"></div><div class="tooltip">'
'<span class="x"></span><br/>'
'<span class="y"></span></div>')
append('<h2>%s</h2><div id="%s" '
'style="width:600px;height:%ipx"></div>' % (title, div_id, height))
append('</script><script type="text/javascript">//<![CDATA[')
append("""
TESTER = document.getElementById('%s');
Plotly.plot( TESTER,""" % div_id)
append(json.dumps(data))
append(',')
append(json.dumps(options))
append(',{modeBarButtonsToRemove: ["sendDataToCloud"]});')
append('//]]></script>')
return ''.join(result)
class APDEXStats(object):
......@@ -275,9 +296,11 @@ class APDEXStats(object):
self.duration_total = 0
self.duration_max = 0
self.getDuration = getDuration
self.distribution_dict = defaultdict(lambda: 0)
def accumulate(self, match):
duration = self.getDuration(match)
self.distribution_dict[int(duration/1000000)] += 1
self.duration_total += duration
self.duration_max = max(self.duration_max, duration)
if not statusIsError(match.group('status')):
......@@ -292,6 +315,8 @@ class APDEXStats(object):
setattr(self, attribute,
getattr(self, attribute) + getattr(other, attribute))
self.duration_max = max(self.duration_max, other.duration_max)
for key,value in other.distribution_dict.iteritems():
self.distribution_dict[key] += value
def getApdex(self):
if self.hit:
......@@ -1105,8 +1130,8 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
else:
out.write('<link rel="stylesheet" type="text/css" '
'href="%s/apachedex.css"/>' % js_path)
for script in ('jquery.js', 'jquery.flot.js', 'jquery.flot.time.js',
'jquery.flot.axislabels.js', 'jquery-ui.js', 'apachedex.js'):
for script in (
'jquery.js', 'jquery-ui.js', 'plotly.min.js', 'apachedex.js'):
if js_embed:
out.write('<script type="text/javascript">//<![CDATA[\n')
out.write(getResource(script))
......@@ -1146,7 +1171,7 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
if apdex_data_list:
x_min = min(x_min, apdex_data_list[0][0])
x_max = max(x_max, apdex_data_list[-1][0])
for hit_date, _, hit, _ in apdex_data_list:
for hit_date, _, hit, _, _ in apdex_data_list:
hit_per_day[decimator(hit_date)] += hit
if x_min == LARGER_THAN_INTEGER_STR:
x_min = None
......@@ -1528,6 +1553,8 @@ def main():
for lineno, line in enumerate(logfile, 1):
if show_progress and lineno % 5000 == 0:
print(lineno, end='\r', file=sys.stderr)
global current_line
current_line = line
match = matchline(line)
if match is None:
match = expensive_matchline(line)
......
......@@ -71,6 +71,8 @@ function toggleGraph(node) {
container.toggle();
if (container.attr("data-rendered-marker") == null) {
container.attr("data-rendered-marker", "rendered");
// XXX We need to render on demand with Plotly like it was done with plot to avoid
// Long time rendering
container.find(".graph").each(function (i){renderGraph(this)});
}
}
......
......@@ -10,24 +10,14 @@ if sys.version_info >= (3, ):
else:
from urllib import urlretrieve
FLOT_SHA = 'aefe4e729b2d14efe6e8c0db359cb0e9aa6aae52'
FLOT_AXISLABELS_SHA = '80453cd7fb8a9cad084cf6b581034ada3339dbf8'
PLOTLY_VERSION = '1.52.1'
JQUERY_VERSION = '1.9.1'
JQUERY_UI_VERSION = '1.10.2'
DEPS = {
'jquery.flot.js': (
'http://raw.github.com/flot/flot/%s/jquery.flot.js' % FLOT_SHA,
'7b599c575f19c33bf0d93a6bbac3af02',
),
'jquery.flot.time.js': (
'http://raw.github.com/flot/flot/%s/jquery.flot.time.js' % FLOT_SHA,
'c0aec1608bf2fbb79f24d1905673e2c3',
),
'jquery.flot.axislabels.js': (
'http://raw.github.com/markrcote/flot-axislabels/%s/'
'jquery.flot.axislabels.js' % FLOT_AXISLABELS_SHA,
'a8526e0c1ed3b5cbc1a6b3ebb22bf334',
'plotly.min.js': (
'https://github.com/plotly/plotly.js/raw/v%s/dist/plotly.min.js' % PLOTLY_VERSION,
'02c8285a64ba86691d6c05eba366438e',
),
'jquery.js': (
'http://code.jquery.com/jquery-%s.min.js' % JQUERY_VERSION,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment