distribution graph and using plotly instead of flotcharts

apachedex/__init__.py

@@ -34,6 +34,7 @@ from datetime import datetime, timedelta, date, tzinfo
 from functools import partial
 from operator import itemgetter
 from urllib import splittype, splithost
+from copy import copy
 import argparse
 import bz2
 import calendar
@@ -57,6 +58,7 @@ try:
 except ImportError:
     pytz = None
 
+global current_line
 
 def getResource(name, encoding='utf-8'):
   return pkgutil.get_data(__name__, name).decode(encoding)
@@ -160,6 +162,7 @@ def getDataPoints(apdex_dict, status_period_dict={}):
       apdex.getApdex() * 100,
       apdex.hit,
       period_error_dict.get(value_date, 0),
+      apdex.distribution_dict,
     ) for value_date, apdex in sorted(apdex_dict.iteritems(), key=ITEMGETTER0)
   ]
 
@@ -168,26 +171,27 @@ def prepareDataForGraph(daily_data, date_format, placeholder_delta,
   current_date = datetime.strptime(x_min or daily_data[0][0], date_format)
   new_daily_data = []
   append = new_daily_data.append
-  for (measure_date_string, apdex, hit, error_hit) in daily_data:
+  for (measure_date_string, apdex, hit, error_hit, distribution_dict) in daily_data:
     measure_date = datetime.strptime(measure_date_string, date_format)
     if current_date < measure_date:
-      append((current_date.strftime(date_format), 100, 0, 0))
+      append((current_date.strftime(date_format), 100, 0, 0, {}))
       placeholder_end_date = measure_date - placeholder_delta
       if placeholder_end_date > current_date:
-        append((placeholder_end_date.strftime(date_format), 100, 0, 0))
+        append((placeholder_end_date.strftime(date_format), 100, 0, 0, {}))
     coef = coefficient_callback(measure_date)
-    append((measure_date_string, apdex, hit * coef, error_hit * coef))
+    append((measure_date_string, apdex, hit * coef, error_hit * coef, distribution_dict))
     current_date = measure_date + placeholder_delta
   if x_max is not None and current_date < datetime.strptime(x_max,
       date_format):
-    append((current_date.strftime(date_format), 100, 0, 0))
-    append((x_max, 100, 0, 0))
+    append((current_date.strftime(date_format), 100, 0, 0, {}))
+    append((x_max, 100, 0, 0, {}))
   return new_daily_data
 
 def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
     hit_y_min=None, hit_y_max=None, apdex_y_scale=None, hit_y_scale=None):
   date_list = [int(calendar.timegm(time.strptime(x[0], date_format)) * 1000)
     for x in daily_data]
+  string_date_list = [x[0].replace("/","-") for x in daily_data]
   timeformat = '%Y/<br/>%m/%d<br/> %H:%M'
   # There is room for about 10 labels on the X axis.
   minTickSize = (max(1,
@@ -195,73 +199,90 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
   # Guesstimation: 6px per digit. If only em were allowed...
   yLabelWidth = max(int(math.log10(max(x[2] for x in daily_data))) + 1,
     3) * 6
-  return graph('apdex',
-    [zip(date_list, (round(x[1], 2) for x in daily_data))],
+  graph_list = []
+  graph_list.append(graph('apdex',
+    [{
+      "x": string_date_list,
+      "y": [round(x[1], 2) for x in daily_data]
+    }],
     {
-      'xaxis': {
-        'mode': 'time',
-        'timeformat': timeformat,
-        'minTickSize': minTickSize,
-      },
-      'yaxis': {
-        'min': apdex_y_min,
-        'max': 100,
-        'axisLabel': 'apdex (%)',
-        'labelWidth': yLabelWidth,
-        'transform': apdex_y_scale,
-      },
-      'lines': {'show': True},
-      'grid': {
-        'hoverable': True,
-      },
-    },
-  ) + graph('Hits (per %s)' % graph_period,
-    [
-      {
-        'label': 'Errors',
-        'data': zip(date_list, (x[3] for x in daily_data)),
-        'color': 'red',
-      },
-      {
-        'label': 'Hits',
-        'data': zip(date_list, (x[2] for x in daily_data)),
+      "margin": { "t": 0 },
+      "yaxis": {"title": "apdex (%)"},
+    }
+  ))
+  graph_list.append(graph('Hits (per %s)' % graph_period,
+    [{
+      "x": string_date_list,
+      "y": [x[2] for x in daily_data],
+      "name": "Hits"
+    },{
+      "x": string_date_list,
+      "y": [x[3] for x in daily_data],
+      "name": "Errors"
     },
     ],
     {
-      'xaxis': {
-        'mode': 'time',
-        'timeformat': timeformat,
-        'minTickSize': minTickSize,
-      },
-      'yaxis': {
-        'min': hit_y_min,
-        'max': hit_y_max,
-        'axisLabel': 'Hits',
-        'labelWidth': yLabelWidth,
-        'tickDecimals': 0,
-        'transform': hit_y_scale,
-      },
-      'lines': {'show': True},
-      'grid': {
-        'hoverable': True,
-      },
-      'legend': {
-        'backgroundOpacity': 0.25,
-      },
+      "margin": { "t": 0 },
+      "yaxis": {"title": "Hits"},
+    }
+  ))
+  x_list = []
+  y_list = []
+  y_list_append = y_list.append
+  size_list = []
+  text_list = []
+  text_list_append = text_list.append
+  size_list_append = size_list.append
+  total_hit = 0
+  log10 = math.log10
+  for date, distribution_dict in zip(string_date_list, (x[4] for x in daily_data)):
+    x_list.extend([date] * len(distribution_dict))
+    for rendering_time, hit in distribution_dict.iteritems():
+      y_list_append(rendering_time)
+      size_list_append(log10(hit+1))
+      text_list_append("Hits: " + str(hit))
+      total_hit += hit
+
+  graph_list.append(graph("Distribution", [{
+      "mode": 'markers',
+      "marker": {
+        "sizemode": "area",
+        "sizeref": 0.01,
+        "size" : size_list
       },
+      "x": x_list,
+      "y": y_list,
+      "text": text_list,
+      }],
+      {
+        "margin": { "t": 0 },
+        "hovermode": "closest",
+        "yaxis": {"title": "Time (seconds)"},
+      }, height=450
       )
+  )
+  return "\n".join(graph_list)
+
 
-def graph(title, data, options={}):
+global count_graph
+count_graph = 0
+def graph(title, data, options={}, height=300):
   result = []
+  global count_graph
+  count_graph += 1
+  div_id = "graph_%i" % count_graph
   append = result.append
-  append('<h2>%s</h2><div class="graph" '
-    'style="width:600px;height:300px" data-points="' % title)
-  append(escape(json.dumps(data), quote=True))
-  append('" data-options="')
-  append(escape(json.dumps(options), quote=True))
-  append('"></div><div class="tooltip">'
-    '<span class="x"></span><br/>'
-    '<span class="y"></span></div>')
+  append('<h2>%s</h2><div id="%s" '
+    'style="width:600px;height:%ipx"></div>' % (title, div_id, height))
+  append('</script><script type="text/javascript">//<![CDATA[')
+  append("""
+TESTER = document.getElementById('%s');
+Plotly.plot( TESTER,""" % div_id)
+  append(json.dumps(data))
+  append(',')
+  append(json.dumps(options))
+  append(',{modeBarButtonsToRemove: ["sendDataToCloud"]});')
+  append('//]]></script>')
   return ''.join(result)
 
 class APDEXStats(object):
@@ -275,9 +296,11 @@ class APDEXStats(object):
     self.duration_total = 0
     self.duration_max = 0
     self.getDuration = getDuration
+    self.distribution_dict = defaultdict(lambda: 0)
 
   def accumulate(self, match):
     duration = self.getDuration(match)
+    self.distribution_dict[int(duration/1000000)] += 1
     self.duration_total += duration
     self.duration_max = max(self.duration_max, duration)
     if not statusIsError(match.group('status')):
@@ -292,6 +315,8 @@ class APDEXStats(object):
       setattr(self, attribute,
         getattr(self, attribute) + getattr(other, attribute))
     self.duration_max = max(self.duration_max, other.duration_max)
+    for key,value in other.distribution_dict.iteritems():
+      self.distribution_dict[key] += value
 
   def getApdex(self):
     if self.hit:
@@ -1105,8 +1130,8 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
   else:
     out.write('<link rel="stylesheet" type="text/css" '
       'href="%s/apachedex.css"/>' % js_path)
-  for script in ('jquery.js', 'jquery.flot.js', 'jquery.flot.time.js',
-      'jquery.flot.axislabels.js', 'jquery-ui.js', 'apachedex.js'):
+  for script in (
+      'jquery.js', 'jquery-ui.js', 'plotly.min.js', 'apachedex.js'):
     if js_embed:
       out.write('<script type="text/javascript">//<![CDATA[\n')
       out.write(getResource(script))
@@ -1146,7 +1171,7 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
     if apdex_data_list:
       x_min = min(x_min, apdex_data_list[0][0])
       x_max = max(x_max, apdex_data_list[-1][0])
-      for hit_date, _, hit, _ in apdex_data_list:
+      for hit_date, _, hit, _, _ in apdex_data_list:
         hit_per_day[decimator(hit_date)] += hit
   if x_min == LARGER_THAN_INTEGER_STR:
     x_min = None
@@ -1528,6 +1553,8 @@ def main():
     for lineno, line in enumerate(logfile, 1):
       if show_progress and lineno % 5000 == 0:
         print(lineno, end='\r', file=sys.stderr)
+      global current_line
+      current_line = line
       match = matchline(line)
       if match is None:
         match = expensive_matchline(line)

apachedex/apachedex.js

@@ -71,6 +71,8 @@ function toggleGraph(node) {
   container.toggle();
   if (container.attr("data-rendered-marker") == null) {
     container.attr("data-rendered-marker", "rendered");
+    // XXX We need to render on demand with Plotly like it was done with plot to avoid
+    // Long time rendering
     container.find(".graph").each(function (i){renderGraph(this)});
   }
 }

setup.py

@@ -10,24 +10,14 @@ if sys.version_info >= (3, ):
 else:
   from urllib import urlretrieve
 
-FLOT_SHA = 'aefe4e729b2d14efe6e8c0db359cb0e9aa6aae52'
-FLOT_AXISLABELS_SHA = '80453cd7fb8a9cad084cf6b581034ada3339dbf8'
+PLOTLY_VERSION = '1.52.1'
 JQUERY_VERSION = '1.9.1'
 JQUERY_UI_VERSION = '1.10.2'
 
 DEPS = {
-  'jquery.flot.js': (
-    'http://raw.github.com/flot/flot/%s/jquery.flot.js' % FLOT_SHA,
-    '7b599c575f19c33bf0d93a6bbac3af02',
-  ),
-  'jquery.flot.time.js': (
-    'http://raw.github.com/flot/flot/%s/jquery.flot.time.js' % FLOT_SHA,
-    'c0aec1608bf2fbb79f24d1905673e2c3',
-  ),
-  'jquery.flot.axislabels.js': (
-    'http://raw.github.com/markrcote/flot-axislabels/%s/'
-      'jquery.flot.axislabels.js' % FLOT_AXISLABELS_SHA,
-    'a8526e0c1ed3b5cbc1a6b3ebb22bf334',
+  'plotly.min.js': (
+    'https://github.com/plotly/plotly.js/raw/v%s/dist/plotly.min.js' % PLOTLY_VERSION,
+    '02c8285a64ba86691d6c05eba366438e',
   ),
   'jquery.js': (
     'http://code.jquery.com/jquery-%s.min.js' % JQUERY_VERSION,