Compute hottest URLs instead of slowest.

Slowest page list is cluttered by temporary issues not representative of
service.

apachedex/__init__.py

@@ -52,7 +52,6 @@ MONTH_VALUE_DICT = dict((y, x) for (x, y) in enumerate(('Jan', 'Feb', 'Mar',
 US_PER_S = 10 ** 6
 
 N_SLOWEST = 20
-N_SLOWEST_THRESHOLD = N_SLOWEST * 4
 N_ERROR_URL = 10
 N_REFERRER_PER_ERROR_URL = 5
 ITEMGETTER0 = itemgetter(0)
@@ -147,7 +146,7 @@ class GenericSiteStats(object):
     self.status = defaultdict(partial(defaultdict, int))
     if error_detail:
       self.error_url_count = defaultdict(partial(defaultdict, list))
-    self.slowest_list = [(-1, None, None, None)]
+    self.url_apdex = defaultdict(partial(APDEXStats, threshold))
     self.apdex = defaultdict(partial(APDEXStats, threshold))
 
   def accumulate(self, match, url_match, date):
@@ -157,23 +156,14 @@ class GenericSiteStats(object):
       url = match.group('request')
     else:
       url = url_match.group('url')
-    if duration > self.slowest_list[0][0]:
-      slowest_list = self.slowest_list
-      slowest_list.append((duration, match.group('timestamp'), url,
-        match.group('referer')))
-      if len(slowest_list) > N_SLOWEST_THRESHOLD:
-        self._housekeeping()
+    # XXX: can eat memory if there are many different urls
+    self.url_apdex[url.split('?', 1)[0]].accumulate(match)
     status = match.group('status')
     self.status[status][date] += 1
     if self.error_detail and statusIsError(status):
       # XXX: can eat memory if there are many errors on many different urls
       self.error_url_count[status][url].append(match.group('referer'))
 
-  def _housekeeping(self):
-    slowest_list = self.slowest_list
-    slowest_list.sort(key=ITEMGETTER0)
-    slowest_list[:] = slowest_list[-N_SLOWEST:]
-
   def getApdexData(self):
     apdex = APDEXStats(self.threshold)
     for data in self.apdex.itervalues():
@@ -184,7 +174,6 @@ class GenericSiteStats(object):
       in sorted(self.apdex.iteritems(), key=ITEMGETTER0)]
 
   def asHTML(self, stat_filter=lambda x: x):
-    self._housekeeping()
     result = []
     append = result.append
     apdex = APDEXStats(self.threshold)
@@ -247,19 +236,14 @@ class GenericSiteStats(object):
           ))
           append('</tr>')
       append('</table>')
-    append('<h2>Slowest pages</h2><table class="stats"><tr>'
-      '<th>duration (s)</th><th>date</th><th>url</th><th>referer</th></tr>')
-    for duration, timestamp, url, referer in reversed(self.slowest_list):
-      if timestamp is None:
-        continue
-      append('<tr><td class="%s">%.2f</td><td>%s</td>'
-        '<td class="text">%s</td><td class="text">%s</td></tr>' % (
-        getClassForDuration(duration, self.threshold),
-        float(duration) / US_PER_S,
-        escape(timestamp),
-        escape(url),
-        escape(referer),
-      ))
+    append('<h2>Hottest pages</h2><table class="stats"><tr>')
+    append(APDEX_TABLE_HEADERS)
+    append('<th>url</th></tr>')
+    for url, data in sorted(self.url_apdex.iteritems(), key=lambda x: x[1].getAverage() * x[1].hit,
+        reverse=True)[:N_SLOWEST]:
+      append('<tr>')
+      append(getApdexStatsAsHtml(data, self.threshold))
+      append('<td class="text">%s</td></tr>' % escape(url))
     append('</table>')
     return '\n'.join(result)