Add stats on most frequent user agents.

To find which user agents should be ignored (bots, monitoring, ...).

apachedex/__init__.py

@@ -79,6 +79,7 @@ US_PER_S = 10 ** 6
 N_SLOWEST = 20
 N_ERROR_URL = 10
 N_REFERRER_PER_ERROR_URL = 5
+N_USER_AGENT = 20
 ITEMGETTER0 = itemgetter(0)
 ITEMGETTER1 = itemgetter(1)
 APDEX_TOLERATING_COEF = 4
@@ -285,6 +286,7 @@ class GenericSiteStats(object):
       self.error_url_count = defaultdict(partial(defaultdict, list))
     self.url_apdex = defaultdict(partial(APDEXStats, threshold, getDuration))
     self.apdex = defaultdict(partial(APDEXStats, threshold, getDuration))
+    self.user_agent_counter = Counter()
 
   def rescale(self, convert, getDuration):
     for status, date_dict in self.status.iteritems():
@@ -310,6 +312,7 @@ class GenericSiteStats(object):
     if self.error_detail and statusIsError(status):
       # XXX: can eat memory if there are many errors on many different urls
       self.error_url_count[status][url].append(match.group('referer'))
+    self.user_agent_counter[match.group('agent')] += 1
 
   def getApdexData(self):
     return getDataPoints(self.apdex)
@@ -334,6 +337,12 @@ class GenericSiteStats(object):
       append(data.asHTML(self.threshold))
       append('<td class="text">%s</td></tr>' % unquoteToHtml(url, encoding))
     append('</table>')
+    append('<h2>User agents</h2><table class="stats"><tr><th>hits</th>'
+      '<th>user agent</th></tr>')
+    for user_agent, hit in self.user_agent_counter.most_common(N_USER_AGENT):
+      # XXX: s/escape/unquoteToHtml/ ?
+      append('<tr><td>%s</td><td class="text">%s</td></tr>' % (hit, escape(user_agent)))
+    append('</table>')
     column_set = set()
     filtered_status = defaultdict(partial(defaultdict, int))
     for status, date_dict in self.status.iteritems():
@@ -416,6 +425,7 @@ class GenericSiteStats(object):
     status = result.status
     for status_code, date_dict in state['status'].iteritems():
       status[status_code].update(date_dict)
+    result.user_agent_counter.update(state['user_agent_counter'])
     return result
 
   def asJSONState(self):
@@ -426,6 +436,7 @@ class GenericSiteStats(object):
       'url_apdex': _APDEXDateDictAsJSONState(self.url_apdex),
       'apdex': _APDEXDateDictAsJSONState(self.apdex),
       'status': self.status,
+      'user_agent_counter': self.user_agent_counter,
     }
 
   def accumulateFrom(self, other):
@@ -445,6 +456,7 @@ class GenericSiteStats(object):
       date_dict = status[status_code]
       for status_date, count in other_date_dict.iteritems():
         date_dict[status_date] += count
+    self.user_agent_counter.update(other.user_agent_counter)
 
 class ERP5SiteStats(GenericSiteStats):
   """