apachedex: Add ``--erp5-expand-other'' to display all 'other' results in 'Stats per module'.

By default (without passing this option), there is a single row grouping
results of all non-modules URLs ('other'). With this new options, one row per
URL is displayed for non-modules results.

apachedex/__init__.py

@@ -361,7 +361,9 @@ _APDEXDateDictAsJSONState = lambda date_dict: dict(((y, z.asJSONState())
 
 class GenericSiteStats(object):
   def __init__(self, threshold, getDuration, suffix, error_detail=False,
-      user_agent_detail=False):
+      user_agent_detail=False,
+      # Non-generic parameters
+      **kw):
     self.threshold = threshold
     self.suffix = suffix
     self.error_detail = error_detail
@@ -560,16 +562,26 @@ class ERP5SiteStats(GenericSiteStats):
     count line as belonging to a document of that module
   """
   def __init__(self, threshold, getDuration, suffix, error_detail=False,
-      user_agent_detail=False):
+      user_agent_detail=False, erp5_expand_other=False):
     super(ERP5SiteStats, self).__init__(threshold, getDuration, suffix,
       error_detail=error_detail, user_agent_detail=user_agent_detail)
+
+    self.expand_other = erp5_expand_other
+
     # Key levels:
     # - module id (string)
     # - is document (bool)
     # - date (string)
     self.module = defaultdict(partial(defaultdict, partial(
       defaultdict, partial(APDEXStats, threshold, getDuration))))
-    self.no_module = defaultdict(partial(APDEXStats, threshold, getDuration))
+
+    # Key levels:
+    # - id (string)
+    #   => 'other' only if expand_other == False
+    # - date (string)
+    self.no_module = defaultdict(partial(
+        defaultdict, partial(APDEXStats, threshold, getDuration)))
+
     self.site_search = defaultdict(partial(APDEXStats, threshold, getDuration))
 
   def rescale(self, convert, getDuration):
@@ -581,11 +593,17 @@ class ERP5SiteStats(GenericSiteStats):
         for value_date, data in date_dict.iteritems():
           new_date_dict[convert(value_date)].accumulateFrom(data)
         document_dict[is_document] = new_date_dict
-    for attribute_id in ('no_module', 'site_search'):
-      attribute = defaultdict(partial(APDEXStats, threshold, getDuration))
-      for value_date, data in getattr(self, attribute_id).iteritems():
-        attribute[convert(value_date)].accumulateFrom(data)
-      setattr(self, attribute_id, attribute)
+
+    for id_, date_dict in self.no_module.iteritems():
+      new_date_dict = defaultdict(partial(APDEXStats, threshold, getDuration))
+      for value_date, data in date_dict.iteritems():
+        new_date_dict[convert(value_date)].accumulateFrom(data)
+      self.no_module[id_] = new_date_dict
+
+    attribute = defaultdict(partial(APDEXStats, threshold, getDuration))
+    for value_date, data in getattr(self, attribute_id).iteritems():
+      attribute[convert(value_date)].accumulateFrom(data)
+    self.site_search = attribute
 
   def accumulate(self, match, url_match, value_date):
     split = self.suffix(url_match.group('url')).split('?', 1)[0].split('/')
@@ -598,8 +616,10 @@ class ERP5SiteStats(GenericSiteStats):
     elif split and split[0] == 'ERP5Site_viewSearchResult':
       super(ERP5SiteStats, self).accumulate(match, url_match, value_date)
       self.site_search[value_date].accumulate(match)
+    elif split and self.expand_other:
+      self.no_module[split[0]][value_date].accumulate(match)
     else:
-      self.no_module[value_date].accumulate(match)
+      self.no_module['other'][value_date].accumulate(match)
 
   def asHTML(self, date_format, placeholder_delta, graph_period, graph_coefficient,
       encoding, stat_filter=lambda x: x, x_min=None, x_max=None,
@@ -615,10 +635,16 @@ class ERP5SiteStats(GenericSiteStats):
       None))
     filtered_module = defaultdict(partial(defaultdict, partial(
       defaultdict, partial(APDEXStats, self.threshold, None))))
-    filtered_no_module = defaultdict(partial(APDEXStats, self.threshold, None))
-    for value_date, value in self.no_module.iteritems():
-      filtered_no_module[stat_filter(value_date)].accumulateFrom(value)
-    column_set = set(filtered_no_module)
+    other_overall = APDEXStats(self.threshold, None)
+    filtered_no_module = defaultdict(partial(
+      defaultdict, partial(APDEXStats, self.threshold, None)))
+    column_set = set()
+    for key, data_dict in self.no_module.iteritems():
+      filtered_id_dict = filtered_no_module[key]
+      for value_date, value in data_dict.iteritems():
+        filtered_id_dict[stat_filter(value_date)].accumulateFrom(value)
+        other_overall.accumulateFrom(value)
+      column_set.update(filtered_id_dict)
     filtered_site_search = defaultdict(partial(APDEXStats, self.threshold,
       None))
     for value_date, value in self.site_search.iteritems():
@@ -691,14 +717,18 @@ class ERP5SiteStats(GenericSiteStats):
       '</th>')
     hiddenGraph(self.site_search, 'site search')
     site_search_overall = apdexAsColumns(filtered_site_search)
-    append('<tr class="group_top group_bottom" title="other"><th colspan="2">other</th>')
-    hiddenGraph(self.no_module, 'other')
-    no_module_overall = apdexAsColumns(filtered_no_module)
-    append('</tr></table><h2>Per-level overall</h2><table class="stats"><tr>'
+    append('</tr>')
+    for id_, date_dict in sorted(filtered_no_module.iteritems()):
+      append('<tr class="group_top group_bottom" title="%s"><th colspan="2">%s</th>'
+             % (id_, id_))
+      hiddenGraph(self.no_module[id_], id_)
+      apdexAsColumns(date_dict)
+      append('</tr>')
+    append('</table><h2>Per-level overall</h2><table class="stats"><tr>'
       '<th>level</th>')
     append(APDEXStats.asHTMLHeader())
     append('</tr><tr><th>other</th>')
-    append(no_module_overall.asHTML(self.threshold))
+    append(other_overall.asHTML(self.threshold))
     append('</tr><tr><th>site search</th>')
     append(site_search_overall.asHTML(self.threshold))
     append('</tr><tr><th>module</th>')
@@ -725,10 +755,17 @@ class ERP5SiteStats(GenericSiteStats):
         date_dict = module_dict[is_document == 'true']
         for value_date, apdex_state in date_dict_state.iteritems():
           date_dict[value_date] = APDEXStats.fromJSONState(apdex_state, getDuration)
-    for attribute_id in ('no_module', 'site_search'):
-      attribute = getattr(result, attribute_id)
-      for value_date, apdex_state in state[attribute_id].iteritems():
-        attribute[value_date] = APDEXStats.fromJSONState(apdex_state, getDuration)
+
+    for id_, date_dict in state['no_module'].iteritems():
+      no_module_dict = result.no_module[id_]
+      for value_date, apdex_state in date_dict.iteritems():
+        no_module_dict[value_date] = APDEXStats.fromJSONState(
+            apdex_state, getDuration)
+
+    for value_date, apdex_state in state['site_search'].iteritems():
+      result.site_search[value_date] = APDEXStats.fromJSONState(
+        apdex_state, getDuration)
+
     return result
 
   def asJSONState(self):
@@ -738,9 +775,12 @@ class ERP5SiteStats(GenericSiteStats):
       module_dict_state = module[module_id] = {}
       for is_document, date_dict in module_dict.iteritems():
         module_dict_state[is_document] = _APDEXDateDictAsJSONState(date_dict)
-    for attribute_id in ('no_module', 'site_search'):
-      result[attribute_id] = _APDEXDateDictAsJSONState(getattr(self,
-        attribute_id))
+
+    result['no_module'] = no_module = {}
+    for id_, date_dict in self.no_module.iteritems():
+      no_module[id_] = _APDEXDateDictAsJSONState(date_dict)
+
+    result['site_search'] = _APDEXDateDictAsJSONState(self.site_search)
     return result
 
   def accumulateFrom(self, other):
@@ -752,10 +792,15 @@ class ERP5SiteStats(GenericSiteStats):
         date_dict = module_dict[is_document]
         for value_date, apdex in other_date_dict.iteritems():
           date_dict[value_date].accumulateFrom(apdex)
-    for attribute_id in ('no_module', 'site_search'):
-      attribute = getattr(self, attribute_id)
-      for value_date, apdex in getattr(other, attribute_id).iteritems():
-        attribute[value_date].accumulateFrom(apdex)
+
+    for id_, other_date_dict in other.no_module.iteritems():
+      date_dict = self.no_module[id_]
+      for value_date, apdex in other_date_dict.iteritems():
+        date_dict.accumulateFrom(apdex)
+
+    attribute = self.site_search
+    for value_date, apdex in other.site_search.iteritems():
+      attribute[value_date].accumulateFrom(apdex)
 
 DURATION_US_FORMAT = '%D'
 DURATION_S_FORMAT = '%T'
@@ -1260,6 +1305,8 @@ def main():
     help='Include detailed report (url & referers) for error statuses.')
   group.add_argument('-u', '--user-agent-detail', action='store_true',
     help='Include report of most frequent user agents.')
+  group.add_argument('--erp5-expand-other', action='store_true',
+    help='Expand ERP5 `other` stats')
 
   group.add_argument('-f', '--format', choices=format_generator,
     default='html', help='Format in which output should be generated.')
@@ -1404,6 +1451,7 @@ def main():
   threshold = args.apdex
   error_detail = args.error_detail
   user_agent_detail = args.user_agent_detail
+  erp5_expand_other = args.erp5_expand_other
   file_count = len(infile_list)
   per_site = {}
   if '-' in args.state_file and '-' in infile_list:
@@ -1536,7 +1584,8 @@ def main():
         site_data = per_site[site]
       except KeyError:
         site_data = per_site[site] = action(threshold, getDuration,
-          error_detail=error_detail, user_agent_detail=user_agent_detail)
+          error_detail=error_detail, user_agent_detail=user_agent_detail,
+          erp5_expand_other=erp5_expand_other)
       try:
         site_data.accumulate(match, url_match, hit_date)
       except Exception:

[Vincent Pelletier]

Traceback (most recent call last):
  File "apachedex/__init__.py", line 1274, in wrapper
    return func(*args, **kw)
  File "apachedex/__init__.py", line 1586, in main
    site_data.rescale(rescale, getDuration)
  File "apachedex/__init__.py", line 605, in rescale
    for value_date, data in getattr(self, attribute_id).iteritems():
NameError: global name 'attribute_id' is not defined

Could you check ?

[Arnaud Fontaine]

My bad, sorry. This should do it:

-- a/apachedex/__init__.py
+++ b/apachedex/__init__.py
@@ -602,7 +602,7 @@ class ERP5SiteStats(GenericSiteStats):
       self.no_module[id_] = new_date_dict
 
     attribute = defaultdict(partial(APDEXStats, threshold, getDuration))
-    for value_date, data in getattr(self, attribute_id).iteritems():
+    for value_date, data in self.site_search.iteritems():
       attribute[convert(value_date)].accumulateFrom(data)
     self.site_search = attribute

[Arnaud Fontaine]

Let me know if that works and I will prepare a commit or you can commit it, it's up to you.

[Vincent Pelletier]

Thanks for the patch, applied and released as 1.7.1 .