Commit 0ba22c47 authored by Arnaud Fontaine's avatar Arnaud Fontaine

apachedex: Add ``--erp5-expand-other'' to display all 'other' results in 'Stats per module'.

By default (without passing this option), there is a single row grouping
results of all non-modules URLs ('other'). With this new options, one row per
URL is displayed for non-modules results.
parent 2f4388dd
......@@ -361,7 +361,9 @@ _APDEXDateDictAsJSONState = lambda date_dict: dict(((y, z.asJSONState())
class GenericSiteStats(object):
def __init__(self, threshold, getDuration, suffix, error_detail=False,
user_agent_detail=False):
user_agent_detail=False,
# Non-generic parameters
**kw):
self.threshold = threshold
self.suffix = suffix
self.error_detail = error_detail
......@@ -560,16 +562,26 @@ class ERP5SiteStats(GenericSiteStats):
count line as belonging to a document of that module
"""
def __init__(self, threshold, getDuration, suffix, error_detail=False,
user_agent_detail=False):
user_agent_detail=False, erp5_expand_other=False):
super(ERP5SiteStats, self).__init__(threshold, getDuration, suffix,
error_detail=error_detail, user_agent_detail=user_agent_detail)
self.expand_other = erp5_expand_other
# Key levels:
# - module id (string)
# - is document (bool)
# - date (string)
self.module = defaultdict(partial(defaultdict, partial(
defaultdict, partial(APDEXStats, threshold, getDuration))))
self.no_module = defaultdict(partial(APDEXStats, threshold, getDuration))
# Key levels:
# - id (string)
# => 'other' only if expand_other == False
# - date (string)
self.no_module = defaultdict(partial(
defaultdict, partial(APDEXStats, threshold, getDuration)))
self.site_search = defaultdict(partial(APDEXStats, threshold, getDuration))
def rescale(self, convert, getDuration):
......@@ -581,11 +593,17 @@ class ERP5SiteStats(GenericSiteStats):
for value_date, data in date_dict.iteritems():
new_date_dict[convert(value_date)].accumulateFrom(data)
document_dict[is_document] = new_date_dict
for attribute_id in ('no_module', 'site_search'):
for id_, date_dict in self.no_module.iteritems():
new_date_dict = defaultdict(partial(APDEXStats, threshold, getDuration))
for value_date, data in date_dict.iteritems():
new_date_dict[convert(value_date)].accumulateFrom(data)
self.no_module[id_] = new_date_dict
attribute = defaultdict(partial(APDEXStats, threshold, getDuration))
for value_date, data in getattr(self, attribute_id).iteritems():
attribute[convert(value_date)].accumulateFrom(data)
setattr(self, attribute_id, attribute)
self.site_search = attribute
def accumulate(self, match, url_match, value_date):
split = self.suffix(url_match.group('url')).split('?', 1)[0].split('/')
......@@ -598,8 +616,10 @@ class ERP5SiteStats(GenericSiteStats):
elif split and split[0] == 'ERP5Site_viewSearchResult':
super(ERP5SiteStats, self).accumulate(match, url_match, value_date)
self.site_search[value_date].accumulate(match)
elif split and self.expand_other:
self.no_module[split[0]][value_date].accumulate(match)
else:
self.no_module[value_date].accumulate(match)
self.no_module['other'][value_date].accumulate(match)
def asHTML(self, date_format, placeholder_delta, graph_period, graph_coefficient,
encoding, stat_filter=lambda x: x, x_min=None, x_max=None,
......@@ -615,10 +635,16 @@ class ERP5SiteStats(GenericSiteStats):
None))
filtered_module = defaultdict(partial(defaultdict, partial(
defaultdict, partial(APDEXStats, self.threshold, None))))
filtered_no_module = defaultdict(partial(APDEXStats, self.threshold, None))
for value_date, value in self.no_module.iteritems():
filtered_no_module[stat_filter(value_date)].accumulateFrom(value)
column_set = set(filtered_no_module)
other_overall = APDEXStats(self.threshold, None)
filtered_no_module = defaultdict(partial(
defaultdict, partial(APDEXStats, self.threshold, None)))
column_set = set()
for key, data_dict in self.no_module.iteritems():
filtered_id_dict = filtered_no_module[key]
for value_date, value in data_dict.iteritems():
filtered_id_dict[stat_filter(value_date)].accumulateFrom(value)
other_overall.accumulateFrom(value)
column_set.update(filtered_id_dict)
filtered_site_search = defaultdict(partial(APDEXStats, self.threshold,
None))
for value_date, value in self.site_search.iteritems():
......@@ -691,14 +717,18 @@ class ERP5SiteStats(GenericSiteStats):
'</th>')
hiddenGraph(self.site_search, 'site search')
site_search_overall = apdexAsColumns(filtered_site_search)
append('<tr class="group_top group_bottom" title="other"><th colspan="2">other</th>')
hiddenGraph(self.no_module, 'other')
no_module_overall = apdexAsColumns(filtered_no_module)
append('</tr></table><h2>Per-level overall</h2><table class="stats"><tr>'
append('</tr>')
for id_, date_dict in sorted(filtered_no_module.iteritems()):
append('<tr class="group_top group_bottom" title="%s"><th colspan="2">%s</th>'
% (id_, id_))
hiddenGraph(self.no_module[id_], id_)
apdexAsColumns(date_dict)
append('</tr>')
append('</table><h2>Per-level overall</h2><table class="stats"><tr>'
'<th>level</th>')
append(APDEXStats.asHTMLHeader())
append('</tr><tr><th>other</th>')
append(no_module_overall.asHTML(self.threshold))
append(other_overall.asHTML(self.threshold))
append('</tr><tr><th>site search</th>')
append(site_search_overall.asHTML(self.threshold))
append('</tr><tr><th>module</th>')
......@@ -725,10 +755,17 @@ class ERP5SiteStats(GenericSiteStats):
date_dict = module_dict[is_document == 'true']
for value_date, apdex_state in date_dict_state.iteritems():
date_dict[value_date] = APDEXStats.fromJSONState(apdex_state, getDuration)
for attribute_id in ('no_module', 'site_search'):
attribute = getattr(result, attribute_id)
for value_date, apdex_state in state[attribute_id].iteritems():
attribute[value_date] = APDEXStats.fromJSONState(apdex_state, getDuration)
for id_, date_dict in state['no_module'].iteritems():
no_module_dict = result.no_module[id_]
for value_date, apdex_state in date_dict.iteritems():
no_module_dict[value_date] = APDEXStats.fromJSONState(
apdex_state, getDuration)
for value_date, apdex_state in state['site_search'].iteritems():
result.site_search[value_date] = APDEXStats.fromJSONState(
apdex_state, getDuration)
return result
def asJSONState(self):
......@@ -738,9 +775,12 @@ class ERP5SiteStats(GenericSiteStats):
module_dict_state = module[module_id] = {}
for is_document, date_dict in module_dict.iteritems():
module_dict_state[is_document] = _APDEXDateDictAsJSONState(date_dict)
for attribute_id in ('no_module', 'site_search'):
result[attribute_id] = _APDEXDateDictAsJSONState(getattr(self,
attribute_id))
result['no_module'] = no_module = {}
for id_, date_dict in self.no_module.iteritems():
no_module[id_] = _APDEXDateDictAsJSONState(date_dict)
result['site_search'] = _APDEXDateDictAsJSONState(self.site_search)
return result
def accumulateFrom(self, other):
......@@ -752,9 +792,14 @@ class ERP5SiteStats(GenericSiteStats):
date_dict = module_dict[is_document]
for value_date, apdex in other_date_dict.iteritems():
date_dict[value_date].accumulateFrom(apdex)
for attribute_id in ('no_module', 'site_search'):
attribute = getattr(self, attribute_id)
for value_date, apdex in getattr(other, attribute_id).iteritems():
for id_, other_date_dict in other.no_module.iteritems():
date_dict = self.no_module[id_]
for value_date, apdex in other_date_dict.iteritems():
date_dict.accumulateFrom(apdex)
attribute = self.site_search
for value_date, apdex in other.site_search.iteritems():
attribute[value_date].accumulateFrom(apdex)
DURATION_US_FORMAT = '%D'
......@@ -1260,6 +1305,8 @@ def main():
help='Include detailed report (url & referers) for error statuses.')
group.add_argument('-u', '--user-agent-detail', action='store_true',
help='Include report of most frequent user agents.')
group.add_argument('--erp5-expand-other', action='store_true',
help='Expand ERP5 `other` stats')
group.add_argument('-f', '--format', choices=format_generator,
default='html', help='Format in which output should be generated.')
......@@ -1404,6 +1451,7 @@ def main():
threshold = args.apdex
error_detail = args.error_detail
user_agent_detail = args.user_agent_detail
erp5_expand_other = args.erp5_expand_other
file_count = len(infile_list)
per_site = {}
if '-' in args.state_file and '-' in infile_list:
......@@ -1536,7 +1584,8 @@ def main():
site_data = per_site[site]
except KeyError:
site_data = per_site[site] = action(threshold, getDuration,
error_detail=error_detail, user_agent_detail=user_agent_detail)
error_detail=error_detail, user_agent_detail=user_agent_detail,
erp5_expand_other=erp5_expand_other)
try:
site_data.accumulate(match, url_match, hit_date)
except Exception:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment