Commit 0ba22c47 authored by Arnaud Fontaine's avatar Arnaud Fontaine

apachedex: Add ``--erp5-expand-other'' to display all 'other' results in 'Stats per module'.

By default (without passing this option), there is a single row grouping
results of all non-modules URLs ('other'). With this new options, one row per
URL is displayed for non-modules results.
parent 2f4388dd
...@@ -361,7 +361,9 @@ _APDEXDateDictAsJSONState = lambda date_dict: dict(((y, z.asJSONState()) ...@@ -361,7 +361,9 @@ _APDEXDateDictAsJSONState = lambda date_dict: dict(((y, z.asJSONState())
class GenericSiteStats(object): class GenericSiteStats(object):
def __init__(self, threshold, getDuration, suffix, error_detail=False, def __init__(self, threshold, getDuration, suffix, error_detail=False,
user_agent_detail=False): user_agent_detail=False,
# Non-generic parameters
**kw):
self.threshold = threshold self.threshold = threshold
self.suffix = suffix self.suffix = suffix
self.error_detail = error_detail self.error_detail = error_detail
...@@ -560,16 +562,26 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -560,16 +562,26 @@ class ERP5SiteStats(GenericSiteStats):
count line as belonging to a document of that module count line as belonging to a document of that module
""" """
def __init__(self, threshold, getDuration, suffix, error_detail=False, def __init__(self, threshold, getDuration, suffix, error_detail=False,
user_agent_detail=False): user_agent_detail=False, erp5_expand_other=False):
super(ERP5SiteStats, self).__init__(threshold, getDuration, suffix, super(ERP5SiteStats, self).__init__(threshold, getDuration, suffix,
error_detail=error_detail, user_agent_detail=user_agent_detail) error_detail=error_detail, user_agent_detail=user_agent_detail)
self.expand_other = erp5_expand_other
# Key levels: # Key levels:
# - module id (string) # - module id (string)
# - is document (bool) # - is document (bool)
# - date (string) # - date (string)
self.module = defaultdict(partial(defaultdict, partial( self.module = defaultdict(partial(defaultdict, partial(
defaultdict, partial(APDEXStats, threshold, getDuration)))) defaultdict, partial(APDEXStats, threshold, getDuration))))
self.no_module = defaultdict(partial(APDEXStats, threshold, getDuration))
# Key levels:
# - id (string)
# => 'other' only if expand_other == False
# - date (string)
self.no_module = defaultdict(partial(
defaultdict, partial(APDEXStats, threshold, getDuration)))
self.site_search = defaultdict(partial(APDEXStats, threshold, getDuration)) self.site_search = defaultdict(partial(APDEXStats, threshold, getDuration))
def rescale(self, convert, getDuration): def rescale(self, convert, getDuration):
...@@ -581,11 +593,17 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -581,11 +593,17 @@ class ERP5SiteStats(GenericSiteStats):
for value_date, data in date_dict.iteritems(): for value_date, data in date_dict.iteritems():
new_date_dict[convert(value_date)].accumulateFrom(data) new_date_dict[convert(value_date)].accumulateFrom(data)
document_dict[is_document] = new_date_dict document_dict[is_document] = new_date_dict
for attribute_id in ('no_module', 'site_search'):
attribute = defaultdict(partial(APDEXStats, threshold, getDuration)) for id_, date_dict in self.no_module.iteritems():
for value_date, data in getattr(self, attribute_id).iteritems(): new_date_dict = defaultdict(partial(APDEXStats, threshold, getDuration))
attribute[convert(value_date)].accumulateFrom(data) for value_date, data in date_dict.iteritems():
setattr(self, attribute_id, attribute) new_date_dict[convert(value_date)].accumulateFrom(data)
self.no_module[id_] = new_date_dict
attribute = defaultdict(partial(APDEXStats, threshold, getDuration))
for value_date, data in getattr(self, attribute_id).iteritems():
attribute[convert(value_date)].accumulateFrom(data)
self.site_search = attribute
def accumulate(self, match, url_match, value_date): def accumulate(self, match, url_match, value_date):
split = self.suffix(url_match.group('url')).split('?', 1)[0].split('/') split = self.suffix(url_match.group('url')).split('?', 1)[0].split('/')
...@@ -598,8 +616,10 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -598,8 +616,10 @@ class ERP5SiteStats(GenericSiteStats):
elif split and split[0] == 'ERP5Site_viewSearchResult': elif split and split[0] == 'ERP5Site_viewSearchResult':
super(ERP5SiteStats, self).accumulate(match, url_match, value_date) super(ERP5SiteStats, self).accumulate(match, url_match, value_date)
self.site_search[value_date].accumulate(match) self.site_search[value_date].accumulate(match)
elif split and self.expand_other:
self.no_module[split[0]][value_date].accumulate(match)
else: else:
self.no_module[value_date].accumulate(match) self.no_module['other'][value_date].accumulate(match)
def asHTML(self, date_format, placeholder_delta, graph_period, graph_coefficient, def asHTML(self, date_format, placeholder_delta, graph_period, graph_coefficient,
encoding, stat_filter=lambda x: x, x_min=None, x_max=None, encoding, stat_filter=lambda x: x, x_min=None, x_max=None,
...@@ -615,10 +635,16 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -615,10 +635,16 @@ class ERP5SiteStats(GenericSiteStats):
None)) None))
filtered_module = defaultdict(partial(defaultdict, partial( filtered_module = defaultdict(partial(defaultdict, partial(
defaultdict, partial(APDEXStats, self.threshold, None)))) defaultdict, partial(APDEXStats, self.threshold, None))))
filtered_no_module = defaultdict(partial(APDEXStats, self.threshold, None)) other_overall = APDEXStats(self.threshold, None)
for value_date, value in self.no_module.iteritems(): filtered_no_module = defaultdict(partial(
filtered_no_module[stat_filter(value_date)].accumulateFrom(value) defaultdict, partial(APDEXStats, self.threshold, None)))
column_set = set(filtered_no_module) column_set = set()
for key, data_dict in self.no_module.iteritems():
filtered_id_dict = filtered_no_module[key]
for value_date, value in data_dict.iteritems():
filtered_id_dict[stat_filter(value_date)].accumulateFrom(value)
other_overall.accumulateFrom(value)
column_set.update(filtered_id_dict)
filtered_site_search = defaultdict(partial(APDEXStats, self.threshold, filtered_site_search = defaultdict(partial(APDEXStats, self.threshold,
None)) None))
for value_date, value in self.site_search.iteritems(): for value_date, value in self.site_search.iteritems():
...@@ -691,14 +717,18 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -691,14 +717,18 @@ class ERP5SiteStats(GenericSiteStats):
'</th>') '</th>')
hiddenGraph(self.site_search, 'site search') hiddenGraph(self.site_search, 'site search')
site_search_overall = apdexAsColumns(filtered_site_search) site_search_overall = apdexAsColumns(filtered_site_search)
append('<tr class="group_top group_bottom" title="other"><th colspan="2">other</th>') append('</tr>')
hiddenGraph(self.no_module, 'other') for id_, date_dict in sorted(filtered_no_module.iteritems()):
no_module_overall = apdexAsColumns(filtered_no_module) append('<tr class="group_top group_bottom" title="%s"><th colspan="2">%s</th>'
append('</tr></table><h2>Per-level overall</h2><table class="stats"><tr>' % (id_, id_))
hiddenGraph(self.no_module[id_], id_)
apdexAsColumns(date_dict)
append('</tr>')
append('</table><h2>Per-level overall</h2><table class="stats"><tr>'
'<th>level</th>') '<th>level</th>')
append(APDEXStats.asHTMLHeader()) append(APDEXStats.asHTMLHeader())
append('</tr><tr><th>other</th>') append('</tr><tr><th>other</th>')
append(no_module_overall.asHTML(self.threshold)) append(other_overall.asHTML(self.threshold))
append('</tr><tr><th>site search</th>') append('</tr><tr><th>site search</th>')
append(site_search_overall.asHTML(self.threshold)) append(site_search_overall.asHTML(self.threshold))
append('</tr><tr><th>module</th>') append('</tr><tr><th>module</th>')
...@@ -725,10 +755,17 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -725,10 +755,17 @@ class ERP5SiteStats(GenericSiteStats):
date_dict = module_dict[is_document == 'true'] date_dict = module_dict[is_document == 'true']
for value_date, apdex_state in date_dict_state.iteritems(): for value_date, apdex_state in date_dict_state.iteritems():
date_dict[value_date] = APDEXStats.fromJSONState(apdex_state, getDuration) date_dict[value_date] = APDEXStats.fromJSONState(apdex_state, getDuration)
for attribute_id in ('no_module', 'site_search'):
attribute = getattr(result, attribute_id) for id_, date_dict in state['no_module'].iteritems():
for value_date, apdex_state in state[attribute_id].iteritems(): no_module_dict = result.no_module[id_]
attribute[value_date] = APDEXStats.fromJSONState(apdex_state, getDuration) for value_date, apdex_state in date_dict.iteritems():
no_module_dict[value_date] = APDEXStats.fromJSONState(
apdex_state, getDuration)
for value_date, apdex_state in state['site_search'].iteritems():
result.site_search[value_date] = APDEXStats.fromJSONState(
apdex_state, getDuration)
return result return result
def asJSONState(self): def asJSONState(self):
...@@ -738,9 +775,12 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -738,9 +775,12 @@ class ERP5SiteStats(GenericSiteStats):
module_dict_state = module[module_id] = {} module_dict_state = module[module_id] = {}
for is_document, date_dict in module_dict.iteritems(): for is_document, date_dict in module_dict.iteritems():
module_dict_state[is_document] = _APDEXDateDictAsJSONState(date_dict) module_dict_state[is_document] = _APDEXDateDictAsJSONState(date_dict)
for attribute_id in ('no_module', 'site_search'):
result[attribute_id] = _APDEXDateDictAsJSONState(getattr(self, result['no_module'] = no_module = {}
attribute_id)) for id_, date_dict in self.no_module.iteritems():
no_module[id_] = _APDEXDateDictAsJSONState(date_dict)
result['site_search'] = _APDEXDateDictAsJSONState(self.site_search)
return result return result
def accumulateFrom(self, other): def accumulateFrom(self, other):
...@@ -752,10 +792,15 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -752,10 +792,15 @@ class ERP5SiteStats(GenericSiteStats):
date_dict = module_dict[is_document] date_dict = module_dict[is_document]
for value_date, apdex in other_date_dict.iteritems(): for value_date, apdex in other_date_dict.iteritems():
date_dict[value_date].accumulateFrom(apdex) date_dict[value_date].accumulateFrom(apdex)
for attribute_id in ('no_module', 'site_search'):
attribute = getattr(self, attribute_id) for id_, other_date_dict in other.no_module.iteritems():
for value_date, apdex in getattr(other, attribute_id).iteritems(): date_dict = self.no_module[id_]
attribute[value_date].accumulateFrom(apdex) for value_date, apdex in other_date_dict.iteritems():
date_dict.accumulateFrom(apdex)
attribute = self.site_search
for value_date, apdex in other.site_search.iteritems():
attribute[value_date].accumulateFrom(apdex)
DURATION_US_FORMAT = '%D' DURATION_US_FORMAT = '%D'
DURATION_S_FORMAT = '%T' DURATION_S_FORMAT = '%T'
...@@ -1260,6 +1305,8 @@ def main(): ...@@ -1260,6 +1305,8 @@ def main():
help='Include detailed report (url & referers) for error statuses.') help='Include detailed report (url & referers) for error statuses.')
group.add_argument('-u', '--user-agent-detail', action='store_true', group.add_argument('-u', '--user-agent-detail', action='store_true',
help='Include report of most frequent user agents.') help='Include report of most frequent user agents.')
group.add_argument('--erp5-expand-other', action='store_true',
help='Expand ERP5 `other` stats')
group.add_argument('-f', '--format', choices=format_generator, group.add_argument('-f', '--format', choices=format_generator,
default='html', help='Format in which output should be generated.') default='html', help='Format in which output should be generated.')
...@@ -1404,6 +1451,7 @@ def main(): ...@@ -1404,6 +1451,7 @@ def main():
threshold = args.apdex threshold = args.apdex
error_detail = args.error_detail error_detail = args.error_detail
user_agent_detail = args.user_agent_detail user_agent_detail = args.user_agent_detail
erp5_expand_other = args.erp5_expand_other
file_count = len(infile_list) file_count = len(infile_list)
per_site = {} per_site = {}
if '-' in args.state_file and '-' in infile_list: if '-' in args.state_file and '-' in infile_list:
...@@ -1536,7 +1584,8 @@ def main(): ...@@ -1536,7 +1584,8 @@ def main():
site_data = per_site[site] site_data = per_site[site]
except KeyError: except KeyError:
site_data = per_site[site] = action(threshold, getDuration, site_data = per_site[site] = action(threshold, getDuration,
error_detail=error_detail, user_agent_detail=user_agent_detail) error_detail=error_detail, user_agent_detail=user_agent_detail,
erp5_expand_other=erp5_expand_other)
try: try:
site_data.accumulate(match, url_match, hit_date) site_data.accumulate(match, url_match, hit_date)
except Exception: except Exception:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment