Commit 466146fb authored by Vincent Pelletier's avatar Vincent Pelletier

Make pylint somewhat happy

parent 3efd68a4
...@@ -52,7 +52,7 @@ ignore=CVS ...@@ -52,7 +52,7 @@ ignore=CVS
# ignore-list. The regex matches against paths and can be in Posix or Windows # ignore-list. The regex matches against paths and can be in Posix or Windows
# format. Because '\\' represents the directory delimiter on Windows systems, # format. Because '\\' represents the directory delimiter on Windows systems,
# it can't be used as an escape character. # it can't be used as an escape character.
ignore-paths= ignore-paths=apachedex/_version.py
# Files or directories matching the regular expression patterns are skipped. # Files or directories matching the regular expression patterns are skipped.
# The regex matches against base names, not paths. The default value ignores # The regex matches against base names, not paths. The default value ignores
...@@ -177,7 +177,7 @@ const-naming-style=UPPER_CASE ...@@ -177,7 +177,7 @@ const-naming-style=UPPER_CASE
docstring-min-length=-1 docstring-min-length=-1
# Naming style matching correct function names. # Naming style matching correct function names.
function-naming-style=snake_case function-naming-style=camelCase
# Regular expression matching correct function names. Overrides function- # Regular expression matching correct function names. Overrides function-
# naming-style. If left empty, function names will be checked with the set # naming-style. If left empty, function names will be checked with the set
......
...@@ -26,6 +26,19 @@ ...@@ -26,6 +26,19 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
# TODO: resolve these
# pylint: disable=line-too-long
# pylint: disable=too-many-lines
# pylint: disable=too-many-locals
# pylint: disable=too-many-arguments
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
# pylint: disable=too-many-instance-attributes
# pylint: disable=missing-function-docstring
# pylint: disable=missing-class-docstring
# pylint: disable=missing-module-docstring
# pylint: disable=invalid-name
from html import escape from html import escape
from collections import defaultdict, Counter from collections import defaultdict, Counter
...@@ -56,11 +69,9 @@ try: ...@@ -56,11 +69,9 @@ try:
import pytz import pytz
except ImportError: except ImportError:
pytz = None pytz = None
from ._version import get_versions
__version__ = get_versions()['version']
def getResource(name, encoding='utf-8'): del get_versions
return pkgutil.get_data(__name__, name).decode(encoding)
gzip_open = gzip.open gzip_open = gzip.open
lzma_open = lzma.open lzma_open = lzma.open
...@@ -120,7 +131,7 @@ def getClassForStatusHit(hit, status): ...@@ -120,7 +131,7 @@ def getClassForStatusHit(hit, status):
return 'problem' return 'problem'
return '' return ''
def getDataPoints(apdex_dict, status_period_dict={}): def getDataPoints(apdex_dict, status_period_dict={}): # pylint: disable=dangerous-default-value
period_error_dict = defaultdict(int) period_error_dict = defaultdict(int)
for status, period_dict in status_period_dict.items(): for status, period_dict in status_period_dict.items():
if statusIsError(status): if statusIsError(status):
...@@ -165,10 +176,10 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None, ...@@ -165,10 +176,10 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
for x in daily_data] for x in daily_data]
timeformat = '%Y/<br/>%m/%d<br/> %H:%M' timeformat = '%Y/<br/>%m/%d<br/> %H:%M'
# There is room for about 10 labels on the X axis. # There is room for about 10 labels on the X axis.
minTickSize = (max(1, min_tick_size = (max(1,
(date_list[-1] - date_list[0]) / (60 * 60 * 1000 * 10)), 'hour') (date_list[-1] - date_list[0]) / (60 * 60 * 1000 * 10)), 'hour')
# Guesstimation: 6px per digit. If only em were allowed... # Guesstimation: 6px per digit. If only em were allowed...
yLabelWidth = max(int(math.log10(max(x[2] for x in daily_data))) + 1, y_label_width = max(int(math.log10(max(x[2] for x in daily_data))) + 1,
3) * 6 3) * 6
return graph('apdex', return graph('apdex',
[list(zip(date_list, (round(x[1], 2) for x in daily_data)))], [list(zip(date_list, (round(x[1], 2) for x in daily_data)))],
...@@ -176,13 +187,13 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None, ...@@ -176,13 +187,13 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
'xaxis': { 'xaxis': {
'mode': 'time', 'mode': 'time',
'timeformat': timeformat, 'timeformat': timeformat,
'minTickSize': minTickSize, 'minTickSize': min_tick_size,
}, },
'yaxis': { 'yaxis': {
'min': apdex_y_min, 'min': apdex_y_min,
'max': 100, 'max': 100,
'axisLabel': 'apdex (%)', 'axisLabel': 'apdex (%)',
'labelWidth': yLabelWidth, 'labelWidth': y_label_width,
'transform': apdex_y_scale, 'transform': apdex_y_scale,
}, },
'lines': {'show': True}, 'lines': {'show': True},
...@@ -190,7 +201,7 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None, ...@@ -190,7 +201,7 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
'hoverable': True, 'hoverable': True,
}, },
}, },
) + graph('Hits (per %s)' % graph_period, ) + graph(f'Hits (per {graph_period})',
[ [
{ {
'label': 'Errors', 'label': 'Errors',
...@@ -206,13 +217,13 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None, ...@@ -206,13 +217,13 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
'xaxis': { 'xaxis': {
'mode': 'time', 'mode': 'time',
'timeformat': timeformat, 'timeformat': timeformat,
'minTickSize': minTickSize, 'minTickSize': min_tick_size,
}, },
'yaxis': { 'yaxis': {
'min': hit_y_min, 'min': hit_y_min,
'max': hit_y_max, 'max': hit_y_max,
'axisLabel': 'Hits', 'axisLabel': 'Hits',
'labelWidth': yLabelWidth, 'labelWidth': y_label_width,
'tickDecimals': 0, 'tickDecimals': 0,
'transform': hit_y_scale, 'transform': hit_y_scale,
}, },
...@@ -226,11 +237,11 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None, ...@@ -226,11 +237,11 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
}, },
) )
def graph(title, data, options={}): def graph(title, data, options={}): # pylint: disable=dangerous-default-value
result = [] result = []
append = result.append append = result.append
append('<h2>%s</h2><div class="graph" ' append(f'<h2>{title}</h2><div class="graph" '
'style="width:600px;height:300px" data-points="' % title) 'style="width:600px;height:300px" data-points="')
append(escape(json.dumps(data), quote=True)) append(escape(json.dumps(data), quote=True))
append('" data-options="') append('" data-options="')
append(escape(json.dumps(options), quote=True)) append(escape(json.dumps(options), quote=True))
...@@ -239,7 +250,7 @@ def graph(title, data, options={}): ...@@ -239,7 +250,7 @@ def graph(title, data, options={}):
'<span class="y"></span></div>') '<span class="y"></span></div>')
return ''.join(result) return ''.join(result)
class APDEXStats(object): class APDEXStats:
def __init__(self, threshold, getDuration): def __init__(self, threshold, getDuration):
threshold *= US_PER_S threshold *= US_PER_S
self.threshold = threshold self.threshold = threshold
...@@ -283,8 +294,10 @@ class APDEXStats(object): ...@@ -283,8 +294,10 @@ class APDEXStats(object):
@staticmethod @staticmethod
def asHTMLHeader(overall=False): def asHTMLHeader(overall=False):
return '<th>apdex</th><th>hits</th><th>avg (s)</th>' \ return (
'<th%s>max (s)</th>' % (overall and ' class="overall_right"' or '') '<th>apdex</th><th>hits</th><th>avg (s)</th>'
'<th' + (' class="overall_right"' if overall else '') + '>max (s)</th>'
)
def asHTML(self, threshold, overall=False): def asHTML(self, threshold, overall=False):
apdex = self.getApdex() apdex = self.getApdex()
...@@ -293,9 +306,9 @@ class APDEXStats(object): ...@@ -293,9 +306,9 @@ class APDEXStats(object):
hit = self.hit hit = self.hit
if hit: if hit:
extra_class = '' extra_class = ''
apdex_style = 'color: #%s; background-color: #%s' % ( apdex_style = (
(apdex < .5 and 'f' or '0') * 3, 'color: #' + (('f' if apdex < .5 else '0') * 3) + ';'
('%x' % int(apdex * 0xf)) * 3, 'background-color: #' + (f'{int(apdex * 0xf):x}' * 3)
) )
else: else:
extra_class = 'no_hit' extra_class = 'no_hit'
...@@ -331,14 +344,17 @@ class APDEXStats(object): ...@@ -331,14 +344,17 @@ class APDEXStats(object):
del result['getDuration'] del result['getDuration']
return result return result
_APDEXDateDictAsJSONState = lambda date_dict: dict(((y, z.asJSONState()) def _APDEXDateDictAsJSONState(date_dict):
for y, z in date_dict.items())) return {
y: z.asJSONState()
for y, z in date_dict.items()
}
class GenericSiteStats(object): class GenericSiteStats:
def __init__(self, threshold, getDuration, suffix, error_detail=False, def __init__(self, threshold, getDuration, suffix, error_detail=False,
user_agent_detail=False, user_agent_detail=False,
# Non-generic parameters # Non-generic parameters
**kw): **_):
self.threshold = threshold self.threshold = threshold
self.suffix = suffix self.suffix = suffix
self.error_detail = error_detail self.error_detail = error_detail
...@@ -387,7 +403,7 @@ class GenericSiteStats(object): ...@@ -387,7 +403,7 @@ class GenericSiteStats(object):
apdex_y_min=None, hit_y_min=None, hit_y_max=None, apdex_y_min=None, hit_y_min=None, hit_y_max=None,
apdex_y_scale=None, hit_y_scale=None, apdex_y_scale=None, hit_y_scale=None,
n_hottest_pages=N_HOTTEST_PAGES_DEFAULT, n_hottest_pages=N_HOTTEST_PAGES_DEFAULT,
): ): # pylint: disable=unused-argument
result = [] result = []
append = result.append append = result.append
apdex = APDEXStats(self.threshold, None) apdex = APDEXStats(self.threshold, None)
...@@ -404,13 +420,13 @@ class GenericSiteStats(object): ...@@ -404,13 +420,13 @@ class GenericSiteStats(object):
reverse=True)[:n_hottest_pages]: reverse=True)[:n_hottest_pages]:
append('<tr>') append('<tr>')
append(data.asHTML(self.threshold)) append(data.asHTML(self.threshold))
append('<td class="text">%s</td></tr>' % escape(url)) append(f'<td class="text">{escape(url)}</td></tr>')
append('</table>') append('</table>')
if self.user_agent_detail: if self.user_agent_detail:
append('<h2>User agents</h2><table class="stats"><tr><th>hits</th>' append('<h2>User agents</h2><table class="stats"><tr><th>hits</th>'
'<th>user agent</th></tr>') '<th>user agent</th></tr>')
for user_agent, hit in self.user_agent_counter.most_common(N_USER_AGENT): for user_agent, hit in self.user_agent_counter.most_common(N_USER_AGENT):
append('<tr><td>%s</td><td class="text">%s</td></tr>' % (hit, escape(user_agent))) append(f'<tr><td>{hit}</td><td class="text">{escape(user_agent)}</td></tr>')
append('</table>') append('</table>')
column_set = set() column_set = set()
filtered_status = defaultdict(partial(defaultdict, int)) filtered_status = defaultdict(partial(defaultdict, int))
...@@ -423,21 +439,20 @@ class GenericSiteStats(object): ...@@ -423,21 +439,20 @@ class GenericSiteStats(object):
append('<h2>Hits per status code</h2><table class="stats"><tr>' append('<h2>Hits per status code</h2><table class="stats"><tr>'
'<th>status</th><th>overall</th>') '<th>status</th><th>overall</th>')
for column in column_list: for column in column_list:
append('<th>%s</th>' % column) append(f'<th>{column}</th>')
append('</tr>') append('</tr>')
def hitTd(hit, status): def hitTd(hit, status):
return '<td class="%s">%s</td>' % (getClassForStatusHit(hit, status), hit) return f'<td class="{getClassForStatusHit(hit, status)}">{hit}</td>'
def statusAsHtml(status): def statusAsHtml(status):
try: try:
definition = HTTP_STATUS_CAPTION_DICT[int(status)] definition = HTTP_STATUS_CAPTION_DICT[int(status)]
except KeyError: except KeyError:
return status return status
else: return f'<abbr title="{definition}">{status}</abbr>'
return '<abbr title="%s">%s</abbr>' % (definition, status)
has_errors = False has_errors = False
for status, data_dict in sorted(filtered_status.items(), key=ITEMGETTER0): for status, data_dict in sorted(filtered_status.items(), key=ITEMGETTER0):
has_errors |= statusIsError(status) has_errors |= statusIsError(status)
append('<tr title="%s"><th>%s</th>' % (status, statusAsHtml(status))) append(f'<tr title="{status}"><th>{statusAsHtml(status)}</th>')
append(hitTd(sum(data_dict.values()), status)) append(hitTd(sum(data_dict.values()), status))
for column in column_list: for column in column_list:
append(hitTd(data_dict[column], status)) append(hitTd(data_dict[column], status))
...@@ -454,22 +469,20 @@ class GenericSiteStats(object): ...@@ -454,22 +469,20 @@ class GenericSiteStats(object):
'<th>hits</th><th>url</th><th>referers</th></tr>') '<th>hits</th><th>url</th><th>referers</th></tr>')
for status, url_list in sorted(filtered_status_url.items(), for status, url_list in sorted(filtered_status_url.items(),
key=ITEMGETTER0): key=ITEMGETTER0):
append('<tr><th rowspan="%s">%s</th>' % (len(url_list), append(f'<tr><th rowspan="{len(url_list)}">{statusAsHtml(status)}</th>')
statusAsHtml(status)))
first_url = True first_url = True
for url, referer_counter in url_list: for url, referer_counter in url_list:
if first_url: if first_url:
first_url = False first_url = False
else: else:
append('<tr>') append('<tr>')
append('<td>%s</td><td class="text">%s</td>' append(
'<td class="text">%s</td>' % ( f'<td>{getHitForUrl(referer_counter)}</td><td class="text">{escape(url)}</td>'
getHitForUrl(referer_counter), '<td class="text">' + '<br/>'.join(
escape(url), f'{hit}: {escape(referer)}'
'<br/>'.join('%i: %s' % (hit, escape(referer)) for referer, hit in referer_counter.most_common(N_REFERRER_PER_ERROR_URL)
for referer, hit in referer_counter.most_common( ) + '</td>'
N_REFERRER_PER_ERROR_URL)), )
))
append('</tr>') append('</tr>')
append('</table>') append('</table>')
return '\n'.join(result) return '\n'.join(result)
...@@ -538,7 +551,7 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -538,7 +551,7 @@ class ERP5SiteStats(GenericSiteStats):
""" """
def __init__(self, threshold, getDuration, suffix, error_detail=False, def __init__(self, threshold, getDuration, suffix, error_detail=False,
user_agent_detail=False, erp5_expand_other=False): user_agent_detail=False, erp5_expand_other=False):
super(ERP5SiteStats, self).__init__(threshold, getDuration, suffix, super().__init__(threshold, getDuration, suffix,
error_detail=error_detail, user_agent_detail=user_agent_detail) error_detail=error_detail, user_agent_detail=user_agent_detail)
self.expand_other = erp5_expand_other self.expand_other = erp5_expand_other
...@@ -560,7 +573,7 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -560,7 +573,7 @@ class ERP5SiteStats(GenericSiteStats):
self.site_search = defaultdict(partial(APDEXStats, threshold, getDuration)) self.site_search = defaultdict(partial(APDEXStats, threshold, getDuration))
def rescale(self, convert, getDuration): def rescale(self, convert, getDuration):
super(ERP5SiteStats, self).rescale(convert, getDuration) super().rescale(convert, getDuration)
threshold = self.threshold threshold = self.threshold
for document_dict in self.module.values(): for document_dict in self.module.values():
for is_document, date_dict in document_dict.items(): for is_document, date_dict in document_dict.items():
...@@ -583,13 +596,13 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -583,13 +596,13 @@ class ERP5SiteStats(GenericSiteStats):
def accumulate(self, match, url_match, value_date): def accumulate(self, match, url_match, value_date):
split = self.suffix(url_match.group('url')).split('?', 1)[0].split('/') split = self.suffix(url_match.group('url')).split('?', 1)[0].split('/')
if split and split[0].endswith('_module'): if split and split[0].endswith('_module'):
super(ERP5SiteStats, self).accumulate(match, url_match, value_date) super().accumulate(match, url_match, value_date)
module = split[0] module = split[0]
self.module[module][ self.module[module][
len(split) > 1 and (split[1] != 'view' and '_view' not in split[1]) len(split) > 1 and (split[1] != 'view' and '_view' not in split[1])
][value_date].accumulate(match) ][value_date].accumulate(match)
elif split and split[0] == 'ERP5Site_viewSearchResult': elif split and split[0] == 'ERP5Site_viewSearchResult':
super(ERP5SiteStats, self).accumulate(match, url_match, value_date) super().accumulate(match, url_match, value_date)
self.site_search[value_date].accumulate(match) self.site_search[value_date].accumulate(match)
elif split and self.expand_other: elif split and self.expand_other:
self.no_module[split[0]][value_date].accumulate(match) self.no_module[split[0]][value_date].accumulate(match)
...@@ -637,7 +650,7 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -637,7 +650,7 @@ class ERP5SiteStats(GenericSiteStats):
column_set.update(filtered_data_dict) column_set.update(filtered_data_dict)
column_list = sorted(column_set) column_list = sorted(column_set)
for column in column_list: for column in column_list:
append('<th colspan="4">%s</th>' % column) append(f'<th colspan="4">{column}</th>')
append('</tr><tr>') append('</tr><tr>')
for i in range(len(column_list) + 1): for i in range(len(column_list) + 1):
append(APDEXStats.asHTMLHeader(i == 0)) append(APDEXStats.asHTMLHeader(i == 0))
...@@ -656,9 +669,8 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -656,9 +669,8 @@ class ERP5SiteStats(GenericSiteStats):
if len(data) > 1: if len(data) > 1:
append('<span class="action" onclick="toggleGraph(this)">+</span>' append('<span class="action" onclick="toggleGraph(this)">+</span>'
'<div class="positioner"><div class="container">' '<div class="positioner"><div class="container">'
'<div class="title">%s</div>' f'<div class="title">{title}</div>'
'<div class="action close" onclick="hideGraph(this)">close</div>' % '<div class="action close" onclick="hideGraph(this)">close</div>'
title
) )
append(graphPair( append(graphPair(
prepareDataForGraph( prepareDataForGraph(
...@@ -680,11 +692,11 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -680,11 +692,11 @@ class ERP5SiteStats(GenericSiteStats):
append('</div></div>') append('</div></div>')
append('</td>') append('</td>')
for module_id, data_dict in sorted(filtered_module.items(), key=ITEMGETTER0): for module_id, data_dict in sorted(filtered_module.items(), key=ITEMGETTER0):
append('<tr class="group_top" title="%s (module)"><th rowspan="2">%s</th>' append(f'<tr class="group_top" title="{module_id} (module)"><th rowspan="2">{module_id}</th>'
'<th>module</th>' % (module_id, module_id)) '<th>module</th>')
hiddenGraph(self.module[module_id][False], module_id + ' (module)') hiddenGraph(self.module[module_id][False], module_id + ' (module)')
apdexAsColumns(data_dict[False]) apdexAsColumns(data_dict[False])
append('</tr><tr class="group_bottom" title="%s (document)"><th>document</th>' % module_id) append(f'</tr><tr class="group_bottom" title="{module_id} (document)"><th>document</th>')
hiddenGraph(self.module[module_id][True], module_id + ' (document)') hiddenGraph(self.module[module_id][True], module_id + ' (document)')
apdexAsColumns(data_dict[True]) apdexAsColumns(data_dict[True])
append('</tr>') append('</tr>')
...@@ -694,8 +706,7 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -694,8 +706,7 @@ class ERP5SiteStats(GenericSiteStats):
site_search_overall = apdexAsColumns(filtered_site_search) site_search_overall = apdexAsColumns(filtered_site_search)
append('</tr>') append('</tr>')
for id_, date_dict in sorted(filtered_no_module.items()): for id_, date_dict in sorted(filtered_no_module.items()):
append('<tr class="group_top group_bottom" title="%s"><th colspan="2">%s</th>' append('<tr class="group_top group_bottom" title="{id_}"><th colspan="2">{id_}</th>')
% (id_, id_))
hiddenGraph(self.no_module[id_], id_) hiddenGraph(self.no_module[id_], id_)
apdexAsColumns(date_dict) apdexAsColumns(date_dict)
append('</tr>') append('</tr>')
...@@ -711,7 +722,7 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -711,7 +722,7 @@ class ERP5SiteStats(GenericSiteStats):
append('</tr><tr><th>document</th>') append('</tr><tr><th>document</th>')
append(module_document_overall[True].asHTML(self.threshold)) append(module_document_overall[True].asHTML(self.threshold))
append('</tr></table>') append('</tr></table>')
append(super(ERP5SiteStats, self).asHTML(date_format, append(super().asHTML(date_format,
placeholder_delta, graph_period, graph_coefficient, encoding, placeholder_delta, graph_period, graph_coefficient, encoding,
stat_filter=stat_filter, stat_filter=stat_filter,
x_min=x_min, x_max=x_max, x_min=x_min, x_max=x_max,
...@@ -724,7 +735,7 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -724,7 +735,7 @@ class ERP5SiteStats(GenericSiteStats):
@classmethod @classmethod
def fromJSONState(cls, state, getDuration, suffix): def fromJSONState(cls, state, getDuration, suffix):
result = super(ERP5SiteStats, cls).fromJSONState(state, getDuration, suffix) result = super().fromJSONState(state, getDuration, suffix)
for module_id, module_dict_state in state['module'].items(): for module_id, module_dict_state in state['module'].items():
module_dict = result.module[module_id] module_dict = result.module[module_id]
for is_document, date_dict_state in module_dict_state.items(): for is_document, date_dict_state in module_dict_state.items():
...@@ -745,7 +756,7 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -745,7 +756,7 @@ class ERP5SiteStats(GenericSiteStats):
return result return result
def asJSONState(self): def asJSONState(self):
result = super(ERP5SiteStats, self).asJSONState() result = super().asJSONState()
result['module'] = module = {} result['module'] = module = {}
for module_id, module_dict in self.module.items(): for module_id, module_dict in self.module.items():
module_dict_state = module[module_id] = {} module_dict_state = module[module_id] = {}
...@@ -760,7 +771,7 @@ class ERP5SiteStats(GenericSiteStats): ...@@ -760,7 +771,7 @@ class ERP5SiteStats(GenericSiteStats):
return result return result
def accumulateFrom(self, other): def accumulateFrom(self, other):
super(ERP5SiteStats, self).accumulateFrom(other) super().accumulateFrom(other)
module = self.module module = self.module
for module_id, other_module_dict in other.module.items(): for module_id, other_module_dict in other.module.items():
module_dict = module[module_id] module_dict = module[module_id]
...@@ -835,13 +846,13 @@ class AggregateSiteUrl(argparse.Action): ...@@ -835,13 +846,13 @@ class AggregateSiteUrl(argparse.Action):
except StopIteration: except StopIteration:
break break
if value in site_caption_dict: if value in site_caption_dict:
raise ValueError('Duplicate base: %r' % value) raise ValueError(f'Duplicate base: {value}')
if action is not None and value[0] == '+': if action is not None and value[0] == '+':
caption = value[1:] caption = value[1:]
try: try:
value = next_value() value = next_value()
except StopIteration: except StopIteration:
raise ValueError('No base follows caption %r' % value) raise ValueError(f'No base follows caption {value}') from None
else: else:
caption = value caption = value
site_caption_dict[value] = caption site_caption_dict[value] = caption
...@@ -878,7 +889,9 @@ class ShlexArgumentParser(argparse.ArgumentParser): ...@@ -878,7 +889,9 @@ class ShlexArgumentParser(argparse.ArgumentParser):
os.path.dirname(filepath), os.path.dirname(filepath),
)) ))
try: try:
with open(os.path.join(new_cwd, os.path.basename(filepath)) with open(
os.path.join(new_cwd, os.path.basename(filepath)),
encoding='utf-8',
) as in_file: ) as in_file:
extend(self.__read_args_from_files( extend(self.__read_args_from_files(
shlex.split(in_file.read(), comments=True), shlex.split(in_file.read(), comments=True),
...@@ -896,7 +909,7 @@ class ShlexArgumentParser(argparse.ArgumentParser): ...@@ -896,7 +909,7 @@ class ShlexArgumentParser(argparse.ArgumentParser):
else: else:
args = list(args) args = list(args)
args = self.__read_args_from_files(args, os.getcwd()) args = self.__read_args_from_files(args, os.getcwd())
return super(ShlexArgumentParser, self).parse_known_args(args=args, return super().parse_known_args(args=args,
namespace=namespace) namespace=namespace)
_month_offset_cache = {} _month_offset_cache = {}
...@@ -919,11 +932,11 @@ def _asWeekString(dt): ...@@ -919,11 +932,11 @@ def _asWeekString(dt):
month -= 1 month -= 1
day += calendar.monthrange(year, month)[1] day += calendar.monthrange(year, month)[1]
assert day > 0 and month > 0, (dt, year, month, day) assert day > 0 and month > 0, (dt, year, month, day)
return '%04i/%02i/%02i' % (year, month, day) return f'{year:04}/{month:02}/{day:02}'
def _weekStringAsQuarterString(timestamp): def _weekStringAsQuarterString(timestamp):
year, month, _ = timestamp.split('/') year, month, _ = timestamp.split('/')
return '%s/%02i' % (year, (int(month) - 1) // 3 * 3 + 1) return f'{year}/{(int(month) - 1) // 3 * 3 + 1:02}'
def _roundWeek(dt): def _roundWeek(dt):
day_of_year = dt.timetuple().tm_yday day_of_year = dt.timetuple().tm_yday
...@@ -946,11 +959,11 @@ def _hourAsWeekString(timestamp): ...@@ -946,11 +959,11 @@ def _hourAsWeekString(timestamp):
def _asHalfDayString(timestamp): def _asHalfDayString(timestamp):
prefix, _ = timestamp.rsplit(':', 1) prefix, _ = timestamp.rsplit(':', 1)
prefix, hours = prefix.split(' ') prefix, hours = prefix.split(' ')
return '%s %02i' % (prefix, int(hours) // 12 * 12) return f'{prefix} {int(hours) // 12 * 12:02}'
def _asQuarterHourString(timestamp): def _asQuarterHourString(timestamp):
prefix, minute = timestamp.rsplit(':', 1) prefix, minute = timestamp.rsplit(':', 1)
return '%s:%02i' % (prefix, int(minute) // 15 * 15) return f'{prefix}:{int(minute) // 15 * 15:02}'
# Key: argument (represents table granularity) # Key: argument (represents table granularity)
# Value: # Value:
...@@ -1004,7 +1017,7 @@ period_parser = { ...@@ -1004,7 +1017,7 @@ period_parser = {
lambda x: 1, lambda x: 1,
), ),
'week': ( 'week': (
lambda x: x.strftime('%Y/%m/%d ') + '%02i' % (x.hour // 6 * 6), lambda x: x.strftime('%Y/%m/%d ') + f'{x.hour // 6 * 6:02}',
_hourAsWeekString, _hourAsWeekString,
'6 hours', '6 hours',
'%Y/%m/%d %H', '%Y/%m/%d %H',
...@@ -1025,7 +1038,7 @@ period_parser = { ...@@ -1025,7 +1038,7 @@ period_parser = {
lambda x: 1, lambda x: 1,
), ),
'halfday': ( 'halfday': (
lambda x: x.strftime('%Y/%m/%d %H:') + '%02i' % (x.minute // 30 * 30), lambda x: x.strftime('%Y/%m/%d %H:') + f'{x.minute // 30 * 30:02}',
_asHalfDayString, _asHalfDayString,
'30 minutes', '30 minutes',
'%Y/%m/%d %H:%M', '%Y/%m/%d %H:%M',
...@@ -1054,8 +1067,16 @@ hit_y_scale_dict = { ...@@ -1054,8 +1067,16 @@ hit_y_scale_dict = {
'log': 'log0ToAny', 'log': 'log0ToAny',
} }
def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict, def asHTML(
stats, site_caption_dict): out,
encoding,
per_site,
args,
default_site,
period_parameter_dict,
stats,
site_caption_dict,
): # pylint: disable=unused-argument
period = period_parameter_dict['period'] period = period_parameter_dict['period']
decimator = period_parameter_dict['decimator'] decimator = period_parameter_dict['decimator']
date_format = period_parameter_dict['date_format'] date_format = period_parameter_dict['date_format']
...@@ -1069,8 +1090,8 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict, ...@@ -1069,8 +1090,8 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
hit_y_max = None hit_y_max = None
else: else:
apdex_y_min = hit_y_min = None apdex_y_min = hit_y_min = None
out.write('<!DOCTYPE html>\n<html><head><meta charset="%s">' out.write(f'<!DOCTYPE html>\n<html><head><meta charset="{encoding}">'
'<title>Stats</title><meta name="generator" content="APacheDEX" />' % encoding) '<title>Stats</title><meta name="generator" content="APacheDEX" />')
js_path = args.js js_path = args.js
js_embed = js_path is None or args.js_embed js_embed = js_path is None or args.js_embed
if js_embed: if js_embed:
...@@ -1079,7 +1100,7 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict, ...@@ -1079,7 +1100,7 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
out.write('</style>') out.write('</style>')
else: else:
out.write('<link rel="stylesheet" type="text/css" ' out.write('<link rel="stylesheet" type="text/css" '
'href="%s/apachedex.css"/>' % js_path) f'href="{js_path}/apachedex.css"/>')
for script in ('jquery.js', 'jquery.flot.js', 'jquery.flot.time.js', for script in ('jquery.js', 'jquery.flot.js', 'jquery.flot.time.js',
'jquery.flot.axislabels.js', 'jquery-ui.js', 'apachedex.js'): 'jquery.flot.axislabels.js', 'jquery-ui.js', 'apachedex.js'):
if js_embed: if js_embed:
...@@ -1087,8 +1108,7 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict, ...@@ -1087,8 +1108,7 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
out.write(getResource(script)) out.write(getResource(script))
out.write('\n//]]></script>') out.write('\n//]]></script>')
else: else:
out.write('<script type="text/javascript" src="%s/%s"></script>' % ( out.write(f'<script type="text/javascript" src="{js_path}/{script}"></script>')
js_path, script))
apdex_y_scale = apdex_y_scale_dict[args.apdex_yscale] apdex_y_scale = apdex_y_scale_dict[args.apdex_yscale]
hit_y_scale = hit_y_scale_dict[args.hit_yscale] hit_y_scale = hit_y_scale_dict[args.hit_yscale]
out.write('</head><body><h1>Overall</h1>') out.write('</head><body><h1>Overall</h1>')
...@@ -1100,19 +1120,17 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict, ...@@ -1100,19 +1120,17 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
if len(per_site) > 1: if len(per_site) > 1:
out.write('<h2>Index</h2><ol>') out.write('<h2>Index</h2><ol>')
for i, (site_id, _) in site_list: for i, (site_id, _) in site_list:
out.write('<li><a href="#%s" title="%s">%s</a></li>' % (i, out.write(f'<li><a href="#{i}" title="{escape(repr(site_id), quote=True)}">{html_site_caption_dict[site_id]}</a></li>')
escape(repr(site_id), quote=True), html_site_caption_dict[site_id]))
out.write('</ol>') out.write('</ol>')
out.write('<h2>Parameters</h2><table class="stats">') out.write('<h2>Parameters</h2><table class="stats">')
for caption, value in ( for caption, value in (
('apdex threshold', '%.2fs' % args.apdex), ('apdex threshold', f'{args.apdex:.2f}s'),
('period', args.period or (period + ' (auto)')), ('period', args.period or (period + ' (auto)')),
('timezone', args.to_timezone or "(input's)") ('timezone', args.to_timezone or "(input's)")
): ):
out.write('<tr><th class="text">%s</th><td>%s</td></tr>' % ( out.write(f'<tr><th class="text">{caption}</th><td>{value}</td></tr>')
caption, value)) out.write(f'</table><h2>Hits per {period}</h2><table class="stats">'
out.write('</table><h2>Hits per %s</h2><table class="stats">' '<tr><th>date</th><th>hits</th></tr>')
'<tr><th>date</th><th>hits</th></tr>' % period)
hit_per_day = defaultdict(int) hit_per_day = defaultdict(int)
x_min = LARGER_THAN_INTEGER_STR x_min = LARGER_THAN_INTEGER_STR
x_max = SMALLER_THAN_INTEGER_STR x_max = SMALLER_THAN_INTEGER_STR
...@@ -1127,12 +1145,11 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict, ...@@ -1127,12 +1145,11 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
x_min = None x_min = None
x_max = None x_max = None
for hit_date, hit in sorted(hit_per_day.items(), key=ITEMGETTER0): for hit_date, hit in sorted(hit_per_day.items(), key=ITEMGETTER0):
out.write('<tr><td>%s</td><td>%s</td></tr>' % (hit_date, hit)) out.write(f'<tr><td>{hit_date}</td><td>{hit}</td></tr>')
out.write('</table>') out.write('</table>')
n_hottest_pages = args.n_hottest_pages n_hottest_pages = args.n_hottest_pages
for i, (site_id, data) in site_list: for i, (site_id, data) in site_list:
out.write('<h1 id="%s" title="%s">%s</h1>' % (i, escape(repr(site_id), out.write(f'<h1 id="{i}" title="{escape(repr(site_id), quote=True)}">{html_site_caption_dict[site_id]}</h1>')
quote=True), html_site_caption_dict[site_id]))
apdex_data = data.getApdexData() apdex_data = data.getApdexData()
if apdex_data: if apdex_data:
out.write( out.write(
...@@ -1171,12 +1188,7 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict, ...@@ -1171,12 +1188,7 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
all_lines = stats['all_lines'] all_lines = stats['all_lines']
for caption, value in ( for caption, value in (
('Execution date', datetime.now().isoformat()), ('Execution date', datetime.now().isoformat()),
('Interpreter', '%s %s build %s (%s)' % ( ('Interpreter', f'{platform.python_implementation()} {platform.python_version()} build {buildno} ({builddate})'),
platform.python_implementation(),
platform.python_version(),
buildno,
builddate,
)),
('State file count', stats['state_file_count']), ('State file count', stats['state_file_count']),
('State loading time', timedelta(seconds=stats['parsing_start_time'] ('State loading time', timedelta(seconds=stats['parsing_start_time']
- stats['loading_start_time'])), - stats['loading_start_time'])),
...@@ -1187,16 +1199,15 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict, ...@@ -1187,16 +1199,15 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
('... skipped (URL)', stats['skipped_lines']), ('... skipped (URL)', stats['skipped_lines']),
('... skipped (user agent)', stats['skipped_user_agent']), ('... skipped (user agent)', stats['skipped_user_agent']),
('Parsing time', timedelta(seconds=parsing_time)), ('Parsing time', timedelta(seconds=parsing_time)),
('Parsing rate', '%i line/s' % (all_lines / parsing_time)), ('Parsing rate', f'{all_lines // parsing_time} line/s'),
('Rendering time', timedelta(seconds=( ('Rendering time', timedelta(seconds=(
end_stat_time - end_parsing_time))), end_stat_time - end_parsing_time))),
): ):
out.write('<tr><th class="text">%s</th><td>%s</td></tr>' % ( out.write('<tr><th class="text">{caption}</th><td>{value}</td></tr>')
caption, value))
out.write('</table>') out.write('</table>')
out.write('</body></html>') out.write('</body></html>')
def asJSON(out, encoding, per_site, *_): def asJSON(out, encoding, per_site, *_): # pylint: disable=unused-argument
json.dump([(x, y.asJSONState()) for x, y in per_site.items()], out) json.dump([(x, y.asJSONState()) for x, y in per_site.items()], out)
format_generator = { format_generator = {
...@@ -1349,16 +1360,19 @@ def main(): ...@@ -1349,16 +1360,19 @@ def main():
parser.error('Either --state-file or logfile arguments ' parser.error('Either --state-file or logfile arguments '
'must be specified.') 'must be specified.')
if DURATION_US_FORMAT in args.logformat: if DURATION_US_FORMAT in args.logformat:
getDuration = lambda x: int(x.group('duration')) def getDuration(x):
return int(x.group('duration'))
elif DURATION_MS_FORMAT in args.logformat: elif DURATION_MS_FORMAT in args.logformat:
getDuration = lambda x: int(x.group('duration_ms')) * US_PER_MS def getDuration(x):
return int(x.group('duration_ms')) * US_PER_MS
elif DURATION_S_FORMAT in args.logformat: elif DURATION_S_FORMAT in args.logformat:
getDuration = lambda x: int(x.group('duration_s')) * US_PER_S def getDuration(x):
return int(x.group('duration_s')) * US_PER_S
else: else:
parser.error('Neither %D nor %T are present in logformat, apdex ' parser.error('Neither %D nor %T are present in logformat, apdex '
'cannot be computed.') 'cannot be computed.')
if args.duration_cap: if args.duration_cap:
def getDuration( def getDuration( # pylint: disable=function-redefined
match, match,
_duration_cap=int(args.duration_cap * US_PER_S), _duration_cap=int(args.duration_cap * US_PER_S),
_getDuration=getDuration, _getDuration=getDuration,
...@@ -1369,8 +1383,8 @@ def main(): ...@@ -1369,8 +1383,8 @@ def main():
return duration return duration
if args.match_servername is not None and \ if args.match_servername is not None and \
args.match_servername not in args.logformat: args.match_servername not in args.logformat:
parser.error('--match-servername %s requested, but missing ' parser.error(f'--match-servername {args.match_servername} requested, but missing '
'from logformat.' % args.match_servername) 'from logformat.')
get_url_prefix = server_name_group_dict.get(args.match_servername, get_url_prefix = server_name_group_dict.get(args.match_servername,
lambda _, path: path) lambda _, path: path)
line_regex = '' line_regex = ''
...@@ -1421,7 +1435,7 @@ def main(): ...@@ -1421,7 +1435,7 @@ def main():
dt, tz = match.group('timestamp').split() dt, tz = match.group('timestamp').split()
day, month, rest = dt.split('/', 2) day, month, rest = dt.split('/', 2)
return datetime.strptime( return datetime.strptime(
'%s/%02i/%s' % (day, MONTH_VALUE_DICT[month], rest), f'{day}/{MONTH_VALUE_DICT[month]:02}/{rest}',
'%d/%m/%Y:%H:%M:%S').replace(tzinfo=getTZInfo(tz)) '%d/%m/%Y:%H:%M:%S').replace(tzinfo=getTZInfo(tz))
if args.to_timezone: if args.to_timezone:
to_timezone = args.to_timezone to_timezone = args.to_timezone
...@@ -1432,7 +1446,8 @@ def main(): ...@@ -1432,7 +1446,8 @@ def main():
raise ValueError('pytz is not available, cannot convert timezone.') raise ValueError('pytz is not available, cannot convert timezone.')
getTimezoneInfo = pytz.timezone getTimezoneInfo = pytz.timezone
tz_info = getTimezoneInfo(to_timezone) tz_info = getTimezoneInfo(to_timezone)
matchToDateTime = lambda x: _matchToDateTime(x).astimezone(tz_info) def matchToDateTime(x):
return _matchToDateTime(x).astimezone(tz_info)
else: else:
matchToDateTime = _matchToDateTime matchToDateTime = _matchToDateTime
asDate, decimator, graph_period, date_format, placeholder_delta, \ asDate, decimator, graph_period, date_format, placeholder_delta, \
...@@ -1459,7 +1474,7 @@ def main(): ...@@ -1459,7 +1474,7 @@ def main():
parser.error('stdin cannot be used both as log and state input.') parser.error('stdin cannot be used both as log and state input.')
loading_start_time = time.time() loading_start_time = time.time()
for state_file_name in args.state_file: for state_file_name in args.state_file:
print('Loading %s...' % state_file_name, end='', file=sys.stderr) print(f'Loading {state_file_name}...', end='', file=sys.stderr)
if state_file_name == '-': if state_file_name == '-':
state_file = sys.stdin state_file = sys.stdin
else: else:
...@@ -1479,7 +1494,7 @@ def main(): ...@@ -1479,7 +1494,7 @@ def main():
site = None site = None
action = default_action action = default_action
if action is None: if action is None:
print('Info: no prefix match %r, stats skipped' % url, print(f'Info: no prefix match {url}, stats skipped',
file='sys.stderr') file='sys.stderr')
continue continue
site_stats = action.func.fromJSONState(site_state, site_stats = action.func.fromJSONState(site_state,
...@@ -1488,7 +1503,7 @@ def main(): ...@@ -1488,7 +1503,7 @@ def main():
per_site[site].accumulateFrom(site_stats) per_site[site].accumulateFrom(site_stats)
else: else:
per_site[site] = site_stats per_site[site] = site_stats
print('done (%s)' % timedelta(seconds=time.time() - load_start), print(f'done ({timedelta(seconds=time.time() - load_start)})',
file=sys.stderr) file=sys.stderr)
skip_user_agent = [re.compile(x).match skip_user_agent = [re.compile(x).match
for x in itertools.chain(*args.skip_user_agent)] for x in itertools.chain(*args.skip_user_agent)]
...@@ -1501,7 +1516,7 @@ def main(): ...@@ -1501,7 +1516,7 @@ def main():
parsing_start_time = time.time() parsing_start_time = time.time()
for fileno, filename in enumerate(infile_list, 1): for fileno, filename in enumerate(infile_list, 1):
if show_progress: if show_progress:
print('Processing %s [%i/%i]' % (filename, fileno, file_count), print(f'Processing {filename} [{fileno}/{file_count}]',
file=sys.stderr) file=sys.stderr)
if filename == '-': if filename == '-':
logfile = sys.stdin logfile = sys.stdin
...@@ -1526,7 +1541,7 @@ def main(): ...@@ -1526,7 +1541,7 @@ def main():
match = expensive_matchline(line) match = expensive_matchline(line)
if match is None: if match is None:
if not quiet: if not quiet:
print('Malformed line at %s:%i: %r' % (filename, lineno, line), print(f'Malformed line at {filename}:{lineno}: {line}',
file=sys.stderr) file=sys.stderr)
malformed_lines += 1 malformed_lines += 1
continue continue
...@@ -1567,19 +1582,26 @@ def main(): ...@@ -1567,19 +1582,26 @@ def main():
if original_period != period: if original_period != period:
original_period = period original_period = period
if show_progress: if show_progress:
print('Increasing period to %s...' % period, end='', print(f'Increasing period to {period}...', end='',
file=sys.stderr) file=sys.stderr)
old_date_format = date_format old_date_format = date_format
asDate, decimator, graph_period, date_format, placeholder_delta, \ (
round_date, graph_coefficient = period_parser[period] asDate,
decimator,
graph_period,
date_format,
placeholder_delta,
round_date,
graph_coefficient,
) = period_parser[period]
latest_date = rescale(latest_date) latest_date = rescale(latest_date)
earliest_date = rescale(earliest_date) earliest_date = rescale(earliest_date)
period_increase_start = time.time() period_increase_start = time.time()
for site_data in per_site.values(): for site_data in per_site.values():
site_data.rescale(rescale, getDuration) site_data.rescale(rescale, getDuration)
if show_progress: if show_progress:
print('done (%s)' % timedelta(seconds=time.time() print(f'done ({timedelta(seconds=time.time() - period_increase_start)})',
- period_increase_start), file=sys.stderr) file=sys.stderr)
hit_date = asDate(matchToDateTime(match)) hit_date = asDate(matchToDateTime(match))
try: try:
site_data = per_site[site] site_data = per_site[site]
...@@ -1589,9 +1611,9 @@ def main(): ...@@ -1589,9 +1611,9 @@ def main():
erp5_expand_other=erp5_expand_other) erp5_expand_other=erp5_expand_other)
try: try:
site_data.accumulate(match, url_match, hit_date) site_data.accumulate(match, url_match, hit_date)
except Exception: except Exception: # pylint: disable=broad-exception-caught
if not quiet: if not quiet:
print('Error analysing line at %s:%i: %r' % (filename, lineno, line), print(f'Error analysing line at {filename}:{lineno}: {line!r}',
file=sys.stderr) file=sys.stderr)
traceback.print_exc(file=sys.stderr) traceback.print_exc(file=sys.stderr)
all_lines += lineno all_lines += lineno
...@@ -1637,7 +1659,6 @@ if __name__ == '__main__': ...@@ -1637,7 +1659,6 @@ if __name__ == '__main__':
return f.read() return f.read()
main() main()
else:
from ._version import get_versions def getResource(name, encoding='utf-8'):
__version__ = get_versions()['version'] return pkgutil.get_data(__name__, name).decode(encoding)
del get_versions
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment