Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
A
apachedex
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Vincent Pelletier
apachedex
Commits
466146fb
Commit
466146fb
authored
Dec 26, 2023
by
Vincent Pelletier
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Make pylint somewhat happy
parent
3efd68a4
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
149 additions
and
128 deletions
+149
-128
.pylintrc
.pylintrc
+3
-3
apachedex/__init__.py
apachedex/__init__.py
+146
-125
No files found.
.pylintrc
View file @
466146fb
...
...
@@ -52,7 +52,7 @@ ignore=CVS
# ignore-list. The regex matches against paths and can be in Posix or Windows
# format. Because '\\' represents the directory delimiter on Windows systems,
# it can't be used as an escape character.
ignore-paths=
ignore-paths=
apachedex/_version.py
# Files or directories matching the regular expression patterns are skipped.
# The regex matches against base names, not paths. The default value ignores
...
...
@@ -177,7 +177,7 @@ const-naming-style=UPPER_CASE
docstring-min-length=-1
# Naming style matching correct function names.
function-naming-style=
snake_c
ase
function-naming-style=
camelC
ase
# Regular expression matching correct function names. Overrides function-
# naming-style. If left empty, function names will be checked with the set
...
...
@@ -334,7 +334,7 @@ indent-after-paren=4
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
# tab).
indent-string='
'
indent-string=' '
# Maximum number of characters on a single line.
max-line-length=100
...
...
apachedex/__init__.py
View file @
466146fb
...
...
@@ -26,6 +26,19 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
# TODO: resolve these
# pylint: disable=line-too-long
# pylint: disable=too-many-lines
# pylint: disable=too-many-locals
# pylint: disable=too-many-arguments
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
# pylint: disable=too-many-instance-attributes
# pylint: disable=missing-function-docstring
# pylint: disable=missing-class-docstring
# pylint: disable=missing-module-docstring
# pylint: disable=invalid-name
from
html
import
escape
from
collections
import
defaultdict
,
Counter
...
...
@@ -53,14 +66,12 @@ import sys
import
time
import
traceback
try
:
import
pytz
import
pytz
except
ImportError
:
pytz
=
None
def
getResource
(
name
,
encoding
=
'utf-8'
):
return
pkgutil
.
get_data
(
__name__
,
name
).
decode
(
encoding
)
pytz
=
None
from
._version
import
get_versions
__version__
=
get_versions
()[
'version'
]
del
get_versions
gzip_open
=
gzip
.
open
lzma_open
=
lzma
.
open
...
...
@@ -120,7 +131,7 @@ def getClassForStatusHit(hit, status):
return
'problem'
return
''
def
getDataPoints
(
apdex_dict
,
status_period_dict
=
{}):
def
getDataPoints
(
apdex_dict
,
status_period_dict
=
{}):
# pylint: disable=dangerous-default-value
period_error_dict
=
defaultdict
(
int
)
for
status
,
period_dict
in
status_period_dict
.
items
():
if
statusIsError
(
status
):
...
...
@@ -165,10 +176,10 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
for
x
in
daily_data
]
timeformat
=
'%Y/<br/>%m/%d<br/> %H:%M'
# There is room for about 10 labels on the X axis.
min
TickS
ize
=
(
max
(
1
,
min
_tick_s
ize
=
(
max
(
1
,
(
date_list
[
-
1
]
-
date_list
[
0
])
/
(
60
*
60
*
1000
*
10
)),
'hour'
)
# Guesstimation: 6px per digit. If only em were allowed...
y
LabelW
idth
=
max
(
int
(
math
.
log10
(
max
(
x
[
2
]
for
x
in
daily_data
)))
+
1
,
y
_label_w
idth
=
max
(
int
(
math
.
log10
(
max
(
x
[
2
]
for
x
in
daily_data
)))
+
1
,
3
)
*
6
return
graph
(
'apdex'
,
[
list
(
zip
(
date_list
,
(
round
(
x
[
1
],
2
)
for
x
in
daily_data
)))],
...
...
@@ -176,13 +187,13 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
'xaxis'
:
{
'mode'
:
'time'
,
'timeformat'
:
timeformat
,
'minTickSize'
:
min
TickS
ize
,
'minTickSize'
:
min
_tick_s
ize
,
},
'yaxis'
:
{
'min'
:
apdex_y_min
,
'max'
:
100
,
'axisLabel'
:
'apdex (%)'
,
'labelWidth'
:
y
LabelW
idth
,
'labelWidth'
:
y
_label_w
idth
,
'transform'
:
apdex_y_scale
,
},
'lines'
:
{
'show'
:
True
},
...
...
@@ -190,7 +201,7 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
'hoverable'
:
True
,
},
},
)
+
graph
(
'Hits (per %s)'
%
graph_period
,
)
+
graph
(
f'Hits (per
{
graph_period
}
)'
,
[
{
'label'
:
'Errors'
,
...
...
@@ -206,13 +217,13 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
'xaxis'
:
{
'mode'
:
'time'
,
'timeformat'
:
timeformat
,
'minTickSize'
:
min
TickS
ize
,
'minTickSize'
:
min
_tick_s
ize
,
},
'yaxis'
:
{
'min'
:
hit_y_min
,
'max'
:
hit_y_max
,
'axisLabel'
:
'Hits'
,
'labelWidth'
:
y
LabelW
idth
,
'labelWidth'
:
y
_label_w
idth
,
'tickDecimals'
:
0
,
'transform'
:
hit_y_scale
,
},
...
...
@@ -226,11 +237,11 @@ def graphPair(daily_data, date_format, graph_period, apdex_y_min=None,
},
)
def
graph
(
title
,
data
,
options
=
{}):
def
graph
(
title
,
data
,
options
=
{}):
# pylint: disable=dangerous-default-value
result
=
[]
append
=
result
.
append
append
(
'<h2>%s
</h2><div class="graph" '
'style="width:600px;height:300px" data-points="'
%
title
)
append
(
f'<h2>
{
title
}
</h2><div class="graph" '
'style="width:600px;height:300px" data-points="'
)
append
(
escape
(
json
.
dumps
(
data
),
quote
=
True
))
append
(
'" data-options="'
)
append
(
escape
(
json
.
dumps
(
options
),
quote
=
True
))
...
...
@@ -239,7 +250,7 @@ def graph(title, data, options={}):
'<span class="y"></span></div>'
)
return
''
.
join
(
result
)
class
APDEXStats
(
object
)
:
class
APDEXStats
:
def
__init__
(
self
,
threshold
,
getDuration
):
threshold
*=
US_PER_S
self
.
threshold
=
threshold
...
...
@@ -283,8 +294,10 @@ class APDEXStats(object):
@
staticmethod
def
asHTMLHeader
(
overall
=
False
):
return
'<th>apdex</th><th>hits</th><th>avg (s)</th>'
\
'<th%s>max (s)</th>'
%
(
overall
and
' class="overall_right"'
or
''
)
return
(
'<th>apdex</th><th>hits</th><th>avg (s)</th>'
'<th'
+
(
' class="overall_right"'
if
overall
else
''
)
+
'>max (s)</th>'
)
def
asHTML
(
self
,
threshold
,
overall
=
False
):
apdex
=
self
.
getApdex
()
...
...
@@ -293,9 +306,9 @@ class APDEXStats(object):
hit
=
self
.
hit
if
hit
:
extra_class
=
''
apdex_style
=
'color: #%s; background-color: #%s'
%
(
(
apdex
<
.
5
and
'f'
or
'0'
)
*
3
,
(
'%x'
%
int
(
apdex
*
0xf
))
*
3
,
apdex_style
=
(
'color: #'
+
((
'f'
if
apdex
<
.
5
else
'0'
)
*
3
)
+
';'
'background-color: #'
+
(
f'
{
int
(
apdex
*
0xf
):
x
}
'
*
3
)
)
else
:
extra_class
=
'no_hit'
...
...
@@ -331,14 +344,17 @@ class APDEXStats(object):
del
result
[
'getDuration'
]
return
result
_APDEXDateDictAsJSONState
=
lambda
date_dict
:
dict
(((
y
,
z
.
asJSONState
())
for
y
,
z
in
date_dict
.
items
()))
def
_APDEXDateDictAsJSONState
(
date_dict
):
return
{
y
:
z
.
asJSONState
()
for
y
,
z
in
date_dict
.
items
()
}
class
GenericSiteStats
(
object
)
:
class
GenericSiteStats
:
def
__init__
(
self
,
threshold
,
getDuration
,
suffix
,
error_detail
=
False
,
user_agent_detail
=
False
,
# Non-generic parameters
**
kw
):
**
_
):
self
.
threshold
=
threshold
self
.
suffix
=
suffix
self
.
error_detail
=
error_detail
...
...
@@ -387,7 +403,7 @@ class GenericSiteStats(object):
apdex_y_min
=
None
,
hit_y_min
=
None
,
hit_y_max
=
None
,
apdex_y_scale
=
None
,
hit_y_scale
=
None
,
n_hottest_pages
=
N_HOTTEST_PAGES_DEFAULT
,
):
):
# pylint: disable=unused-argument
result
=
[]
append
=
result
.
append
apdex
=
APDEXStats
(
self
.
threshold
,
None
)
...
...
@@ -404,13 +420,13 @@ class GenericSiteStats(object):
reverse
=
True
)[:
n_hottest_pages
]:
append
(
'<tr>'
)
append
(
data
.
asHTML
(
self
.
threshold
))
append
(
'<td class="text">%s</td></tr>'
%
escape
(
url
)
)
append
(
f'<td class="text">
{
escape
(
url
)
}
</td></tr>'
)
append
(
'</table>'
)
if
self
.
user_agent_detail
:
append
(
'<h2>User agents</h2><table class="stats"><tr><th>hits</th>'
'<th>user agent</th></tr>'
)
for
user_agent
,
hit
in
self
.
user_agent_counter
.
most_common
(
N_USER_AGENT
):
append
(
'<tr><td>%s</td><td class="text">%s</td></tr>'
%
(
hit
,
escape
(
user_agent
))
)
append
(
f'<tr><td>
{
hit
}
</td><td class="text">
{
escape
(
user_agent
)
}
</td></tr>'
)
append
(
'</table>'
)
column_set
=
set
()
filtered_status
=
defaultdict
(
partial
(
defaultdict
,
int
))
...
...
@@ -423,21 +439,20 @@ class GenericSiteStats(object):
append
(
'<h2>Hits per status code</h2><table class="stats"><tr>'
'<th>status</th><th>overall</th>'
)
for
column
in
column_list
:
append
(
'<th>%s</th>'
%
column
)
append
(
f'<th>
{
column
}
</th>'
)
append
(
'</tr>'
)
def
hitTd
(
hit
,
status
):
return
'<td class="%s">%s</td>'
%
(
getClassForStatusHit
(
hit
,
status
),
hit
)
return
f'<td class="
{
getClassForStatusHit
(
hit
,
status
)
}
">
{
hit
}
</td>'
def
statusAsHtml
(
status
):
try
:
definition
=
HTTP_STATUS_CAPTION_DICT
[
int
(
status
)]
except
KeyError
:
return
status
else
:
return
'<abbr title="%s">%s</abbr>'
%
(
definition
,
status
)
return
f'<abbr title="
{
definition
}
">
{
status
}
</abbr>'
has_errors
=
False
for
status
,
data_dict
in
sorted
(
filtered_status
.
items
(),
key
=
ITEMGETTER0
):
has_errors
|=
statusIsError
(
status
)
append
(
'<tr title="%s"><th>%s</th>'
%
(
status
,
statusAsHtml
(
status
))
)
append
(
f'<tr title="
{
status
}
"><th>
{
statusAsHtml
(
status
)
}
</th>'
)
append
(
hitTd
(
sum
(
data_dict
.
values
()),
status
))
for
column
in
column_list
:
append
(
hitTd
(
data_dict
[
column
],
status
))
...
...
@@ -454,22 +469,20 @@ class GenericSiteStats(object):
'<th>hits</th><th>url</th><th>referers</th></tr>'
)
for
status
,
url_list
in
sorted
(
filtered_status_url
.
items
(),
key
=
ITEMGETTER0
):
append
(
'<tr><th rowspan="%s">%s</th>'
%
(
len
(
url_list
),
statusAsHtml
(
status
)))
append
(
f'<tr><th rowspan="
{
len
(
url_list
)
}
">
{
statusAsHtml
(
status
)
}
</th>'
)
first_url
=
True
for
url
,
referer_counter
in
url_list
:
if
first_url
:
first_url
=
False
else
:
append
(
'<tr>'
)
append
(
'<td>%s</td><td class="text">%s</td>'
'<td class="text">%s</td>'
%
(
getHitForUrl
(
referer_counter
),
escape
(
url
),
'<br/>'
.
join
(
'%i: %s'
%
(
hit
,
escape
(
referer
))
for
referer
,
hit
in
referer_counter
.
most_common
(
N_REFERRER_PER_ERROR_URL
)),
))
append
(
f'<td>
{
getHitForUrl
(
referer_counter
)
}
</td><td class="text">
{
escape
(
url
)
}
</td>'
'<td class="text">'
+
'<br/>'
.
join
(
f'
{
hit
}
:
{
escape
(
referer
)
}
'
for
referer
,
hit
in
referer_counter
.
most_common
(
N_REFERRER_PER_ERROR_URL
)
)
+
'</td>'
)
append
(
'</tr>'
)
append
(
'</table>'
)
return
'
\
n
'
.
join
(
result
)
...
...
@@ -538,7 +551,7 @@ class ERP5SiteStats(GenericSiteStats):
"""
def
__init__
(
self
,
threshold
,
getDuration
,
suffix
,
error_detail
=
False
,
user_agent_detail
=
False
,
erp5_expand_other
=
False
):
super
(
ERP5SiteStats
,
self
).
__init__
(
threshold
,
getDuration
,
suffix
,
super
().
__init__
(
threshold
,
getDuration
,
suffix
,
error_detail
=
error_detail
,
user_agent_detail
=
user_agent_detail
)
self
.
expand_other
=
erp5_expand_other
...
...
@@ -560,7 +573,7 @@ class ERP5SiteStats(GenericSiteStats):
self
.
site_search
=
defaultdict
(
partial
(
APDEXStats
,
threshold
,
getDuration
))
def
rescale
(
self
,
convert
,
getDuration
):
super
(
ERP5SiteStats
,
self
).
rescale
(
convert
,
getDuration
)
super
().
rescale
(
convert
,
getDuration
)
threshold
=
self
.
threshold
for
document_dict
in
self
.
module
.
values
():
for
is_document
,
date_dict
in
document_dict
.
items
():
...
...
@@ -583,13 +596,13 @@ class ERP5SiteStats(GenericSiteStats):
def
accumulate
(
self
,
match
,
url_match
,
value_date
):
split
=
self
.
suffix
(
url_match
.
group
(
'url'
)).
split
(
'?'
,
1
)[
0
].
split
(
'/'
)
if
split
and
split
[
0
].
endswith
(
'_module'
):
super
(
ERP5SiteStats
,
self
).
accumulate
(
match
,
url_match
,
value_date
)
super
().
accumulate
(
match
,
url_match
,
value_date
)
module
=
split
[
0
]
self
.
module
[
module
][
len
(
split
)
>
1
and
(
split
[
1
]
!=
'view'
and
'_view'
not
in
split
[
1
])
][
value_date
].
accumulate
(
match
)
elif
split
and
split
[
0
]
==
'ERP5Site_viewSearchResult'
:
super
(
ERP5SiteStats
,
self
).
accumulate
(
match
,
url_match
,
value_date
)
super
().
accumulate
(
match
,
url_match
,
value_date
)
self
.
site_search
[
value_date
].
accumulate
(
match
)
elif
split
and
self
.
expand_other
:
self
.
no_module
[
split
[
0
]][
value_date
].
accumulate
(
match
)
...
...
@@ -637,7 +650,7 @@ class ERP5SiteStats(GenericSiteStats):
column_set
.
update
(
filtered_data_dict
)
column_list
=
sorted
(
column_set
)
for
column
in
column_list
:
append
(
'<th colspan="4">%s</th>'
%
column
)
append
(
f'<th colspan="4">
{
column
}
</th>'
)
append
(
'</tr><tr>'
)
for
i
in
range
(
len
(
column_list
)
+
1
):
append
(
APDEXStats
.
asHTMLHeader
(
i
==
0
))
...
...
@@ -656,9 +669,8 @@ class ERP5SiteStats(GenericSiteStats):
if
len
(
data
)
>
1
:
append
(
'<span class="action" onclick="toggleGraph(this)">+</span>'
'<div class="positioner"><div class="container">'
'<div class="title">%s</div>'
'<div class="action close" onclick="hideGraph(this)">close</div>'
%
title
f'<div class="title">
{
title
}
</div>'
'<div class="action close" onclick="hideGraph(this)">close</div>'
)
append
(
graphPair
(
prepareDataForGraph
(
...
...
@@ -680,11 +692,11 @@ class ERP5SiteStats(GenericSiteStats):
append
(
'</div></div>'
)
append
(
'</td>'
)
for
module_id
,
data_dict
in
sorted
(
filtered_module
.
items
(),
key
=
ITEMGETTER0
):
append
(
'<tr class="group_top" title="%s (module)"><th rowspan="2">%s
</th>'
'<th>module</th>'
%
(
module_id
,
module_id
)
)
append
(
f'<tr class="group_top" title="
{
module_id
}
(module)"><th rowspan="2">
{
module_id
}
</th>'
'<th>module</th>'
)
hiddenGraph
(
self
.
module
[
module_id
][
False
],
module_id
+
' (module)'
)
apdexAsColumns
(
data_dict
[
False
])
append
(
'</tr><tr class="group_bottom" title="%s (document)"><th>document</th>'
%
module_id
)
append
(
f'</tr><tr class="group_bottom" title="
{
module_id
}
(document)"><th>document</th>'
)
hiddenGraph
(
self
.
module
[
module_id
][
True
],
module_id
+
' (document)'
)
apdexAsColumns
(
data_dict
[
True
])
append
(
'</tr>'
)
...
...
@@ -694,8 +706,7 @@ class ERP5SiteStats(GenericSiteStats):
site_search_overall
=
apdexAsColumns
(
filtered_site_search
)
append
(
'</tr>'
)
for
id_
,
date_dict
in
sorted
(
filtered_no_module
.
items
()):
append
(
'<tr class="group_top group_bottom" title="%s"><th colspan="2">%s</th>'
%
(
id_
,
id_
))
append
(
'<tr class="group_top group_bottom" title="{id_}"><th colspan="2">{id_}</th>'
)
hiddenGraph
(
self
.
no_module
[
id_
],
id_
)
apdexAsColumns
(
date_dict
)
append
(
'</tr>'
)
...
...
@@ -711,7 +722,7 @@ class ERP5SiteStats(GenericSiteStats):
append
(
'</tr><tr><th>document</th>'
)
append
(
module_document_overall
[
True
].
asHTML
(
self
.
threshold
))
append
(
'</tr></table>'
)
append
(
super
(
ERP5SiteStats
,
self
).
asHTML
(
date_format
,
append
(
super
().
asHTML
(
date_format
,
placeholder_delta
,
graph_period
,
graph_coefficient
,
encoding
,
stat_filter
=
stat_filter
,
x_min
=
x_min
,
x_max
=
x_max
,
...
...
@@ -724,7 +735,7 @@ class ERP5SiteStats(GenericSiteStats):
@
classmethod
def
fromJSONState
(
cls
,
state
,
getDuration
,
suffix
):
result
=
super
(
ERP5SiteStats
,
cls
).
fromJSONState
(
state
,
getDuration
,
suffix
)
result
=
super
().
fromJSONState
(
state
,
getDuration
,
suffix
)
for
module_id
,
module_dict_state
in
state
[
'module'
].
items
():
module_dict
=
result
.
module
[
module_id
]
for
is_document
,
date_dict_state
in
module_dict_state
.
items
():
...
...
@@ -745,7 +756,7 @@ class ERP5SiteStats(GenericSiteStats):
return
result
def
asJSONState
(
self
):
result
=
super
(
ERP5SiteStats
,
self
).
asJSONState
()
result
=
super
().
asJSONState
()
result
[
'module'
]
=
module
=
{}
for
module_id
,
module_dict
in
self
.
module
.
items
():
module_dict_state
=
module
[
module_id
]
=
{}
...
...
@@ -760,7 +771,7 @@ class ERP5SiteStats(GenericSiteStats):
return
result
def
accumulateFrom
(
self
,
other
):
super
(
ERP5SiteStats
,
self
).
accumulateFrom
(
other
)
super
().
accumulateFrom
(
other
)
module
=
self
.
module
for
module_id
,
other_module_dict
in
other
.
module
.
items
():
module_dict
=
module
[
module_id
]
...
...
@@ -835,13 +846,13 @@ class AggregateSiteUrl(argparse.Action):
except
StopIteration
:
break
if
value
in
site_caption_dict
:
raise
ValueError
(
'Duplicate base: %r'
%
value
)
raise
ValueError
(
f'Duplicate base:
{
value
}
'
)
if
action
is
not
None
and
value
[
0
]
==
'+'
:
caption
=
value
[
1
:]
try
:
value
=
next_value
()
except
StopIteration
:
raise
ValueError
(
'No base follows caption %r'
%
value
)
raise
ValueError
(
f'No base follows caption
{
value
}
'
)
from
None
else
:
caption
=
value
site_caption_dict
[
value
]
=
caption
...
...
@@ -878,8 +889,10 @@ class ShlexArgumentParser(argparse.ArgumentParser):
os
.
path
.
dirname
(
filepath
),
))
try
:
with
open
(
os
.
path
.
join
(
new_cwd
,
os
.
path
.
basename
(
filepath
))
)
as
in_file
:
with
open
(
os
.
path
.
join
(
new_cwd
,
os
.
path
.
basename
(
filepath
)),
encoding
=
'utf-8'
,
)
as
in_file
:
extend
(
self
.
__read_args_from_files
(
shlex
.
split
(
in_file
.
read
(),
comments
=
True
),
new_cwd
,
...
...
@@ -896,7 +909,7 @@ class ShlexArgumentParser(argparse.ArgumentParser):
else
:
args
=
list
(
args
)
args
=
self
.
__read_args_from_files
(
args
,
os
.
getcwd
())
return
super
(
ShlexArgumentParser
,
self
).
parse_known_args
(
args
=
args
,
return
super
().
parse_known_args
(
args
=
args
,
namespace
=
namespace
)
_month_offset_cache
=
{}
...
...
@@ -919,11 +932,11 @@ def _asWeekString(dt):
month
-=
1
day
+=
calendar
.
monthrange
(
year
,
month
)[
1
]
assert
day
>
0
and
month
>
0
,
(
dt
,
year
,
month
,
day
)
return
'%04i/%02i/%02i'
%
(
year
,
month
,
day
)
return
f'
{
year
:
04
}
/
{
month
:
02
}
/
{
day
:
02
}
'
def
_weekStringAsQuarterString
(
timestamp
):
year
,
month
,
_
=
timestamp
.
split
(
'/'
)
return
'%s/%02i'
%
(
year
,
(
int
(
month
)
-
1
)
//
3
*
3
+
1
)
return
f'
{
year
}
/
{
(
int
(
month
)
-
1
)
//
3
*
3
+
1
:
02
}
'
def
_roundWeek
(
dt
):
day_of_year
=
dt
.
timetuple
().
tm_yday
...
...
@@ -946,11 +959,11 @@ def _hourAsWeekString(timestamp):
def
_asHalfDayString
(
timestamp
):
prefix
,
_
=
timestamp
.
rsplit
(
':'
,
1
)
prefix
,
hours
=
prefix
.
split
(
' '
)
return
'%s %02i'
%
(
prefix
,
int
(
hours
)
//
12
*
12
)
return
f'
{
prefix
}
{
int
(
hours
)
//
12
*
12
:
02
}
'
def
_asQuarterHourString
(
timestamp
):
prefix
,
minute
=
timestamp
.
rsplit
(
':'
,
1
)
return
'%s:%02i'
%
(
prefix
,
int
(
minute
)
//
15
*
15
)
return
f'
{
prefix
}
:
{
int
(
minute
)
//
15
*
15
:
02
}
'
# Key: argument (represents table granularity)
# Value:
...
...
@@ -1004,7 +1017,7 @@ period_parser = {
lambda
x
:
1
,
),
'week'
:
(
lambda
x
:
x
.
strftime
(
'%Y/%m/%d '
)
+
'%02i'
%
(
x
.
hour
//
6
*
6
)
,
lambda
x
:
x
.
strftime
(
'%Y/%m/%d '
)
+
f'
{
x
.
hour
//
6
*
6
:
02
}
'
,
_hourAsWeekString
,
'6 hours'
,
'%Y/%m/%d %H'
,
...
...
@@ -1025,7 +1038,7 @@ period_parser = {
lambda
x
:
1
,
),
'halfday'
:
(
lambda
x
:
x
.
strftime
(
'%Y/%m/%d %H:'
)
+
'%02i'
%
(
x
.
minute
//
30
*
30
)
,
lambda
x
:
x
.
strftime
(
'%Y/%m/%d %H:'
)
+
f'
{
x
.
minute
//
30
*
30
:
02
}
'
,
_asHalfDayString
,
'30 minutes'
,
'%Y/%m/%d %H:%M'
,
...
...
@@ -1054,8 +1067,16 @@ hit_y_scale_dict = {
'log'
:
'log0ToAny'
,
}
def
asHTML
(
out
,
encoding
,
per_site
,
args
,
default_site
,
period_parameter_dict
,
stats
,
site_caption_dict
):
def
asHTML
(
out
,
encoding
,
per_site
,
args
,
default_site
,
period_parameter_dict
,
stats
,
site_caption_dict
,
):
# pylint: disable=unused-argument
period
=
period_parameter_dict
[
'period'
]
decimator
=
period_parameter_dict
[
'decimator'
]
date_format
=
period_parameter_dict
[
'date_format'
]
...
...
@@ -1069,8 +1090,8 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
hit_y_max
=
None
else
:
apdex_y_min
=
hit_y_min
=
None
out
.
write
(
'<!DOCTYPE html>
\
n
<html><head><meta charset="%s
">'
'<title>Stats</title><meta name="generator" content="APacheDEX" />'
%
encoding
)
out
.
write
(
f'<!DOCTYPE html>
\
n
<html><head><meta charset="
{
encoding
}
">'
'<title>Stats</title><meta name="generator" content="APacheDEX" />'
)
js_path
=
args
.
js
js_embed
=
js_path
is
None
or
args
.
js_embed
if
js_embed
:
...
...
@@ -1079,7 +1100,7 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
out
.
write
(
'</style>'
)
else
:
out
.
write
(
'<link rel="stylesheet" type="text/css" '
'href="%s/apachedex.css"/>'
%
js_path
)
f'href="
{
js_path
}
/apachedex.css"/>'
)
for
script
in
(
'jquery.js'
,
'jquery.flot.js'
,
'jquery.flot.time.js'
,
'jquery.flot.axislabels.js'
,
'jquery-ui.js'
,
'apachedex.js'
):
if
js_embed
:
...
...
@@ -1087,8 +1108,7 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
out
.
write
(
getResource
(
script
))
out
.
write
(
'
\
n
//]]></script>'
)
else
:
out
.
write
(
'<script type="text/javascript" src="%s/%s"></script>'
%
(
js_path
,
script
))
out
.
write
(
f'<script type="text/javascript" src="
{
js_path
}
/
{
script
}
"></script>'
)
apdex_y_scale
=
apdex_y_scale_dict
[
args
.
apdex_yscale
]
hit_y_scale
=
hit_y_scale_dict
[
args
.
hit_yscale
]
out
.
write
(
'</head><body><h1>Overall</h1>'
)
...
...
@@ -1100,19 +1120,17 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
if
len
(
per_site
)
>
1
:
out
.
write
(
'<h2>Index</h2><ol>'
)
for
i
,
(
site_id
,
_
)
in
site_list
:
out
.
write
(
'<li><a href="#%s" title="%s">%s</a></li>'
%
(
i
,
escape
(
repr
(
site_id
),
quote
=
True
),
html_site_caption_dict
[
site_id
]))
out
.
write
(
f'<li><a href="#
{
i
}
" title="
{
escape
(
repr
(
site_id
),
quote
=
True
)
}
">
{
html_site_caption_dict
[
site_id
]
}
</a></li>'
)
out
.
write
(
'</ol>'
)
out
.
write
(
'<h2>Parameters</h2><table class="stats">'
)
for
caption
,
value
in
(
(
'apdex threshold'
,
'%.2fs'
%
args
.
apdex
),
(
'apdex threshold'
,
f'
{
args
.
apdex
:.
2
f
}
s'
),
(
'period'
,
args
.
period
or
(
period
+
' (auto)'
)),
(
'timezone'
,
args
.
to_timezone
or
"(input's)"
)
):
out
.
write
(
'<tr><th class="text">%s</th><td>%s</td></tr>'
%
(
caption
,
value
))
out
.
write
(
'</table><h2>Hits per %s</h2><table class="stats">'
'<tr><th>date</th><th>hits</th></tr>'
%
period
)
out
.
write
(
f'<tr><th class="text">
{
caption
}
</th><td>
{
value
}
</td></tr>'
)
out
.
write
(
f'</table><h2>Hits per
{
period
}
</h2><table class="stats">'
'<tr><th>date</th><th>hits</th></tr>'
)
hit_per_day
=
defaultdict
(
int
)
x_min
=
LARGER_THAN_INTEGER_STR
x_max
=
SMALLER_THAN_INTEGER_STR
...
...
@@ -1127,12 +1145,11 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
x_min
=
None
x_max
=
None
for
hit_date
,
hit
in
sorted
(
hit_per_day
.
items
(),
key
=
ITEMGETTER0
):
out
.
write
(
'<tr><td>%s</td><td>%s</td></tr>'
%
(
hit_date
,
hit
)
)
out
.
write
(
f'<tr><td>
{
hit_date
}
</td><td>
{
hit
}
</td></tr>'
)
out
.
write
(
'</table>'
)
n_hottest_pages
=
args
.
n_hottest_pages
for
i
,
(
site_id
,
data
)
in
site_list
:
out
.
write
(
'<h1 id="%s" title="%s">%s</h1>'
%
(
i
,
escape
(
repr
(
site_id
),
quote
=
True
),
html_site_caption_dict
[
site_id
]))
out
.
write
(
f'<h1 id="
{
i
}
" title="
{
escape
(
repr
(
site_id
),
quote
=
True
)
}
">
{
html_site_caption_dict
[
site_id
]
}
</h1>'
)
apdex_data
=
data
.
getApdexData
()
if
apdex_data
:
out
.
write
(
...
...
@@ -1171,12 +1188,7 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
all_lines
=
stats
[
'all_lines'
]
for
caption
,
value
in
(
(
'Execution date'
,
datetime
.
now
().
isoformat
()),
(
'Interpreter'
,
'%s %s build %s (%s)'
%
(
platform
.
python_implementation
(),
platform
.
python_version
(),
buildno
,
builddate
,
)),
(
'Interpreter'
,
f'
{
platform
.
python_implementation
()
}
{
platform
.
python_version
()
}
build
{
buildno
}
(
{
builddate
}
)'
),
(
'State file count'
,
stats
[
'state_file_count'
]),
(
'State loading time'
,
timedelta
(
seconds
=
stats
[
'parsing_start_time'
]
-
stats
[
'loading_start_time'
])),
...
...
@@ -1187,16 +1199,15 @@ def asHTML(out, encoding, per_site, args, default_site, period_parameter_dict,
(
'... skipped (URL)'
,
stats
[
'skipped_lines'
]),
(
'... skipped (user agent)'
,
stats
[
'skipped_user_agent'
]),
(
'Parsing time'
,
timedelta
(
seconds
=
parsing_time
)),
(
'Parsing rate'
,
'%i line/s'
%
(
all_lines
/
parsing_time
)
),
(
'Parsing rate'
,
f'
{
all_lines
//
parsing_time
}
line/s'
),
(
'Rendering time'
,
timedelta
(
seconds
=
(
end_stat_time
-
end_parsing_time
))),
):
out
.
write
(
'<tr><th class="text">%s</th><td>%s</td></tr>'
%
(
caption
,
value
))
out
.
write
(
'<tr><th class="text">{caption}</th><td>{value}</td></tr>'
)
out
.
write
(
'</table>'
)
out
.
write
(
'</body></html>'
)
def
asJSON
(
out
,
encoding
,
per_site
,
*
_
):
def
asJSON
(
out
,
encoding
,
per_site
,
*
_
):
# pylint: disable=unused-argument
json
.
dump
([(
x
,
y
.
asJSONState
())
for
x
,
y
in
per_site
.
items
()],
out
)
format_generator
=
{
...
...
@@ -1349,16 +1360,19 @@ def main():
parser
.
error
(
'Either --state-file or logfile arguments '
'must be specified.'
)
if
DURATION_US_FORMAT
in
args
.
logformat
:
getDuration
=
lambda
x
:
int
(
x
.
group
(
'duration'
))
def
getDuration
(
x
):
return
int
(
x
.
group
(
'duration'
))
elif
DURATION_MS_FORMAT
in
args
.
logformat
:
getDuration
=
lambda
x
:
int
(
x
.
group
(
'duration_ms'
))
*
US_PER_MS
def
getDuration
(
x
):
return
int
(
x
.
group
(
'duration_ms'
))
*
US_PER_MS
elif
DURATION_S_FORMAT
in
args
.
logformat
:
getDuration
=
lambda
x
:
int
(
x
.
group
(
'duration_s'
))
*
US_PER_S
def
getDuration
(
x
):
return
int
(
x
.
group
(
'duration_s'
))
*
US_PER_S
else
:
parser
.
error
(
'Neither %D nor %T are present in logformat, apdex '
'cannot be computed.'
)
if
args
.
duration_cap
:
def
getDuration
(
def
getDuration
(
# pylint: disable=function-redefined
match
,
_duration_cap
=
int
(
args
.
duration_cap
*
US_PER_S
),
_getDuration
=
getDuration
,
...
...
@@ -1369,8 +1383,8 @@ def main():
return
duration
if
args
.
match_servername
is
not
None
and
\
args
.
match_servername
not
in
args
.
logformat
:
parser
.
error
(
'--match-servername %s
requested, but missing '
'from logformat.'
%
args
.
match_servername
)
parser
.
error
(
f'--match-servername
{
args
.
match_servername
}
requested, but missing '
'from logformat.'
)
get_url_prefix
=
server_name_group_dict
.
get
(
args
.
match_servername
,
lambda
_
,
path
:
path
)
line_regex
=
''
...
...
@@ -1421,7 +1435,7 @@ def main():
dt, tz = match.group('
timestamp
').split()
day, month, rest = dt.split('
/
', 2)
return datetime.strptime(
'
%
s
/%
02
i
/%
s
' % (day, MONTH_VALUE_DICT[month], rest)
,
f'
{
day
}
/
{
MONTH_VALUE_DICT
[
month
]:
02
}
/
{
rest
}
'
,
'
%
d
/%
m
/%
Y
:
%
H
:
%
M
:
%
S
').replace(tzinfo=getTZInfo(tz))
if args.to_timezone:
to_timezone = args.to_timezone
...
...
@@ -1432,7 +1446,8 @@ def main():
raise ValueError('
pytz
is
not
available
,
cannot
convert
timezone
.
')
getTimezoneInfo = pytz.timezone
tz_info = getTimezoneInfo(to_timezone)
matchToDateTime = lambda x: _matchToDateTime(x).astimezone(tz_info)
def matchToDateTime(x):
return _matchToDateTime(x).astimezone(tz_info)
else:
matchToDateTime = _matchToDateTime
asDate, decimator, graph_period, date_format, placeholder_delta,
\
...
...
@@ -1459,7 +1474,7 @@ def main():
parser.error('
stdin
cannot
be
used
both
as
log
and
state
input
.
')
loading_start_time = time.time()
for state_file_name in args.state_file:
print(
'
Loading
%
s
...
' % state_file_name
, end='', file=sys.stderr)
print(
f'
Loading
{
state_file_name
}...
'
, end='', file=sys.stderr)
if state_file_name == '
-
':
state_file = sys.stdin
else:
...
...
@@ -1479,7 +1494,7 @@ def main():
site = None
action = default_action
if action is None:
print(
'
Info
:
no
prefix
match
%
r
,
stats
skipped
' % url
,
print(
f'
Info
:
no
prefix
match
{
url
},
stats
skipped
'
,
file='
sys
.
stderr
')
continue
site_stats = action.func.fromJSONState(site_state,
...
...
@@ -1488,7 +1503,7 @@ def main():
per_site[site].accumulateFrom(site_stats)
else:
per_site[site] = site_stats
print(
'
done
(
%
s
)
' % timedelta(seconds=time.time() - load_start)
,
print(
f'
done
({
timedelta
(
seconds
=
time
.
time
()
-
load_start
)})
'
,
file=sys.stderr)
skip_user_agent = [re.compile(x).match
for x in itertools.chain(*args.skip_user_agent)]
...
...
@@ -1501,7 +1516,7 @@ def main():
parsing_start_time = time.time()
for fileno, filename in enumerate(infile_list, 1):
if show_progress:
print(
'
Processing
%
s
[
%
i
/%
i
]
' % (filename, fileno, file_count)
,
print(
f'
Processing
{
filename
}
[{
fileno
}
/
{
file_count
}]
'
,
file=sys.stderr)
if filename == '
-
':
logfile = sys.stdin
...
...
@@ -1526,7 +1541,7 @@ def main():
match = expensive_matchline(line)
if match is None:
if not quiet:
print(
'
Malformed
line
at
%
s
:
%
i
:
%
r' % (filename, lineno, line)
,
print(
f'
Malformed
line
at
{
filename
}:{
lineno
}:
{
line
}
'
,
file=sys.stderr)
malformed_lines += 1
continue
...
...
@@ -1567,19 +1582,26 @@ def main():
if original_period != period:
original_period = period
if show_progress:
print(
'
Increasing
period
to
%
s
...
' % period
, end='',
print(
f'
Increasing
period
to
{
period
}...
'
, end='',
file=sys.stderr)
old_date_format = date_format
asDate, decimator, graph_period, date_format, placeholder_delta,
\
round_date, graph_coefficient = period_parser[period]
(
asDate,
decimator,
graph_period,
date_format,
placeholder_delta,
round_date,
graph_coefficient,
) = period_parser[period]
latest_date = rescale(latest_date)
earliest_date = rescale(earliest_date)
period_increase_start = time.time()
for site_data in per_site.values():
site_data.rescale(rescale, getDuration)
if show_progress:
print(
'
done
(
%
s
)
' % timedelta(seconds=time.time()
- period_increase_start),
file=sys.stderr)
print(
f'
done
({
timedelta
(
seconds
=
time
.
time
()
-
period_increase_start
)})
',
file=sys.stderr)
hit_date = asDate(matchToDateTime(match))
try:
site_data = per_site[site]
...
...
@@ -1589,9 +1611,9 @@ def main():
erp5_expand_other=erp5_expand_other)
try:
site_data.accumulate(match, url_match, hit_date)
except Exception:
except Exception:
# pylint: disable=broad-exception-caught
if not quiet:
print(
'
Error
analysing
line
at
%
s
:
%
i
:
%
r' % (filename, lineno, line)
,
print(
f'
Error
analysing
line
at
{
filename
}:{
lineno
}:
{
line
!
r
}
'
,
file=sys.stderr)
traceback.print_exc(file=sys.stderr)
all_lines += lineno
...
...
@@ -1637,7 +1659,6 @@ if __name__ == '__main__':
return f.read()
main()
from ._version import get_versions
__version__ = get_versions()['
version
']
del get_versions
else:
def getResource(name, encoding='
utf
-
8
'):
return pkgutil.get_data(__name__, name).decode(encoding)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment