Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
erp5
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
144
Merge Requests
144
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
nexedi
erp5
Commits
e772b97e
Commit
e772b97e
authored
Sep 13, 2018
by
Bryton Lacquement
🚪
Committed by
Julien Muchembled
Aug 13, 2019
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
fixup! erp5.util: add support for Python 3
31804f68
was merged too soon. /reviewed-on
nexedi/erp5!913
parent
e7ca6787
Changes
15
Hide whitespace changes
Inline
Side-by-side
Showing
15 changed files
with
59 additions
and
56 deletions
+59
-56
erp5/util/benchmark/argument.py
erp5/util/benchmark/argument.py
+1
-1
erp5/util/benchmark/performance_tester.py
erp5/util/benchmark/performance_tester.py
+2
-2
erp5/util/benchmark/process.py
erp5/util/benchmark/process.py
+6
-6
erp5/util/benchmark/report.py
erp5/util/benchmark/report.py
+11
-8
erp5/util/benchmark/result.py
erp5/util/benchmark/result.py
+1
-1
erp5/util/benchmark/scalability_tester.py
erp5/util/benchmark/scalability_tester.py
+5
-5
erp5/util/scalability/runScalabilityTestSuite.py
erp5/util/scalability/runScalabilityTestSuite.py
+5
-3
erp5/util/taskdistribution/__init__.py
erp5/util/taskdistribution/__init__.py
+1
-1
erp5/util/testbrowser/browser.py
erp5/util/testbrowser/browser.py
+6
-10
erp5/util/testbrowser/examples/createPerson.py
erp5/util/testbrowser/examples/createPerson.py
+2
-1
erp5/util/testnode/ScalabilityTestRunner.py
erp5/util/testnode/ScalabilityTestRunner.py
+2
-2
erp5/util/testnode/SlapOSMasterCommunicator.py
erp5/util/testnode/SlapOSMasterCommunicator.py
+1
-1
erp5/util/timinglogparser/__init__.py
erp5/util/timinglogparser/__init__.py
+11
-11
erp5/util/timinglogplotter/__init__.py
erp5/util/timinglogplotter/__init__.py
+3
-3
erp5/util/webchecker/__init__.py
erp5/util/webchecker/__init__.py
+2
-1
No files found.
erp5/util/benchmark/argument.py
View file @
e772b97e
...
...
@@ -57,7 +57,7 @@ class ArgumentType(object):
try
:
module
=
__import__
(
module_name
,
globals
(),
locals
(),
[
object_name
],
-
1
)
except
Exception
,
e
:
except
Exception
as
e
:
raise
argparse
.
ArgumentTypeError
(
"Cannot import '%s.%s': %s"
%
\
(
module_name
,
object_name
,
str
(
e
)))
...
...
erp5/util/benchmark/performance_tester.py
View file @
e772b97e
...
...
@@ -264,7 +264,7 @@ class PerformanceTester(object):
try
:
error_message
=
exit_msg_queue
.
get
()
except
KeyboardInterrupt
,
e
:
except
KeyboardInterrupt
as
e
:
print
(
"
\
n
Interrupted by user, stopping gracefully..."
,
file
=
sys
.
stderr
)
exit_status
=
2
...
...
@@ -272,7 +272,7 @@ class PerformanceTester(object):
# blocking system call above and the system call should not be restarted
# (using siginterrupt), otherwise the process will stall forever as its
# child has already exited
except
IOError
,
e
:
except
IOError
as
e
:
if
e
.
errno
==
errno
.
EINTR
:
continue
...
...
erp5/util/benchmark/process.py
View file @
e772b97e
...
...
@@ -88,13 +88,13 @@ class BenchmarkProcess(multiprocessing.Process):
target
(
result
,
self
.
_browser
)
except
StopIteration
:
raise
except
Exception
,
e
:
except
Exception
as
e
:
self
.
_logger
.
info
(
"Exception while running target suite for user %s: %s"
%
(
self
.
_browser
.
_username
,
str
(
e
)))
msg
=
"%s: %s"
%
(
target
,
traceback
.
format_exc
())
try
:
msg
+=
"Last response headers:
\
n
%s
\
n
Last response contents:
\
n
%s"
%
\
(
self
.
_browser
.
headers
,
self
.
_browser
.
contents
)
except
:
except
Exception
:
pass
self
.
_error_counter
+=
1
...
...
@@ -123,7 +123,7 @@ class BenchmarkProcess(multiprocessing.Process):
try
:
self
.
_logger
.
info
(
str
(
result
.
getCurrentSuiteUseCaseStat
()))
except
:
except
Exception
:
pass
result
.
iterationFinished
()
...
...
@@ -183,14 +183,14 @@ class BenchmarkProcess(multiprocessing.Process):
runIteration
(
result
)
self
.
_current_repeat
+=
1
except
StopIteration
,
e
:
except
StopIteration
as
e
:
self
.
_logger
.
error
(
e
)
except
RuntimeError
,
e
:
except
RuntimeError
as
e
:
exit_msg
=
str
(
e
)
exit_status
=
1
except
BaseException
,
e
:
except
BaseException
as
e
:
exit_msg
=
traceback
.
format_exc
()
self
.
_logger
.
error
(
exit_msg
)
exit_status
=
2
...
...
erp5/util/benchmark/report.py
View file @
e772b97e
...
...
@@ -34,6 +34,7 @@
from
__future__
import
print_function
import
argparse
import
re
import
six
def
parseArguments
():
parser
=
argparse
.
ArgumentParser
(
...
...
@@ -98,13 +99,15 @@ def computeStatisticFromFilenameList(argument_namespace, filename_list,
merged_label_dict
=
{}
for
filename
in
filename_list
:
reader
=
csv
.
reader
(
open
(
filename
,
'r
b
'
),
delimiter
=
','
,
reader
=
csv
.
reader
(
open
(
filename
,
'r'
),
delimiter
=
','
,
quoting
=
csv
.
QUOTE_MINIMAL
)
reader_list
.
append
(
reader
)
# Get headers
row_list
=
[
unicode
(
row
,
'utf-8'
)
for
row
in
reader
.
next
()
]
if
str
is
bytes
:
row_list
=
[
row
.
decode
(
'utf-8'
)
for
row
in
next
(
reader
)]
else
:
row_list
=
[
list
(
next
(
reader
))]
if
not
label_list
:
label_list
=
row_list
label_merged_index
=
0
...
...
@@ -156,8 +159,8 @@ def computeStatisticFromFilenameList(argument_namespace, filename_list,
report_dict
[
'results'
].
setdefault
(
stat
.
full_label
,
[]).
append
(
stat
)
if
row_list
!=
label_list
:
raise
AssertionError
,
"ERROR: Result labels: %s != %s"
%
\
(
label_list
,
row_list
)
raise
AssertionError
(
"ERROR: Result labels: %s != %s"
%
(
label_list
,
row_list
)
)
iteration_index
=
0
for
row_list
in
reader
:
...
...
@@ -169,7 +172,7 @@ def computeStatisticFromFilenameList(argument_namespace, filename_list,
use_case_suite
=
row_use_case_mapping_dict
.
get
(
idx
,
None
)
if
use_case_suite
:
current_count
=
int
(
row
)
current_duration
=
float
(
row_iter
.
next
()[
1
])
/
3600.
0
current_duration
=
float
(
next
(
row_iter
)[
1
])
/
360
0
if
not
current_count
:
continue
...
...
@@ -587,8 +590,8 @@ def generateReport():
(
nb_users_list
[
0
],
nb_users_list
[
-
1
])
for
suite_name
,
report_dict
in
range_user_report_dict
.
iteritems
(
):
for
label
,
stat_list
in
report_dict
[
'results'
].
iteritems
(
):
for
suite_name
,
report_dict
in
six
.
iteritems
(
range_user_report_dict
):
for
label
,
stat_list
in
six
.
iteritems
(
report_dict
[
'results'
]
):
drawConcurrentUsersPlot
(
pdf
,
title_fmt
%
label
,
...
...
erp5/util/benchmark/result.py
View file @
e772b97e
...
...
@@ -334,7 +334,7 @@ class CSVBenchmarkResult(BenchmarkResult):
from
cStringIO
import
StringIO
import
xmlrpclib
from
six.moves
import
xmlrpc_client
as
xmlrpclib
import
datetime
class
ERP5BenchmarkResult
(
BenchmarkResult
):
...
...
erp5/util/benchmark/scalability_tester.py
View file @
e772b97e
...
...
@@ -51,16 +51,16 @@ class ScalabilityTester(PerformanceTester):
def
postRun
(
self
,
error_message_set
):
from
logging
import
Formatter
import
sys
import
urllib
import
urllib2
from
six.moves.urllib.request
import
urlencode
from
six.moves.urllib.parse
import
urlopen
try
:
url
lib2
.
url
open
(
"http://[%s]:%d/report"
%
\
urlopen
(
"http://[%s]:%d/report"
%
\
(
self
.
_argument_namespace
.
manager_address
,
self
.
_argument_namespace
.
manager_port
),
url
lib
.
url
encode
({
'error_message_set'
:
'|'
.
join
(
error_message_set
)})).
close
()
urlencode
({
'error_message_set'
:
'|'
.
join
(
error_message_set
)})).
close
()
except
:
except
Exception
:
print
(
"ERROR: %s"
%
Formatter
().
formatException
(
sys
.
exc_info
()),
file
=
sys
.
stderr
)
def
getResultClass
(
self
):
...
...
erp5/util/scalability/runScalabilityTestSuite.py
View file @
e772b97e
#!/usr/bin/env python
from
__future__
import
division
import
argparse
import
os
import
shutil
...
...
@@ -253,7 +255,7 @@ class ScalabilityLauncher(object):
log_file_name_prefix
=
"%s_%s_suite_%s"
%
(
LOG_FILE_PREFIX
,
current_test
.
title
,
test_suite
)
command_list
.
append
([
tester_path
,
instance_url
,
str
(
user_quantity
/
len
(
test_suite_list
)),
str
(
user_quantity
/
/
len
(
test_suite_list
)),
test_suite
,
'--benchmark-path-list'
,
benchmarks_path
,
'--users-file-path'
,
user_file_path
,
...
...
@@ -264,7 +266,7 @@ class ScalabilityLauncher(object):
'--user-index'
,
str
(
user_index
),
"--duration"
,
"%d"
%
test_duration
,
])
user_index
+=
user_quantity
/
len
(
test_suite_list
)
user_index
+=
user_quantity
/
/
len
(
test_suite_list
)
# Launch commands
exec_env
=
os
.
environ
.
copy
()
exec_env
.
update
({
'raise_error_if_fail'
:
False
})
...
...
@@ -298,7 +300,7 @@ class ScalabilityLauncher(object):
"tests=%s
\
n
"
\
"duration=%d
\
n
"
\
%
(
(
user_quantity
/
len
(
test_suite_list
))
*
len
(
test_suite_list
),
(
user_quantity
/
/
len
(
test_suite_list
))
*
len
(
test_suite_list
),
len
(
test_suite_list
),
'_'
.
join
(
test_suite_list
),
test_duration
...
...
erp5/util/taskdistribution/__init__.py
View file @
e772b97e
...
...
@@ -170,7 +170,7 @@ class TestResultLineProxy(RPCRetry):
try
:
return
bool
(
self
.
_retryRPC
(
'isTestCaseAlive'
,
(
self
.
_test_result_line_path
,)))
except
:
except
Exception
:
raise
ValueError
(
'isTestCaseAlive Failed.'
)
def
stop
(
self
,
test_count
=
None
,
error_count
=
None
,
failure_count
=
None
,
...
...
erp5/util/testbrowser/browser.py
View file @
e772b97e
...
...
@@ -32,8 +32,8 @@
import
logging
import
sys
import
urllib
import
Cookie
from
six.moves.urllib.parse
import
urlencode
from
six.moves
import
http_cookies
as
Cookie
import
re
from
zope.testbrowser._compat
import
urlparse
...
...
@@ -205,7 +205,7 @@ class Browser(ExtendedTestBrowser):
url_or_path
=
urlparse
.
urljoin
(
self
.
_erp5_base_url
,
url_or_path
)
if
isinstance
(
data
,
dict
):
data
=
url
lib
.
url
encode
(
data
)
data
=
urlencode
(
data
)
self
.
_logger
.
debug
(
"Opening: "
+
url_or_path
)
super
(
Browser
,
self
).
open
(
url_or_path
,
data
)
...
...
@@ -274,7 +274,7 @@ class Browser(ExtendedTestBrowser):
location_without_query_string
,
query_string
=
location
.
split
(
'?'
)
location
=
(
location_without_query_string
+
'?'
+
url
lib
.
url
encode
(
urlparse
.
parse_qs
(
query_string
,
'?'
+
urlencode
(
urlparse
.
parse_qs
(
query_string
,
strict_parsing
=
True
),
doseq
=
True
))
# END: Bugfix
...
...
@@ -318,7 +318,7 @@ class Browser(ExtendedTestBrowser):
url_or_path
=
urlparse
.
urljoin
(
self
.
_erp5_base_url
,
url_or_path
)
if
isinstance
(
data
,
dict
):
data
=
url
lib
.
url
encode
(
data
)
data
=
urlencode
(
data
)
url
=
self
.
_absoluteUrl
(
url_or_path
)
self
.
_logger
.
debug
(
"Opening: "
+
url
)
...
...
@@ -365,11 +365,7 @@ class Browser(ExtendedTestBrowser):
@return: Cookie value
@rtype: str
"""
for
cookie_name
,
cookie_value
in
self
.
cookies
.
iteritems
():
if
name
==
cookie_name
:
return
cookie_value
return
default
return
self
.
cookies
.
get
(
name
,
default
)
@
property
def
mainForm
(
self
):
...
...
erp5/util/testbrowser/examples/createPerson.py
View file @
e772b97e
...
...
@@ -2,6 +2,7 @@
# -*- coding: utf-8 -*-
from
__future__
import
division
,
print_function
import
six
from
erp5.util.testbrowser.browser
import
Browser
ITERATION
=
20
...
...
@@ -89,5 +90,5 @@ if __name__ == '__main__':
benchmarkAddPerson
(
counter
,
result_dict
)
counter
+=
1
for
title
,
time_list
in
result_dict
.
iteritems
(
):
for
title
,
time_list
in
six
.
iteritems
(
result_dict
):
print
(
"%s: %.4fs"
%
(
title
,
sum
(
time_list
)
/
ITERATION
))
erp5/util/testnode/ScalabilityTestRunner.py
View file @
e772b97e
...
...
@@ -104,7 +104,7 @@ class ScalabilityTestRunner():
self
.
slapos_url
=
self
.
testnode
.
taskdistribution
.
getSlaposUrl
()
if
not
self
.
slapos_url
:
self
.
slapos_url
=
self
.
testnode
.
config
[
'server_url'
]
except
:
except
Exception
:
self
.
slapos_url
=
self
.
testnode
.
config
[
'server_url'
]
# Get Slapos Master url used for api rest (using hateoas)
...
...
@@ -458,7 +458,7 @@ Require valid-user
suite_class
=
getattr
(
module
,
test_suite
)
suite
=
suite_class
(
**
kwargs
)
repo_location
=
"%s/%s/"
%
(
location
,
SCALABILITY_TEST
)
except
:
except
Exception
:
pass
return
suite
,
repo_location
...
...
erp5/util/testnode/SlapOSMasterCommunicator.py
View file @
e772b97e
...
...
@@ -363,7 +363,7 @@ class SlapOSTester(SlapOSMasterCommunicator):
def
getInstanceGuid
():
try
:
return
self
.
instance
.
getInstanceGuid
()
except
:
except
Exception
:
return
None
frontend_master_ipv6
=
None
instance_guid
=
None
...
...
erp5/util/timinglogparser/__init__.py
View file @
e772b97e
...
...
@@ -35,6 +35,7 @@ import imp
import
gzip
import
getopt
from
time
import
time
import
six
PROFILING_ENABLED
=
False
if
PROFILING_ENABLED
:
...
...
@@ -150,7 +151,7 @@ def parseFile(filename, measure_dict):
if
line_number
>
0
:
duration
=
time
()
-
begin
print
(
"Matched %i lines (%.2f%%), %i skipped (%.2f%%), %i unmatched (%.2f%%) in %.2fs (%i lines per second)."
%
\
(
match_count
,
(
match_count
/
line_number
)
*
100
,
skip_count
,
(
skip_count
/
line_number
)
*
100
,
(
line_number
-
match_count
-
skip_count
),
(
1
-
(
match_count
+
skip_count
)
/
line_number
)
)
*
100
,
duration
,
line_number
//
duration
),
(
match_count
,
(
match_count
/
line_number
)
*
100
,
skip_count
,
(
skip_count
/
line_number
)
*
100
,
(
line_number
-
match_count
-
skip_count
),
(
1
-
(
match_count
+
skip_count
)
/
line_number
)
*
100
,
duration
,
line_number
//
duration
),
file
=
sys
.
stderr
)
debug
=
False
...
...
@@ -209,9 +210,9 @@ if len(load_file_name_list):
with
open
(
load_file_name
)
as
load_file
:
temp_measure_dict
=
eval
(
load_file
.
read
(),
{})
assert
isinstance
(
measure_dict
,
dict
)
for
filter_id
,
result_dict
in
temp_measure_dict
.
iteritems
(
):
for
result
,
date_dict
in
result_dict
.
iteritems
(
):
for
date
,
duration_list
in
date_dict
.
iteritems
(
):
for
filter_id
,
result_dict
in
six
.
iteritems
(
temp_measure_dict
):
for
result
,
date_dict
in
six
.
iteritems
(
result_dict
):
for
date
,
duration_list
in
six
.
iteritems
(
date_dict
):
measure_dict
.
setdefault
(
filter_id
,
{}).
setdefault
(
result
,
{}).
setdefault
(
date
,
[]).
extend
(
duration_list
)
print
(
'Previous processing result restored from %r'
%
(
load_file_name
,
),
file
=
sys
.
stderr
)
...
...
@@ -231,18 +232,17 @@ if outfile_prefix is not None:
append
=
measure_id_list
.
append
sheet_dict
=
{}
line_dict
=
{}
for
match_id
,
match_dict
in
measure_dict
.
iteritems
(
):
for
result_id
,
result_dict
in
match_dict
.
iteritems
(
):
for
match_id
,
match_dict
in
six
.
iteritems
(
measure_dict
):
for
result_id
,
result_dict
in
six
.
iteritems
(
match_dict
):
measure_id
=
(
match_id
,
result_id
)
sheet_dict
.
setdefault
(
match_id
,
[]).
append
((
result_id
,
measure_id
))
append
(
measure_id
)
for
date
,
measure_list
in
result_dict
.
iteritems
(
):
for
date
,
measure_list
in
six
.
iteritems
(
result_dict
):
first_level_dict
=
line_dict
.
setdefault
(
date
,
{})
assert
measure_id
not
in
first_level_dict
first_level_dict
[
measure_id
]
=
measure_list
date_list
=
line_dict
.
keys
()
date_list
.
sort
(
key
=
date_key
)
date_list
=
sorted
(
line_dict
,
key
=
date_key
)
def
render_cell
(
value_list
,
format
):
if
isinstance
(
value_list
,
(
list
,
tuple
)):
...
...
@@ -251,7 +251,7 @@ if outfile_prefix is not None:
return
value_list
def
renderOutput
(
data_format
,
filename_suffix
):
for
sheet_id
,
sheet_column_list
in
s
heet_dict
.
iteritems
(
):
for
sheet_id
,
sheet_column_list
in
s
ix
.
iteritems
(
sheet_dict
):
outfile_name
=
'%s_%s_%s.csv'
%
(
outfile_prefix
,
sheet_id
,
filename_suffix
)
print
(
'Writing to %r...'
%
(
outfile_name
,
),
file
=
sys
.
stderr
)
with
open
(
outfile_name
,
'w'
)
as
outfile
:
...
...
@@ -259,7 +259,7 @@ if outfile_prefix is not None:
decimate_dict
=
{}
decimate
=
0
for
date
in
date_list
:
for
key
,
value
in
line_dict
[
date
].
iteritems
(
):
for
key
,
value
in
six
.
iteritems
(
line_dict
[
date
]
):
decimate_dict
.
setdefault
(
key
,
[]).
extend
(
value
)
decimate
+=
1
if
decimate
==
decimate_count
:
...
...
erp5/util/timinglogplotter/__init__.py
View file @
e772b97e
...
...
@@ -84,9 +84,9 @@ class CSVFile(object):
if
cell
>
value_max
.
get
(
key
,
0
):
value_max
[
key
]
=
cell
column_dict
[
key
].
append
(
cell
)
line_num
=
line_num
/
100
line_num
/=
100
for
key
in
ratio_dict
:
ratio_dict
[
key
]
/
/
=
line_num
ratio_dict
[
key
]
/=
line_num
def
getColumn
(
self
,
column_id
):
return
self
.
column_dict
[
self
.
column_list
[
column_id
]]
...
...
@@ -136,7 +136,7 @@ def main():
# date_list will be like ['2009/07/01', '2009/07/05', '2009/07/10', ...]
factor
=
1
if
len
(
date_string_list
)
>
20
:
factor
=
int
(
len
(
date_string_list
)
//
20
)
factor
=
len
(
date_string_list
)
//
20
i
=
0
for
date_string
in
date_string_list
:
if
i
%
factor
==
0
:
...
...
erp5/util/webchecker/__init__.py
View file @
e772b97e
...
...
@@ -39,6 +39,7 @@ import tempfile
from
datetime
import
datetime
import
threading
import
signal
import
six
_MARKER
=
[]
...
...
@@ -517,7 +518,7 @@ class HTTPCacheCheckerTestSuite(object):
logging.info('End of second pass
\
n
')
if self.report_dict:
report_message_list = ['*Errors*:']
for url, message_list in s
elf.report_dict.iteritems(
):
for url, message_list in s
ix.iteritems(self.report_dict
):
unique_message_list = []
for message in message_list:
if message not in unique_message_list:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment