Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
erp5
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Lukas Niegsch
erp5
Commits
31804f68
Commit
31804f68
authored
Sep 13, 2018
by
Bryton Lacquement
🚪
Committed by
Julien Muchembled
Jul 31, 2019
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
erp5.util: add support for Python 3
/reviewed-on
nexedi/erp5!830
parent
5abb074d
Changes
23
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
23 changed files
with
307 additions
and
259 deletions
+307
-259
erp5/tests/testERP5TestNode.py
erp5/tests/testERP5TestNode.py
+63
-64
erp5/util/benchmark/performance_tester.py
erp5/util/benchmark/performance_tester.py
+3
-2
erp5/util/benchmark/report.py
erp5/util/benchmark/report.py
+4
-5
erp5/util/benchmark/scalability_tester.py
erp5/util/benchmark/scalability_tester.py
+2
-1
erp5/util/scalability/requestUrl.py
erp5/util/scalability/requestUrl.py
+2
-2
erp5/util/scalability/runScalabilityTestSuite.py
erp5/util/scalability/runScalabilityTestSuite.py
+11
-22
erp5/util/taskdistribution/__init__.py
erp5/util/taskdistribution/__init__.py
+16
-6
erp5/util/testbrowser/examples/createERP5User.py
erp5/util/testbrowser/examples/createERP5User.py
+3
-5
erp5/util/testbrowser/examples/createPerson.py
erp5/util/testbrowser/examples/createPerson.py
+2
-1
erp5/util/testnode/NodeTestSuite.py
erp5/util/testnode/NodeTestSuite.py
+5
-3
erp5/util/testnode/ProcessManager.py
erp5/util/testnode/ProcessManager.py
+8
-7
erp5/util/testnode/ScalabilityTestRunner.py
erp5/util/testnode/ScalabilityTestRunner.py
+15
-16
erp5/util/testnode/SlapOSControler.py
erp5/util/testnode/SlapOSControler.py
+7
-5
erp5/util/testnode/SlapOSMasterCommunicator.py
erp5/util/testnode/SlapOSMasterCommunicator.py
+12
-6
erp5/util/testnode/Updater.py
erp5/util/testnode/Updater.py
+4
-3
erp5/util/testnode/Utils.py
erp5/util/testnode/Utils.py
+26
-11
erp5/util/testnode/__init__.py
erp5/util/testnode/__init__.py
+2
-2
erp5/util/testnode/testnode.py
erp5/util/testnode/testnode.py
+1
-1
erp5/util/testsuite/__init__.py
erp5/util/testsuite/__init__.py
+38
-33
erp5/util/timinglogparser/__init__.py
erp5/util/timinglogparser/__init__.py
+35
-34
erp5/util/timinglogplotter/__init__.py
erp5/util/timinglogplotter/__init__.py
+11
-9
erp5/util/webchecker/__init__.py
erp5/util/webchecker/__init__.py
+8
-7
product/ERP5/bin/genbt5list
product/ERP5/bin/genbt5list
+29
-14
No files found.
erp5/tests/testERP5TestNode.py
View file @
31804f68
This diff is collapsed.
Click to expand it.
erp5/util/benchmark/performance_tester.py
View file @
31804f68
...
...
@@ -28,6 +28,7 @@
#
##############################################################################
from
__future__
import
print_function
import
argparse
import
os
import
sys
...
...
@@ -264,7 +265,7 @@ class PerformanceTester(object):
error_message
=
exit_msg_queue
.
get
()
except
KeyboardInterrupt
,
e
:
print
>>
sys
.
stderr
,
"
\
n
Interrupted by user, stopping gracefully..."
print
(
"
\
n
Interrupted by user, stopping gracefully..."
,
file
=
sys
.
stderr
)
exit_status
=
2
# An IOError may be raised when receiving a SIGINT which interrupts the
...
...
@@ -337,7 +338,7 @@ class PerformanceTester(object):
def
main
():
error_message_set
,
exit_status
=
PerformanceTester
().
run
()
for
error_message
in
error_message_set
:
print
>>
sys
.
stderr
,
"ERROR: %s"
%
error_message
print
(
"ERROR: %s"
%
error_message
,
file
=
sys
.
stderr
)
sys
.
exit
(
exit_status
)
...
...
erp5/util/benchmark/report.py
View file @
31804f68
...
...
@@ -31,6 +31,7 @@
#
##############################################################################
from
__future__
import
print_function
import
argparse
import
re
...
...
@@ -537,7 +538,7 @@ def generateReport():
for
filename
in
filename_iter
:
# There may be no results at all in case of errors
if
not
os
.
stat
(
filename
).
st_size
:
print
>>
sys
.
stderr
,
"Ignoring empty file %s"
%
filename
print
(
"Ignoring empty file %s"
%
filename
,
file
=
sys
.
stderr
)
continue
report_dict
=
per_nb_users_report_dict
.
setdefault
(
...
...
@@ -546,10 +547,8 @@ def generateReport():
report_dict
[
'filename'
].
append
(
filename
)
if
not
per_nb_users_report_dict
:
print
>>
sys
.
stderr
,
"ERROR: No result file found, perhaps "
\
"``--filename-prefix'' should be specified?"
sys
.
exit
(
1
)
sys
.
exit
(
"ERROR: No result file found, perhaps ``--filename-prefix'' should"
"be specified?"
)
pdf
=
PdfPages
(
argument_namespace
.
output_filename
)
...
...
erp5/util/benchmark/scalability_tester.py
View file @
31804f68
...
...
@@ -28,6 +28,7 @@
#
##############################################################################
from
__future__
import
print_function
from
.result
import
CSVBenchmarkResult
,
NothingFlushedException
class
CSVScalabilityBenchmarkResult
(
CSVBenchmarkResult
):
...
...
@@ -60,7 +61,7 @@ class ScalabilityTester(PerformanceTester):
urllib
.
urlencode
({
'error_message_set'
:
'|'
.
join
(
error_message_set
)})).
close
()
except
:
print
>>
sys
.
stderr
,
"ERROR: %s"
%
Formatter
().
formatException
(
sys
.
exc_info
()
)
print
(
"ERROR: %s"
%
Formatter
().
formatException
(
sys
.
exc_info
()),
file
=
sys
.
stderr
)
def
getResultClass
(
self
):
if
not
self
.
_argument_namespace
.
erp5_publish_url
:
...
...
erp5/util/scalability/requestUrl.py
View file @
31804f68
...
...
@@ -27,7 +27,7 @@ def main():
if
error_message_set
:
exit_status
=
1
for
error
in
error_message_set
:
print
error
print
(
error
)
elif
result
:
print
result
print
(
result
)
sys
.
exit
(
exit_status
)
erp5/util/scalability/runScalabilityTestSuite.py
View file @
31804f68
...
...
@@ -5,23 +5,15 @@ import os
import
shutil
import
time
import
sys
import
multiprocessing
import
signal
import
errno
import
json
import
logging
import
logging.handlers
import
glob
import
urlparse
import
httplib
import
base64
import
threading
from
erp5.util.benchmark.argument
import
ArgumentType
from
erp5.util.benchmark.performance_tester
import
PerformanceTester
from
erp5.util.benchmark.thread
import
TestThread
,
TestMetricThread
from
erp5.util
import
taskdistribution
from
erp5.util.testnode
import
Utils
from
erp5.util.testnode.ProcessManager
import
SubprocessError
,
ProcessManager
,
CancellationErro
r
from
erp5.util.testnode.ProcessManager
import
ProcessManage
r
import
datetime
MAX_INSTALLATION_TIME
=
60
*
50
...
...
@@ -179,31 +171,28 @@ class ScalabilityLauncher(object):
"""
data_array
=
self
.
__argumentNamespace
.
current_test_data
.
split
(
','
)
data
=
json
.
dumps
({
"count"
:
data_array
[
0
],
"title"
:
data_array
[
1
],
"relative_path"
:
data_array
[
2
]})
de
coded_data
=
Utils
.
deunicodeData
(
json
.
loads
(
data
))
return
ScalabilityTest
(
de
coded_data
,
self
.
test_result
)
en
coded_data
=
Utils
.
deunicodeData
(
json
.
loads
(
data
))
return
ScalabilityTest
(
en
coded_data
,
self
.
test_result
)
def
clearUsersFile
(
self
,
user_file_path
):
self
.
log
(
"Clearing users file: %s"
%
user_file_path
)
os
.
remove
(
user_file_path
)
users_file
=
open
(
user_file_path
,
"w"
)
for
line
in
self
.
users_file_original_content
:
users_file
.
write
(
line
)
users_file
.
close
()
with
open
(
user_file_path
,
"w"
)
as
users_file
:
for
line
in
self
.
users_file_original_content
:
users_file
.
write
(
line
)
def
updateUsersFile
(
self
,
user_quantity
,
password
,
user_file_path
):
self
.
log
(
"Updating users file: %s"
%
user_file_path
)
users_file
=
open
(
user_file_path
,
"r"
)
file_content
=
users_file
.
readlines
()
with
open
(
user_file_path
,
"r"
)
as
users_file
:
file_content
=
users_file
.
readlines
()
self
.
users_file_original_content
=
file_content
new_file_content
=
[]
for
line
in
file_content
:
new_file_content
.
append
(
line
.
replace
(
'<password>'
,
password
).
replace
(
'<user_quantity>'
,
str
(
user_quantity
)))
users_file
.
close
()
os
.
remove
(
user_file_path
)
users_file
=
open
(
user_file_path
,
"w"
)
for
line
in
new_file_content
:
users_file
.
write
(
line
)
users_file
.
close
()
with
open
(
user_file_path
,
"w"
)
as
users_file
:
for
line
in
new_file_content
:
users_file
.
write
(
line
)
def
run
(
self
):
self
.
log
(
"Scalability Launcher started, with:"
)
...
...
erp5/util/taskdistribution/__init__.py
View file @
31804f68
...
...
@@ -40,11 +40,15 @@ Example use:
test_line.stop()
"""
from
__future__
import
print_function
import
httplib
import
six
from
six.moves
import
(
map
,
http_client
as
httplib
,
xmlrpc_client
as
xmlrpclib
,
)
import
socket
import
threading
import
time
import
xmlrpclib
__all__
=
[
'TaskDistributor'
,
'TestResultProxy'
,
'TestResultLineProxy'
,
'patchRPCParser'
]
...
...
@@ -89,11 +93,17 @@ def patchRPCParser(error_handler):
def
verbose_feed
(
self
,
data
):
try
:
return
original_feed
(
self
,
data
)
except
Exception
,
exc
:
except
Exception
as
exc
:
if
not
error_handler
(
data
,
exc
):
raise
parser_klass
.
feed
=
verbose_feed
try
:
# PY3
basestring
except
NameError
:
basestring
=
bytes
,
str
unicode
=
str
def
binarize_args
(
arg
):
# Converts recursively basestring arg into xmlrpclib.Binary, as they can
# contain non-XML allowed characters
...
...
@@ -102,9 +112,9 @@ def binarize_args(arg):
arg
=
arg
.
encode
(
'utf-8'
)
return
xmlrpclib
.
Binary
(
arg
)
if
isinstance
(
arg
,
(
list
,
tuple
,
set
)):
return
map
(
binarize_args
,
arg
)
return
list
(
map
(
binarize_args
,
arg
)
)
if
isinstance
(
arg
,
dict
):
return
{
k
:
binarize_args
(
v
)
for
k
,
v
in
arg
.
iteritems
(
)}
return
{
k
:
binarize_args
(
v
)
for
k
,
v
in
six
.
iteritems
(
arg
)}
return
arg
class
RPCRetry
(
object
):
...
...
@@ -350,7 +360,7 @@ class TestResultProxy(RPCRetry):
caption_list
=
[]
append
=
caption_list
.
append
for
name
,
(
stream
,
max_history_bytes
)
in
\
s
elf
.
_watcher_dict
.
iteritems
(
):
s
ix
.
iteritems
(
self
.
_watcher_dict
):
append
(
'==> %s <=='
%
(
name
,
))
start
=
stream
.
tell
()
stream
.
seek
(
0
,
2
)
...
...
erp5/util/testbrowser/examples/createERP5User.py
View file @
31804f68
...
...
@@ -8,6 +8,7 @@
#
# TODO: There must be a better way than the code below to do that though...
from
__future__
import
print_function
import
sys
from
erp5.util.testbrowser.browser
import
Browser
...
...
@@ -19,11 +20,8 @@ try:
user_nbr
=
int
(
user_nbr
)
except
ValueError
:
print
>>
sys
.
stderr
,
"ERROR: Missing arguments: %s URL USERNAME "
\
"PASSWORD NUMBER_OF_USERS NEW_USERNAME_PREFIX NEW_USERS_PASSWORD"
%
\
sys
.
argv
[
0
]
sys
.
exit
(
1
)
sys
.
exit
(
"ERROR: Missing arguments: %s URL USERNAME PASSWORD NUMBER_OF_USERS "
"NEW_USERNAME_PREFIX NEW_USERS_PASSWORD"
%
sys
.
argv
[
0
])
# Create a browser instance
browser
=
Browser
(
url
,
username
,
password
)
...
...
erp5/util/testbrowser/examples/createPerson.py
View file @
31804f68
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from
__future__
import
division
,
print_function
from
erp5.util.testbrowser.browser
import
Browser
ITERATION
=
20
...
...
@@ -89,4 +90,4 @@ if __name__ == '__main__':
counter
+=
1
for
title
,
time_list
in
result_dict
.
iteritems
():
print
"%s: %.4fs"
%
(
title
,
float
(
sum
(
time_list
))
/
ITERATION
)
print
(
"%s: %.4fs"
%
(
title
,
sum
(
time_list
)
/
ITERATION
)
)
erp5/util/testnode/NodeTestSuite.py
View file @
31804f68
...
...
@@ -32,6 +32,8 @@ import string
import
random
from
.Utils
import
createFolder
from
six.moves
import
range
class
SlapOSInstance
(
object
):
"""
Base of an software instance,
...
...
@@ -69,14 +71,14 @@ class NodeTestSuite(SlapOSInstance):
def
createSuiteLog
(
self
):
# /srv/slapgrid/slappartXX/srv/var/log/testnode/az-D27KqX7FxJ/suite.log
alphabets
=
string
.
digits
+
string
.
letters
alphabets
=
string
.
digits
+
string
.
ascii_
letters
while
1
:
log_folder_name
=
'%s-%s'
%
(
self
.
reference
,
''
.
join
(
random
.
choice
(
alphabets
)
for
i
in
x
range
(
10
)))
''
.
join
(
random
.
choice
(
alphabets
)
for
i
in
range
(
10
)))
log_folder_path
=
os
.
path
.
join
(
self
.
log_directory
,
log_folder_name
)
try
:
os
.
makedirs
(
log_folder_path
)
except
OSError
,
e
:
except
OSError
as
e
:
if
e
.
errno
!=
errno
.
EEXIST
:
raise
else
:
...
...
erp5/util/testnode/ProcessManager.py
View file @
31804f68
...
...
@@ -79,7 +79,8 @@ def subprocess_capture(p, log_prefix, get_output=True):
break
if
get_output
:
buffer
.
append
(
data
)
log
(
log_prefix
+
data
.
rstrip
(
'
\
n
'
))
log
(
log_prefix
+
(
data
if
str
is
bytes
else
data
.
decode
(
'utf-8'
,
errors
=
'replace'
)).
rstrip
(
'
\
n
'
))
if
p
.
stdout
:
stdout
=
[]
stdout_thread
=
threading
.
Thread
(
target
=
readerthread
,
...
...
@@ -97,8 +98,8 @@ def subprocess_capture(p, log_prefix, get_output=True):
stdout_thread
.
join
()
if
p
.
stderr
:
stderr_thread
.
join
()
return
(
p
.
stdout
and
''
.
join
(
stdout
),
p
.
stderr
and
''
.
join
(
stderr
))
return
(
p
.
stdout
and
b
''
.
join
(
stdout
),
p
.
stderr
and
b
''
.
join
(
stderr
))
def
killCommand
(
pid
):
"""
...
...
@@ -109,7 +110,7 @@ def killCommand(pid):
try
:
process
=
psutil
.
Process
(
pid
)
process
.
suspend
()
except
psutil
.
Error
,
e
:
except
psutil
.
Error
as
e
:
return
process_list
=
[
process
]
new_list
=
process
.
children
(
recursive
=
True
)
...
...
@@ -118,19 +119,19 @@ def killCommand(pid):
for
child
in
new_list
:
try
:
child
.
suspend
()
except
psutil
.
Error
,
e
:
except
psutil
.
Error
as
e
:
logger
.
debug
(
"killCommand/suspend: %s"
,
e
)
time
.
sleep
(
1
)
new_list
=
set
(
process
.
children
(
recursive
=
True
)).
difference
(
process_list
)
for
process
in
process_list
:
try
:
process
.
kill
()
except
psutil
.
Error
,
e
:
except
psutil
.
Error
as
e
:
logger
.
debug
(
"killCommand/kill: %s"
,
e
)
class
ProcessManager
(
object
):
stdin
=
file
(
os
.
devnull
)
stdin
=
open
(
os
.
devnull
)
def
__init__
(
self
,
max_timeout
=
MAX_TIMEOUT
):
self
.
process_pid_set
=
set
()
...
...
erp5/util/testnode/ScalabilityTestRunner.py
View file @
31804f68
...
...
@@ -30,30 +30,31 @@ import subprocess
import
sys
import
time
import
glob
import
SlapOSControler
import
SlapOSMasterCommunicator
from
.
import
SlapOSControler
,
SlapOSMasterCommunicator
import
json
import
time
import
shutil
import
logging
import
string
import
random
import
urlparse
from
six.moves.urllib.parse
import
urlparse
import
base64
import
httplib
import
Utils
from
six.moves
import
http_client
as
httplib
from
.
import
Utils
import
requests
import
slapos.slap
import
cPickle
as
pickle
from
ProcessManager
import
SubprocessError
,
ProcessManager
,
CancellationError
from
six.moves
import
cPickle
as
pickle
from
.
ProcessManager
import
SubprocessError
,
ProcessManager
,
CancellationError
from
subprocess
import
CalledProcessError
from
Updater
import
Updater
from
.
Updater
import
Updater
from
erp5.util
import
taskdistribution
from
erp5.util.benchmark.thread
import
TestThread
# for dummy slapos answer
import
signal
from
.
import
logger
from
six.moves
import
range
# max time to generate frontend instance: 1.5 hour
MAX_FRONTEND_TIME
=
60
*
90
# max time to register instance to slapOSMaster: 5 minutes
...
...
@@ -322,18 +323,16 @@ ces or already launched.")
software_hash_directory
=
self
.
testnode
.
config
[
'slapos_binary'
].
rsplit
(
"bin/slapos"
,
1
)[
0
]
apache_htpasswd
=
software_hash_directory
+
"parts/apache/bin/htpasswd"
testsuite_directory
=
self
.
testnode
.
config
[
'repository_path_list'
][
0
].
rsplit
(
'/'
,
1
)[
0
]
htaccess_file
=
open
(
testsuite_directory
+
HTACCESS
,
"w"
)
file_content
=
"""
with
open
(
testsuite_directory
+
HTACCESS
,
"w"
)
as
htaccess_file
:
htaccess_file
.
write
(
"""
AuthType Basic
AuthName "Password Protected Area"
AuthUserFile "%s%s"
Require valid-user
"""
%
(
testsuite_directory
,
HTPASSWD
)
htaccess_file
.
write
(
file_content
)
htaccess_file
.
close
()
"""
%
(
testsuite_directory
,
HTPASSWD
))
password_path
=
testsuite_directory
+
PASSWORD_FILE
with
open
(
password_path
,
"w"
)
as
password_file
:
password
=
''
.
join
(
random
.
choice
(
string
.
digits
+
string
.
letters
)
for
i
in
x
range
(
PASSWORD_LENGTH
))
password
=
''
.
join
(
random
.
choice
(
string
.
digits
+
string
.
ascii_letters
)
for
i
in
range
(
PASSWORD_LENGTH
))
password_file
.
write
(
password
)
user
=
TESTNODE_USER
command
=
[
apache_htpasswd
,
"-bc"
,
testsuite_directory
+
HTPASSWD
,
user
,
password
]
...
...
@@ -363,7 +362,7 @@ Require valid-user
user
,
password
=
self
.
generateProfilePasswordAccess
()
logger
.
info
(
"Software Profile password: %s"
%
password
)
self
.
reachable_profile
=
"https://%s:%s@%s"
%
(
user
,
password
,
os
.
path
.
join
(
urlparse
.
urlparse
(
self
.
testnode
.
config
[
'frontend_url'
]).
netloc
,
os
.
path
.
join
(
urlparse
(
self
.
testnode
.
config
[
'frontend_url'
]).
netloc
,
"software"
,
self
.
randomized_path
,
"software.cfg"
))
def
prepareSlapOSForTestSuite
(
self
,
node_test_suite
):
...
...
@@ -526,7 +525,7 @@ Require valid-user
if
not
self
.
launchable
:
return
{
'status_code'
:
1
,
'error_message'
:
"Current test_suite is not actually launchable."
}
configuration_list
=
node_test_suite
.
configuration_list
test_list
=
range
(
0
,
len
(
configuration_list
))
test_list
=
list
(
range
(
len
(
configuration_list
)
))
try
:
test_result_proxy
=
self
.
testnode
.
taskdistribution
.
createTestResult
(
node_test_suite
.
revision
,
test_list
,
...
...
erp5/util/testnode/SlapOSControler.py
View file @
31804f68
...
...
@@ -35,6 +35,8 @@ from slapos import client
from
.
import
logger
from
.Utils
import
createFolder
from
six.moves
import
range
MAX_PARTITIONS
=
10
MAX_SR_RETRIES
=
3
...
...
@@ -243,7 +245,7 @@ class SlapOSControler(object):
computer
=
slap
.
registerComputer
(
config
[
'computer_id'
])
# Call a method to ensure connection to master can be established
computer
.
getComputerPartitionList
()
except
slapos
.
slap
.
ConnectionError
,
e
:
except
slapos
.
slap
.
ConnectionError
as
e
:
retries
+=
1
if
retries
>=
60
:
raise
...
...
@@ -270,7 +272,7 @@ class SlapOSControler(object):
# MySQL DB content) from previous runs. To support changes of partition
# naming scheme (which already happened), do this at instance_root level.
createFolder
(
instance_root
,
True
)
for
i
in
x
range
(
MAX_PARTITIONS
):
for
i
in
range
(
MAX_PARTITIONS
):
# create partition and configure computer
# XXX: at the moment all partitions do share same virtual interface address
# this is not a problem as usually all services are on different ports
...
...
@@ -278,7 +280,7 @@ class SlapOSControler(object):
partition_path
=
os
.
path
.
join
(
instance_root
,
partition_reference
)
if
not
(
os
.
path
.
exists
(
partition_path
)):
os
.
mkdir
(
partition_path
)
os
.
chmod
(
partition_path
,
0750
)
os
.
chmod
(
partition_path
,
0
o
750
)
computer
.
updateConfiguration
(
xml_marshaller
.
xml_marshaller
.
dumps
({
'address'
:
config
[
'ipv4_address'
],
'instance_root'
:
instance_root
,
...
...
@@ -318,7 +320,7 @@ class SlapOSControler(object):
os
.
environ
[
'PATH'
]
=
environment
[
'PATH'
]
# a SR may fail for number of reasons (incl. network failures)
# so be tolerant and run it a few times before giving up
for
_
in
x
range
(
MAX_SR_RETRIES
):
for
_
in
range
(
MAX_SR_RETRIES
):
status_dict
=
self
.
spawn
(
config
[
'slapos_binary'
],
'node'
,
'software'
,
'--all'
,
'--pidfile'
,
os
.
path
.
join
(
self
.
software_root
,
'slapos-node.pid'
),
...
...
@@ -346,7 +348,7 @@ class SlapOSControler(object):
# try to run for all partitions as one partition may in theory request another one
# this not always is required but curently no way to know how "tree" of partitions
# may "expand"
for
_
in
x
range
(
max_quantity
):
for
_
in
range
(
max_quantity
):
status_dict
=
self
.
spawn
(
config
[
'slapos_binary'
],
'node'
,
'instance'
,
'--pidfile'
,
os
.
path
.
join
(
self
.
instance_root
,
'slapos-node.pid'
),
'--cfg'
,
self
.
slapos_config
,
raise_error_if_fail
=
False
,
...
...
erp5/util/testnode/SlapOSMasterCommunicator.py
View file @
31804f68
from
__future__
import
print_function
import
datetime
import
json
import
traceback
...
...
@@ -12,6 +14,8 @@ from requests.exceptions import HTTPError
from
..taskdistribution
import
SAFE_RPC_EXCEPTION_LIST
from
.
import
logger
import
six
# max time to instance changing state: 3 hour
MAX_INSTANCE_TIME
=
60
*
60
*
3
...
...
@@ -52,7 +56,7 @@ def retryOnNetworkFailure(func,
except
_except_list
:
traceback
.
print_exc
()
print
'Network failure. Retry method %s in %i seconds'
%
(
func
,
retry_time
)
print
(
'Network failure. Retry method %s in %i seconds'
%
(
func
,
retry_time
)
)
time
.
sleep
(
retry_time
)
retry_time
=
min
(
retry_time
*
1.5
,
640
)
...
...
@@ -92,8 +96,9 @@ class SlapOSMasterCommunicator(object):
if
instance_title
is
not
None
:
self
.
name
=
instance_title
if
request_kw
is
not
None
:
if
isinstance
(
request_kw
,
basestring
)
or
\
isinstance
(
request_kw
,
unicode
):
if
isinstance
(
request_kw
,
bytes
):
self
.
request_kw
=
json
.
loads
(
request_kw
.
decode
(
'utf-8'
))
elif
isinstance
(
request_kw
,
six
.
text_type
):
self
.
request_kw
=
json
.
loads
(
request_kw
)
else
:
self
.
request_kw
=
request_kw
...
...
@@ -214,7 +219,7 @@ class SlapOSMasterCommunicator(object):
result
=
self
.
hateoas_navigator
.
GET
(
url
)
result
=
json
.
loads
(
result
)
if
result
[
'_links'
].
get
(
'action_object_slap'
,
None
)
is
None
:
print
result
[
'links'
]
print
(
result
[
'links'
])
return
None
object_link
=
self
.
hateoas_navigator
.
hateoasGetLinkFromLinks
(
...
...
@@ -385,8 +390,9 @@ class SlapOSTester(SlapOSMasterCommunicator):
self
.
name
=
name
self
.
computer_guid
=
computer_guid
if
isinstance
(
request_kw
,
str
)
or
\
isinstance
(
request_kw
,
unicode
):
if
isinstance
(
request_kw
,
bytes
):
self
.
request_kw
=
json
.
loads
(
request_kw
.
decode
(
'utf-8'
))
elif
isinstance
(
request_kw
,
six
.
text_type
):
self
.
request_kw
=
json
.
loads
(
request_kw
)
else
:
self
.
request_kw
=
request_kw
...
...
erp5/util/testnode/Updater.py
View file @
31804f68
...
...
@@ -30,6 +30,7 @@ import re
from
.
import
logger
from
.ProcessManager
import
SubprocessError
from
.Utils
import
rmtree
from
slapos.util
import
bytes2str
,
str2bytes
SVN_UP_REV
=
re
.
compile
(
r'^(?:At|Updated to) revision (\
d+).$
')
SVN_CHANGED_REV = re.compile(r'
^
Last
Changed
Rev
.
*
:
\
s
*
(
\
d
+
)
', re.MULTILINE)
...
...
@@ -82,7 +83,7 @@ class Updater(object):
# allow several processes clean the same folder at the same time
try:
os.remove(os.path.join(path, file))
except OSError
,
e:
except OSError
as
e:
if e.errno != errno.ENOENT:
raise
...
...
@@ -96,7 +97,7 @@ class Updater(object):
**kw)
def _git(self, *args, **kw):
return
self.spawn(self.git_binary, *args, **kw)['
stdout
'].strip(
)
return
bytes2str(self.spawn(self.git_binary, *args, **kw)['
stdout
'].strip()
)
def git_update_server_info(self):
return self._git('
update
-
server
-
info
', '
-
f')
...
...
@@ -219,7 +220,7 @@ class Updater(object):
self.deletePycFiles(path)
try:
status_dict = self.spawn(*args)
except SubprocessError
,
e:
except SubprocessError
as
e:
if '
cleanup
' not in e.stderr:
raise
self.spawn('
svn
', '
cleanup
', path)
...
...
erp5/util/testnode/Utils.py
View file @
31804f68
...
...
@@ -3,6 +3,9 @@ import stat
import
shutil
import
errno
import
six
from
six.moves
import
map
def
rmtree
(
path
):
"""Delete a path recursively.
...
...
@@ -11,14 +14,22 @@ def rmtree(path):
def
chmod_retry
(
func
,
failed_path
,
exc_info
):
"""Make sure the directories are executable and writable.
"""
# Depending on the Python version, the following items differ.
if
six
.
PY3
:
expected_error_type
=
PermissionError
expected_func
=
os
.
lstat
else
:
expected_error_type
=
OSError
expected_func
=
os
.
listdir
e
=
exc_info
[
1
]
if
isinstance
(
e
,
OSError
):
if
isinstance
(
e
,
expected_error_type
):
if
e
.
errno
==
errno
.
ENOENT
:
# because we are calling again rmtree on listdir errors, this path might
# have been already deleted by the recursive call to rmtree.
return
if
e
.
errno
==
errno
.
EACCES
:
if
func
is
os
.
listdir
:
if
func
is
expected_func
:
os
.
chmod
(
failed_path
,
0o700
)
# corner case to handle errors in listing directories.
# https://bugs.python.org/issue8523
...
...
@@ -39,12 +50,16 @@ def createFolder(folder, clean=False):
rmtree
(
folder
)
os
.
mkdir
(
folder
)
def
deunicodeData
(
data
):
if
isinstance
(
data
,
list
):
return
map
(
deunicodeData
,
data
)
if
isinstance
(
data
,
unicode
):
return
data
.
encode
(
'utf8'
)
if
isinstance
(
data
,
dict
):
return
{
deunicodeData
(
key
):
deunicodeData
(
value
)
for
key
,
value
in
data
.
iteritems
()}
return
data
if
six
.
PY3
:
def
deunicodeData
(
data
):
return
data
else
:
def
deunicodeData
(
data
):
if
isinstance
(
data
,
list
):
return
list
(
map
(
deunicodeData
,
data
))
if
isinstance
(
data
,
unicode
):
return
data
.
encode
(
'utf8'
)
if
isinstance
(
data
,
dict
):
return
{
deunicodeData
(
key
):
deunicodeData
(
value
)
for
key
,
value
in
six
.
iteritems
(
data
)}
return
data
erp5/util/testnode/__init__.py
View file @
31804f68
...
...
@@ -24,7 +24,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import
ConfigP
arser
from
six.moves
import
configp
arser
import
argparse
import
logging
import
logging.handlers
...
...
@@ -64,7 +64,7 @@ def main(*args):
CONFIG
=
{
'partition_reference'
:
'test0'
,
}
config
=
ConfigP
arser
.
SafeConfigParser
()
config
=
configp
arser
.
SafeConfigParser
()
# do not change case of option keys
config
.
optionxform
=
str
config
.
readfp
(
parsed_argument
.
configuration_file
[
0
])
...
...
erp5/util/testnode/testnode.py
View file @
31804f68
...
...
@@ -171,7 +171,7 @@ shared = true
# only limit to particular error, if we run that code for all errors,
# then if server having most repositories is down for some time, we would
# erase all repositories and facing later hours of downloads
if
getattr
(
error
,
'stderr'
,
''
).
find
(
'index'
)
>=
0
:
if
b'index'
in
getattr
(
error
,
'stderr'
,
b''
)
:
rmtree
(
repository_path
)
logger
.
warning
(
"Error while getting repository, ignoring this test suite"
,
exc_info
=
1
)
...
...
erp5/util/testsuite/__init__.py
View file @
31804f68
from
__future__
import
print_function
import
argparse
import
re
,
os
,
shlex
,
glob
import
sys
,
threading
,
subprocess
import
traceback
import
errno
import
pprint
import
six
from
six.moves
import
range
from
erp5.util
import
taskdistribution
from
pprint
import
pprint
if
six
.
PY3
:
stdbin
=
lambda
x
:
x
.
buffer
else
:
stdbin
=
lambda
x
:
x
# PY3: use shlex.quote
_format_command_search
=
re
.
compile
(
"[[
\
\
s $({?*
\
\
`#~';<>&|]"
).
search
_format_command_escape
=
lambda
s
:
"'%s'"
%
r"'\''"
.
join
(
s
.
split
(
"'"
))
def
format_command
(
*
args
,
**
kw
):
...
...
@@ -31,7 +41,7 @@ def subprocess_capture(p, quiet=False):
buffer
.
append
(
data
)
if
p
.
stdout
:
stdout
=
[]
output
=
quiet
and
(
lambda
data
:
None
)
or
sys
.
stdout
.
write
output
=
(
lambda
data
:
None
)
if
quiet
else
stdbin
(
sys
.
stdout
)
.
write
stdout_thread
=
threading
.
Thread
(
target
=
readerthread
,
args
=
(
p
.
stdout
,
output
,
stdout
))
stdout_thread
.
setDaemon
(
True
)
...
...
@@ -39,7 +49,7 @@ def subprocess_capture(p, quiet=False):
if
p
.
stderr
:
stderr
=
[]
stderr_thread
=
threading
.
Thread
(
target
=
readerthread
,
args
=
(
p
.
stderr
,
sys
.
stderr
.
write
,
stderr
))
args
=
(
p
.
stderr
,
stdbin
(
sys
.
stderr
)
.
write
,
stderr
))
stderr_thread
.
setDaemon
(
True
)
stderr_thread
.
start
()
if
p
.
stdout
:
...
...
@@ -47,8 +57,8 @@ def subprocess_capture(p, quiet=False):
if
p
.
stderr
:
stderr_thread
.
join
()
p
.
wait
()
return
(
p
.
stdout
and
''
.
join
(
stdout
),
p
.
stderr
and
''
.
join
(
stderr
))
return
(
p
.
stdout
and
b
''
.
join
(
stdout
),
p
.
stderr
and
b
''
.
join
(
stderr
))
class
SubprocessError
(
EnvironmentError
):
def
__init__
(
self
,
status_dict
):
...
...
@@ -72,15 +82,15 @@ class Persistent(object):
def
__getattr__
(
self
,
attr
):
if
attr
==
'_db'
:
try
:
db
=
file
(
self
.
_filename
,
'r+'
)
except
IOError
,
e
:
db
=
open
(
self
.
_filename
,
'r+'
)
except
IOError
as
e
: