Commit 31804f68 authored by Bryton Lacquement's avatar Bryton Lacquement 🚪 Committed by Julien Muchembled

erp5.util: add support for Python 3

/reviewed-on nexedi/erp5!830
parent 5abb074d
This diff is collapsed.
......@@ -28,6 +28,7 @@
#
##############################################################################
from __future__ import print_function
import argparse
import os
import sys
......@@ -264,7 +265,7 @@ class PerformanceTester(object):
error_message = exit_msg_queue.get()
except KeyboardInterrupt, e:
print >>sys.stderr, "\nInterrupted by user, stopping gracefully..."
print("\nInterrupted by user, stopping gracefully...", file=sys.stderr)
exit_status = 2
# An IOError may be raised when receiving a SIGINT which interrupts the
......@@ -337,7 +338,7 @@ class PerformanceTester(object):
def main():
error_message_set, exit_status = PerformanceTester().run()
for error_message in error_message_set:
print >>sys.stderr, "ERROR: %s" % error_message
print("ERROR: %s" % error_message, file=sys.stderr)
sys.exit(exit_status)
......
......@@ -31,6 +31,7 @@
#
##############################################################################
from __future__ import print_function
import argparse
import re
......@@ -537,7 +538,7 @@ def generateReport():
for filename in filename_iter:
# There may be no results at all in case of errors
if not os.stat(filename).st_size:
print >>sys.stderr, "Ignoring empty file %s" % filename
print("Ignoring empty file %s" % filename, file=sys.stderr)
continue
report_dict = per_nb_users_report_dict.setdefault(
......@@ -546,10 +547,8 @@ def generateReport():
report_dict['filename'].append(filename)
if not per_nb_users_report_dict:
print >>sys.stderr, "ERROR: No result file found, perhaps "\
"``--filename-prefix'' should be specified?"
sys.exit(1)
sys.exit("ERROR: No result file found, perhaps ``--filename-prefix'' should"
"be specified?")
pdf = PdfPages(argument_namespace.output_filename)
......
......@@ -28,6 +28,7 @@
#
##############################################################################
from __future__ import print_function
from .result import CSVBenchmarkResult, NothingFlushedException
class CSVScalabilityBenchmarkResult(CSVBenchmarkResult):
......@@ -60,7 +61,7 @@ class ScalabilityTester(PerformanceTester):
urllib.urlencode({'error_message_set': '|'.join(error_message_set)})).close()
except:
print >>sys.stderr, "ERROR: %s" % Formatter().formatException(sys.exc_info())
print("ERROR: %s" % Formatter().formatException(sys.exc_info()), file=sys.stderr)
def getResultClass(self):
if not self._argument_namespace.erp5_publish_url:
......
......@@ -27,7 +27,7 @@ def main():
if error_message_set:
exit_status = 1
for error in error_message_set:
print error
print(error)
elif result:
print result
print(result)
sys.exit(exit_status)
......@@ -5,23 +5,15 @@ import os
import shutil
import time
import sys
import multiprocessing
import signal
import errno
import json
import logging
import logging.handlers
import glob
import urlparse
import httplib
import base64
import threading
from erp5.util.benchmark.argument import ArgumentType
from erp5.util.benchmark.performance_tester import PerformanceTester
from erp5.util.benchmark.thread import TestThread, TestMetricThread
from erp5.util import taskdistribution
from erp5.util.testnode import Utils
from erp5.util.testnode.ProcessManager import SubprocessError, ProcessManager, CancellationError
from erp5.util.testnode.ProcessManager import ProcessManager
import datetime
MAX_INSTALLATION_TIME = 60*50
......@@ -179,31 +171,28 @@ class ScalabilityLauncher(object):
"""
data_array = self.__argumentNamespace.current_test_data.split(',')
data = json.dumps({"count": data_array[0], "title": data_array[1], "relative_path": data_array[2]})
decoded_data = Utils.deunicodeData(json.loads(data))
return ScalabilityTest(decoded_data, self.test_result)
encoded_data = Utils.deunicodeData(json.loads(data))
return ScalabilityTest(encoded_data, self.test_result)
def clearUsersFile(self, user_file_path):
self.log("Clearing users file: %s" % user_file_path)
os.remove(user_file_path)
users_file = open(user_file_path, "w")
for line in self.users_file_original_content:
users_file.write(line)
users_file.close()
with open(user_file_path, "w") as users_file:
for line in self.users_file_original_content:
users_file.write(line)
def updateUsersFile(self, user_quantity, password, user_file_path):
self.log("Updating users file: %s" % user_file_path)
users_file = open(user_file_path, "r")
file_content = users_file.readlines()
with open(user_file_path, "r") as users_file:
file_content = users_file.readlines()
self.users_file_original_content = file_content
new_file_content = []
for line in file_content:
new_file_content.append(line.replace('<password>', password).replace('<user_quantity>', str(user_quantity)))
users_file.close()
os.remove(user_file_path)
users_file = open(user_file_path, "w")
for line in new_file_content:
users_file.write(line)
users_file.close()
with open(user_file_path, "w") as users_file:
for line in new_file_content:
users_file.write(line)
def run(self):
self.log("Scalability Launcher started, with:")
......
......@@ -40,11 +40,15 @@ Example use:
test_line.stop()
"""
from __future__ import print_function
import httplib
import six
from six.moves import (
map,
http_client as httplib,
xmlrpc_client as xmlrpclib,
)
import socket
import threading
import time
import xmlrpclib
__all__ = ['TaskDistributor', 'TestResultProxy', 'TestResultLineProxy', 'patchRPCParser']
......@@ -89,11 +93,17 @@ def patchRPCParser(error_handler):
def verbose_feed(self, data):
try:
return original_feed(self, data)
except Exception, exc:
except Exception as exc:
if not error_handler(data, exc):
raise
parser_klass.feed = verbose_feed
try: # PY3
basestring
except NameError:
basestring = bytes, str
unicode = str
def binarize_args(arg):
# Converts recursively basestring arg into xmlrpclib.Binary, as they can
# contain non-XML allowed characters
......@@ -102,9 +112,9 @@ def binarize_args(arg):
arg = arg.encode('utf-8')
return xmlrpclib.Binary(arg)
if isinstance(arg, (list, tuple, set)):
return map(binarize_args, arg)
return list(map(binarize_args, arg))
if isinstance(arg, dict):
return {k: binarize_args(v) for k, v in arg.iteritems()}
return {k: binarize_args(v) for k, v in six.iteritems(arg)}
return arg
class RPCRetry(object):
......@@ -350,7 +360,7 @@ class TestResultProxy(RPCRetry):
caption_list = []
append = caption_list.append
for name, (stream, max_history_bytes) in \
self._watcher_dict.iteritems():
six.iteritems(self._watcher_dict):
append('==> %s <==' % (name, ))
start = stream.tell()
stream.seek(0, 2)
......
......@@ -8,6 +8,7 @@
#
# TODO: There must be a better way than the code below to do that though...
from __future__ import print_function
import sys
from erp5.util.testbrowser.browser import Browser
......@@ -19,11 +20,8 @@ try:
user_nbr = int(user_nbr)
except ValueError:
print >>sys.stderr, "ERROR: Missing arguments: %s URL USERNAME " \
"PASSWORD NUMBER_OF_USERS NEW_USERNAME_PREFIX NEW_USERS_PASSWORD" % \
sys.argv[0]
sys.exit(1)
sys.exit("ERROR: Missing arguments: %s URL USERNAME PASSWORD NUMBER_OF_USERS "
"NEW_USERNAME_PREFIX NEW_USERS_PASSWORD" % sys.argv[0])
# Create a browser instance
browser = Browser(url, username, password)
......
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import division, print_function
from erp5.util.testbrowser.browser import Browser
ITERATION = 20
......@@ -89,4 +90,4 @@ if __name__ == '__main__':
counter += 1
for title, time_list in result_dict.iteritems():
print "%s: %.4fs" % (title, float(sum(time_list)) / ITERATION)
print("%s: %.4fs" % (title, sum(time_list) / ITERATION))
......@@ -32,6 +32,8 @@ import string
import random
from .Utils import createFolder
from six.moves import range
class SlapOSInstance(object):
"""
Base of an software instance,
......@@ -69,14 +71,14 @@ class NodeTestSuite(SlapOSInstance):
def createSuiteLog(self):
# /srv/slapgrid/slappartXX/srv/var/log/testnode/az-D27KqX7FxJ/suite.log
alphabets = string.digits + string.letters
alphabets = string.digits + string.ascii_letters
while 1:
log_folder_name = '%s-%s' % (self.reference,
''.join(random.choice(alphabets) for i in xrange(10)))
''.join(random.choice(alphabets) for i in range(10)))
log_folder_path = os.path.join(self.log_directory, log_folder_name)
try:
os.makedirs(log_folder_path)
except OSError, e:
except OSError as e:
if e.errno != errno.EEXIST:
raise
else:
......
......@@ -79,7 +79,8 @@ def subprocess_capture(p, log_prefix, get_output=True):
break
if get_output:
buffer.append(data)
log(log_prefix + data.rstrip('\n'))
log(log_prefix + (data if str is bytes else
data.decode('utf-8', errors='replace')).rstrip('\n'))
if p.stdout:
stdout = []
stdout_thread = threading.Thread(target=readerthread,
......@@ -97,8 +98,8 @@ def subprocess_capture(p, log_prefix, get_output=True):
stdout_thread.join()
if p.stderr:
stderr_thread.join()
return (p.stdout and ''.join(stdout),
p.stderr and ''.join(stderr))
return (p.stdout and b''.join(stdout),
p.stderr and b''.join(stderr))
def killCommand(pid):
"""
......@@ -109,7 +110,7 @@ def killCommand(pid):
try:
process = psutil.Process(pid)
process.suspend()
except psutil.Error, e:
except psutil.Error as e:
return
process_list = [process]
new_list = process.children(recursive=True)
......@@ -118,19 +119,19 @@ def killCommand(pid):
for child in new_list:
try:
child.suspend()
except psutil.Error, e:
except psutil.Error as e:
logger.debug("killCommand/suspend: %s", e)
time.sleep(1)
new_list = set(process.children(recursive=True)).difference(process_list)
for process in process_list:
try:
process.kill()
except psutil.Error, e:
except psutil.Error as e:
logger.debug("killCommand/kill: %s", e)
class ProcessManager(object):
stdin = file(os.devnull)
stdin = open(os.devnull)
def __init__(self, max_timeout=MAX_TIMEOUT):
self.process_pid_set = set()
......
......@@ -30,30 +30,31 @@ import subprocess
import sys
import time
import glob
import SlapOSControler
import SlapOSMasterCommunicator
from . import SlapOSControler, SlapOSMasterCommunicator
import json
import time
import shutil
import logging
import string
import random
import urlparse
from six.moves.urllib.parse import urlparse
import base64
import httplib
import Utils
from six.moves import http_client as httplib
from . import Utils
import requests
import slapos.slap
import cPickle as pickle
from ProcessManager import SubprocessError, ProcessManager, CancellationError
from six.moves import cPickle as pickle
from .ProcessManager import SubprocessError, ProcessManager, CancellationError
from subprocess import CalledProcessError
from Updater import Updater
from .Updater import Updater
from erp5.util import taskdistribution
from erp5.util.benchmark.thread import TestThread
# for dummy slapos answer
import signal
from . import logger
from six.moves import range
# max time to generate frontend instance: 1.5 hour
MAX_FRONTEND_TIME = 60*90
# max time to register instance to slapOSMaster: 5 minutes
......@@ -322,18 +323,16 @@ ces or already launched.")
software_hash_directory = self.testnode.config['slapos_binary'].rsplit("bin/slapos", 1)[0]
apache_htpasswd = software_hash_directory + "parts/apache/bin/htpasswd"
testsuite_directory = self.testnode.config['repository_path_list'][0].rsplit('/', 1)[0]
htaccess_file = open(testsuite_directory + HTACCESS, "w")
file_content = """
with open(testsuite_directory + HTACCESS, "w") as htaccess_file:
htaccess_file.write("""
AuthType Basic
AuthName "Password Protected Area"
AuthUserFile "%s%s"
Require valid-user
""" % (testsuite_directory, HTPASSWD)
htaccess_file.write(file_content)
htaccess_file.close()
""" % (testsuite_directory, HTPASSWD))
password_path = testsuite_directory + PASSWORD_FILE
with open(password_path, "w") as password_file:
password = ''.join(random.choice(string.digits + string.letters) for i in xrange(PASSWORD_LENGTH))
password = ''.join(random.choice(string.digits + string.ascii_letters) for i in range(PASSWORD_LENGTH))
password_file.write(password)
user = TESTNODE_USER
command = [apache_htpasswd, "-bc", testsuite_directory + HTPASSWD, user, password]
......@@ -363,7 +362,7 @@ Require valid-user
user, password = self.generateProfilePasswordAccess()
logger.info("Software Profile password: %s" % password)
self.reachable_profile = "https://%s:%s@%s" % (user, password,
os.path.join(urlparse.urlparse(self.testnode.config['frontend_url']).netloc,
os.path.join(urlparse(self.testnode.config['frontend_url']).netloc,
"software", self.randomized_path, "software.cfg"))
def prepareSlapOSForTestSuite(self, node_test_suite):
......@@ -526,7 +525,7 @@ Require valid-user
if not self.launchable:
return {'status_code' : 1, 'error_message': "Current test_suite is not actually launchable." }
configuration_list = node_test_suite.configuration_list
test_list = range(0, len(configuration_list))
test_list = list(range(len(configuration_list)))
try:
test_result_proxy = self.testnode.taskdistribution.createTestResult(
node_test_suite.revision, test_list,
......
......@@ -35,6 +35,8 @@ from slapos import client
from . import logger
from .Utils import createFolder
from six.moves import range
MAX_PARTITIONS = 10
MAX_SR_RETRIES = 3
......@@ -243,7 +245,7 @@ class SlapOSControler(object):
computer = slap.registerComputer(config['computer_id'])
# Call a method to ensure connection to master can be established
computer.getComputerPartitionList()
except slapos.slap.ConnectionError, e:
except slapos.slap.ConnectionError as e:
retries += 1
if retries >= 60:
raise
......@@ -270,7 +272,7 @@ class SlapOSControler(object):
# MySQL DB content) from previous runs. To support changes of partition
# naming scheme (which already happened), do this at instance_root level.
createFolder(instance_root, True)
for i in xrange(MAX_PARTITIONS):
for i in range(MAX_PARTITIONS):
# create partition and configure computer
# XXX: at the moment all partitions do share same virtual interface address
# this is not a problem as usually all services are on different ports
......@@ -278,7 +280,7 @@ class SlapOSControler(object):
partition_path = os.path.join(instance_root, partition_reference)
if not(os.path.exists(partition_path)):
os.mkdir(partition_path)
os.chmod(partition_path, 0750)
os.chmod(partition_path, 0o750)
computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({
'address': config['ipv4_address'],
'instance_root': instance_root,
......@@ -318,7 +320,7 @@ class SlapOSControler(object):
os.environ['PATH'] = environment['PATH']
# a SR may fail for number of reasons (incl. network failures)
# so be tolerant and run it a few times before giving up
for _ in xrange(MAX_SR_RETRIES):
for _ in range(MAX_SR_RETRIES):
status_dict = self.spawn(config['slapos_binary'],
'node', 'software', '--all',
'--pidfile', os.path.join(self.software_root, 'slapos-node.pid'),
......@@ -346,7 +348,7 @@ class SlapOSControler(object):
# try to run for all partitions as one partition may in theory request another one
# this not always is required but curently no way to know how "tree" of partitions
# may "expand"
for _ in xrange(max_quantity):
for _ in range(max_quantity):
status_dict = self.spawn(config['slapos_binary'], 'node', 'instance',
'--pidfile', os.path.join(self.instance_root, 'slapos-node.pid'),
'--cfg', self.slapos_config, raise_error_if_fail=False,
......
from __future__ import print_function
import datetime
import json
import traceback
......@@ -12,6 +14,8 @@ from requests.exceptions import HTTPError
from ..taskdistribution import SAFE_RPC_EXCEPTION_LIST
from . import logger
import six
# max time to instance changing state: 3 hour
MAX_INSTANCE_TIME = 60*60*3
......@@ -52,7 +56,7 @@ def retryOnNetworkFailure(func,
except _except_list:
traceback.print_exc()
print 'Network failure. Retry method %s in %i seconds' % (func, retry_time)
print('Network failure. Retry method %s in %i seconds' % (func, retry_time))
time.sleep(retry_time)
retry_time = min(retry_time*1.5, 640)
......@@ -92,8 +96,9 @@ class SlapOSMasterCommunicator(object):
if instance_title is not None:
self.name = instance_title
if request_kw is not None:
if isinstance(request_kw, basestring) or \
isinstance(request_kw, unicode):
if isinstance(request_kw, bytes):
self.request_kw = json.loads(request_kw.decode('utf-8'))
elif isinstance(request_kw, six.text_type):
self.request_kw = json.loads(request_kw)
else:
self.request_kw = request_kw
......@@ -214,7 +219,7 @@ class SlapOSMasterCommunicator(object):
result = self.hateoas_navigator.GET(url)
result = json.loads(result)
if result['_links'].get('action_object_slap', None) is None:
print result['links']
print(result['links'])
return None
object_link = self.hateoas_navigator.hateoasGetLinkFromLinks(
......@@ -385,8 +390,9 @@ class SlapOSTester(SlapOSMasterCommunicator):
self.name = name
self.computer_guid = computer_guid
if isinstance(request_kw, str) or \
isinstance(request_kw, unicode):
if isinstance(request_kw, bytes):
self.request_kw = json.loads(request_kw.decode('utf-8'))
elif isinstance(request_kw, six.text_type):
self.request_kw = json.loads(request_kw)
else:
self.request_kw = request_kw
......
......@@ -30,6 +30,7 @@ import re
from . import logger
from .ProcessManager import SubprocessError
from .Utils import rmtree
from slapos.util import bytes2str, str2bytes
SVN_UP_REV = re.compile(r'^(?:At|Updated to) revision (\d+).$')
SVN_CHANGED_REV = re.compile(r'^Last Changed Rev.*:\s*(\d+)', re.MULTILINE)
......@@ -82,7 +83,7 @@ class Updater(object):
# allow several processes clean the same folder at the same time
try:
os.remove(os.path.join(path, file))
except OSError, e:
except OSError as e:
if e.errno != errno.ENOENT:
raise
......@@ -96,7 +97,7 @@ class Updater(object):
**kw)
def _git(self, *args, **kw):
return self.spawn(self.git_binary, *args, **kw)['stdout'].strip()
return bytes2str(self.spawn(self.git_binary, *args, **kw)['stdout'].strip())
def git_update_server_info(self):
return self._git('update-server-info', '-f')
......@@ -219,7 +220,7 @@ class Updater(object):
self.deletePycFiles(path)
try:
status_dict = self.spawn(*args)
except SubprocessError, e:
except SubprocessError as e:
if 'cleanup' not in e.stderr:
raise
self.spawn('svn', 'cleanup', path)
......
......@@ -3,6 +3,9 @@ import stat
import shutil
import errno
import six
from six.moves import map
def rmtree(path):
"""Delete a path recursively.
......@@ -11,14 +14,22 @@ def rmtree(path):
def chmod_retry(func, failed_path, exc_info):
"""Make sure the directories are executable and writable.
"""
# Depending on the Python version, the following items differ.
if six.PY3:
expected_error_type = PermissionError
expected_func = os.lstat
else:
expected_error_type = OSError
expected_func = os.listdir
e = exc_info[1]
if isinstance(e, OSError):
if isinstance(e, expected_error_type):
if e.errno == errno.ENOENT:
# because we are calling again rmtree on listdir errors, this path might
# have been already deleted by the recursive call to rmtree.
return
if e.errno == errno.EACCES:
if func is os.listdir:
if func is expected_func:
os.chmod(failed_path, 0o700)
# corner case to handle errors in listing directories.
# https://bugs.python.org/issue8523
......@@ -39,12 +50,16 @@ def createFolder(folder, clean=False):
rmtree(folder)
os.mkdir(folder)
def deunicodeData(data):
if isinstance(data, list):
return map(deunicodeData, data)
if isinstance(data, unicode):
return data.encode('utf8')
if isinstance(data, dict):
return {deunicodeData(key): deunicodeData(value)
for key, value in data.iteritems()}
return data
if six.PY3:
def deunicodeData(data):
return data
else:
def deunicodeData(data):
if isinstance(data, list):
return list(map(deunicodeData, data))
if isinstance(data, unicode):
return data.encode('utf8')
if isinstance(data, dict):
return {deunicodeData(key): deunicodeData(value)
for key, value in six.iteritems(data)}
return data
......@@ -24,7 +24,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import ConfigParser
from six.moves import configparser
import argparse
import logging
import logging.handlers
......@@ -64,7 +64,7 @@ def main(*args):
CONFIG = {
'partition_reference': 'test0',
}
config = ConfigParser.SafeConfigParser()
config = configparser.SafeConfigParser()
# do not change case of option keys
config.optionxform = str
config.readfp(parsed_argument.configuration_file[0])
......
......@@ -171,7 +171,7 @@ shared = true
# only limit to particular error, if we run that code for all errors,
# then if server having most repositories is down for some time, we would
# erase all repositories and facing later hours of downloads
if getattr(error, 'stderr', '').find('index') >= 0:
if b'index' in getattr(error, 'stderr', b''):
rmtree(repository_path)
logger.warning("Error while getting repository, ignoring this test suite",
exc_info=1)
......
from __future__ import print_function
import argparse
import re, os, shlex, glob
import sys, threading, subprocess
import traceback
import errno
import pprint
import six
from six.moves import range
from erp5.util import taskdistribution
from pprint import pprint
if six.PY3:
stdbin = lambda x: x.buffer
else:
stdbin = lambda x: x
# PY3: use shlex.quote
_format_command_search = re.compile("[[\\s $({?*\\`#~';<>&|]").search
_format_command_escape = lambda s: "'%s'" % r"'\''".join(s.split("'"))
def format_command(*args, **kw):
......@@ -31,7 +41,7 @@ def subprocess_capture(p, quiet=False):
buffer.append(data)
if p.stdout:
stdout = []
output = quiet and (lambda data: None) or sys.stdout.write
output = (lambda data: None) if quiet else stdbin(sys.stdout).write
stdout_thread = threading.Thread(target=readerthread,
args=(p.stdout, output, stdout))
stdout_thread.setDaemon(True)
......@@ -39,7 +49,7 @@ def subprocess_capture(p, quiet=False):
if p.stderr:
stderr = []
stderr_thread = threading.Thread(target=readerthread,
args=(p.stderr, sys.stderr.write, stderr))
args=(p.stderr, stdbin(sys.stderr).write, stderr))
stderr_thread.setDaemon(True)
stderr_thread.start()
if p.stdout:
......@@ -47,8 +57,8 @@ def subprocess_capture(p, quiet=False):
if p.stderr:
stderr_thread.join()
p.wait()
return (p.stdout and ''.join(stdout),
p.stderr and ''.join(stderr))
return (p.stdout and b''.join(stdout),
p.stderr and b''.join(stderr))
class SubprocessError(EnvironmentError):
def __init__(self, status_dict):
......@@ -72,15 +82,15 @@ class Persistent(object):
def __getattr__(self, attr):
if attr == '_db':
try:
db = file(self._filename, 'r+')
except IOError, e:
db = open(self._filename, 'r+')
except IOError as e: