Skip to content

erp5
erp5
Commit 90d37ca8 authored by Bryton Lacquement's avatar Bryton Lacquement 🚪
Browse files
Options

erp5.util: add support for Python 3

parent e9389f1a
Hide whitespace changes
Inline Side-by-side
Showing with 192 additions and 164 deletions
erp5/tests/testERP5TestNode.py
View file @ 90d37ca8
......@@ -23,7 +23,6 @@ import sys
import tempfile
import json
import time
import types
import re
@contextmanager
......@@ -144,7 +143,8 @@ class ERP5TestNode(TestCase):
self.__dict__.update(**kw)
def __call__(self, command):
return subprocess.check_output(command, **self.__dict__)
return subprocess.check_output(command, universal_newlines=True,
**self.__dict__)
return Caller(**kw)
def generateTestRepositoryList(self, add_third_repository=False):
......@@ -338,7 +338,7 @@ shared = true
vcs_repository_info['branch'] = 'master'
rev_list = self.getAndUpdateFullRevisionList(test_node, node_test_suite)
output = call("git branch".split()).strip()
print output
print(output)
self.assertTrue("* master" in output.split('\n'))
# Add a third branch on remote, make sure we could switch to it
remote_call = self.getCaller(cwd=self.remote_repository2)
......@@ -489,35 +489,21 @@ shared = true
test_node.purgeOldTestSuite(test_suite_data)
self.assertEquals(['foo'], os.listdir(self.working_directory))
def test_purgeOldTestSuiteChmodNonWriteable(self):
def test_purgeOldTestSuiteChmod(self):
"""Old test suites can be deleted even when some files/directories have
been chmod'd to make non-writeable """
been chmod'd to make read only. """
test_node = self.getTestNode()
test_suite_data = self.getTestSuiteData(add_third_repository=True)
os.mkdir(os.path.join(self.working_directory, 'bar'))
non_writable_file = open(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 'w')
non_writable_file.close()
os.chmod(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 0o400) # -r--------
os.chmod(os.path.join(self.working_directory, 'bar'), 0o500) # dr-x------
# make this file and directory non writeable
os.chmod(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 0o000)
os.chmod(os.path.join(self.working_directory, 'bar'), 0o000)
test_node.purgeOldTestSuite(test_suite_data) # should not fail
self.assertEqual([], os.listdir(self.working_directory))
def test_purgeOldTestSuiteChmodNonWriteableNonReadable(self):
"""Old test suites can be deleted even when some files/directories have
been chmod'd to make them non readable and non writeable. """
test_node = self.getTestNode()
test_suite_data = self.getTestSuiteData(add_third_repository=True)
os.mkdir(os.path.join(self.working_directory, 'bar'))
non_writable_file = open(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 'w')
non_writable_file.close()
os.chmod(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 0o000) # ----------
os.chmod(os.path.join(self.working_directory, 'bar'), 0o000) # d---------
test_node.purgeOldTestSuite(test_suite_data) # should not fail
self.assertEqual([], os.listdir(self.working_directory))
def test_09_runTestSuite(self, my_test_type='UnitTest'):
"""
......@@ -986,7 +972,7 @@ shared = true
self.assertRaises(SubprocessError, callPrepareSlapOS)
self.assertEquals(node_test_suite.retry_software_count, 0)
for x in xrange(0,11):
for x in range(11):
callRaisingPrepareSlapos()
self.assertEquals(len(init_call_kw_list), 11)
self.assertEquals(init_call_kw_list[-1]['reset_software'], False)
......
erp5/util/scalability/requestUrl.py
View file @ 90d37ca8
......@@ -27,7 +27,7 @@ def main():
if error_message_set:
exit_status = 1
for error in error_message_set:
print error
print(error)
elif result:
print result
print(result)
sys.exit(exit_status)
erp5/util/scalability/runScalabilityTestSuite.py
View file @ 90d37ca8
......@@ -5,23 +5,15 @@ import os
import shutil
import time
import sys
import multiprocessing
import signal
import errno
import json
import logging
import logging.handlers
import glob
import urlparse
import httplib
import base64
import threading
from erp5.util.benchmark.argument import ArgumentType
from erp5.util.benchmark.performance_tester import PerformanceTester
from erp5.util.benchmark.thread import TestThread, TestMetricThread
from erp5.util import taskdistribution
from erp5.util.testnode import Utils
from erp5.util.testnode.ProcessManager import SubprocessError, ProcessManager, CancellationError
from erp5.util.testnode.ProcessManager import ProcessManager
import datetime
MAX_INSTALLATION_TIME = 60*50
......@@ -179,8 +171,8 @@ class ScalabilityLauncher(object):
"""
data_array = self.__argumentNamespace.current_test_data.split(',')
data = json.dumps({"count": data_array[0], "title": data_array[1], "relative_path": data_array[2]})
decoded_data = Utils.deunicodeData(json.loads(data))
return ScalabilityTest(decoded_data, self.test_result)
encoded_data = Utils.deunicodeData(json.loads(data))
return ScalabilityTest(encoded_data, self.test_result)
def clearUsersFile(self, user_file_path):
self.log("Clearing users file: %s" % user_file_path)
......
erp5/util/taskdistribution/__init__.py
View file @ 90d37ca8
......@@ -40,11 +40,15 @@ Example use:
test_line.stop()
"""
from __future__ import print_function
import httplib
import six
from six.moves import (
map,
http_client as httplib,
xmlrpc_client as xmlrpclib,
)
import socket
import threading
import time
import xmlrpclib
__all__ = ['TaskDistributor', 'TestResultProxy', 'TestResultLineProxy', 'patchRPCParser']
......@@ -89,11 +93,17 @@ def patchRPCParser(error_handler):
def verbose_feed(self, data):
try:
return original_feed(self, data)
except Exception, exc:
except Exception as exc:
if not error_handler(data, exc):
raise
parser_klass.feed = verbose_feed
try: # PY3
basestring
except NameError:
basestring = bytes, str
unicode = str
def binarize_args(arg):
# Converts recursively basestring arg into xmlrpclib.Binary, as they can
# contain non-XML allowed characters
......@@ -102,9 +112,9 @@ def binarize_args(arg):
arg = arg.encode('utf-8')
return xmlrpclib.Binary(arg)
if isinstance(arg, (list, tuple, set)):
return map(binarize_args, arg)
return list(map(binarize_args, arg))
if isinstance(arg, dict):
return {k: binarize_args(v) for k, v in arg.iteritems()}
return {k: binarize_args(v) for k, v in six.iteritems(arg)}
return arg
class RPCRetry(object):
......@@ -349,7 +359,7 @@ class TestResultProxy(RPCRetry):
caption_list = []
append = caption_list.append
for name, (stream, max_history_bytes) in \
self._watcher_dict.iteritems():
six.iteritems(self._watcher_dict):
append('==> %s <==' % (name, ))
start = stream.tell()
stream.seek(0, 2)
......
erp5/util/testnode/NodeTestSuite.py
View file @ 90d37ca8
......@@ -32,6 +32,8 @@ import string
import random
from .Utils import createFolder
from six.moves import range
class SlapOSInstance(object):
"""
Base of an software instance,
......@@ -68,14 +70,14 @@ class NodeTestSuite(SlapOSInstance):
def createSuiteLog(self):
# /srv/slapgrid/slappartXX/srv/var/log/testnode/az-D27KqX7FxJ/suite.log
alphabets = string.digits + string.letters
alphabets = string.digits + string.ascii_letters
while 1:
log_folder_name = '%s-%s' % (self.reference,
''.join(random.choice(alphabets) for i in xrange(10)))
''.join(random.choice(alphabets) for i in range(10)))
log_folder_path = os.path.join(self.log_directory, log_folder_name)
try:
os.makedirs(log_folder_path)
except OSError, e:
except OSError as e:
if e.errno != errno.EEXIST:
raise
else:
......
erp5/util/testnode/ProcessManager.py
View file @ 90d37ca8
......@@ -33,6 +33,7 @@ import signal
import sys
import time
from . import logger
from slapos.util import bytes2str
MAX_TIMEOUT = 3600 * 4
......@@ -79,7 +80,7 @@ def subprocess_capture(p, log_prefix, get_output=True):
break
if get_output:
buffer.append(data)
log(log_prefix + data.rstrip('\n'))
log(log_prefix + bytes2str(data).rstrip('\n'))
if p.stdout:
stdout = []
stdout_thread = threading.Thread(target=readerthread,
......@@ -97,8 +98,8 @@ def subprocess_capture(p, log_prefix, get_output=True):
stdout_thread.join()
if p.stderr:
stderr_thread.join()
return (p.stdout and ''.join(stdout),
p.stderr and ''.join(stderr))
return (p.stdout and b''.join(stdout),
p.stderr and b''.join(stderr))
def killCommand(pid):
"""
......@@ -109,7 +110,7 @@ def killCommand(pid):
try:
process = psutil.Process(pid)
process.suspend()
except psutil.Error, e:
except psutil.Error as e:
return
process_list = [process]
new_list = process.children(recursive=True)
......@@ -118,19 +119,19 @@ def killCommand(pid):
for child in new_list:
try:
child.suspend()
except psutil.Error, e:
except psutil.Error as e:
logger.debug("killCommand/suspend: %s", e)
time.sleep(1)
new_list = set(process.children(recursive=True)).difference(process_list)
for process in process_list:
try:
process.kill()
except psutil.Error, e:
except psutil.Error as e:
logger.debug("killCommand/kill: %s", e)
class ProcessManager(object):
stdin = file(os.devnull)
stdin = open(os.devnull)
def __init__(self, max_timeout=MAX_TIMEOUT):
self.process_pid_set = set()
......
erp5/util/testnode/ScalabilityTestRunner.py
View file @ 90d37ca8
......@@ -30,30 +30,31 @@ import subprocess
import sys
import time
import glob
import SlapOSControler
import SlapOSMasterCommunicator
from . import SlapOSControler, SlapOSMasterCommunicator
import json
import time
import shutil
import logging
import string
import random
import urlparse
from six.moves.urllib import parse
import base64
import httplib
import Utils
from six.moves import http_client as httplib
from . import Utils
import requests
import slapos.slap
import cPickle as pickle
from ProcessManager import SubprocessError, ProcessManager, CancellationError
from six.moves import cPickle as pickle
from .ProcessManager import SubprocessError, ProcessManager, CancellationError
from subprocess import CalledProcessError
from Updater import Updater
from .Updater import Updater
from erp5.util import taskdistribution
from erp5.util.benchmark.thread import TestThread
# for dummy slapos answer
import signal
from . import logger
from six.moves import range
# max time to generate frontend instance: 1.5 hour
MAX_FRONTEND_TIME = 60*90
# max time to register instance to slapOSMaster: 5 minutes
......@@ -333,7 +334,7 @@ Require valid-user
htaccess_file.close()
password_path = testsuite_directory + PASSWORD_FILE
with open(password_path, "w") as password_file:
password = ''.join(random.choice(string.digits + string.letters) for i in xrange(PASSWORD_LENGTH))
password = ''.join(random.choice(string.digits + string.letters) for i in range(PASSWORD_LENGTH))
password_file.write(password)
user = TESTNODE_USER
command = [apache_htpasswd, "-bc", testsuite_directory + HTPASSWD, user, password]
......@@ -363,7 +364,7 @@ Require valid-user
user, password = self.generateProfilePasswordAccess()
logger.info("Software Profile password: %s" % password)
self.reachable_profile = "https://%s:%s@%s" % (user, password,
os.path.join(urlparse.urlparse(self.testnode.config['frontend_url']).netloc,
os.path.join(parse.urlparse(self.testnode.config['frontend_url']).netloc,
"software", self.randomized_path, "software.cfg"))
def prepareSlapOSForTestSuite(self, node_test_suite):
......@@ -526,7 +527,7 @@ Require valid-user
if not self.launchable:
return {'status_code' : 1, 'error_message': "Current test_suite is not actually launchable." }
configuration_list = node_test_suite.configuration_list
test_list = range(0, len(configuration_list))
test_list = list(range(len(configuration_list)))
try:
test_result_proxy = self.testnode.taskdistribution.createTestResult(
node_test_suite.revision, test_list,
......
erp5/util/testnode/SlapOSControler.py
View file @ 90d37ca8
......@@ -35,6 +35,8 @@ from slapos import client
from . import logger
from .Utils import createFolder
from six.moves import range
MAX_PARTITIONS = 10
MAX_SR_RETRIES = 3
......@@ -245,7 +247,7 @@ class SlapOSControler(object):
computer = slap.registerComputer(config['computer_id'])
# Call a method to ensure connection to master can be established
computer.getComputerPartitionList()
except slapos.slap.ConnectionError, e:
except slapos.slap.ConnectionError as e:
retries += 1
if retries >= 60:
raise
......@@ -272,7 +274,7 @@ class SlapOSControler(object):
# MySQL DB content) from previous runs. To support changes of partition
# naming scheme (which already happened), do this at instance_root level.
createFolder(instance_root, True)
for i in xrange(MAX_PARTITIONS):
for i in range(MAX_PARTITIONS):
# create partition and configure computer
# XXX: at the moment all partitions do share same virtual interface address
# this is not a problem as usually all services are on different ports
......@@ -280,7 +282,7 @@ class SlapOSControler(object):
partition_path = os.path.join(instance_root, partition_reference)
if not(os.path.exists(partition_path)):
os.mkdir(partition_path)
os.chmod(partition_path, 0750)
os.chmod(partition_path, 0o750)
computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({
'address': config['ipv4_address'],
'instance_root': instance_root,
......@@ -320,7 +322,7 @@ class SlapOSControler(object):
os.environ['PATH'] = environment['PATH']
# a SR may fail for number of reasons (incl. network failures)
# so be tolerant and run it a few times before giving up
for _ in xrange(MAX_SR_RETRIES):
for _ in range(MAX_SR_RETRIES):
status_dict = self.spawn(config['slapos_binary'],
'node', 'software', '--all',
'--pidfile', os.path.join(self.software_root, 'slapos-node.pid'),
......@@ -348,7 +350,7 @@ class SlapOSControler(object):
# try to run for all partitions as one partition may in theory request another one
# this not always is required but curently no way to know how "tree" of partitions
# may "expand"
for _ in xrange(max_quantity):
for _ in range(max_quantity):
status_dict = self.spawn(config['slapos_binary'], 'node', 'instance',
'--pidfile', os.path.join(self.instance_root, 'slapos-node.pid'),
'--cfg', self.slapos_config, raise_error_if_fail=False,
......
erp5/util/testnode/SlapOSMasterCommunicator.py
View file @ 90d37ca8
from __future__ import print_function
import datetime
import json
import traceback
......@@ -9,9 +11,11 @@ from uritemplate import expand
import slapos.slap
from slapos.slap import SoftwareProductCollection
from requests.exceptions import HTTPError
from ..taskdistribution import SAFE_RPC_EXCEPTION_LIST
from erp5.util.taskdistribution import SAFE_RPC_EXCEPTION_LIST
from . import logger
import six
# max time to instance changing state: 3 hour
MAX_INSTANCE_TIME = 60*60*3
......@@ -52,7 +56,7 @@ def retryOnNetworkFailure(func,
except _except_list:
traceback.print_exc()
print 'Network failure. Retry method %s in %i seconds' % (func, retry_time)
print('Network failure. Retry method %s in %i seconds' % (func, retry_time))
time.sleep(retry_time)
retry_time = min(retry_time*1.5, 640)
......@@ -92,8 +96,7 @@ class SlapOSMasterCommunicator(object):
if instance_title is not None:
self.name = instance_title
if request_kw is not None:
if isinstance(request_kw, basestring) or \
isinstance(request_kw, unicode):
if isinstance(request_kw, (six.binary_type, six.text_type)):
self.request_kw = json.loads(request_kw)
else:
self.request_kw = request_kw
......@@ -214,7 +217,7 @@ class SlapOSMasterCommunicator(object):
result = self.hateoas_navigator.GET(url)
result = json.loads(result)
if result['_links'].get('action_object_slap', None) is None:
print result['links']
print(result['links'])
return None
object_link = self.hateoas_navigator.hateoasGetLinkFromLinks(
......@@ -385,8 +388,7 @@ class SlapOSTester(SlapOSMasterCommunicator):
self.name = name
self.computer_guid = computer_guid
if isinstance(request_kw, str) or \
isinstance(request_kw, unicode):
if isinstance(request_kw, (six.binary_type, six.text_type)):
self.request_kw = json.loads(request_kw)
else:
self.request_kw = request_kw
......
erp5/util/testnode/Updater.py
View file @ 90d37ca8
......@@ -30,6 +30,7 @@ import re
from . import logger
from .ProcessManager import SubprocessError
from .Utils import rmtree
from slapos.util import bytes2str, str2bytes
SVN_UP_REV = re.compile(r'^(?:At|Updated to) revision (\d+).$')
SVN_CHANGED_REV = re.compile(r'^Last Changed Rev.*:\s*(\d+)', re.MULTILINE)
......@@ -82,7 +83,7 @@ class Updater(object):
# allow several processes clean the same folder at the same time
try:
os.remove(os.path.join(path, file))
except OSError, e:
except OSError as e:
if e.errno != errno.ENOENT:
raise
......@@ -96,7 +97,7 @@ class Updater(object):
**kw)
def _git(self, *args, **kw):
return self.spawn(self.git_binary, *args, **kw)['stdout'].strip()
return bytes2str(self.spawn(self.git_binary, *args, **kw)['stdout'].strip())
def git_update_server_info(self):
return self._git('update-server-info', '-f')
......@@ -219,7 +220,7 @@ class Updater(object):
self.deletePycFiles(path)
try:
status_dict = self.spawn(*args)
except SubprocessError, e:
except SubprocessError as e:
if 'cleanup' not in e.stderr:
raise
self.spawn('svn', 'cleanup', path)
......
erp5/util/testnode/Utils.py
View file @ 90d37ca8
......@@ -3,6 +3,9 @@ import stat
import shutil
import errno
import six
from six.moves import map
def rmtree(path):
"""Delete a path recursively.
......@@ -11,14 +14,22 @@ def rmtree(path):
def chmod_retry(func, failed_path, exc_info):
"""Make sure the directories are executable and writable.
"""
# Depending on the Python version, the following items differ.
if six.PY3:
expected_error_type = PermissionError
expected_func = os.lstat
else:
expected_error_type = OSError
expected_func = os.listdir
e = exc_info[1]
if isinstance(e, OSError):
if isinstance(e, expected_error_type):
if e.errno == errno.ENOENT:
# because we are calling again rmtree on listdir errors, this path might
# have been already deleted by the recursive call to rmtree.
return
if e.errno == errno.EACCES:
if func is os.listdir:
if func is expected_func:
os.chmod(failed_path, 0o700)
# corner case to handle errors in listing directories.
# https://bugs.python.org/issue8523
......@@ -39,12 +50,16 @@ def createFolder(folder, clean=False):
rmtree(folder)
os.mkdir(folder)
def deunicodeData(data):
if isinstance(data, list):
return map(deunicodeData, data)
if isinstance(data, unicode):
return data.encode('utf8')
if isinstance(data, dict):
return {deunicodeData(key): deunicodeData(value)
for key, value in data.iteritems()}
return data
if six.PY3:
def deunicodeData(data):
return data
else:
def deunicodeData(data):
if isinstance(data, list):
return list(map(deunicodeData, data))
if isinstance(data, unicode):
return data.encode('utf8')
if isinstance(data, dict):
return {deunicodeData(key): deunicodeData(value)
for key, value in six.iteritems(data)}
return data
erp5/util/testnode/__init__.py
View file @ 90d37ca8
......@@ -24,7 +24,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import ConfigParser
from six.moves import configparser
import argparse
import logging
import logging.handlers
......@@ -64,7 +64,7 @@ def main(*args):
CONFIG = {
'partition_reference': 'test0',
}
config = ConfigParser.SafeConfigParser()
config = configparser.SafeConfigParser()
# do not change case of option keys
config.optionxform = str
config.readfp(parsed_argument.configuration_file[0])
......
erp5/util/testnode/testnode.py
View file @ 90d37ca8
......@@ -39,7 +39,7 @@ from .ScalabilityTestRunner import ScalabilityTestRunner
from .UnitTestRunner import UnitTestRunner
from .Utils import deunicodeData
from .Utils import rmtree
from .. import taskdistribution
from erp5.util import taskdistribution
MAX_LOG_TIME = 15 # time in days we should keep logs that we can see through
# httd
......
erp5/util/testsuite/__init__.py
View file @ 90d37ca8
from __future__ import print_function
import argparse
import re, os, shlex, glob
import sys, threading, subprocess
import traceback
import errno
import pprint
import six
from six.moves import range
from erp5.util import taskdistribution
from pprint import pprint
if six.PY3:
stdbin = lambda x: x.buffer
else:
stdbin = lambda x: x
# PY3: use shlex.quote
_format_command_search = re.compile("[[\\s $({?*\\`#~';<>&|]").search
_format_command_escape = lambda s: "'%s'" % r"'\''".join(s.split("'"))
def format_command(*args, **kw):
......@@ -31,7 +41,7 @@ def subprocess_capture(p, quiet=False):
buffer.append(data)
if p.stdout:
stdout = []
output = quiet and (lambda data: None) or sys.stdout.write
output = (lambda data: None) if quiet else stdbin(sys.stdout).write
stdout_thread = threading.Thread(target=readerthread,
args=(p.stdout, output, stdout))
stdout_thread.setDaemon(True)
......@@ -39,7 +49,7 @@ def subprocess_capture(p, quiet=False):
if p.stderr:
stderr = []
stderr_thread = threading.Thread(target=readerthread,
args=(p.stderr, sys.stderr.write, stderr))
args=(p.stderr, stdbin(sys.stderr).write, stderr))
stderr_thread.setDaemon(True)
stderr_thread.start()
if p.stdout:
......@@ -47,8 +57,8 @@ def subprocess_capture(p, quiet=False):
if p.stderr:
stderr_thread.join()
p.wait()
return (p.stdout and ''.join(stdout),
p.stderr and ''.join(stderr))
return (p.stdout and b''.join(stdout),
p.stderr and b''.join(stderr))
class SubprocessError(EnvironmentError):
def __init__(self, status_dict):
......@@ -72,15 +82,15 @@ class Persistent(object):
def __getattr__(self, attr):
if attr == '_db':
try:
db = file(self._filename, 'r+')
except IOError, e:
db = open(self._filename, 'r+')
except IOError as e:
if e.errno != errno.ENOENT:
raise
db = file(self._filename, 'w+')
db = open(self._filename, 'w+')
else:
try:
self.__dict__.update(eval(db.read()))
except StandardError:
except Exception:
pass
self._db = db
return db
......@@ -89,7 +99,7 @@ class Persistent(object):
def sync(self):
self._db.seek(0)
db = dict(x for x in self.__dict__.iteritems() if x[0][:1] != '_')
db = dict(x for x in six.iteritems(self.__dict__) if x[0][:1] != '_')
pprint.pprint(db, self._db)
self._db.truncate()
......@@ -103,10 +113,10 @@ class TestSuite(object):
"""
RUN_RE = re.compile(
r'Ran (?P<all_tests>\d+) tests? in (?P<seconds>\d+\.\d+)s',
br'Ran (?P<all_tests>\d+) tests? in (?P<seconds>\d+\.\d+)s',
re.DOTALL)
STATUS_RE = re.compile(r"""
STATUS_RE = re.compile(br"""
(OK|FAILED)\s+\(
(failures=(?P<failures>\d+),?\s*)?
(errors=(?P<errors>\d+),?\s*)?
......@@ -117,7 +127,7 @@ class TestSuite(object):
""", re.DOTALL | re.VERBOSE)
SUB_STATUS_RE = re.compile(
r"""SUB\s+RESULT:\s+(?P<all_tests>\d+)\s+Tests,\s+
br"""SUB\s+RESULT:\s+(?P<all_tests>\d+)\s+Tests,\s+
(?P<failures>\d+)\s+Failures\s*
\(?
(skipped=(?P<skips>\d+),?\s*)?
......@@ -130,7 +140,10 @@ class TestSuite(object):
mysql_db_count = 1
allow_restart = False
realtime_output = True
stdin = file(os.devnull)
try: # PY3
stdin = subprocess.DEVNULL
except AttributeError:
stdin = open(os.devnull, 'rb')
def __init__(self, max_instance_count, **kw):
self.__dict__.update(kw)
......@@ -139,8 +152,8 @@ class TestSuite(object):
self.acquire = pool.acquire
self.release = pool.release
self._instance = threading.local()
self._pool = max_instance_count == 1 and [None] or \
range(1, max_instance_count + 1)
self._pool = [None] if max_instance_count == 1 else \
list(range(1, max_instance_count + 1))
self._ready = set()
self.running = {}
if max_instance_count != 1:
......@@ -185,13 +198,14 @@ class TestSuite(object):
def spawn(self, *args, **kw):
quiet = kw.pop('quiet', False)
cwd = kw.pop('cwd', None)
env = kw and dict(os.environ, **kw) or None
command = format_command(*args, **kw)
print '\n$ ' + command
print('\n$ ' + command)
sys.stdout.flush()
try:
p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env)
stderr=subprocess.PIPE, env=env, cwd=cwd)
except Exception:
# Catch any exception here, to warn user instead of beeing silent,
# by generating fake error result
......@@ -229,20 +243,11 @@ class EggTestSuite(TestSuite):
The python interpreter is ``python_interpreter``
"""
def run(self, test):
print test
original_dir = os.getcwd()
try:
os.chdir(self.egg_test_path_dict[test])
return self.runUnitTest(test)
finally:
os.chdir(original_dir)
def runUnitTest(self, *args, **kw):
print(test)
try:
runUnitTest = "{python} setup.py test".format(python=self.python_interpreter)
args = tuple(shlex.split(runUnitTest))
status_dict = self.spawn(*args, **kw)
except SubprocessError, e:
status_dict = self.spawn(self.python_interpreter, 'setup.py', 'test',
cwd=self.egg_test_path_dict[test])
except SubprocessError as e:
status_dict = e.status_dict
test_log = status_dict['stderr']
search = self.RUN_RE.search(test_log)
......@@ -270,7 +275,7 @@ class EggTestSuite(TestSuite):
return status_dict
def getTestList(self):
return self.egg_test_path_dict.keys()
return list(self.egg_test_path_dict)
def runTestSuite():
parser = argparse.ArgumentParser(description='Run a test suite.')
......@@ -327,7 +332,7 @@ def runTestSuite():
if test_result is not None:
assert revision == test_result.revision, (revision, test_result.revision)
while suite.acquire():
test = test_result.start(suite.running.keys())
test = test_result.start(list(suite.running))
if test is not None:
suite.start(test.name, lambda status_dict, __test=test:
__test.stop(**status_dict))
......
erp5/util/timinglogparser/__init__.py
View file @ 90d37ca8
......@@ -27,6 +27,8 @@
#
##############################################################################
from __future__ import print_function
import os
import sys
import imp
......@@ -126,7 +128,7 @@ def parseFile(filename, measure_dict):
sys.stderr.flush()
match_list = LINE_PATTERN.findall(line)
if len(match_list) != 1:
print >>sys.stderr, 'Unparseable line: %s:%i %r' % (filename, line_number, line)
print('Unparseable line: %s:%i %r' % (filename, line_number, line), file=sys.stderr)
else:
result, filter_id, date, duration = processLine(match_list[0], filename, line_number)
# Possible result values & meaning:
......@@ -135,20 +137,21 @@ def parseFile(filename, measure_dict):
# (string): use & skip to next line
if result is False:
if debug:
print >>sys.stderr, '? %s:%i %r' % (filename, line_number, match_list[0])
print('? %s:%i %r' % (filename, line_number, match_list[0]), file=sys.stderr)
elif result is True:
if debug:
print >>sys.stderr, '- %s:%i %r' % (filename, line_number, match_list[0])
print('- %s:%i %r' % (filename, line_number, match_list[0]), file=sys.stderr)
skip_count += 1
else:
measure_dict.setdefault(filter_id, {}).setdefault(result, {}).setdefault(date, []).append(int(duration))
match_count += 1
line = logfile.readline()
print >>sys.stderr, '%i' % (line_number, )
print('%i' % (line_number, ), file=sys.stderr)
if line_number > 0:
duration = time() - begin
print >>sys.stderr, "Matched %i lines (%.2f%%), %i skipped (%.2f%%), %i unmatched (%.2f%%) in %.2fs (%i lines per second)." % \
(match_count, (float(match_count) / line_number) * 100, skip_count, (float(skip_count) / line_number) * 100, (line_number - match_count - skip_count), (1 - (float(match_count + skip_count) / line_number)) * 100, duration, line_number / duration)
print("Matched %i lines (%.2f%%), %i skipped (%.2f%%), %i unmatched (%.2f%%) in %.2fs (%i lines per second)." % \
(match_count, (float(match_count) / line_number) * 100, skip_count, (float(skip_count) / line_number) * 100, (line_number - match_count - skip_count), (1 - (float(match_count + skip_count) / line_number)) * 100, duration, line_number / duration),
file=sys.stderr)
debug = False
outfile_prefix = None
......@@ -161,9 +164,9 @@ decimate_count = 1
try:
opts, file_list = getopt.getopt(sys.argv[1:], '', ['debug', 'config=', 'prefix=', 'no-average', 'sum', 'load=', 'save=', 'decimate='])
except Exception, reason:
print >>sys.stderr, reason
print >>sys.stderr, usage
except Exception as reason:
print(reason, file=sys.stderr)
print(usage, file=sys.stderr)
sys.exit(1)
for name, value in opts:
......@@ -185,7 +188,7 @@ for name, value in opts:
decimate_count = int(value)
if configuration is None:
raise ValueError, '--config is mandatory'
raise ValueError('--config is mandatory')
config_file = os.path.splitext(os.path.basename(configuration))[0]
config_path = [os.path.dirname(os.path.abspath(configuration))] + sys.path
......@@ -211,18 +214,18 @@ if len(load_file_name_list):
for result, date_dict in result_dict.iteritems():
for date, duration_list in date_dict.iteritems():
measure_dict.setdefault(filter_id, {}).setdefault(result, {}).setdefault(date, []).extend(duration_list)
print >>sys.stderr, 'Previous processing result restored from %r' % (load_file_name, )
print('Previous processing result restored from %r' % (load_file_name, ), file=sys.stderr)
for filename in file_list:
file_number += 1
print >>sys.stderr, 'Processing %s [%i/%i]...' % (filename, file_number, file_count)
print('Processing %s [%i/%i]...' % (filename, file_number, file_count), file=sys.stderr)
parseFile(filename, measure_dict)
if save_file_name is not None:
save_file = open(save_file_name, 'w')
save_file.write(repr(measure_dict))
save_file.close()
print >>sys.stderr, 'Processing result saved to %r' % (save_file_name, )
print('Processing result saved to %r' % (save_file_name, ), file=sys.stderr)
if outfile_prefix is not None:
## Generate a list of all measures and a 2-levels dictionnary with date as key and measure dictionnary as value
......@@ -252,9 +255,9 @@ if outfile_prefix is not None:
def renderOutput(data_format, filename_suffix):
for sheet_id, sheet_column_list in sheet_dict.iteritems():
outfile_name = '%s_%s_%s.csv' % (outfile_prefix, sheet_id, filename_suffix)
print >>sys.stderr, 'Writing to %r...' % (outfile_name, )
print('Writing to %r...' % (outfile_name, ), file=sys.stderr)
outfile = open(outfile_name, 'w')
print >>outfile, '"date",%s' % (','.join(['"%s"' % (x[0], ) for x in sheet_column_list]), )
print('"date",%s' % (','.join(['"%s"' % (x[0], ) for x in sheet_column_list]), ), file=outfile)
decimate_dict = {}
decimate = 0
for date in date_list:
......@@ -262,11 +265,11 @@ if outfile_prefix is not None:
decimate_dict.setdefault(key, []).extend(value)
decimate += 1
if decimate == decimate_count:
print >>outfile, '"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list]))
print('"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list])), file=outfile)
decimate_dict = {}
decimate = 0
if len(decimate_dict):
print >>outfile, '"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list]))
print('"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list])), file=outfile)
if do_average:
renderOutput('=%(sum)i/%(count)i', 'avg')
......
erp5/util/timinglogplotter/__init__.py
View file @ 90d37ca8
......@@ -27,6 +27,8 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import print_function
from datetime import date
from os import path
import rpy2.robjects as robjects
......@@ -121,7 +123,7 @@ def main():
current_dir = os.getcwd()
for file_name in file_name_list:
print 'Loading %s...' % (file_name, )
print('Loading %s...' % (file_name, ))
file = CSVFile(file_name)
date_string_list = file.getColumn(0)
......@@ -183,13 +185,13 @@ def main():
y_data.append(value)
i += 1
if len(x_data) == 0:
print 'Nothing to plot for %s...' % (out_file_name, )
print('Nothing to plot for %s...' % (out_file_name, ))
continue
if options.minimal_non_empty_ratio is not None:
column_len = len(column)
if column_len:
if float(len(x_data))/column_len < options.minimal_non_empty_ratio:
print 'Not enough values to plot for %s...' % (out_file_name, )
print('Not enough values to plot for %s...' % (out_file_name, ))
continue
r_y_data = robjects.FloatVector(y_data)
r_x_data = robjects.FloatVector(x_data)
......@@ -220,7 +222,7 @@ def main():
# stop changing the out-type file
r("""dev.off()""")
print 'Saving %s...' % (out_file_name, )
print('Saving %s...' % (out_file_name, ))
if __name__ == '__main__':
main()
......
erp5/util/webchecker/__init__.py
View file @ 90d37ca8
......@@ -26,6 +26,8 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import print_function
import os
import shutil
import sys
......@@ -543,7 +545,7 @@ class HTTPCacheCheckerTestSuite(object):
from optparse import OptionParser
import ConfigParser
from six.moves import configparser
def _formatConfiguration(configuration):
""" format the configuration"""
......@@ -559,11 +561,11 @@ def web_checker_utility():
(options, args) = parser.parse_args()
if len(args) != 1 :
print parser.print_help()
print(parser.print_help())
parser.error('incorrect number of arguments')
config_path = args[0]
config = ConfigParser.RawConfigParser()
config = configparser.RawConfigParser()
config.read(config_path)
working_directory = config.get('web_checker', 'working_directory')
url = config.get('web_checker', 'url')
......@@ -619,6 +621,6 @@ def web_checker_utility():
file_object.write(result)
file_object.close()
else:
print result
print(result)
product/ERP5/bin/genbt5list
View file @ 90d37ca8
......@@ -35,12 +35,15 @@ import posixpath
import tarfile
import os
import sys
import cgi
try:
from html import escape
except ImportError:
from cgi import escape
from base64 import b64encode
from cStringIO import StringIO
from six import BytesIO
from hashlib import sha1
from urllib import unquote
from six.moves.urllib.parse import unquote
import six
# Order is important for installation
# We want to have:
......@@ -107,11 +110,11 @@ item_name_list = tuple('_%s_item' % x for x in item_name_list)
class BusinessTemplateRevision(list):
def hash(self, path, text):
self.append((path, sha1(text).digest()))
self.append((path.encode('utf-8'), sha1(text).digest()))
def digest(self):
self.sort()
return b64encode(sha1('\0'.join(h + p for (h, p) in self)).digest())
return b64encode(sha1(b'\0'.join(h + p for (h, p) in self)).digest())
class BusinessTemplate(dict):
......@@ -149,7 +152,7 @@ force_install
def __iter__(self):
self['revision'] = self.revision.digest()
return iter(sorted(self.iteritems()))
return iter(sorted(six.iteritems(self)))
@classmethod
def fromTar(cls, tar):
......@@ -177,8 +180,8 @@ force_install
return iter(self)
def generateInformation(dir, info=id, err=None):
xml = StringIO()
xml.write('<?xml version="1.0"?>\n<repository>\n')
xml = BytesIO()
xml.write(b'<?xml version="1.0"?>\n<repository>\n')
for name in sorted(os.listdir(dir)):
path = os.path.join(dir, name)
if name.endswith('.bt5'):
......@@ -199,13 +202,14 @@ def generateInformation(dir, info=id, err=None):
property_list = BusinessTemplate.fromDir(path)
else:
continue
xml.write(' <template id="%s">\n' % name)
xml.write(b' <template id="%s">\n' % name.encode('utf-8'))
for k, v in property_list:
for v in (v,) if type(v) is str else v:
xml.write(' <%s>%s</%s>\n' % (k, cgi.escape(v), k))
xml.write(' </template>\n')
k = k.encode('utf-8')
for v in (v,) if type(v) is bytes else v:
xml.write(b' <%s>%s</%s>\n' % (k, escape(v.decode('utf-8')).encode('utf-8'), k))
xml.write(b' </template>\n')
info('done\n')
xml.write('</repository>\n')
xml.write(b'</repository>\n')
return xml
def main(dir_list=None, **kw):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7