From 22a3bb66bfc4448b6229b26d99b0db383f1ad18b Mon Sep 17 00:00:00 2001 From: Bryton Lacquement Date: Thu, 13 Sep 2018 16:31:27 +0200 Subject: [PATCH 01/14] erp5.util: add support for Python 3 --- erp5/tests/testERP5TestNode.py | 32 +++------ erp5/util/scalability/requestUrl.py | 4 +- .../scalability/runScalabilityTestSuite.py | 14 +--- erp5/util/taskdistribution/__init__.py | 22 ++++-- erp5/util/testnode/NodeTestSuite.py | 8 ++- erp5/util/testnode/ProcessManager.py | 15 ++-- erp5/util/testnode/ScalabilityTestRunner.py | 23 +++--- erp5/util/testnode/SlapOSControler.py | 12 ++-- .../util/testnode/SlapOSMasterCommunicator.py | 16 +++-- erp5/util/testnode/Updater.py | 7 +- erp5/util/testnode/Utils.py | 37 +++++++--- erp5/util/testnode/__init__.py | 4 +- erp5/util/testnode/testnode.py | 2 +- erp5/util/testsuite/__init__.py | 71 ++++++++++--------- erp5/util/timinglogparser/__init__.py | 37 +++++----- erp5/util/timinglogplotter/__init__.py | 10 +-- erp5/util/webchecker/__init__.py | 10 +-- product/ERP5/bin/genbt5list | 32 +++++---- 18 files changed, 192 insertions(+), 164 deletions(-) diff --git a/erp5/tests/testERP5TestNode.py b/erp5/tests/testERP5TestNode.py index 7bf7037cc3a..6e926457596 100644 --- a/erp5/tests/testERP5TestNode.py +++ b/erp5/tests/testERP5TestNode.py @@ -23,7 +23,6 @@ import sys import tempfile import json import time -import types import re @contextmanager @@ -144,7 +143,8 @@ class ERP5TestNode(TestCase): self.__dict__.update(**kw) def __call__(self, command): - return subprocess.check_output(command, **self.__dict__) + return subprocess.check_output(command, universal_newlines=True, + **self.__dict__) return Caller(**kw) def generateTestRepositoryList(self, add_third_repository=False): @@ -338,7 +338,7 @@ shared = true vcs_repository_info['branch'] = 'master' rev_list = self.getAndUpdateFullRevisionList(test_node, node_test_suite) output = call("git branch".split()).strip() - print output + print(output) self.assertTrue("* master" in output.split('\n')) # Add a third branch on remote, make sure we could switch to it remote_call = self.getCaller(cwd=self.remote_repository2) @@ -521,35 +521,21 @@ shared = true test_node.purgeOldTestSuite(test_suite_data) self.assertEquals(['foo'], os.listdir(self.working_directory)) - def test_purgeOldTestSuiteChmodNonWriteable(self): + def test_purgeOldTestSuiteChmod(self): """Old test suites can be deleted even when some files/directories have - been chmod'd to make non-writeable """ + been chmod'd to make read only. """ test_node = self.getTestNode() test_suite_data = self.getTestSuiteData(add_third_repository=True) os.mkdir(os.path.join(self.working_directory, 'bar')) non_writable_file = open(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 'w') non_writable_file.close() - - os.chmod(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 0o400) # -r-------- - os.chmod(os.path.join(self.working_directory, 'bar'), 0o500) # dr-x------ + # make this file and directory non writeable + os.chmod(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 0o000) + os.chmod(os.path.join(self.working_directory, 'bar'), 0o000) test_node.purgeOldTestSuite(test_suite_data) # should not fail self.assertEqual([], os.listdir(self.working_directory)) - def test_purgeOldTestSuiteChmodNonWriteableNonReadable(self): - """Old test suites can be deleted even when some files/directories have - been chmod'd to make them non readable and non writeable. """ - test_node = self.getTestNode() - test_suite_data = self.getTestSuiteData(add_third_repository=True) - os.mkdir(os.path.join(self.working_directory, 'bar')) - non_writable_file = open(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 'w') - non_writable_file.close() - - os.chmod(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 0o000) # ---------- - os.chmod(os.path.join(self.working_directory, 'bar'), 0o000) # d--------- - - test_node.purgeOldTestSuite(test_suite_data) # should not fail - self.assertEqual([], os.listdir(self.working_directory)) def test_09_runTestSuite(self, my_test_type='UnitTest'): """ @@ -1022,7 +1008,7 @@ shared = true self.assertRaises(SubprocessError, callPrepareSlapOS) self.assertEquals(node_test_suite.retry_software_count, 0) - for x in xrange(0,11): + for x in range(11): callRaisingPrepareSlapos() self.assertEquals(len(init_call_kw_list), 11) self.assertEquals(init_call_kw_list[-1]['reset_software'], False) diff --git a/erp5/util/scalability/requestUrl.py b/erp5/util/scalability/requestUrl.py index 5f0d9d96b93..98dc9bbf7f2 100644 --- a/erp5/util/scalability/requestUrl.py +++ b/erp5/util/scalability/requestUrl.py @@ -27,7 +27,7 @@ def main(): if error_message_set: exit_status = 1 for error in error_message_set: - print error + print(error) elif result: - print result + print(result) sys.exit(exit_status) diff --git a/erp5/util/scalability/runScalabilityTestSuite.py b/erp5/util/scalability/runScalabilityTestSuite.py index 9c7bd955d93..2749fe42f00 100644 --- a/erp5/util/scalability/runScalabilityTestSuite.py +++ b/erp5/util/scalability/runScalabilityTestSuite.py @@ -5,23 +5,15 @@ import os import shutil import time import sys -import multiprocessing -import signal -import errno import json import logging import logging.handlers import glob -import urlparse -import httplib -import base64 import threading -from erp5.util.benchmark.argument import ArgumentType -from erp5.util.benchmark.performance_tester import PerformanceTester from erp5.util.benchmark.thread import TestThread, TestMetricThread from erp5.util import taskdistribution from erp5.util.testnode import Utils -from erp5.util.testnode.ProcessManager import SubprocessError, ProcessManager, CancellationError +from erp5.util.testnode.ProcessManager import ProcessManager import datetime MAX_INSTALLATION_TIME = 60*50 @@ -179,8 +171,8 @@ class ScalabilityLauncher(object): """ data_array = self.__argumentNamespace.current_test_data.split(',') data = json.dumps({"count": data_array[0], "title": data_array[1], "relative_path": data_array[2]}) - decoded_data = Utils.deunicodeData(json.loads(data)) - return ScalabilityTest(decoded_data, self.test_result) + encoded_data = Utils.deunicodeData(json.loads(data)) + return ScalabilityTest(encoded_data, self.test_result) def clearUsersFile(self, user_file_path): self.log("Clearing users file: %s" % user_file_path) diff --git a/erp5/util/taskdistribution/__init__.py b/erp5/util/taskdistribution/__init__.py index 4eef9bd3458..f0e6262a561 100644 --- a/erp5/util/taskdistribution/__init__.py +++ b/erp5/util/taskdistribution/__init__.py @@ -40,11 +40,15 @@ Example use: test_line.stop() """ from __future__ import print_function -import httplib +import six +from six.moves import ( + map, + http_client as httplib, + xmlrpc_client as xmlrpclib, +) import socket import threading import time -import xmlrpclib __all__ = ['TaskDistributor', 'TestResultProxy', 'TestResultLineProxy', 'patchRPCParser'] @@ -89,11 +93,17 @@ def patchRPCParser(error_handler): def verbose_feed(self, data): try: return original_feed(self, data) - except Exception, exc: + except Exception as exc: if not error_handler(data, exc): raise parser_klass.feed = verbose_feed +try: # PY3 + basestring +except NameError: + basestring = bytes, str + unicode = str + def binarize_args(arg): # Converts recursively basestring arg into xmlrpclib.Binary, as they can # contain non-XML allowed characters @@ -102,9 +112,9 @@ def binarize_args(arg): arg = arg.encode('utf-8') return xmlrpclib.Binary(arg) if isinstance(arg, (list, tuple, set)): - return map(binarize_args, arg) + return list(map(binarize_args, arg)) if isinstance(arg, dict): - return {k: binarize_args(v) for k, v in arg.iteritems()} + return {k: binarize_args(v) for k, v in six.iteritems(arg)} return arg class RPCRetry(object): @@ -350,7 +360,7 @@ class TestResultProxy(RPCRetry): caption_list = [] append = caption_list.append for name, (stream, max_history_bytes) in \ - self._watcher_dict.iteritems(): + six.iteritems(self._watcher_dict): append('==> %s <==' % (name, )) start = stream.tell() stream.seek(0, 2) diff --git a/erp5/util/testnode/NodeTestSuite.py b/erp5/util/testnode/NodeTestSuite.py index 0d198d18b98..6916d36bf50 100644 --- a/erp5/util/testnode/NodeTestSuite.py +++ b/erp5/util/testnode/NodeTestSuite.py @@ -32,6 +32,8 @@ import string import random from .Utils import createFolder +from six.moves import range + class SlapOSInstance(object): """ Base of an software instance, @@ -69,14 +71,14 @@ class NodeTestSuite(SlapOSInstance): def createSuiteLog(self): # /srv/slapgrid/slappartXX/srv/var/log/testnode/az-D27KqX7FxJ/suite.log - alphabets = string.digits + string.letters + alphabets = string.digits + string.ascii_letters while 1: log_folder_name = '%s-%s' % (self.reference, - ''.join(random.choice(alphabets) for i in xrange(10))) + ''.join(random.choice(alphabets) for i in range(10))) log_folder_path = os.path.join(self.log_directory, log_folder_name) try: os.makedirs(log_folder_path) - except OSError, e: + except OSError as e: if e.errno != errno.EEXIST: raise else: diff --git a/erp5/util/testnode/ProcessManager.py b/erp5/util/testnode/ProcessManager.py index 0dc0d1cb34b..17f3024bae1 100644 --- a/erp5/util/testnode/ProcessManager.py +++ b/erp5/util/testnode/ProcessManager.py @@ -33,6 +33,7 @@ import signal import sys import time from . import logger +from slapos.util import bytes2str MAX_TIMEOUT = 3600 * 4 @@ -79,7 +80,7 @@ def subprocess_capture(p, log_prefix, get_output=True): break if get_output: buffer.append(data) - log(log_prefix + data.rstrip('\n')) + log(log_prefix + bytes2str(data).rstrip('\n')) if p.stdout: stdout = [] stdout_thread = threading.Thread(target=readerthread, @@ -97,8 +98,8 @@ def subprocess_capture(p, log_prefix, get_output=True): stdout_thread.join() if p.stderr: stderr_thread.join() - return (p.stdout and ''.join(stdout), - p.stderr and ''.join(stderr)) + return (p.stdout and b''.join(stdout), + p.stderr and b''.join(stderr)) def killCommand(pid): """ @@ -109,7 +110,7 @@ def killCommand(pid): try: process = psutil.Process(pid) process.suspend() - except psutil.Error, e: + except psutil.Error as e: return process_list = [process] new_list = process.children(recursive=True) @@ -118,19 +119,19 @@ def killCommand(pid): for child in new_list: try: child.suspend() - except psutil.Error, e: + except psutil.Error as e: logger.debug("killCommand/suspend: %s", e) time.sleep(1) new_list = set(process.children(recursive=True)).difference(process_list) for process in process_list: try: process.kill() - except psutil.Error, e: + except psutil.Error as e: logger.debug("killCommand/kill: %s", e) class ProcessManager(object): - stdin = file(os.devnull) + stdin = open(os.devnull) def __init__(self, max_timeout=MAX_TIMEOUT): self.process_pid_set = set() diff --git a/erp5/util/testnode/ScalabilityTestRunner.py b/erp5/util/testnode/ScalabilityTestRunner.py index 975071eee45..a10adc2146e 100644 --- a/erp5/util/testnode/ScalabilityTestRunner.py +++ b/erp5/util/testnode/ScalabilityTestRunner.py @@ -30,30 +30,31 @@ import subprocess import sys import time import glob -import SlapOSControler -import SlapOSMasterCommunicator +from . import SlapOSControler, SlapOSMasterCommunicator import json import time import shutil import logging import string import random -import urlparse +from six.moves.urllib import parse import base64 -import httplib -import Utils +from six.moves import http_client as httplib +from . import Utils import requests import slapos.slap -import cPickle as pickle -from ProcessManager import SubprocessError, ProcessManager, CancellationError +from six.moves import cPickle as pickle +from .ProcessManager import SubprocessError, ProcessManager, CancellationError from subprocess import CalledProcessError -from Updater import Updater +from .Updater import Updater from erp5.util import taskdistribution from erp5.util.benchmark.thread import TestThread # for dummy slapos answer import signal from . import logger +from six.moves import range + # max time to generate frontend instance: 1.5 hour MAX_FRONTEND_TIME = 60*90 # max time to register instance to slapOSMaster: 5 minutes @@ -333,7 +334,7 @@ Require valid-user htaccess_file.close() password_path = testsuite_directory + PASSWORD_FILE with open(password_path, "w") as password_file: - password = ''.join(random.choice(string.digits + string.letters) for i in xrange(PASSWORD_LENGTH)) + password = ''.join(random.choice(string.digits + string.letters) for i in range(PASSWORD_LENGTH)) password_file.write(password) user = TESTNODE_USER command = [apache_htpasswd, "-bc", testsuite_directory + HTPASSWD, user, password] @@ -363,7 +364,7 @@ Require valid-user user, password = self.generateProfilePasswordAccess() logger.info("Software Profile password: %s" % password) self.reachable_profile = "https://%s:%s@%s" % (user, password, - os.path.join(urlparse.urlparse(self.testnode.config['frontend_url']).netloc, + os.path.join(parse.urlparse(self.testnode.config['frontend_url']).netloc, "software", self.randomized_path, "software.cfg")) def prepareSlapOSForTestSuite(self, node_test_suite): @@ -526,7 +527,7 @@ Require valid-user if not self.launchable: return {'status_code' : 1, 'error_message': "Current test_suite is not actually launchable." } configuration_list = node_test_suite.configuration_list - test_list = range(0, len(configuration_list)) + test_list = list(range(len(configuration_list))) try: test_result_proxy = self.testnode.taskdistribution.createTestResult( node_test_suite.revision, test_list, diff --git a/erp5/util/testnode/SlapOSControler.py b/erp5/util/testnode/SlapOSControler.py index e70d8751473..352b017b0f9 100644 --- a/erp5/util/testnode/SlapOSControler.py +++ b/erp5/util/testnode/SlapOSControler.py @@ -35,6 +35,8 @@ from slapos import client from . import logger from .Utils import createFolder +from six.moves import range + MAX_PARTITIONS = 10 MAX_SR_RETRIES = 3 @@ -243,7 +245,7 @@ class SlapOSControler(object): computer = slap.registerComputer(config['computer_id']) # Call a method to ensure connection to master can be established computer.getComputerPartitionList() - except slapos.slap.ConnectionError, e: + except slapos.slap.ConnectionError as e: retries += 1 if retries >= 60: raise @@ -270,7 +272,7 @@ class SlapOSControler(object): # MySQL DB content) from previous runs. To support changes of partition # naming scheme (which already happened), do this at instance_root level. createFolder(instance_root, True) - for i in xrange(MAX_PARTITIONS): + for i in range(MAX_PARTITIONS): # create partition and configure computer # XXX: at the moment all partitions do share same virtual interface address # this is not a problem as usually all services are on different ports @@ -278,7 +280,7 @@ class SlapOSControler(object): partition_path = os.path.join(instance_root, partition_reference) if not(os.path.exists(partition_path)): os.mkdir(partition_path) - os.chmod(partition_path, 0750) + os.chmod(partition_path, 0o750) computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({ 'address': config['ipv4_address'], 'instance_root': instance_root, @@ -318,7 +320,7 @@ class SlapOSControler(object): os.environ['PATH'] = environment['PATH'] # a SR may fail for number of reasons (incl. network failures) # so be tolerant and run it a few times before giving up - for _ in xrange(MAX_SR_RETRIES): + for _ in range(MAX_SR_RETRIES): status_dict = self.spawn(config['slapos_binary'], 'node', 'software', '--all', '--pidfile', os.path.join(self.software_root, 'slapos-node.pid'), @@ -346,7 +348,7 @@ class SlapOSControler(object): # try to run for all partitions as one partition may in theory request another one # this not always is required but curently no way to know how "tree" of partitions # may "expand" - for _ in xrange(max_quantity): + for _ in range(max_quantity): status_dict = self.spawn(config['slapos_binary'], 'node', 'instance', '--pidfile', os.path.join(self.instance_root, 'slapos-node.pid'), '--cfg', self.slapos_config, raise_error_if_fail=False, diff --git a/erp5/util/testnode/SlapOSMasterCommunicator.py b/erp5/util/testnode/SlapOSMasterCommunicator.py index 8ffeadfd15b..8634585d425 100644 --- a/erp5/util/testnode/SlapOSMasterCommunicator.py +++ b/erp5/util/testnode/SlapOSMasterCommunicator.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import datetime import json import traceback @@ -9,9 +11,11 @@ from uritemplate import expand import slapos.slap from slapos.slap import SoftwareProductCollection from requests.exceptions import HTTPError -from ..taskdistribution import SAFE_RPC_EXCEPTION_LIST +from erp5.util.taskdistribution import SAFE_RPC_EXCEPTION_LIST from . import logger +import six + # max time to instance changing state: 3 hour MAX_INSTANCE_TIME = 60*60*3 @@ -52,7 +56,7 @@ def retryOnNetworkFailure(func, except _except_list: traceback.print_exc() - print 'Network failure. Retry method %s in %i seconds' % (func, retry_time) + print('Network failure. Retry method %s in %i seconds' % (func, retry_time)) time.sleep(retry_time) retry_time = min(retry_time*1.5, 640) @@ -92,8 +96,7 @@ class SlapOSMasterCommunicator(object): if instance_title is not None: self.name = instance_title if request_kw is not None: - if isinstance(request_kw, basestring) or \ - isinstance(request_kw, unicode): + if isinstance(request_kw, (six.binary_type, six.text_type)): self.request_kw = json.loads(request_kw) else: self.request_kw = request_kw @@ -214,7 +217,7 @@ class SlapOSMasterCommunicator(object): result = self.hateoas_navigator.GET(url) result = json.loads(result) if result['_links'].get('action_object_slap', None) is None: - print result['links'] + print(result['links']) return None object_link = self.hateoas_navigator.hateoasGetLinkFromLinks( @@ -385,8 +388,7 @@ class SlapOSTester(SlapOSMasterCommunicator): self.name = name self.computer_guid = computer_guid - if isinstance(request_kw, str) or \ - isinstance(request_kw, unicode): + if isinstance(request_kw, (six.binary_type, six.text_type)): self.request_kw = json.loads(request_kw) else: self.request_kw = request_kw diff --git a/erp5/util/testnode/Updater.py b/erp5/util/testnode/Updater.py index b3287fbddbe..8278113229f 100644 --- a/erp5/util/testnode/Updater.py +++ b/erp5/util/testnode/Updater.py @@ -30,6 +30,7 @@ import re from . import logger from .ProcessManager import SubprocessError from .Utils import rmtree +from slapos.util import bytes2str, str2bytes SVN_UP_REV = re.compile(r'^(?:At|Updated to) revision (\d+).$') SVN_CHANGED_REV = re.compile(r'^Last Changed Rev.*:\s*(\d+)', re.MULTILINE) @@ -82,7 +83,7 @@ class Updater(object): # allow several processes clean the same folder at the same time try: os.remove(os.path.join(path, file)) - except OSError, e: + except OSError as e: if e.errno != errno.ENOENT: raise @@ -96,7 +97,7 @@ class Updater(object): **kw) def _git(self, *args, **kw): - return self.spawn(self.git_binary, *args, **kw)['stdout'].strip() + return bytes2str(self.spawn(self.git_binary, *args, **kw)['stdout'].strip()) def git_update_server_info(self): return self._git('update-server-info', '-f') @@ -219,7 +220,7 @@ class Updater(object): self.deletePycFiles(path) try: status_dict = self.spawn(*args) - except SubprocessError, e: + except SubprocessError as e: if 'cleanup' not in e.stderr: raise self.spawn('svn', 'cleanup', path) diff --git a/erp5/util/testnode/Utils.py b/erp5/util/testnode/Utils.py index b7e879d9eeb..87b2df8ae8b 100644 --- a/erp5/util/testnode/Utils.py +++ b/erp5/util/testnode/Utils.py @@ -3,6 +3,9 @@ import stat import shutil import errno +import six +from six.moves import map + def rmtree(path): """Delete a path recursively. @@ -11,14 +14,22 @@ def rmtree(path): def chmod_retry(func, failed_path, exc_info): """Make sure the directories are executable and writable. """ + # Depending on the Python version, the following items differ. + if six.PY3: + expected_error_type = PermissionError + expected_func = os.lstat + else: + expected_error_type = OSError + expected_func = os.listdir + e = exc_info[1] - if isinstance(e, OSError): + if isinstance(e, expected_error_type): if e.errno == errno.ENOENT: # because we are calling again rmtree on listdir errors, this path might # have been already deleted by the recursive call to rmtree. return if e.errno == errno.EACCES: - if func is os.listdir: + if func is expected_func: os.chmod(failed_path, 0o700) # corner case to handle errors in listing directories. # https://bugs.python.org/issue8523 @@ -39,12 +50,16 @@ def createFolder(folder, clean=False): rmtree(folder) os.mkdir(folder) -def deunicodeData(data): - if isinstance(data, list): - return map(deunicodeData, data) - if isinstance(data, unicode): - return data.encode('utf8') - if isinstance(data, dict): - return {deunicodeData(key): deunicodeData(value) - for key, value in data.iteritems()} - return data +if six.PY3: + def deunicodeData(data): + return data +else: + def deunicodeData(data): + if isinstance(data, list): + return list(map(deunicodeData, data)) + if isinstance(data, unicode): + return data.encode('utf8') + if isinstance(data, dict): + return {deunicodeData(key): deunicodeData(value) + for key, value in six.iteritems(data)} + return data diff --git a/erp5/util/testnode/__init__.py b/erp5/util/testnode/__init__.py index 480df94edff..e84a7a8760b 100644 --- a/erp5/util/testnode/__init__.py +++ b/erp5/util/testnode/__init__.py @@ -24,7 +24,7 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # ############################################################################## -import ConfigParser +from six.moves import configparser import argparse import logging import logging.handlers @@ -64,7 +64,7 @@ def main(*args): CONFIG = { 'partition_reference': 'test0', } - config = ConfigParser.SafeConfigParser() + config = configparser.SafeConfigParser() # do not change case of option keys config.optionxform = str config.readfp(parsed_argument.configuration_file[0]) diff --git a/erp5/util/testnode/testnode.py b/erp5/util/testnode/testnode.py index b863248cf6a..3cd3aa4cc70 100644 --- a/erp5/util/testnode/testnode.py +++ b/erp5/util/testnode/testnode.py @@ -39,7 +39,7 @@ from .ScalabilityTestRunner import ScalabilityTestRunner from .UnitTestRunner import UnitTestRunner from .Utils import deunicodeData from .Utils import rmtree -from .. import taskdistribution +from erp5.util import taskdistribution MAX_LOG_TIME = 15 # time in days we should keep logs that we can see through # httd diff --git a/erp5/util/testsuite/__init__.py b/erp5/util/testsuite/__init__.py index 0d1b8a3f169..b3bacf0a6ae 100644 --- a/erp5/util/testsuite/__init__.py +++ b/erp5/util/testsuite/__init__.py @@ -1,12 +1,22 @@ +from __future__ import print_function + import argparse import re, os, shlex, glob import sys, threading, subprocess import traceback import errno +import pprint +import six +from six.moves import range from erp5.util import taskdistribution -from pprint import pprint +if six.PY3: + stdbin = lambda x: x.buffer +else: + stdbin = lambda x: x + +# PY3: use shlex.quote _format_command_search = re.compile("[[\\s $({?*\\`#~';<>&|]").search _format_command_escape = lambda s: "'%s'" % r"'\''".join(s.split("'")) def format_command(*args, **kw): @@ -31,7 +41,7 @@ def subprocess_capture(p, quiet=False): buffer.append(data) if p.stdout: stdout = [] - output = quiet and (lambda data: None) or sys.stdout.write + output = (lambda data: None) if quiet else stdbin(sys.stdout).write stdout_thread = threading.Thread(target=readerthread, args=(p.stdout, output, stdout)) stdout_thread.setDaemon(True) @@ -39,7 +49,7 @@ def subprocess_capture(p, quiet=False): if p.stderr: stderr = [] stderr_thread = threading.Thread(target=readerthread, - args=(p.stderr, sys.stderr.write, stderr)) + args=(p.stderr, stdbin(sys.stderr).write, stderr)) stderr_thread.setDaemon(True) stderr_thread.start() if p.stdout: @@ -47,8 +57,8 @@ def subprocess_capture(p, quiet=False): if p.stderr: stderr_thread.join() p.wait() - return (p.stdout and ''.join(stdout), - p.stderr and ''.join(stderr)) + return (p.stdout and b''.join(stdout), + p.stderr and b''.join(stderr)) class SubprocessError(EnvironmentError): def __init__(self, status_dict): @@ -72,15 +82,15 @@ class Persistent(object): def __getattr__(self, attr): if attr == '_db': try: - db = file(self._filename, 'r+') - except IOError, e: + db = open(self._filename, 'r+') + except IOError as e: if e.errno != errno.ENOENT: raise - db = file(self._filename, 'w+') + db = open(self._filename, 'w+') else: try: self.__dict__.update(eval(db.read())) - except StandardError: + except Exception: pass self._db = db return db @@ -89,7 +99,7 @@ class Persistent(object): def sync(self): self._db.seek(0) - db = dict(x for x in self.__dict__.iteritems() if x[0][:1] != '_') + db = dict(x for x in six.iteritems(self.__dict__) if x[0][:1] != '_') pprint.pprint(db, self._db) self._db.truncate() @@ -103,10 +113,10 @@ class TestSuite(object): """ RUN_RE = re.compile( - r'Ran (?P\d+) tests? in (?P\d+\.\d+)s', + br'Ran (?P\d+) tests? in (?P\d+\.\d+)s', re.DOTALL) - STATUS_RE = re.compile(r""" + STATUS_RE = re.compile(br""" (OK|FAILED)\s+\( (failures=(?P\d+),?\s*)? (errors=(?P\d+),?\s*)? @@ -117,7 +127,7 @@ class TestSuite(object): """, re.DOTALL | re.VERBOSE) SUB_STATUS_RE = re.compile( - r"""SUB\s+RESULT:\s+(?P\d+)\s+Tests,\s+ + br"""SUB\s+RESULT:\s+(?P\d+)\s+Tests,\s+ (?P\d+)\s+Failures\s* \(? (skipped=(?P\d+),?\s*)? @@ -130,7 +140,10 @@ class TestSuite(object): mysql_db_count = 1 allow_restart = False realtime_output = True - stdin = file(os.devnull) + try: # PY3 + stdin = subprocess.DEVNULL + except AttributeError: + stdin = open(os.devnull, 'rb') def __init__(self, max_instance_count, **kw): self.__dict__.update(kw) @@ -139,8 +152,8 @@ class TestSuite(object): self.acquire = pool.acquire self.release = pool.release self._instance = threading.local() - self._pool = max_instance_count == 1 and [None] or \ - range(1, max_instance_count + 1) + self._pool = [None] if max_instance_count == 1 else \ + list(range(1, max_instance_count + 1)) self._ready = set() self.running = {} if max_instance_count != 1: @@ -185,13 +198,14 @@ class TestSuite(object): def spawn(self, *args, **kw): quiet = kw.pop('quiet', False) + cwd = kw.pop('cwd', None) env = kw and dict(os.environ, **kw) or None command = format_command(*args, **kw) - print '\n$ ' + command + print('\n$ ' + command) sys.stdout.flush() try: p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, env=env) + stderr=subprocess.PIPE, env=env, cwd=cwd) except Exception: # Catch any exception here, to warn user instead of beeing silent, # by generating fake error result @@ -229,20 +243,11 @@ class EggTestSuite(TestSuite): The python interpreter is ``python_interpreter`` """ def run(self, test): - print test - original_dir = os.getcwd() - try: - os.chdir(self.egg_test_path_dict[test]) - return self.runUnitTest(test) - finally: - os.chdir(original_dir) - - def runUnitTest(self, *args, **kw): + print(test) try: - runUnitTest = "{python} setup.py test".format(python=self.python_interpreter) - args = tuple(shlex.split(runUnitTest)) - status_dict = self.spawn(*args, **kw) - except SubprocessError, e: + status_dict = self.spawn(self.python_interpreter, 'setup.py', 'test', + cwd=self.egg_test_path_dict[test]) + except SubprocessError as e: status_dict = e.status_dict test_log = status_dict['stderr'] search = self.RUN_RE.search(test_log) @@ -270,7 +275,7 @@ class EggTestSuite(TestSuite): return status_dict def getTestList(self): - return self.egg_test_path_dict.keys() + return list(self.egg_test_path_dict) def runTestSuite(): parser = argparse.ArgumentParser(description='Run a test suite.') @@ -327,7 +332,7 @@ def runTestSuite(): if test_result is not None: assert revision == test_result.revision, (revision, test_result.revision) while suite.acquire(): - test = test_result.start(suite.running.keys()) + test = test_result.start(list(suite.running)) if test is not None: suite.start(test.name, lambda status_dict, __test=test: __test.stop(**status_dict)) diff --git a/erp5/util/timinglogparser/__init__.py b/erp5/util/timinglogparser/__init__.py index 1679598c72b..7dbc5505616 100755 --- a/erp5/util/timinglogparser/__init__.py +++ b/erp5/util/timinglogparser/__init__.py @@ -27,6 +27,8 @@ # ############################################################################## +from __future__ import print_function + import os import sys import imp @@ -126,7 +128,7 @@ def parseFile(filename, measure_dict): sys.stderr.flush() match_list = LINE_PATTERN.findall(line) if len(match_list) != 1: - print >>sys.stderr, 'Unparseable line: %s:%i %r' % (filename, line_number, line) + print('Unparseable line: %s:%i %r' % (filename, line_number, line), file=sys.stderr) else: result, filter_id, date, duration = processLine(match_list[0], filename, line_number) # Possible result values & meaning: @@ -135,20 +137,21 @@ def parseFile(filename, measure_dict): # (string): use & skip to next line if result is False: if debug: - print >>sys.stderr, '? %s:%i %r' % (filename, line_number, match_list[0]) + print('? %s:%i %r' % (filename, line_number, match_list[0]), file=sys.stderr) elif result is True: if debug: - print >>sys.stderr, '- %s:%i %r' % (filename, line_number, match_list[0]) + print('- %s:%i %r' % (filename, line_number, match_list[0]), file=sys.stderr) skip_count += 1 else: measure_dict.setdefault(filter_id, {}).setdefault(result, {}).setdefault(date, []).append(int(duration)) match_count += 1 line = logfile.readline() - print >>sys.stderr, '%i' % (line_number, ) + print('%i' % (line_number, ), file=sys.stderr) if line_number > 0: duration = time() - begin - print >>sys.stderr, "Matched %i lines (%.2f%%), %i skipped (%.2f%%), %i unmatched (%.2f%%) in %.2fs (%i lines per second)." % \ - (match_count, (float(match_count) / line_number) * 100, skip_count, (float(skip_count) / line_number) * 100, (line_number - match_count - skip_count), (1 - (float(match_count + skip_count) / line_number)) * 100, duration, line_number / duration) + print("Matched %i lines (%.2f%%), %i skipped (%.2f%%), %i unmatched (%.2f%%) in %.2fs (%i lines per second)." % \ + (match_count, (float(match_count) / line_number) * 100, skip_count, (float(skip_count) / line_number) * 100, (line_number - match_count - skip_count), (1 - (float(match_count + skip_count) / line_number)) * 100, duration, line_number / duration), + file=sys.stderr) debug = False outfile_prefix = None @@ -161,9 +164,9 @@ decimate_count = 1 try: opts, file_list = getopt.getopt(sys.argv[1:], '', ['debug', 'config=', 'prefix=', 'no-average', 'sum', 'load=', 'save=', 'decimate=']) -except Exception, reason: - print >>sys.stderr, reason - print >>sys.stderr, usage +except Exception as reason: + print(reason, file=sys.stderr) + print(usage, file=sys.stderr) sys.exit(1) for name, value in opts: @@ -185,7 +188,7 @@ for name, value in opts: decimate_count = int(value) if configuration is None: - raise ValueError, '--config is mandatory' + raise ValueError('--config is mandatory') config_file = os.path.splitext(os.path.basename(configuration))[0] config_path = [os.path.dirname(os.path.abspath(configuration))] + sys.path @@ -211,18 +214,18 @@ if len(load_file_name_list): for result, date_dict in result_dict.iteritems(): for date, duration_list in date_dict.iteritems(): measure_dict.setdefault(filter_id, {}).setdefault(result, {}).setdefault(date, []).extend(duration_list) - print >>sys.stderr, 'Previous processing result restored from %r' % (load_file_name, ) + print('Previous processing result restored from %r' % (load_file_name, ), file=sys.stderr) for filename in file_list: file_number += 1 - print >>sys.stderr, 'Processing %s [%i/%i]...' % (filename, file_number, file_count) + print('Processing %s [%i/%i]...' % (filename, file_number, file_count), file=sys.stderr) parseFile(filename, measure_dict) if save_file_name is not None: save_file = open(save_file_name, 'w') save_file.write(repr(measure_dict)) save_file.close() - print >>sys.stderr, 'Processing result saved to %r' % (save_file_name, ) + print('Processing result saved to %r' % (save_file_name, ), file=sys.stderr) if outfile_prefix is not None: ## Generate a list of all measures and a 2-levels dictionnary with date as key and measure dictionnary as value @@ -252,9 +255,9 @@ if outfile_prefix is not None: def renderOutput(data_format, filename_suffix): for sheet_id, sheet_column_list in sheet_dict.iteritems(): outfile_name = '%s_%s_%s.csv' % (outfile_prefix, sheet_id, filename_suffix) - print >>sys.stderr, 'Writing to %r...' % (outfile_name, ) + print('Writing to %r...' % (outfile_name, ), file=sys.stderr) outfile = open(outfile_name, 'w') - print >>outfile, '"date",%s' % (','.join(['"%s"' % (x[0], ) for x in sheet_column_list]), ) + print('"date",%s' % (','.join(['"%s"' % (x[0], ) for x in sheet_column_list]), ), file=outfile) decimate_dict = {} decimate = 0 for date in date_list: @@ -262,11 +265,11 @@ if outfile_prefix is not None: decimate_dict.setdefault(key, []).extend(value) decimate += 1 if decimate == decimate_count: - print >>outfile, '"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list])) + print('"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list])), file=outfile) decimate_dict = {} decimate = 0 if len(decimate_dict): - print >>outfile, '"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list])) + print('"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list])), file=outfile) if do_average: renderOutput('=%(sum)i/%(count)i', 'avg') diff --git a/erp5/util/timinglogplotter/__init__.py b/erp5/util/timinglogplotter/__init__.py index 50595b487e8..5c8fb02301b 100755 --- a/erp5/util/timinglogplotter/__init__.py +++ b/erp5/util/timinglogplotter/__init__.py @@ -27,6 +27,8 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # ############################################################################## +from __future__ import print_function + from datetime import date from os import path import rpy2.robjects as robjects @@ -121,7 +123,7 @@ def main(): current_dir = os.getcwd() for file_name in file_name_list: - print 'Loading %s...' % (file_name, ) + print('Loading %s...' % (file_name, )) file = CSVFile(file_name) date_string_list = file.getColumn(0) @@ -183,13 +185,13 @@ def main(): y_data.append(value) i += 1 if len(x_data) == 0: - print 'Nothing to plot for %s...' % (out_file_name, ) + print('Nothing to plot for %s...' % (out_file_name, )) continue if options.minimal_non_empty_ratio is not None: column_len = len(column) if column_len: if float(len(x_data))/column_len < options.minimal_non_empty_ratio: - print 'Not enough values to plot for %s...' % (out_file_name, ) + print('Not enough values to plot for %s...' % (out_file_name, )) continue r_y_data = robjects.FloatVector(y_data) r_x_data = robjects.FloatVector(x_data) @@ -220,7 +222,7 @@ def main(): # stop changing the out-type file r("""dev.off()""") - print 'Saving %s...' % (out_file_name, ) + print('Saving %s...' % (out_file_name, )) if __name__ == '__main__': main() diff --git a/erp5/util/webchecker/__init__.py b/erp5/util/webchecker/__init__.py index 33c78ec36ef..6155869b5f3 100644 --- a/erp5/util/webchecker/__init__.py +++ b/erp5/util/webchecker/__init__.py @@ -26,6 +26,8 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # ############################################################################## +from __future__ import print_function + import os import shutil import sys @@ -543,7 +545,7 @@ class HTTPCacheCheckerTestSuite(object): from optparse import OptionParser -import ConfigParser +from six.moves import configparser def _formatConfiguration(configuration): """ format the configuration""" @@ -559,11 +561,11 @@ def web_checker_utility(): (options, args) = parser.parse_args() if len(args) != 1 : - print parser.print_help() + print(parser.print_help()) parser.error('incorrect number of arguments') config_path = args[0] - config = ConfigParser.RawConfigParser() + config = configparser.RawConfigParser() config.read(config_path) working_directory = config.get('web_checker', 'working_directory') url = config.get('web_checker', 'url') @@ -619,6 +621,6 @@ def web_checker_utility(): file_object.write(result) file_object.close() else: - print result + print(result) diff --git a/product/ERP5/bin/genbt5list b/product/ERP5/bin/genbt5list index f4cbfb0fa3b..240a8a2dcd0 100755 --- a/product/ERP5/bin/genbt5list +++ b/product/ERP5/bin/genbt5list @@ -35,12 +35,15 @@ import posixpath import tarfile import os import sys -import cgi +try: + from html import escape +except ImportError: + from cgi import escape from base64 import b64encode -from cStringIO import StringIO +from six import BytesIO from hashlib import sha1 -from urllib import unquote - +from six.moves.urllib.parse import unquote +import six # Order is important for installation # We want to have: @@ -109,11 +112,11 @@ item_name_list = tuple('_%s_item' % x for x in item_name_list) class BusinessTemplateRevision(list): def hash(self, path, text): - self.append((path, sha1(text).digest())) + self.append((path.encode('utf-8'), sha1(text).digest())) def digest(self): self.sort() - return b64encode(sha1('\0'.join(h + p for (h, p) in self)).digest()) + return b64encode(sha1(b'\0'.join(h + p for (h, p) in self)).digest()) class BusinessTemplate(dict): @@ -151,7 +154,7 @@ force_install def __iter__(self): self['revision'] = self.revision.digest() - return iter(sorted(self.iteritems())) + return iter(sorted(six.iteritems(self))) @classmethod def fromTar(cls, tar): @@ -179,8 +182,8 @@ force_install return iter(self) def generateInformation(dir, info=id, err=None): - xml = StringIO() - xml.write('\n\n') + xml = BytesIO() + xml.write(b'\n\n') for name in sorted(os.listdir(dir)): path = os.path.join(dir, name) if name.endswith('.bt5'): @@ -201,13 +204,14 @@ def generateInformation(dir, info=id, err=None): property_list = BusinessTemplate.fromDir(path) else: continue - xml.write(' \n') info('done\n') - xml.write('\n') + xml.write(b'\n') return xml def main(dir_list=None, **kw): -- 2.30.9 From d72c2f9989c104d4ea8644c0316212dc1964aa40 Mon Sep 17 00:00:00 2001 From: Bryton Lacquement Date: Tue, 5 Feb 2019 11:22:04 +0100 Subject: [PATCH 02/14] fixup! erp5.util: add support for Python 3 Undo accidental revert --- erp5/tests/testERP5TestNode.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/erp5/tests/testERP5TestNode.py b/erp5/tests/testERP5TestNode.py index 6e926457596..9500602b3cb 100644 --- a/erp5/tests/testERP5TestNode.py +++ b/erp5/tests/testERP5TestNode.py @@ -521,21 +521,35 @@ shared = true test_node.purgeOldTestSuite(test_suite_data) self.assertEquals(['foo'], os.listdir(self.working_directory)) - def test_purgeOldTestSuiteChmod(self): + def test_purgeOldTestSuiteChmodNonWriteable(self): """Old test suites can be deleted even when some files/directories have - been chmod'd to make read only. """ + been chmod'd to make non-writeable """ test_node = self.getTestNode() test_suite_data = self.getTestSuiteData(add_third_repository=True) os.mkdir(os.path.join(self.working_directory, 'bar')) non_writable_file = open(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 'w') non_writable_file.close() - # make this file and directory non writeable - os.chmod(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 0o000) - os.chmod(os.path.join(self.working_directory, 'bar'), 0o000) + + os.chmod(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 0o400) # -r-------- + os.chmod(os.path.join(self.working_directory, 'bar'), 0o500) # dr-x------ test_node.purgeOldTestSuite(test_suite_data) # should not fail self.assertEqual([], os.listdir(self.working_directory)) + def test_purgeOldTestSuiteChmodNonWriteableNonReadable(self): + """Old test suites can be deleted even when some files/directories have + been chmod'd to make them non readable and non writeable. """ + test_node = self.getTestNode() + test_suite_data = self.getTestSuiteData(add_third_repository=True) + os.mkdir(os.path.join(self.working_directory, 'bar')) + non_writable_file = open(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 'w') + non_writable_file.close() + + os.chmod(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 0o000) # ---------- + os.chmod(os.path.join(self.working_directory, 'bar'), 0o000) # d--------- + + test_node.purgeOldTestSuite(test_suite_data) # should not fail + self.assertEqual([], os.listdir(self.working_directory)) def test_09_runTestSuite(self, my_test_type='UnitTest'): """ -- 2.30.9 From 7ef79ec55fb2d5d0d2d2561a81df7599d73129de Mon Sep 17 00:00:00 2001 From: Bryton Lacquement Date: Tue, 5 Feb 2019 13:27:27 +0100 Subject: [PATCH 03/14] genbt5list: remove the use of six This script should only depend on the standard library, for convenience. --- product/ERP5/bin/genbt5list | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/product/ERP5/bin/genbt5list b/product/ERP5/bin/genbt5list index 240a8a2dcd0..15e678ee2f3 100755 --- a/product/ERP5/bin/genbt5list +++ b/product/ERP5/bin/genbt5list @@ -40,10 +40,19 @@ try: except ImportError: from cgi import escape from base64 import b64encode -from six import BytesIO +from io import BytesIO from hashlib import sha1 -from six.moves.urllib.parse import unquote -import six +try: + from urllib.parse import unquote +except ImportError: + from urllib import unquote + +if sys.version_info[0] == 3: + def iteritems(d): + return iter(d.items()) +else: + def iteritems(d): + return d.iteritems() # Order is important for installation # We want to have: @@ -154,7 +163,7 @@ force_install def __iter__(self): self['revision'] = self.revision.digest() - return iter(sorted(six.iteritems(self))) + return iter(sorted(iteritems(self))) @classmethod def fromTar(cls, tar): -- 2.30.9 From a90d4958354cdba296195fbc556909a9deb9f843 Mon Sep 17 00:00:00 2001 From: Bryton Lacquement Date: Fri, 8 Feb 2019 10:05:21 +0100 Subject: [PATCH 04/14] Change assertNotEquals to assertNotEqual `assertNotEquals` is a deprecated alias in Python 3. It warns about this, which creates noise in logs. --- erp5/tests/testERP5TestNode.py | 112 ++++++++++++++++----------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/erp5/tests/testERP5TestNode.py b/erp5/tests/testERP5TestNode.py index 9500602b3cb..13a85952995 100644 --- a/erp5/tests/testERP5TestNode.py +++ b/erp5/tests/testERP5TestNode.py @@ -172,10 +172,10 @@ class ERP5TestNode(TestCase): output = call(['git', 'log', '--format=%H %s']) output = output.strip() output_line_list = output.split("\n") - self.assertEquals(2, len(output_line_list)) + self.assertEqual(2, len(output_line_list)) expected_commit_subject_list = ["next_commit", "first_commit"] commit_subject_list = [x.split()[1] for x in output_line_list] - self.assertEquals(expected_commit_subject_list, commit_subject_list) + self.assertEqual(expected_commit_subject_list, commit_subject_list) commit_dict['rep%i' % i] = [x.split() for x in output_line_list] if repository_path == self.remote_repository2: output = call('git checkout master -b foo'.split()) @@ -192,13 +192,13 @@ class ERP5TestNode(TestCase): """ test_node = self.getTestNode() node_test_suite = test_node.getNodeTestSuite('foo') - self.assertEquals(0, node_test_suite.retry_software_count) + self.assertEqual(0, node_test_suite.retry_software_count) node_test_suite.retry_software_count = 2 self.assertIs(node_test_suite, test_node.getNodeTestSuite('foo')) - self.assertEquals(2, node_test_suite.retry_software_count) + self.assertEqual(2, node_test_suite.retry_software_count) del test_node.node_test_suite_dict['foo'] node_test_suite = test_node.getNodeTestSuite('foo') - self.assertEquals(0, node_test_suite.retry_software_count) + self.assertEqual(0, node_test_suite.retry_software_count) def test_02_NodeTestSuiteWorkingDirectory(self): """ @@ -206,9 +206,9 @@ class ERP5TestNode(TestCase): """ test_node = self.getTestNode() node_test_suite = test_node.getNodeTestSuite('foo') - self.assertEquals("%s/foo" % self.working_directory, + self.assertEqual("%s/foo" % self.working_directory, node_test_suite.working_directory) - self.assertEquals("%s/foo/test_suite" % self.working_directory, + self.assertEqual("%s/foo/test_suite" % self.working_directory, node_test_suite.test_suite_directory) def test_03_NodeTestSuiteCheckDataAfterEdit(self): @@ -219,13 +219,13 @@ class ERP5TestNode(TestCase): test_node = self.getTestNode() node_test_suite = test_node.getNodeTestSuite('foo') self.updateNodeTestSuiteData(node_test_suite) - self.assertEquals(2, len(node_test_suite.vcs_repository_list)) + self.assertEqual(2, len(node_test_suite.vcs_repository_list)) repository_path_list = [] for vcs_repository in node_test_suite.vcs_repository_list: repository_path_list.append(vcs_repository['repository_path']) expected_list = ["%s/rep0" % node_test_suite.working_directory, "%s/rep1" % node_test_suite.working_directory] - self.assertEquals(expected_list, repository_path_list) + self.assertEqual(expected_list, repository_path_list) def test_04_constructProfile(self, my_test_type='UnitTest'): """ @@ -239,7 +239,7 @@ class ERP5TestNode(TestCase): node_test_suite.revision_list = (('rep1', (1234, 'azerty')), ('rep2', (3456, 'qwerty'))) test_node.constructProfile(node_test_suite,my_test_type) - self.assertEquals("%s/software.cfg" % (node_test_suite.working_directory,), + self.assertEqual("%s/software.cfg" % (node_test_suite.working_directory,), node_test_suite.custom_profile_path) profile = open(node_test_suite.custom_profile_path, 'r') if my_test_type=='UnitTest': @@ -282,7 +282,7 @@ ignore-ssl-certificate = true develop = false shared = true """ % {'temp_dir': self._temp_dir, 'revision1': revision1, 'revision2': revision2} - self.assertEquals(expected_profile, profile.read()) + self.assertEqual(expected_profile, profile.read()) profile.close() def getAndUpdateFullRevisionList(self, test_node, node_test_suite): @@ -298,9 +298,9 @@ shared = true node_test_suite = test_node.getNodeTestSuite('foo') self.updateNodeTestSuiteData(node_test_suite) rev_list = self.getAndUpdateFullRevisionList(test_node, node_test_suite) - self.assertEquals(2, len(rev_list)) - self.assertEquals(rev_list[0], 'rep0=2-%s' % commit_dict['rep0'][0][0]) - self.assertEquals(rev_list[1], 'rep1=2-%s' % commit_dict['rep1'][0][0]) + self.assertEqual(2, len(rev_list)) + self.assertEqual(rev_list[0], 'rep0=2-%s' % commit_dict['rep0'][0][0]) + self.assertEqual(rev_list[1], 'rep1=2-%s' % commit_dict['rep1'][0][0]) my_file = open(os.path.join(self.remote_repository1, 'first_file'), 'w') my_file.write("next_content") my_file.close() @@ -309,7 +309,7 @@ shared = true rev_list = self.getAndUpdateFullRevisionList(test_node, node_test_suite) self.assertTrue(rev_list[0].startswith('rep0=2-')) self.assertTrue(rev_list[1].startswith('rep1=3-')) - self.assertEquals(2, len(node_test_suite.vcs_repository_list)) + self.assertEqual(2, len(node_test_suite.vcs_repository_list)) for vcs_repository in node_test_suite.vcs_repository_list: self.assertTrue(os.path.exists(vcs_repository['repository_path'])) @@ -323,8 +323,8 @@ shared = true node_test_suite = test_node.getNodeTestSuite('foo') self.updateNodeTestSuiteData(node_test_suite, add_third_repository=True) rev_list = self.getAndUpdateFullRevisionList(test_node, node_test_suite) - self.assertEquals(3, len(rev_list)) - self.assertEquals(3, len(node_test_suite.vcs_repository_list)) + self.assertEqual(3, len(rev_list)) + self.assertEqual(3, len(node_test_suite.vcs_repository_list)) rep2_clone_path = [x['repository_path'] for x in \ node_test_suite.vcs_repository_list \ if x['repository_path'].endswith("rep2")][0] @@ -332,8 +332,8 @@ shared = true output = call("git branch".split()).strip() self.assertTrue("* foo" in output.split('\n')) vcs_repository_info = node_test_suite.vcs_repository_list[0] - self.assertEquals(vcs_repository_info['repository_id'], 'rep2') - self.assertEquals(vcs_repository_info['branch'], 'foo') + self.assertEqual(vcs_repository_info['repository_id'], 'rep2') + self.assertEqual(vcs_repository_info['branch'], 'foo') # change it to master vcs_repository_info['branch'] = 'master' rev_list = self.getAndUpdateFullRevisionList(test_node, node_test_suite) @@ -368,8 +368,8 @@ shared = true node_test_suite = test_node.getNodeTestSuite('foo') self.updateNodeTestSuiteData(node_test_suite) rev_list = self.getAndUpdateFullRevisionList(test_node, node_test_suite) - self.assertEquals(2, len(rev_list)) - self.assertEquals(2, len(node_test_suite.vcs_repository_list)) + self.assertEqual(2, len(rev_list)) + self.assertEqual(2, len(node_test_suite.vcs_repository_list)) # patch deleteRepository to make sure it will be called once for the wrong # repos, and not for the repos which has not changed deleted_repository_path_list = [] @@ -386,12 +386,12 @@ shared = true node_test_suite.vcs_repository_list \ if x['repository_path'].endswith("rep0")][0] call = self.getCaller(cwd=rep0_clone_path) - self.assertEquals(call("git config --get remote.origin.url".split()).strip(), + self.assertEqual(call("git config --get remote.origin.url".split()).strip(), self.remote_repository0) rev_list = self.getAndUpdateFullRevisionList(test_node, node_test_suite) - self.assertEquals(call("git config --get remote.origin.url".split()).strip(), + self.assertEqual(call("git config --get remote.origin.url".split()).strip(), self.remote_repository2) - self.assertEquals([rep0_clone_path], deleted_repository_path_list) + self.assertEqual([rep0_clone_path], deleted_repository_path_list) finally: Updater.deleteRepository = original_deleteRepository @@ -407,8 +407,8 @@ shared = true node_test_suite = test_node.getNodeTestSuite('foo') self.updateNodeTestSuiteData(node_test_suite) rev_list = self.getAndUpdateFullRevisionList(test_node, node_test_suite) - self.assertEquals(2, len(rev_list)) - self.assertEquals(2, len(node_test_suite.vcs_repository_list)) + self.assertEqual(2, len(rev_list)) + self.assertEqual(2, len(node_test_suite.vcs_repository_list)) rep0_clone_path = [x['repository_path'] for x in \ node_test_suite.vcs_repository_list \ if x['repository_path'].endswith("rep0")][0] @@ -490,8 +490,8 @@ shared = true info_list.append( call("git log -n1 --format=%H".split()).strip()) return info_list - self.assertEquals(['2', '2'], getRepInfo(count=1)) - self.assertEquals([commit_dict['rep0'][0][0],commit_dict['rep1'][0][0]], + self.assertEqual(['2', '2'], getRepInfo(count=1)) + self.assertEqual([commit_dict['rep0'][0][0],commit_dict['rep1'][0][0]], getRepInfo(hash=1)) class TestResult(object): revision = NodeTestSuite.revision @@ -501,25 +501,25 @@ shared = true test_result.revision_list = (('rep0', (2, commit_dict['rep0'][0][0])), ('rep1', (1, commit_dict['rep1'][1][0]))) test_node.checkRevision(test_result, node_test_suite) - self.assertEquals(['2', '1'], getRepInfo(count=1)) - self.assertEquals([commit_dict['rep0'][0][0],commit_dict['rep1'][1][0]], + self.assertEqual(['2', '1'], getRepInfo(count=1)) + self.assertEqual([commit_dict['rep0'][0][0],commit_dict['rep1'][1][0]], getRepInfo(hash=1)) def test_07_checkExistingTestSuite(self): test_node = self.getTestNode() test_suite_data = self.getTestSuiteData(add_third_repository=True) - self.assertEquals([], os.listdir(self.working_directory)) + self.assertEqual([], os.listdir(self.working_directory)) test_node.purgeOldTestSuite(test_suite_data) - self.assertEquals([], os.listdir(self.working_directory)) + self.assertEqual([], os.listdir(self.working_directory)) os.mkdir(os.path.join(self.working_directory, 'foo')) - self.assertEquals(['foo'], os.listdir(self.working_directory)) + self.assertEqual(['foo'], os.listdir(self.working_directory)) test_node.purgeOldTestSuite(test_suite_data) - self.assertEquals(['foo'], os.listdir(self.working_directory)) + self.assertEqual(['foo'], os.listdir(self.working_directory)) os.mkdir(os.path.join(self.working_directory, 'bar')) - self.assertEquals(set(['bar','foo']), + self.assertEqual(set(['bar','foo']), set(os.listdir(self.working_directory))) test_node.purgeOldTestSuite(test_suite_data) - self.assertEquals(['foo'], os.listdir(self.working_directory)) + self.assertEqual(['foo'], os.listdir(self.working_directory)) def test_purgeOldTestSuiteChmodNonWriteable(self): """Old test suites can be deleted even when some files/directories have @@ -633,11 +633,11 @@ shared = true method_list_for_prepareSlapOSForTestSuite = ["initializeSlapOSControler", "runSoftwareRelease", "runComputerPartition"] runner.prepareSlapOSForTestNode(test_node_slapos) - self.assertEquals(method_list_for_prepareSlapOSForTestNode, + self.assertEqual(method_list_for_prepareSlapOSForTestNode, [x["method_name"] for x in call_list]) call_list = [] runner.prepareSlapOSForTestSuite(node_test_suite) - self.assertEquals(method_list_for_prepareSlapOSForTestSuite, + self.assertEqual(method_list_for_prepareSlapOSForTestSuite, [x["method_name"] for x in call_list]) call_list = [] SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease", status_code=1) @@ -681,7 +681,7 @@ shared = true return json.dumps([]) def _checkExistingTestSuite(reference_set): - test_self.assertEquals(set(reference_set), + test_self.assertEqual(set(reference_set), set(os.listdir(test_node.working_directory))) for x in reference_set: test_self.assertTrue(os.path.exists(os.path.join( @@ -761,7 +761,7 @@ shared = true SlapOSControler.initializeSlapOSControler = doNothing # Inside test_node a runner is created using new UnitTestRunner methods test_node.run() - self.assertEquals(5, counter) + self.assertEqual(5, counter) time.sleep = original_sleep # Restore old class methods if my_test_type == "ScalabilityTest": @@ -797,23 +797,23 @@ shared = true file_name = 'AC_Ra\xc3\xadzertic\xc3\xa1ma' non_ascii_file = open(os.path.join(controler.software_root, file_name), 'w') non_ascii_file.close() - self.assertEquals([file_name], os.listdir(controler.software_root)) + self.assertEqual([file_name], os.listdir(controler.software_root)) controler._resetSoftware() - self.assertEquals([], os.listdir(controler.software_root)) + self.assertEqual([], os.listdir(controler.software_root)) def test_14_createFolder(self): test_node = self.getTestNode() node_test_suite = test_node.getNodeTestSuite('foo') folder = node_test_suite.test_suite_directory - self.assertEquals(False, os.path.exists(folder)) + self.assertEqual(False, os.path.exists(folder)) createFolder(folder) - self.assertEquals(True, os.path.exists(folder)) + self.assertEqual(True, os.path.exists(folder)) to_drop_path = os.path.join(folder, 'drop') to_drop = open(to_drop_path, 'w') to_drop.close() - self.assertEquals(True, os.path.exists(to_drop_path)) + self.assertEqual(True, os.path.exists(to_drop_path)) createFolder(folder, clean=True) - self.assertEquals(False, os.path.exists(to_drop_path)) + self.assertEqual(False, os.path.exists(to_drop_path)) def test_15_suite_log_directory(self, my_test_type='UnitTest', grade='master'): def doNothing(self, *args, **kw): @@ -861,7 +861,7 @@ shared = true def checkTestSuite(test_node): test_node.node_test_suite_dict rand_part_set = set() - self.assertEquals(2, len(test_node.node_test_suite_dict)) + self.assertEqual(2, len(test_node.node_test_suite_dict)) for ref, suite in test_node.node_test_suite_dict.items(): self.assertTrue('var/log/testnode/%s' % suite.reference in \ suite.suite_log_path, @@ -925,7 +925,7 @@ shared = true RunnerClass._prepareSlapOS = patch_prepareSlapOS SlapOSControler.initializeSlapOSControler = doNothing test_node.run() - self.assertEquals(counter, 3) + self.assertEqual(counter, 3) checkTestSuite(test_node) time.sleep = original_sleep # Restore old class methods @@ -1021,18 +1021,18 @@ shared = true def callRaisingPrepareSlapos(): self.assertRaises(SubprocessError, callPrepareSlapOS) - self.assertEquals(node_test_suite.retry_software_count, 0) + self.assertEqual(node_test_suite.retry_software_count, 0) for x in range(11): callRaisingPrepareSlapos() - self.assertEquals(len(init_call_kw_list), 11) - self.assertEquals(init_call_kw_list[-1]['reset_software'], False) - self.assertEquals(node_test_suite.retry_software_count, 11) + self.assertEqual(len(init_call_kw_list), 11) + self.assertEqual(init_call_kw_list[-1]['reset_software'], False) + self.assertEqual(node_test_suite.retry_software_count, 11) callRaisingPrepareSlapos() - self.assertEquals(init_call_kw_list[-1]['reset_software'], True) - self.assertEquals(node_test_suite.retry_software_count, 1) + self.assertEqual(init_call_kw_list[-1]['reset_software'], True) + self.assertEqual(node_test_suite.retry_software_count, 1) callRaisingPrepareSlapos() - self.assertEquals(init_call_kw_list[-1]['reset_software'], False) - self.assertEquals(node_test_suite.retry_software_count, 2) + self.assertEqual(init_call_kw_list[-1]['reset_software'], False) + self.assertEqual(node_test_suite.retry_software_count, 2) SlapOSControler.initializeSlapOSControler = \ initial_initializeSlapOSControler SlapOSControler.runSoftwareRelease = initial_runSoftwareRelease -- 2.30.9 From a0914ca340cd83d931faadbbb0b3ea1a088a036f Mon Sep 17 00:00:00 2001 From: Bryton Lacquement Date: Tue, 18 Jun 2019 11:13:42 +0200 Subject: [PATCH 05/14] fixup! erp5.util: add support for Python 3 --- erp5/tests/testERP5TestNode.py | 7 +++---- erp5/util/testnode/testnode.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/erp5/tests/testERP5TestNode.py b/erp5/tests/testERP5TestNode.py index 13a85952995..c6b7c46e9b4 100644 --- a/erp5/tests/testERP5TestNode.py +++ b/erp5/tests/testERP5TestNode.py @@ -457,10 +457,9 @@ shared = true node_test_suite.vcs_repository_list \ if x['repository_path'].endswith("rep0")][0] # simulate a data corruption on rep0's index - index_file = open(os.path.join(rep0_clone_path, '.git', 'index'), 'a') - index_file.seek(10, os.SEEK_END) - index_file.truncate() - index_file.close() + with open(os.path.join(rep0_clone_path, '.git', 'index'), 'ab') as index_file: + index_file.seek(10, os.SEEK_END) + index_file.truncate() # we get rev list with corrupted repository, we get None, but in the same # time the bad repository is deleted rev_list = self.getAndUpdateFullRevisionList(test_node, node_test_suite) diff --git a/erp5/util/testnode/testnode.py b/erp5/util/testnode/testnode.py index 3cd3aa4cc70..9afdffaa8d0 100644 --- a/erp5/util/testnode/testnode.py +++ b/erp5/util/testnode/testnode.py @@ -171,7 +171,7 @@ shared = true # only limit to particular error, if we run that code for all errors, # then if server having most repositories is down for some time, we would # erase all repositories and facing later hours of downloads - if getattr(error, 'stderr', '').find('index') >= 0: + if getattr(error, 'stderr', b'').find(b'index') >= 0: rmtree(repository_path) logger.warning("Error while getting repository, ignoring this test suite", exc_info=1) -- 2.30.9 From ff5db8a007daae32619b20951913e007e6535262 Mon Sep 17 00:00:00 2001 From: Bryton Lacquement Date: Fri, 21 Jun 2019 16:26:04 +0200 Subject: [PATCH 06/14] wip --- erp5/util/testnode/SlapOSMasterCommunicator.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/erp5/util/testnode/SlapOSMasterCommunicator.py b/erp5/util/testnode/SlapOSMasterCommunicator.py index 8634585d425..9d086c64586 100644 --- a/erp5/util/testnode/SlapOSMasterCommunicator.py +++ b/erp5/util/testnode/SlapOSMasterCommunicator.py @@ -96,7 +96,9 @@ class SlapOSMasterCommunicator(object): if instance_title is not None: self.name = instance_title if request_kw is not None: - if isinstance(request_kw, (six.binary_type, six.text_type)): + if isinstance(request_kw, bytes): + self.request_kw = json.loads(request_kw.decode('utf-8')) + elif isinstance(request_kw, six.text_type): self.request_kw = json.loads(request_kw) else: self.request_kw = request_kw @@ -388,7 +390,9 @@ class SlapOSTester(SlapOSMasterCommunicator): self.name = name self.computer_guid = computer_guid - if isinstance(request_kw, (six.binary_type, six.text_type)): + if isinstance(request_kw, bytes): + self.request_kw = json.loads(request_kw.decode('utf-8')) + elif isinstance(request_kw, six.text_type): self.request_kw = json.loads(request_kw) else: self.request_kw = request_kw -- 2.30.9 From 6a7c266727544bbb8e057574a660cfe7f7333275 Mon Sep 17 00:00:00 2001 From: Bryton Lacquement Date: Sat, 20 Jul 2019 09:59:13 +0200 Subject: [PATCH 07/14] wip --- erp5/util/benchmark/performance_tester.py | 5 +++-- erp5/util/benchmark/report.py | 7 ++++--- erp5/util/benchmark/scalability_tester.py | 3 ++- erp5/util/testbrowser/examples/createERP5User.py | 5 +++-- erp5/util/testbrowser/examples/createPerson.py | 3 ++- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/erp5/util/benchmark/performance_tester.py b/erp5/util/benchmark/performance_tester.py index f5258a6ab1b..c821849b760 100755 --- a/erp5/util/benchmark/performance_tester.py +++ b/erp5/util/benchmark/performance_tester.py @@ -28,6 +28,7 @@ # ############################################################################## +from __future__ import print_function import argparse import os import sys @@ -264,7 +265,7 @@ class PerformanceTester(object): error_message = exit_msg_queue.get() except KeyboardInterrupt, e: - print >>sys.stderr, "\nInterrupted by user, stopping gracefully..." + print("\nInterrupted by user, stopping gracefully...", file=sys.stderr) exit_status = 2 # An IOError may be raised when receiving a SIGINT which interrupts the @@ -337,7 +338,7 @@ class PerformanceTester(object): def main(): error_message_set, exit_status = PerformanceTester().run() for error_message in error_message_set: - print >>sys.stderr, "ERROR: %s" % error_message + print("ERROR: %s" % error_message, file=sys.stderr) sys.exit(exit_status) diff --git a/erp5/util/benchmark/report.py b/erp5/util/benchmark/report.py index 9fc12c0e94d..6ec642fb682 100755 --- a/erp5/util/benchmark/report.py +++ b/erp5/util/benchmark/report.py @@ -31,6 +31,7 @@ # ############################################################################## +from __future__ import print_function import argparse import re @@ -537,7 +538,7 @@ def generateReport(): for filename in filename_iter: # There may be no results at all in case of errors if not os.stat(filename).st_size: - print >>sys.stderr, "Ignoring empty file %s" % filename + print("Ignoring empty file %s" % filename, file=sys.stderr) continue report_dict = per_nb_users_report_dict.setdefault( @@ -546,8 +547,8 @@ def generateReport(): report_dict['filename'].append(filename) if not per_nb_users_report_dict: - print >>sys.stderr, "ERROR: No result file found, perhaps "\ - "``--filename-prefix'' should be specified?" + print("ERROR: No result file found, perhaps "\ + "``--filename-prefix'' should be specified?", file=sys.stderr) sys.exit(1) diff --git a/erp5/util/benchmark/scalability_tester.py b/erp5/util/benchmark/scalability_tester.py index 13cd4f753b6..37e47b39635 100644 --- a/erp5/util/benchmark/scalability_tester.py +++ b/erp5/util/benchmark/scalability_tester.py @@ -28,6 +28,7 @@ # ############################################################################## +from __future__ import print_function from .result import CSVBenchmarkResult, NothingFlushedException class CSVScalabilityBenchmarkResult(CSVBenchmarkResult): @@ -60,7 +61,7 @@ class ScalabilityTester(PerformanceTester): urllib.urlencode({'error_message_set': '|'.join(error_message_set)})).close() except: - print >>sys.stderr, "ERROR: %s" % Formatter().formatException(sys.exc_info()) + print("ERROR: %s" % Formatter().formatException(sys.exc_info()), file=sys.stderr) def getResultClass(self): if not self._argument_namespace.erp5_publish_url: diff --git a/erp5/util/testbrowser/examples/createERP5User.py b/erp5/util/testbrowser/examples/createERP5User.py index 437f20e0b99..1d021283c53 100755 --- a/erp5/util/testbrowser/examples/createERP5User.py +++ b/erp5/util/testbrowser/examples/createERP5User.py @@ -8,6 +8,7 @@ # # TODO: There must be a better way than the code below to do that though... +from __future__ import print_function import sys from erp5.util.testbrowser.browser import Browser @@ -19,9 +20,9 @@ try: user_nbr = int(user_nbr) except ValueError: - print >>sys.stderr, "ERROR: Missing arguments: %s URL USERNAME " \ + print("ERROR: Missing arguments: %s URL USERNAME " \ "PASSWORD NUMBER_OF_USERS NEW_USERNAME_PREFIX NEW_USERS_PASSWORD" % \ - sys.argv[0] + sys.argv[0], file=sys.stderr) sys.exit(1) diff --git a/erp5/util/testbrowser/examples/createPerson.py b/erp5/util/testbrowser/examples/createPerson.py index 0b884def196..7db95f8102b 100755 --- a/erp5/util/testbrowser/examples/createPerson.py +++ b/erp5/util/testbrowser/examples/createPerson.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import print_function from erp5.util.testbrowser.browser import Browser ITERATION = 20 @@ -89,4 +90,4 @@ if __name__ == '__main__': counter += 1 for title, time_list in result_dict.iteritems(): - print "%s: %.4fs" % (title, float(sum(time_list)) / ITERATION) + print("%s: %.4fs" % (title, float(sum(time_list)) / ITERATION)) -- 2.30.9 From c56d634cbd8f6789976b5d4ee50b9344335f2777 Mon Sep 17 00:00:00 2001 From: Bryton Lacquement Date: Mon, 22 Jul 2019 16:51:44 +0200 Subject: [PATCH 08/14] wip --- erp5/util/testnode/ScalabilityTestRunner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erp5/util/testnode/ScalabilityTestRunner.py b/erp5/util/testnode/ScalabilityTestRunner.py index a10adc2146e..176f8a4b2d1 100644 --- a/erp5/util/testnode/ScalabilityTestRunner.py +++ b/erp5/util/testnode/ScalabilityTestRunner.py @@ -334,7 +334,7 @@ Require valid-user htaccess_file.close() password_path = testsuite_directory + PASSWORD_FILE with open(password_path, "w") as password_file: - password = ''.join(random.choice(string.digits + string.letters) for i in range(PASSWORD_LENGTH)) + password = ''.join(random.choice(string.digits + string.ascii_letters) for i in range(PASSWORD_LENGTH)) password_file.write(password) user = TESTNODE_USER command = [apache_htpasswd, "-bc", testsuite_directory + HTPASSWD, user, password] -- 2.30.9 From db74098258105069fd739a766d41111997838589 Mon Sep 17 00:00:00 2001 From: Bryton Lacquement Date: Mon, 22 Jul 2019 17:03:38 +0200 Subject: [PATCH 09/14] wip --- product/ERP5/bin/genbt5list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/ERP5/bin/genbt5list b/product/ERP5/bin/genbt5list index 15e678ee2f3..d6edd7f22c0 100755 --- a/product/ERP5/bin/genbt5list +++ b/product/ERP5/bin/genbt5list @@ -38,7 +38,7 @@ import sys try: from html import escape except ImportError: - from cgi import escape + from cgi import escape # Deprecated since version 3.2 from base64 import b64encode from io import BytesIO from hashlib import sha1 -- 2.30.9 From c46ba4e0c24165bac6c969cd357ddae302dc1973 Mon Sep 17 00:00:00 2001 From: Bryton Lacquement Date: Mon, 29 Jul 2019 17:26:28 +0200 Subject: [PATCH 10/14] wip --- erp5/tests/testERP5TestNode.py | 8 ++++---- erp5/util/benchmark/report.py | 6 ++---- erp5/util/testbrowser/examples/createERP5User.py | 7 ++----- erp5/util/testbrowser/examples/createPerson.py | 4 ++-- erp5/util/testnode/ProcessManager.py | 3 +-- erp5/util/testnode/ScalabilityTestRunner.py | 4 ++-- erp5/util/testnode/SlapOSMasterCommunicator.py | 2 +- erp5/util/testnode/testnode.py | 4 ++-- erp5/util/timinglogparser/__init__.py | 4 ++-- erp5/util/timinglogplotter/__init__.py | 12 ++++++------ product/ERP5/bin/genbt5list | 8 +++++--- 11 files changed, 29 insertions(+), 33 deletions(-) diff --git a/erp5/tests/testERP5TestNode.py b/erp5/tests/testERP5TestNode.py index c6b7c46e9b4..b9e4747d876 100644 --- a/erp5/tests/testERP5TestNode.py +++ b/erp5/tests/testERP5TestNode.py @@ -804,15 +804,15 @@ shared = true test_node = self.getTestNode() node_test_suite = test_node.getNodeTestSuite('foo') folder = node_test_suite.test_suite_directory - self.assertEqual(False, os.path.exists(folder)) + self.assertFalse(os.path.exists(folder)) createFolder(folder) - self.assertEqual(True, os.path.exists(folder)) + self.assertTrue(os.path.exists(folder)) to_drop_path = os.path.join(folder, 'drop') to_drop = open(to_drop_path, 'w') to_drop.close() - self.assertEqual(True, os.path.exists(to_drop_path)) + self.assertTrue(os.path.exists(to_drop_path)) createFolder(folder, clean=True) - self.assertEqual(False, os.path.exists(to_drop_path)) + self.assertFalse(os.path.exists(to_drop_path)) def test_15_suite_log_directory(self, my_test_type='UnitTest', grade='master'): def doNothing(self, *args, **kw): diff --git a/erp5/util/benchmark/report.py b/erp5/util/benchmark/report.py index 6ec642fb682..0755b422a41 100755 --- a/erp5/util/benchmark/report.py +++ b/erp5/util/benchmark/report.py @@ -547,10 +547,8 @@ def generateReport(): report_dict['filename'].append(filename) if not per_nb_users_report_dict: - print("ERROR: No result file found, perhaps "\ - "``--filename-prefix'' should be specified?", file=sys.stderr) - - sys.exit(1) + sys.exit("ERROR: No result file found, perhaps ``--filename-prefix'' should" + "be specified?") pdf = PdfPages(argument_namespace.output_filename) diff --git a/erp5/util/testbrowser/examples/createERP5User.py b/erp5/util/testbrowser/examples/createERP5User.py index 1d021283c53..4f94ee167a3 100755 --- a/erp5/util/testbrowser/examples/createERP5User.py +++ b/erp5/util/testbrowser/examples/createERP5User.py @@ -20,11 +20,8 @@ try: user_nbr = int(user_nbr) except ValueError: - print("ERROR: Missing arguments: %s URL USERNAME " \ - "PASSWORD NUMBER_OF_USERS NEW_USERNAME_PREFIX NEW_USERS_PASSWORD" % \ - sys.argv[0], file=sys.stderr) - - sys.exit(1) + sys.exit("ERROR: Missing arguments: %s URL USERNAME PASSWORD NUMBER_OF_USERS " + "NEW_USERNAME_PREFIX NEW_USERS_PASSWORD" % sys.argv[0]) # Create a browser instance browser = Browser(url, username, password) diff --git a/erp5/util/testbrowser/examples/createPerson.py b/erp5/util/testbrowser/examples/createPerson.py index 7db95f8102b..1dade9efc2a 100755 --- a/erp5/util/testbrowser/examples/createPerson.py +++ b/erp5/util/testbrowser/examples/createPerson.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import print_function +from __future__ import division, print_function from erp5.util.testbrowser.browser import Browser ITERATION = 20 @@ -90,4 +90,4 @@ if __name__ == '__main__': counter += 1 for title, time_list in result_dict.iteritems(): - print("%s: %.4fs" % (title, float(sum(time_list)) / ITERATION)) + print("%s: %.4fs" % (title, sum(time_list) / ITERATION)) diff --git a/erp5/util/testnode/ProcessManager.py b/erp5/util/testnode/ProcessManager.py index 17f3024bae1..445d68d1b21 100644 --- a/erp5/util/testnode/ProcessManager.py +++ b/erp5/util/testnode/ProcessManager.py @@ -33,7 +33,6 @@ import signal import sys import time from . import logger -from slapos.util import bytes2str MAX_TIMEOUT = 3600 * 4 @@ -80,7 +79,7 @@ def subprocess_capture(p, log_prefix, get_output=True): break if get_output: buffer.append(data) - log(log_prefix + bytes2str(data).rstrip('\n')) + log(log_prefix + data.decode('utf-8', errors='replace').rstrip('\n')) if p.stdout: stdout = [] stdout_thread = threading.Thread(target=readerthread, diff --git a/erp5/util/testnode/ScalabilityTestRunner.py b/erp5/util/testnode/ScalabilityTestRunner.py index 176f8a4b2d1..feace45962f 100644 --- a/erp5/util/testnode/ScalabilityTestRunner.py +++ b/erp5/util/testnode/ScalabilityTestRunner.py @@ -37,7 +37,7 @@ import shutil import logging import string import random -from six.moves.urllib import parse +from six.moves.urllib.parse import urlparse import base64 from six.moves import http_client as httplib from . import Utils @@ -364,7 +364,7 @@ Require valid-user user, password = self.generateProfilePasswordAccess() logger.info("Software Profile password: %s" % password) self.reachable_profile = "https://%s:%s@%s" % (user, password, - os.path.join(parse.urlparse(self.testnode.config['frontend_url']).netloc, + os.path.join(urlparse(self.testnode.config['frontend_url']).netloc, "software", self.randomized_path, "software.cfg")) def prepareSlapOSForTestSuite(self, node_test_suite): diff --git a/erp5/util/testnode/SlapOSMasterCommunicator.py b/erp5/util/testnode/SlapOSMasterCommunicator.py index 9d086c64586..7723c75a6b0 100644 --- a/erp5/util/testnode/SlapOSMasterCommunicator.py +++ b/erp5/util/testnode/SlapOSMasterCommunicator.py @@ -11,7 +11,7 @@ from uritemplate import expand import slapos.slap from slapos.slap import SoftwareProductCollection from requests.exceptions import HTTPError -from erp5.util.taskdistribution import SAFE_RPC_EXCEPTION_LIST +from ..taskdistribution import SAFE_RPC_EXCEPTION_LIST from . import logger import six diff --git a/erp5/util/testnode/testnode.py b/erp5/util/testnode/testnode.py index 9afdffaa8d0..42504cedbc6 100644 --- a/erp5/util/testnode/testnode.py +++ b/erp5/util/testnode/testnode.py @@ -39,7 +39,7 @@ from .ScalabilityTestRunner import ScalabilityTestRunner from .UnitTestRunner import UnitTestRunner from .Utils import deunicodeData from .Utils import rmtree -from erp5.util import taskdistribution +from .. import taskdistribution MAX_LOG_TIME = 15 # time in days we should keep logs that we can see through # httd @@ -171,7 +171,7 @@ shared = true # only limit to particular error, if we run that code for all errors, # then if server having most repositories is down for some time, we would # erase all repositories and facing later hours of downloads - if getattr(error, 'stderr', b'').find(b'index') >= 0: + if b'index' in getattr(error, 'stderr', b''): rmtree(repository_path) logger.warning("Error while getting repository, ignoring this test suite", exc_info=1) diff --git a/erp5/util/timinglogparser/__init__.py b/erp5/util/timinglogparser/__init__.py index 7dbc5505616..78429648c15 100755 --- a/erp5/util/timinglogparser/__init__.py +++ b/erp5/util/timinglogparser/__init__.py @@ -27,7 +27,7 @@ # ############################################################################## -from __future__ import print_function +from __future__ import division, print_function import os import sys @@ -150,7 +150,7 @@ def parseFile(filename, measure_dict): if line_number > 0: duration = time() - begin print("Matched %i lines (%.2f%%), %i skipped (%.2f%%), %i unmatched (%.2f%%) in %.2fs (%i lines per second)." % \ - (match_count, (float(match_count) / line_number) * 100, skip_count, (float(skip_count) / line_number) * 100, (line_number - match_count - skip_count), (1 - (float(match_count + skip_count) / line_number)) * 100, duration, line_number / duration), + (match_count, (match_count / line_number) * 100, skip_count, (skip_count / line_number) * 100, (line_number - match_count - skip_count), (1 - (match_count + skip_count) / line_number)) * 100, duration, line_number // duration), file=sys.stderr) debug = False diff --git a/erp5/util/timinglogplotter/__init__.py b/erp5/util/timinglogplotter/__init__.py index 5c8fb02301b..2351ae60198 100755 --- a/erp5/util/timinglogplotter/__init__.py +++ b/erp5/util/timinglogplotter/__init__.py @@ -27,7 +27,7 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # ############################################################################## -from __future__ import print_function +from __future__ import division, print_function from datetime import date from os import path @@ -84,9 +84,9 @@ class CSVFile(object): if cell > value_max.get(key, 0): value_max[key] = cell column_dict[key].append(cell) - line_num = float(line_num) / 100 + line_num = line_num / 100 for key in ratio_dict: - ratio_dict[key] /= line_num + ratio_dict[key] //= line_num def getColumn(self, column_id): return self.column_dict[self.column_list[column_id]] @@ -103,7 +103,7 @@ def computeExpr(expr): if expr: assert expr[0] == '=' num, denom = expr[1:].split('/') - result = float(int(num)) / int(denom) + result = int(num) / int(denom) else: result = None return result @@ -136,7 +136,7 @@ def main(): # date_list will be like ['2009/07/01', '2009/07/05', '2009/07/10', ...] factor = 1 if len(date_string_list) > 20: - factor = int(len(date_string_list) / 20) + factor = int(len(date_string_list) // 20) i = 0 for date_string in date_string_list: if i % factor == 0: @@ -190,7 +190,7 @@ def main(): if options.minimal_non_empty_ratio is not None: column_len = len(column) if column_len: - if float(len(x_data))/column_len < options.minimal_non_empty_ratio: + if len(x_data) / column_len < options.minimal_non_empty_ratio: print('Not enough values to plot for %s...' % (out_file_name, )) continue r_y_data = robjects.FloatVector(y_data) diff --git a/product/ERP5/bin/genbt5list b/product/ERP5/bin/genbt5list index d6edd7f22c0..c2a6dfd526b 100755 --- a/product/ERP5/bin/genbt5list +++ b/product/ERP5/bin/genbt5list @@ -213,11 +213,13 @@ def generateInformation(dir, info=id, err=None): property_list = BusinessTemplate.fromDir(path) else: continue - xml.write(b'