From 16e44e2f6a6dce896bcde31a5bad01349109e1b0 Mon Sep 17 00:00:00 2001 From: Bryton Lacquement Date: Mon, 4 Feb 2019 10:00:16 +0100 Subject: [PATCH] Add support for Python 3 --- slapos/agent/agent.py | 10 +- slapos/agent/tester.py | 24 ++-- slapos/apachedex.py | 16 ++- slapos/cachechecker.py | 13 ++- slapos/container/__init__.py | 10 +- slapos/equeue/__init__.py | 20 ++-- slapos/generatefeed.py | 2 +- slapos/lamp/__init__.py | 23 ++-- slapos/monitor/collect.py | 77 ++++++------- slapos/monitor/globalstate.py | 27 +++-- slapos/monitor/monitor.py | 61 +++++----- slapos/monitor/monitor_config_write.py | 36 +++--- slapos/monitor/runpromise.py | 6 +- slapos/networkbench/__init__.py | 14 ++- slapos/networkbench/http.py | 7 +- slapos/networkbench/ping.py | 5 +- slapos/onetimedownload.py | 6 +- slapos/onetimeupload/__init__.py | 2 +- .../promise/apache_mpm_watchdog/__init__.py | 6 +- .../check_apachedex_result/__init__.py | 2 +- .../promise/check_computer_memory/__init__.py | 22 ++-- .../__init__.py | 6 +- slapos/promise/check_user_memory/__init__.py | 6 +- .../check_web_page_http_cache_hit/__init__.py | 21 ++-- .../is_local_tcp_port_opened/__init__.py | 3 +- .../__init__.py | 12 +- .../plugin/backupserver_check_backup.py | 34 +++--- .../promise/plugin/check_error_on_http_log.py | 22 ++-- .../check_error_on_zope_longrequest_log.py | 12 +- slapos/promise/plugin/check_file_state.py | 6 +- .../promise/plugin/check_free_disk_space.py | 13 +-- .../promise/plugin/check_icmp_packet_lost.py | 6 +- .../check_partition_deployment_state.py | 5 +- .../plugin/check_re6st_optimal_status.py | 6 +- .../promise/plugin/check_server_cpu_load.py | 9 +- slapos/promise/plugin/check_url_available.py | 10 +- .../plugin/monitor_bootstrap_status.py | 7 +- slapos/pubsub/__init__.py | 2 +- slapos/qemuqmpclient/__init__.py | 80 ++++++------- slapos/resilient/runner_exporter.py | 6 +- slapos/resilient/runner_utils.py | 9 +- slapos/runner/runnertest.py | 21 ++-- slapos/runner/sup_process.py | 2 +- slapos/runner/utils.py | 47 ++++---- slapos/runner/views.py | 30 +++-- slapos/securedelete.py | 7 +- slapos/systool.py | 6 +- slapos/test/monitor/test_config_document.py | 22 ++-- slapos/test/monitor/testbootstrap.py | 8 +- slapos/test/monitor/testglobalstate.py | 9 +- slapos/test/monitor/testrunpromise.py | 70 +++++++----- .../promise/plugin/test_check_file_state.py | 13 ++- .../plugin/test_check_url_available.py | 48 ++++---- .../test/promise/test_apache_mpm_watchdog.py | 2 +- slapos/test/test_agent.py | 108 +++++++++--------- slapos/test/test_checkfeedaspromise.py | 8 +- slapos/test/test_generatefeed.py | 5 +- slapos/test/test_qemuqmpclient.py | 4 +- slapos/test/test_runner.py | 9 -- slapos/test/test_runner_exporter.py | 11 +- slapos/test/test_securedelete.py | 26 ++--- 61 files changed, 590 insertions(+), 530 deletions(-) diff --git a/slapos/agent/agent.py b/slapos/agent/agent.py index 5e56037..7c4b1d8 100644 --- a/slapos/agent/agent.py +++ b/slapos/agent/agent.py @@ -25,7 +25,7 @@ # ############################################################################## -import ConfigParser +from six.moves import configparser import argparse import collections import json @@ -57,7 +57,7 @@ class AutoSTemp(object): def __init__(self, value): fd, self.__name = tempfile.mkstemp() - os.write(fd, value) + os.write(fd, value.encode('utf-8')) os.close(fd) @property @@ -67,7 +67,7 @@ class AutoSTemp(object): def __del__(self): self.__unlink(self.__name) -from tester import SoftwareReleaseTester +from .tester import SoftwareReleaseTester class TestMap(object): # tell pytest to skip this class (even if name starts with Test) @@ -94,7 +94,7 @@ class TestMap(object): return set(exclude_list + list(self.ran_test_set)) def getGroupList(self): - return self.test_map_dict.keys() + return list(self.test_map_dict) def dropGroup(self, group): del self.test_map_dict[group] @@ -217,7 +217,7 @@ def main(): logger, log_file = getLogger(log, args.verbose) - configuration = ConfigParser.SafeConfigParser() + configuration = configparser.SafeConfigParser() configuration.readfp(args.configuration_file) pidfile = args.pidfile diff --git a/slapos/agent/tester.py b/slapos/agent/tester.py index 5894789..71ef78d 100644 --- a/slapos/agent/tester.py +++ b/slapos/agent/tester.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import datetime import json import sys @@ -50,16 +52,16 @@ def retryOnNetworkFailure(func): while True: try: return func(*args, **kwargs) - except SAFE_RPC_EXCEPTION_LIST, e: - print 'Network failure: %s , %s' % (sys.exc_info(), e) - except HTTPError, e: - print 'Network failure: %s , %s' % (sys.exc_info(), e) - except ConnectionError, e: - print 'Network failure: %s , %s' % (sys.exc_info(), e) - except slapos.slap.ConnectionError, e: - print 'Network failure: %s , %s' % (sys.exc_info(), e) - - print 'Retry method %s in %i seconds' % (func, retry_time) + except SAFE_RPC_EXCEPTION_LIST as e: + print('Network failure: %s , %s' % (sys.exc_info(), e)) + except HTTPError as e: + print('Network failure: %s , %s' % (sys.exc_info(), e)) + except ConnectionError as e: + print('Network failure: %s , %s' % (sys.exc_info(), e)) + except slapos.slap.ConnectionError as e: + print('Network failure: %s , %s' % (sys.exc_info(), e)) + + print('Retry method %s in %i seconds' % (func, retry_time)) time.sleep(retry_time) retry_time = min(retry_time*1.5, 640) @@ -218,7 +220,7 @@ class SlapOSMasterCommunicator(object): result = self.hateoas_navigator.GET(url) result = json.loads(result) if result['_links'].get('action_object_slap', None) is None: - print result['links'] + print(result['links']) return None object_link = self.hateoas_navigator.hateoasGetLinkFromLinks( diff --git a/slapos/apachedex.py b/slapos/apachedex.py index 99f1a41..0f40228 100644 --- a/slapos/apachedex.py +++ b/slapos/apachedex.py @@ -27,6 +27,8 @@ # ############################################################################## +from __future__ import print_function + import os, errno import subprocess import argparse @@ -52,7 +54,7 @@ def build_command(apachedex_executable, output_file, # Automaticaly replace variable 'date'. apache_log = logfile.strip() % {'date': today} if not os.path.exists(apache_log): - print "WARNING: File %s not found..." % apache_log + print("WARNING: File %s not found..." % apache_log) continue log_list.append(apache_log) if not log_list: @@ -81,7 +83,7 @@ def main(): base_url = args.base_url.strip() if not os.path.exists(output_folder) or not os.path.isdir(output_folder): - print "ERROR: Output folder is not a directory. Exiting..." + print("ERROR: Output folder is not a directory. Exiting...") return 1 today = date.today().strftime("%Y-%m-%d") @@ -93,21 +95,23 @@ def main(): args.apache_log_list, config) except ValueError as e: - print e + print(e) return 1 process_handler = subprocess.Popen(argument_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, ) stdout, stderr = process_handler.communicate() if process_handler.returncode != 0: if stderr: - print stderr + print(stderr) return 1 - with open(output_file, 'r') as f: - print base_url + '/ApacheDex-%s.html' % today + # Check that output_file is a readable file. + with open(output_file, 'r'): + print(base_url + '/ApacheDex-%s.html' % today) return 0 if __name__ == "__main__": diff --git a/slapos/cachechecker.py b/slapos/cachechecker.py index 5edde5e..ef61fc1 100644 --- a/slapos/cachechecker.py +++ b/slapos/cachechecker.py @@ -27,6 +27,8 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # ############################################################################## +from __future__ import print_function + import os import shutil import sys @@ -35,6 +37,7 @@ import shlex from subprocess import Popen, PIPE, STDOUT import logging from datetime import datetime +import six _MARKER = [] WGET = 'wget' @@ -428,7 +431,7 @@ class HTTPCacheCheckerTestSuite(object): if self.report_dict: report_message_list = ['*Errors*:'] - for url, message_list in self.report_dict.iteritems(): + for url, message_list in six.iteritems(self.report_dict): unique_message_list = [] for message in message_list: if message not in unique_message_list: @@ -447,7 +450,7 @@ class HTTPCacheCheckerTestSuite(object): from optparse import OptionParser -import ConfigParser +from six.moves.configparser import RawConfigParser def _formatConfiguration(configuration): """ format the configuration""" @@ -463,11 +466,11 @@ def web_checker_utility(): (options, args) = parser.parse_args() if len(args) != 1 : - print parser.print_help() + print(parser.print_help()) parser.error('incorrect number of arguments') config_path = args[0] - config = ConfigParser.RawConfigParser() + config = RawConfigParser() config.read(config_path) working_directory = config.get('web_checker', 'working_directory') @@ -525,7 +528,7 @@ def web_checker_utility(): file_object.write(result) file_object.close() else: - print result + print(result) if __name__ == '__main__': sys.exit(web_checker_utility()) diff --git a/slapos/container/__init__.py b/slapos/container/__init__.py index bce323d..51c05bd 100644 --- a/slapos/container/__init__.py +++ b/slapos/container/__init__.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -import ConfigParser +from six.moves import configparser import argparse -import gdbm +from six.moves import dbm_gnu as gdbm import sys import os @@ -41,7 +41,7 @@ def main(): run(args) def run(args): - slapos_conf = ConfigParser.ConfigParser() + slapos_conf = configparser.ConfigParser() slapos_conf.read(args.configuration_file) current_binary = os.path.join(os.getcwd(), sys.argv[0]) @@ -52,7 +52,7 @@ def run(args): partition_base_name = slapos_conf.get('slapformat', 'partition_base_name') try: bridge_name = slapos_conf.get('slapformat', 'interface_name') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: bridge_name = slapos_conf.get('slapformat', 'bridge_name') instance_root = slapos_conf.get('slapos', 'instance_root') partition_base_path = os.path.join(instance_root, partition_base_name) @@ -61,7 +61,7 @@ def run(args): logging.basicConfig(level=logging.getLevelName(args.log[0])) - database = gdbm.open(args.database, 'c', 0600) + database = gdbm.open(args.database, 'c', 0o600) try: process.main(sr_directory, partition_list, database, bridge_name) finally: diff --git a/slapos/equeue/__init__.py b/slapos/equeue/__init__.py index fd3be4c..084bf73 100644 --- a/slapos/equeue/__init__.py +++ b/slapos/equeue/__init__.py @@ -28,7 +28,7 @@ import argparse import errno -import gdbm +from six.moves import dbm_gnu as gdbm import json from lockfile import LockFile import logging @@ -38,8 +38,8 @@ import signal import socket import subprocess import sys -import SocketServer -import StringIO +from six.moves import socketserver +import io import threading # Copied from erp5.util:erp5/util/testnode/ProcessManager.py @@ -75,15 +75,15 @@ def subprocess_capture(p, log, log_prefix, get_output=True): return (p.stdout and ''.join(stdout), p.stderr and ''.join(stderr)) -class EqueueServer(SocketServer.ThreadingUnixStreamServer): +class EqueueServer(socketserver.ThreadingUnixStreamServer): daemon_threads = True def __init__(self, *args, **kw): self.options = kw.pop('equeue_options') - SocketServer.ThreadingUnixStreamServer.__init__(self, - RequestHandlerClass=None, - *args, **kw) + super(EqueueServer, self).__init__(self, + RequestHandlerClass=None, + *args, **kw) # Equeue Specific elements self.setLogger(self.options.logfile[0], self.options.loglevel[0]) self.setDB(self.options.database[0]) @@ -106,7 +106,7 @@ class EqueueServer(SocketServer.ThreadingUnixStreamServer): self.logger.addHandler(handler) def setDB(self, database): - self.db = gdbm.open(database, 'cs', 0700) + self.db = gdbm.open(database, 'cs', 0o700) def _hasTakeoverBeenTriggered(self): if hasattr(self, 'takeover_triggered_file_path') and \ @@ -149,7 +149,7 @@ class EqueueServer(SocketServer.ThreadingUnixStreamServer): # Handle request self.logger.debug("Connection with file descriptor %d", request.fileno()) request.settimeout(self.options.timeout) - request_string = StringIO.StringIO() + request_string = io.StringIO() segment = None try: while segment != '': @@ -181,7 +181,7 @@ class EqueueServer(SocketServer.ThreadingUnixStreamServer): def remove_existing_file(path): try: os.remove(path) - except OSError, e: + except OSError as e: if e.errno != errno.ENOENT: raise diff --git a/slapos/generatefeed.py b/slapos/generatefeed.py index 8eb32c2..3d1d5cd 100644 --- a/slapos/generatefeed.py +++ b/slapos/generatefeed.py @@ -76,7 +76,7 @@ def generateFeed(option): # Reduces feed if number of items exceeds max_item if len(item_dict) > option.max_item: - outdated_key_list = sorted_item_dict.keys()[:-option.max_item] + outdated_key_list = list(sorted_item_dict)[:-option.max_item] for outdated_key in outdated_key_list: del sorted_item_dict[outdated_key] deleteFileList(outdated_key_list) diff --git a/slapos/lamp/__init__.py b/slapos/lamp/__init__.py index 6db4707..f252652 100644 --- a/slapos/lamp/__init__.py +++ b/slapos/lamp/__init__.py @@ -12,6 +12,7 @@ # ############################################################################## +from __future__ import print_function import os import time @@ -76,7 +77,7 @@ def run(): result = parser.parse_args() arguments = dict(result._get_kwargs()) if arguments['token'] == None and arguments['file_token'] == None: - print "lampconfigure: Error: Please specify where condition will be taken, use -d or -f option" + print("lampconfigure: Error: Please specify where condition will be taken, use -d or -f option") return setup(arguments) @@ -84,7 +85,7 @@ def setup(arguments): timeout = 5; while True: if not checkAction(arguments): - print "Waiting for 3s and retrying" + print("Waiting for 3s and retrying") time.sleep(3) continue time.sleep(timeout) @@ -115,9 +116,9 @@ def checkAction(arguments): user = arguments['mysql_user'], passwd = arguments['mysql_password'], db = arguments['token']) - except Exception, ex: + except Exception as e: #Mysql is not ready yet?... - print ex.message + print(e) return False if arguments['table'] == "**": #only detect if mysql has been started @@ -145,7 +146,7 @@ def rename(arguments): source = os.path.join(arguments['target_directory'], arguments['source']) destination = os.path.join(arguments['target_directory'], arguments['destination']) if not os.path.exists(source): - print "Error when moving: '%s': no such file or directory" % source + print("Error when moving: '%s': no such file or directory" % source) return os.rename(source, destination) if arguments['mode'] != None: @@ -155,7 +156,7 @@ def delete(arguments): for path in arguments['delete_target']: path = os.path.join(arguments['target_directory'], path) if not os.path.exists(path): - print "Error when deleting: '%s': no such file or directory" % path + print("Error when deleting: '%s': no such file or directory" % path) continue if os.path.isdir(path): shutil.rmtree(path) @@ -164,7 +165,7 @@ def delete(arguments): def run_script(arguments): script = os.path.join(arguments['target_directory'], arguments['script']) - print 'Running script: %s' % script + print('Running script: %s' % script) if os.path.exists(script): import subprocess #run python script with predefined data @@ -176,12 +177,12 @@ def run_script(arguments): result = subprocess.Popen(data, env={'PYTHONPATH': ':'.join(sys.path)}) result.wait() else: - print "Error: can not read file '%s'" % script + print("Error: can not read file '%s'" % script) def run_sql_script(arguments): script = os.path.join(arguments['target_directory'], arguments['sql_script']) - print 'Running SQL script: %s' % script + print('Running SQL script: %s' % script) if os.path.exists(script): conn = MySQLdb.connect(host=arguments['mysql_host'], port=int(arguments['mysql_port']), @@ -196,7 +197,7 @@ def run_sql_script(arguments): conn.close() else: - print "Error: can not read file '%s'" % script + print("Error: can not read file '%s'" % script) @@ -204,6 +205,6 @@ def chmod(arguments): for path in arguments['chmod_target']: path = os.path.join(arguments['target_directory'], path) if not os.path.exists(path): - print "Error when changing mode: '%s': no such file or directory" % path + print("Error when changing mode: '%s': no such file or directory" % path) continue os.chmod(path, int(arguments['mode'], 8)) diff --git a/slapos/monitor/collect.py b/slapos/monitor/collect.py index d3faeba..acb7c24 100644 --- a/slapos/monitor/collect.py +++ b/slapos/monitor/collect.py @@ -27,6 +27,8 @@ # ############################################################################## +from __future__ import division + import sqlite3 import os import pwd @@ -80,19 +82,17 @@ class ResourceCollect: table="sqlite_master", columns='name', where="type='table' AND name='%s'" % name) - table_exists_result = zip(*check_result_cursor) - if not len(table_exists_result) or table_exists_result[0][0] is None: - return False - return True + r = check_result_cursor.fetchone() + return r and r[0] is not None def getPartitionCPULoadAverage(self, partition_id, date_scope): return self.consumption_utils.getPartitionCPULoadAverage(partition_id, date_scope) def getPartitionUsedMemoryAverage(self, partition_id, date_scope): - return self.consumption_utils.getPartitionUsedMemoryAverage(partition_id, date_scope)/(1024*1024.0) + return self.consumption_utils.getPartitionUsedMemoryAverage(partition_id, date_scope)/(1024*1024) def getPartitionDiskUsedAverage(self, partition_id, date_scope): - return self.consumption_utils.getPartitionDiskUsedAverage(partition_id, date_scope)/1024.0 + return self.consumption_utils.getPartitionDiskUsedAverage(partition_id, date_scope)/1024 def getPartitionConsumption(self, partition_id, where="", date_scope=None, min_time=None, max_time=None): """ @@ -123,10 +123,10 @@ class ResourceCollect: resource_dict = { 'pid': result[6], 'cpu_percent': round(result[1]/count, 2), - 'cpu_time': round((result[2] or 0)/(60.0), 2), + 'cpu_time': round((result[2] or 0)/(60), 2), 'cpu_num_threads': round(result[3]/count, 2), 'memory_percent': round(result[4]/count, 2), - 'memory_rss': round((result[5] or 0)/(1024*1024.0), 2), + 'memory_rss': round((result[5] or 0)/(1024*1024), 2), 'io_rw_counter': round(result[7]/count, 2), 'io_cycles_counter': round(result[8]/count, 2) } @@ -159,38 +159,35 @@ class ResourceCollect: query_result = self.db.select('user', date_scope, colums, where="partition='%s' and (time between '%s' and '%s') %s" % (partition_id, min_time, max_time, where)) - result_list = zip(*query_result) - - process_dict = memory_dict = io_dict = {} - if len(result_list): - result = result_list - process_dict = {'total_process': result[0][0], - 'cpu_percent': round((result[1][0] or 0), 2), - 'cpu_time': round((result[2][0] or 0)/(60.0), 2), - 'cpu_num_threads': round((result[3][0] or 0), 2), - 'date': '%s %s' % (date_scope, min_time) - } - memory_dict = {'memory_percent': round((result[4][0] or 0), 2), - 'memory_rss': round((result[5][0] or 0)/(1024*1024.0), 2), - 'date': '%s %s' % (date_scope, min_time) - } - io_dict = {'io_rw_counter': round((result[6][0] or 0), 2), - 'io_cycles_counter': round((result[7][0] or 0), 2), - 'disk_used': 0, - 'date': '%s %s' % (date_scope, min_time) - } - if self.has_table('folder'): - disk_result_cursor = self.db.select( - "folder", date_scope, - columns="SUM(disk_used)", - where="partition='%s' and (time between '%s' and '%s') %s" % ( - partition_id, min_time, max_time, where - ) + result = query_result.fetchone() + + process_dict = {'total_process': result[0], + 'cpu_percent': round((result[1] or 0), 2), + 'cpu_time': round((result[2] or 0)/(60), 2), + 'cpu_num_threads': round((result[3] or 0), 2), + 'date': '%s %s' % (date_scope, min_time) + } + memory_dict = {'memory_percent': round((result[4] or 0), 2), + 'memory_rss': round((result[5] or 0)/(1024*1024), 2), + 'date': '%s %s' % (date_scope, min_time) + } + io_dict = {'io_rw_counter': round((result[6] or 0), 2), + 'io_cycles_counter': round((result[7] or 0), 2), + 'disk_used': 0, + 'date': '%s %s' % (date_scope, min_time) + } + if self.has_table('folder'): + disk_result_cursor = self.db.select( + "folder", date_scope, + columns="SUM(disk_used)", + where="partition='%s' and (time between '%s' and '%s') %s" % ( + partition_id, min_time, max_time, where ) - - disk_used_sum = zip(*disk_result_cursor) - if len(disk_used_sum) and disk_used_sum[0][0] is not None: - io_dict['disk_used'] = round(disk_used_sum[0][0]/1024.0, 2) + ) + + disk_used_sum, = disk_result_cursor.fetchone() + if disk_used_sum is not None: + io_dict['disk_used'] = round(disk_used_sum/1024, 2) self.db.close() return (process_dict, memory_dict, io_dict) @@ -252,7 +249,7 @@ def main(): status_file = os.path.join(parser.output_folder, 'monitor_resource.status.json') if not os.path.exists(parser.collector_db): - print "Collector database not found..." + print("Collector database not found...") initProcessDataFile(process_file) initMemoryDataFile(mem_file) initIODataFile(io_file) diff --git a/slapos/monitor/globalstate.py b/slapos/monitor/globalstate.py index f79d579..45521a5 100644 --- a/slapos/monitor/globalstate.py +++ b/slapos/monitor/globalstate.py @@ -1,16 +1,20 @@ #!/usr/bin/env python +from __future__ import print_function + import sys import os import glob import json -import ConfigParser +from six.moves import configparser import time from datetime import datetime import base64 import hashlib import PyRSS2Gen +from slapos.util import bytes2str, str2bytes + def getKey(item): return item.source.name @@ -30,6 +34,8 @@ class MonitorFeed(object): event_date = item_dict['result']['change-date'] report_date = item_dict['result']['date'] description = item_dict['result'].get('message', '') + guid = base64.b64encode(str2bytes("%s, %s, %s, %s" % (self.hosting_name, + item_dict['title'], has_string, event_date))) rss_item = PyRSS2Gen.RSSItem( categories = [item_dict['status']], source = PyRSS2Gen.Source(item_dict['title'], self.public_url), @@ -37,9 +43,7 @@ class MonitorFeed(object): description = "\n%s" % (description,), link = self.private_url, pubDate = event_date, - guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s, %s, %s" % (self.hosting_name, - item_dict['title'], has_string, event_date)), - isPermaLink=False) + guid = PyRSS2Gen.Guid(bytes2str(guid), isPermaLink=False) ) self.rss_item_list.append(rss_item) @@ -69,7 +73,7 @@ def generateStatisticsData(stat_file_path, content): fstat.write(json.dumps(data_dict)) current_state = '' - if content.has_key('state'): + if 'state' in content: current_state = '%s, %s, %s, %s' % ( content['date'], content['state']['success'], @@ -131,13 +135,14 @@ def generateMonitoringData(config, public_folder, private_folder, public_url, promise_status = "OK" success += 1 tmp_json['result']['change-date'] = tmp_json['result']['date'] - if previous_state_dict.has_key(tmp_json['name']): + if tmp_json['name'] in previous_state_dict: status, change_date, _ = previous_state_dict[tmp_json['name']] if promise_status == status: tmp_json['result']['change-date'] = change_date tmp_json['status'] = promise_status - message_hash = hashlib.md5(tmp_json['result'].get('message', '')).hexdigest() + message_hash = hashlib.md5( + str2bytes(tmp_json['result'].get('message', ''))).hexdigest() new_state_dict[tmp_json['name']] = [ promise_status, tmp_json['result']['change-date'], @@ -150,9 +155,9 @@ def generateMonitoringData(config, public_folder, private_folder, public_url, previous_state_dict.get(tmp_json['name']), public_folder ) - except ValueError, e: + except ValueError as e: # bad json file - print "ERROR: Bad json file at: %s\n%s" % (file, str(e)) + print("ERROR: Bad json file at: %s\n%s" % (file, e)) continue with open(promises_status_file, "w") as f: @@ -187,7 +192,7 @@ def savePromiseHistory(promise_name, state_dict, previous_state_list, else: if previous_state_list is not None: _, change_date, checksum = previous_state_list - current_sum = hashlib.md5(state_dict.get('message', '')).hexdigest() + current_sum = hashlib.md5(str2bytes(state_dict.get('message', ''))).hexdigest() if state_dict['change-date'] == change_date and \ current_sum == checksum: # Only save the changes and not the same info @@ -202,7 +207,7 @@ def savePromiseHistory(promise_name, state_dict, previous_state_list, def run(monitor_conf_file): - config = ConfigParser.ConfigParser() + config = configparser.ConfigParser() config.read(monitor_conf_file) base_folder = config.get('monitor', 'private-folder') diff --git a/slapos/monitor/monitor.py b/slapos/monitor/monitor.py index efcfe3a..dcf2e83 100644 --- a/slapos/monitor/monitor.py +++ b/slapos/monitor/monitor.py @@ -1,20 +1,25 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import print_function + import sys import os import stat import json -import ConfigParser +from six.moves import configparser import traceback import argparse -import urllib2 +from six.moves.urllib.request import urlopen +from six.moves.urllib.error import HTTPError import ssl import glob import socket from datetime import datetime from xml.sax.saxutils import escape +from slapos.util import bytes2str + OPML_START = """ @@ -47,7 +52,7 @@ def parseArguments(): def mkdirAll(path): try: os.makedirs(path) - except OSError, e: + except OSError as e: if e.errno == os.errno.EEXIST and os.path.isdir(path): pass else: raise @@ -55,13 +60,13 @@ def mkdirAll(path): def softConfigGet(config, *args, **kwargs): try: return config.get(*args, **kwargs) - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + except (configparser.NoOptionError, configparser.NoSectionError): return None def createSymlink(source, destination): try: os.symlink(source, destination) - except OSError, e: + except OSError as e: if e.errno != os.errno.EEXIST: raise @@ -99,10 +104,10 @@ class Monitoring(object): def loadConfig(self, pathes, config=None): if config is None: - config = ConfigParser.ConfigParser() + config = configparser.ConfigParser() try: config.read(pathes) - except ConfigParser.MissingSectionHeaderError: + except configparser.MissingSectionHeaderError: traceback.print_exc() return config @@ -131,8 +136,8 @@ class Monitoring(object): try: with open(config_list[2]) as cfile: param_value = cfile.read() - except OSError, e: - print 'Cannot read file %s, Error is: %s' % (config_list[2], str(e)) + except OSError as e: + print('Cannot read file %s, Error is: %s' % (config_list[2], e)) pass else: param_value = "" @@ -147,7 +152,7 @@ class Monitoring(object): ) if config_list[0] == 'htpasswd': if len(config_list) != 5 or not os.path.exists(config_list[4]): - print 'htpasswd file is not specified: %s' % str(config_list) + print('htpasswd file is not specified: %s' % config_list) continue parameter['description']['user'] = config_list[3] parameter['description']['htpasswd'] = config_list[4] @@ -178,8 +183,8 @@ class Monitoring(object): } ) configuration_list.append(parameter) - except OSError, e: - print 'Cannot read file at %s, Error is: %s' % (old_cors_file, str(e)) + except OSError as e: + print('Cannot read file at %s, Error is: %s' % (old_cors_file, e)) pass return configuration_list @@ -192,7 +197,7 @@ class Monitoring(object): try: mkdirAll(dirname) # could also raise OSError os.symlink(path, os.path.join(dirname, os.path.basename(path))) - except OSError, e: + except OSError as e: if e.errno != os.errno.EEXIST: raise @@ -212,20 +217,20 @@ class Monitoring(object): # XXX - working here with public url if hasattr(ssl, '_create_unverified_context'): context = ssl._create_unverified_context() - response = urllib2.urlopen(url, context=context, timeout=timeout) + response = urlopen(url, context=context, timeout=timeout) else: - response = urllib2.urlopen(url, timeout=timeout) - except urllib2.HTTPError: - print "ERROR: Failed to get Monitor configuration file at %s " % url + response = urlopen(url, timeout=timeout) + except HTTPError: + print("ERROR: Failed to get Monitor configuration file at %s " % url) except (socket.timeout, ssl.SSLError) as e: - print "ERROR: Timeout with %r while downloading monitor config at %s " % (e, url) + print("ERROR: Timeout with %r while downloading monitor config at %s " % (e, url)) else: try: - monitor_dict = json.loads(response.read()) + monitor_dict = json.loads(bytes2str(response.read())) monitor_title = monitor_dict.get('title', 'Unknown Instance') success = True - except ValueError, e: - print "ERROR: Json file at %s is not valid" % url + except ValueError as e: + print("ERROR: Json file at %s is not valid" % url) self.bootstrap_is_ok = success return monitor_title @@ -267,8 +272,8 @@ class Monitoring(object): for parameter in parameter_list: if parameter['key']: pfile.write('%s = %s\n' % (parameter['key'], parameter['value'])) - except OSError, e: - print "Error failed to create file %s" % self.parameter_cfg_file + except OSError as e: + print("Error failed to create file %s" % self.parameter_cfg_file) pass @@ -317,8 +322,8 @@ class Monitoring(object): try: if os.path.exists(file): os.unlink(file) - except OSError, e: - print "failed to remove file %s." % file, str(e) + except OSError as e: + print("failed to remove file %s." % file, e) # cleanup result of promises that was removed promise_list = os.listdir(self.legacy_promise_folder) @@ -336,8 +341,8 @@ class Monitoring(object): if os.path.exists(status_path): try: os.unlink(status_path) - except OSError, e: - print "Error: Failed to delete %s" % status_path, str(e) + except OSError as e: + print("Error: Failed to delete %s" % status_path, e) else: promise_list.pop(position) @@ -366,7 +371,7 @@ class Monitoring(object): if self.bootstrap_is_ok: with open(self.promise_output_file, 'w') as promise_file: promise_file.write("") - print "SUCCESS: bootstrap is OK" + print("SUCCESS: bootstrap is OK") return 0 diff --git a/slapos/monitor/monitor_config_write.py b/slapos/monitor/monitor_config_write.py index fad7be4..985982b 100644 --- a/slapos/monitor/monitor_config_write.py +++ b/slapos/monitor/monitor_config_write.py @@ -1,5 +1,7 @@ #!/usr/bin/env python +from __future__ import print_function + import sys import os import re @@ -38,11 +40,11 @@ class MonitorConfigWrite(object): def _fileWrite(self, file_path, content): try: with open(file_path, 'w') as wf: - print file_path, content + print(file_path, content) wf.write(content.strip()) return True - except OSError, e: - print "ERROR while writing changes to %s.\n %s" % (file_path, str(e)) + except OSError as e: + print("ERROR while writing changes to %s.\n %s" % (file_path, e)) return False def _htpasswdWrite(self, htpasswd_bin, parameter_dict, value): @@ -55,7 +57,7 @@ class MonitorConfigWrite(object): ) result = process.communicate()[0] if process.returncode != 0: - print result + print(result) return False with open(parameter_dict['file'], 'w') as pfile: pfile.write(value) @@ -76,31 +78,31 @@ class MonitorConfigWrite(object): or (cors_domain == "" and os.stat(httpd_cors_file).st_size == 0)): # Skip if cors file is not empty return True - except OSError, e: - print "Failed to open file at %s. \n%s" % (old_httpd_cors_file, str(e)) + except OSError as e: + print("Failed to open file at %s. \n%s" % (old_httpd_cors_file, e)) try: with open(self.monitor_https_cors, 'r') as cors_template: template = jinja2.Template(cors_template.read()) rendered_string = template.render(domain=cors_domain) with open(httpd_cors_file, 'w') as file: file.write(rendered_string) - except OSError, e: - print "ERROR while writing CORS changes to %s.\n %s" % (httpd_cors_file, str(e)) + except OSError as e: + print("ERROR while writing CORS changes to %s.\n %s" % (httpd_cors_file, e)) return False # Save current cors domain list try: with open(old_httpd_cors_file, 'w') as cors_file: cors_file.write(cors_domain) - except OSError, e: - print "Failed to open file at %s. \n%s" % (old_httpd_cors_file, str(e)) + except OSError as e: + print("Failed to open file at %s. \n%s" % (old_httpd_cors_file, e)) return False # Restart httpd process try: subprocess.call(httpd_gracefull_bin) - except OSError, e: - print "Failed to execute command %s.\n %s" % (httpd_gracefull_bin, str(e)) + except OSError as e: + print("Failed to execute command %s.\n %s" % (httpd_gracefull_bin, e)) return False return True @@ -122,7 +124,7 @@ class MonitorConfigWrite(object): with open(self.config_json_file) as tmpfile: new_parameter_list = json.loads(tmpfile.read()) except ValueError: - print "Error: Couldn't parse json file %s" % self.config_json_file + print("Error: Couldn't parse json file %s" % self.config_json_file) with open(parameter_config_file) as tmpfile: description_dict = json.loads(tmpfile.read()) @@ -156,8 +158,8 @@ class MonitorConfigWrite(object): for parameter in new_parameter_list: if parameter['key']: pfile.write('%s = %s\n' % (parameter['key'], parameter['value'])) - except OSError, e: - print "Error failed to create file %s" % self.output_cfg_file + except OSError as e: + print("Error failed to create file %s" % self.output_cfg_file) pass return result_dict @@ -190,8 +192,8 @@ def main(): if status and os.path.exists(parameter_tmp_file): try: os.unlink(config_file) - except OSError, e: - print "ERROR cannot remove file: %s" % parameter_tmp_file + except OSError as e: + print("ERROR cannot remove file: %s" % parameter_tmp_file) else: os.rename(parameter_tmp_file, config_file) if run_counter == max_runn: diff --git a/slapos/monitor/runpromise.py b/slapos/monitor/runpromise.py index 87d3276..767c766 100644 --- a/slapos/monitor/runpromise.py +++ b/slapos/monitor/runpromise.py @@ -13,7 +13,7 @@ import glob import argparse import traceback import logging -import ConfigParser +from six.moves import configparser from slapos.grid.promise import PromiseLauncher, PromiseQueueResult, PromiseError from slapos.grid.promise.generic import PROMISE_LOG_FOLDER_NAME from slapos.util import mkdir_p @@ -92,7 +92,7 @@ class MonitorPromiseLauncher(object): def _loadConfigFromFile(self, config_file): - config = ConfigParser.ConfigParser() + config = configparser.ConfigParser() config.read([config_file]) known_key_list = ['partition-cert', 'partition-key', 'partition-id', 'pid-path', 'computer-id', 'check-anomaly', @@ -159,7 +159,7 @@ class MonitorPromiseLauncher(object): exit_code = 0 try: promise_launcher.run() - except PromiseError, e: + except PromiseError as e: # error was already logged exit_code = 1 os.remove(self.config.pid_path) diff --git a/slapos/networkbench/__init__.py b/slapos/networkbench/__init__.py index e575c18..61198fe 100644 --- a/slapos/networkbench/__init__.py +++ b/slapos/networkbench/__init__.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import socket import logging import time @@ -11,10 +13,10 @@ import random import pycurl import argparse import json -from StringIO import StringIO -from ping import ping, ping6 -from dnsbench import resolve -from http import get_curl, request +from io import StringIO +from .ping import ping, ping6 +from .dnsbench import resolve +from .http import get_curl, request import textwrap class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter): @@ -62,11 +64,11 @@ def download_external_configuration(url): try: return json.loads(buffer.getvalue()) except ValueError: - print "Unable to parse external configuration, error:" + print("Unable to parse external configuration, error:") import traceback traceback.print_exc(file=sys.stderr) sys.stderr.flush() - print "Ignoring external configuration" + print("Ignoring external configuration") finally: curl.close() diff --git a/slapos/networkbench/http.py b/slapos/networkbench/http.py index 095c365..2e9c4b9 100644 --- a/slapos/networkbench/http.py +++ b/slapos/networkbench/http.py @@ -1,6 +1,7 @@ import sys import pycurl -from StringIO import StringIO +from io import BytesIO +from slapos.util import bytes2str def get_curl(buffer, url): curl = pycurl.Curl() @@ -22,7 +23,7 @@ def get_curl(buffer, url): def request(url, expected_dict): - buffer = StringIO() + buffer = BytesIO() curl, result = get_curl(buffer, url) body = buffer.getvalue() @@ -43,7 +44,7 @@ def request(url, expected_dict): expected_text = expected_dict.get("expected_text", None) if expected_text is not None and \ - str(expected_text) not in str(body): + str(expected_text) not in bytes2str(body): result = "UNEXPECTED (%s not in page content)" % (expected_text) curl.close() diff --git a/slapos/networkbench/ping.py b/slapos/networkbench/ping.py index 23b95b0..e738089 100644 --- a/slapos/networkbench/ping.py +++ b/slapos/networkbench/ping.py @@ -22,14 +22,15 @@ def ping(host, timeout=10, protocol="4", count=10): test_title = 'PING6' proc = subprocess.Popen((ping_bin, '-c', str(count), '-w', str(timeout), host), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + universal_newlines=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) out, err = proc.communicate() if 'Network is unreachable' in err: return (test_title, host, 600, 'failed', 100, "Network is unreachable") try: packet_loss_line, summary_line = (out.splitlines() or [''])[-2:] - except: + except Exception: return (test_title, host, 600, 'failed', -1, "Fail to parser ping output") m = ping_re.match(summary_line) match = re.search('(\d*)% packet loss', packet_loss_line) diff --git a/slapos/onetimedownload.py b/slapos/onetimedownload.py index 623ec35..208b38b 100644 --- a/slapos/onetimedownload.py +++ b/slapos/onetimedownload.py @@ -27,7 +27,7 @@ import os import sys from optparse import OptionParser, Option -import urllib +from six.moves.urllib.request import urlopen class Parser(OptionParser): @@ -55,7 +55,7 @@ class Config: self.file_path = file_path def onetimedownload(url, file_path): - url_file = urllib.urlopen(url) + url_file = urlopen(url) data = url_file.read() file_object = open(file_path, 'w') file_object.write(data) @@ -72,7 +72,7 @@ def main(): onetimedownload(config.url, config.file_path) return_code = 0 - except SystemExit, err: + except SystemExit as err: # Catch exception raise by optparse return_code = err diff --git a/slapos/onetimeupload/__init__.py b/slapos/onetimeupload/__init__.py index 19dd6eb..29ffb5b 100644 --- a/slapos/onetimeupload/__init__.py +++ b/slapos/onetimeupload/__init__.py @@ -124,7 +124,7 @@ def main(): run(config) return_code = 0 - except SystemExit, err: + except SystemExit as err: # Catch exception raise by optparse return_code = err diff --git a/slapos/promise/apache_mpm_watchdog/__init__.py b/slapos/promise/apache_mpm_watchdog/__init__.py index 28b1df5..cd27dcf 100644 --- a/slapos/promise/apache_mpm_watchdog/__init__.py +++ b/slapos/promise/apache_mpm_watchdog/__init__.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import requests import re import signal @@ -56,11 +58,11 @@ def watchServerStatus(pid_dict, server_status, timeout): if process.cmdline()[0].endswith("/httpd"): _pid_dict.setdefault(i, time.time() + timeout) if _pid_dict[i] < time.time(): - print "Sending signal -%s to %s" % (signal.SIGKILL, i) + print("Sending signal -%s to %s" % (signal.SIGKILL, i)) try: process.kill() except psutil.NoSuchProcess: - print "Process is not there anymore" + print("Process is not there anymore") continue return _pid_dict diff --git a/slapos/promise/check_apachedex_result/__init__.py b/slapos/promise/check_apachedex_result/__init__.py index 1663e6b..ae8a050 100644 --- a/slapos/promise/check_apachedex_result/__init__.py +++ b/slapos/promise/check_apachedex_result/__init__.py @@ -47,7 +47,7 @@ def checkApachedexResult(apachedex_path, apachedex_report_status_file, desired_t with open(apachedex_report_status_file) as f: try: json_content = json.load(f) - except ValueError, e: + except ValueError as e: json_content = '' if json_content: message += "\n" + json_content["message"] diff --git a/slapos/promise/check_computer_memory/__init__.py b/slapos/promise/check_computer_memory/__init__.py index c1cce45..4ec1405 100755 --- a/slapos/promise/check_computer_memory/__init__.py +++ b/slapos/promise/check_computer_memory/__init__.py @@ -7,6 +7,8 @@ Uses: - /proc/meminfo """ +from __future__ import print_function + import sys import sqlite3 import argparse @@ -21,21 +23,21 @@ def getMemoryInfo(database, time, date): try: database.connect() query_result = database.select("computer", date, "memory_size", limit=1) - result = zip(*query_result) - if not result or not result[0][0]: + r = query_result.fetchone() + if not r or not r[0]: return (None, "couldn't fetch total memory, collectordb is empty?") - memory_info['total'] = int(result[0][0]) # in byte + memory_info['total'] = int(r[0]) # in byte # fetch free and used memory where_query = "time between '%s:00' and '%s:30' " % (time, time) query_result = database.select("system", date, "memory_free, memory_used", where=where_query) - result = zip(*query_result) - if not result or not result[0][0]: + r = query_result.fetchone() + if not r or not r[0]: return (None, "couldn't fetch free memory") - memory_info['free'] = int(result[0][0]) # in byte - if not result or not result[1][0]: + memory_info['free'] = int(r[0]) # in byte + if not r or not r[1]: return (None, "couldn't fetch used memory") - memory_info['used'] = int(result[1][0]) # in byte + memory_info['used'] = int(r[1]) # in byte finally: database.close() @@ -95,9 +97,9 @@ def main(): unit=args.unit, ) if error: - print error + print(error) return 0 - print message + print(message) return 0 if result else 1 if __name__ == "__main__": diff --git a/slapos/promise/check_slow_queries_digest_result/__init__.py b/slapos/promise/check_slow_queries_digest_result/__init__.py index 7004c37..814dacc 100644 --- a/slapos/promise/check_slow_queries_digest_result/__init__.py +++ b/slapos/promise/check_slow_queries_digest_result/__init__.py @@ -4,6 +4,8 @@ Check if a mariadb result matches the desired threshold or raises an error. """ +from __future__ import print_function + import json import os import re @@ -58,7 +60,7 @@ def checkMariadbDigestResult(mariadbdex_path, mariadbdex_report_status_file, with open(mariadbdex_report_status_file) as f: try: json_content = json.load(f) - except ValueError, e: + except ValueError as e: json_content = '' if json_content: message += "\n" + json_content["message"] @@ -76,5 +78,5 @@ def main(): args.max_queries_threshold, args.slowest_query_threshold ) - print message + print(message) sys.exit(status) diff --git a/slapos/promise/check_user_memory/__init__.py b/slapos/promise/check_user_memory/__init__.py index e1ec9dc..d318563 100755 --- a/slapos/promise/check_user_memory/__init__.py +++ b/slapos/promise/check_user_memory/__init__.py @@ -4,6 +4,8 @@ Check user memory usage according to a given threshold. """ +from __future__ import print_function + import sys import os import argparse @@ -87,9 +89,9 @@ def main(): unit=args.unit, ) if error: - print error + print(error) return 0 - print message + print(message) return 0 if result else 1 if __name__ == "__main__": diff --git a/slapos/promise/check_web_page_http_cache_hit/__init__.py b/slapos/promise/check_web_page_http_cache_hit/__init__.py index a9a3ddf..08ceda9 100755 --- a/slapos/promise/check_web_page_http_cache_hit/__init__.py +++ b/slapos/promise/check_web_page_http_cache_hit/__init__.py @@ -7,14 +7,14 @@ import sys import tempfile import os import argparse -import ConfigParser +from six.moves import configparser import re import pycurl -from mimetools import Message -from cStringIO import StringIO -from HTMLParser import HTMLParser +from email.message import Message +from io import BytesIO +from six.moves.html_parser import HTMLParser begins_by_known_protocol_re = re.compile("^https?://") get_protocol_re = re.compile("^([a-z]+)://") @@ -63,8 +63,8 @@ def checkWebpageHttpCacheHit(url_list, resolve_list=[], cookie_jar_path=None): parsed_url_dict.add(url) print("Checking cache hit for " + url) c = pycurl.Curl() - response_headers = StringIO() - output = StringIO() + response_headers = BytesIO() + output = BytesIO() c.setopt(c.URL, url) c.setopt(c.RESOLVE, resolve_list) c.setopt(c.WRITEFUNCTION, output.write) @@ -76,8 +76,9 @@ def checkWebpageHttpCacheHit(url_list, resolve_list=[], cookie_jar_path=None): response_headers.truncate(0) output.truncate(0) c.perform() - if str(c.getinfo(pycurl.HTTP_CODE))[0:1] != "2": - if c.getinfo(pycurl.HTTP_CODE) >= 400: + code = c.getinfo(pycurl.HTTP_CODE) + if not (200 <= code < 300): + if code >= 400: report_line_list.append("Status code %s received for %s" % (c.getinfo(pycurl.HTTP_CODE), url)) else: print("Status code %s not handled" % c.getinfo(pycurl.HTTP_CODE)) @@ -119,7 +120,7 @@ def checkWebpageHttpCacheHit(url_list, resolve_list=[], cookie_jar_path=None): def getConfig(config_parser, section, option, default=None, raw=False, vars=None): try: return config_parser.get(section, option, raw=raw, vars=vars) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: return default def main(): @@ -132,7 +133,7 @@ def main(): args.url_list = getattr(args, "url-list") if args.config is not None: - parser = ConfigParser.ConfigParser() + parser = configparser.ConfigParser() parser.read(args.config) if args.url_list == []: args.url_list = getConfig(parser, "public", "url-list", "").split() diff --git a/slapos/promise/is_local_tcp_port_opened/__init__.py b/slapos/promise/is_local_tcp_port_opened/__init__.py index 28c71bf..d18b128 100755 --- a/slapos/promise/is_local_tcp_port_opened/__init__.py +++ b/slapos/promise/is_local_tcp_port_opened/__init__.py @@ -28,7 +28,8 @@ def isLocalTcpPortOpened(ip_address, port): ip_addr_hex = ('%08X' * int_count) % struct.unpack('I' * int_count, socket.inet_pton(family, ip_address)) full_addr_hex = ip_addr_hex + ":%04X" % port - return any(full_addr_hex == line.split()[1] for line in open(tcp_path).readlines()) + with open(tcp_path) as f: + return any(full_addr_hex == line.split()[1] for line in f.readlines()) def main(): if isLocalTcpPortOpened(sys.argv[1], int(sys.argv[2])): diff --git a/slapos/promise/is_process_older_than_dependency_set/__init__.py b/slapos/promise/is_process_older_than_dependency_set/__init__.py index 5083d3f..3897943 100755 --- a/slapos/promise/is_process_older_than_dependency_set/__init__.py +++ b/slapos/promise/is_process_older_than_dependency_set/__init__.py @@ -8,6 +8,8 @@ a file modification date is greater than the start date of the process. """ +from __future__ import print_function + import sys import os import errno @@ -29,10 +31,10 @@ def moduleIsModifiedSince(top, since, followlinks=False): if ext in ignored_extension_set: continue if since < os.stat(os.path.join(root, name)).st_mtime: - print "%s was modified since the process started." % \ - os.path.join(root, name) - print "Process Time %s < Last modidified file %s" % (time.ctime(since), - time.ctime(os.stat(os.path.join(root, name)).st_mtime)) + print("%s was modified since the process started." % + os.path.join(root, name)) + print("Process Time %s < Last modidified file %s" % (time.ctime(since), + time.ctime(os.stat(os.path.join(root, name)).st_mtime))) return True return False @@ -41,7 +43,7 @@ def isProcessOlderThanDependencySet(pid, python_path_list, kill=False): start_time = process.create_time() if any(moduleIsModifiedSince(product_path, start_time) for product_path in python_path_list): if kill: - print "Terminating process %s with pid %s" % (process.name(), pid) + print("Terminating process %s with pid %s" % (process.name(), pid)) process.terminate() return True return False diff --git a/slapos/promise/plugin/backupserver_check_backup.py b/slapos/promise/plugin/backupserver_check_backup.py index 96a5b26..24ac46b 100644 --- a/slapos/promise/plugin/backupserver_check_backup.py +++ b/slapos/promise/plugin/backupserver_check_backup.py @@ -1,4 +1,4 @@ -from zope import interface as zope_interface +from zope.interface import implementer from slapos.grid.promise import interface from slapos.grid.promise.generic import GenericPromise from slapos.grid.promise.generic import TestResult @@ -12,10 +12,9 @@ from croniter import croniter from dateutil.parser import parse from tzlocal import get_localzone +@implementer(interface.IPromise) class RunPromise(GenericPromise): - zope_interface.implements(interface.IPromise) - def __init__(self, config): GenericPromise.__init__(self, config) # check backup ran OK every 5 minutes @@ -52,20 +51,21 @@ class RunPromise(GenericPromise): # First, parse the log file backup_started = False backup_ended = False - for line in open(status, 'r'): - m = re.match(r"(.*), (.*), (.*), backup (.*)$", line) - if m: - if m.group(4) == "running": - backup_started = True - backup_start = parse(m.group(1)) - elif m.group(4) == "failed": - backup_ended = True - backup_failed = True - backup_end = parse(m.group(1)) - elif m.group(4) == "success": - backup_ended = True - backup_failed = False - backup_end = parse(m.group(1)) + with open(status, 'r') as f: + for line in f: + m = re.match(r"(.*), (.*), (.*), backup (.*)$", line) + if m: + if m.group(4) == "running": + backup_started = True + backup_start = parse(m.group(1)) + elif m.group(4) == "failed": + backup_ended = True + backup_failed = True + backup_end = parse(m.group(1)) + elif m.group(4) == "success": + backup_ended = True + backup_failed = False + backup_end = parse(m.group(1)) # Then check result if backup_ended and backup_failed: diff --git a/slapos/promise/plugin/check_error_on_http_log.py b/slapos/promise/plugin/check_error_on_http_log.py index 7189e6c..561c199 100644 --- a/slapos/promise/plugin/check_error_on_http_log.py +++ b/slapos/promise/plugin/check_error_on_http_log.py @@ -1,14 +1,12 @@ -from zope import interface as zope_interface +from zope.interface import implementer from slapos.grid.promise import interface from slapos.grid.promise.generic import GenericPromise, TestResult import re import time import os +@implementer(interface.IPromise) class RunPromise(GenericPromise): - - zope_interface.implements(interface.IPromise) - def __init__(self, config): GenericPromise.__init__(self, config) # set periodicity to run the promise twice per day @@ -26,7 +24,7 @@ class RunPromise(GenericPromise): if not log_file: raise ValueError("log file was not set in promise parameters.") - regex = re.compile("^(\[[^\]]+\]) (\[[^\]]+\]) (.*)$") + regex = re.compile(br"^(\[[^\]]+\]) (\[[^\]]+\]) (.*)$") error_amount = 0 no_route_error = 0 network_is_unreachable = 0 @@ -38,7 +36,7 @@ class RunPromise(GenericPromise): self.logger.info("OK") return - with open(log_file) as f: + with open(log_file, "rb") as f: f.seek(0, 2) block_end_byte = f.tell() f.seek(-min(block_end_byte, 4096), 1) @@ -50,10 +48,10 @@ class RunPromise(GenericPromise): dt, level, msg = m.groups() try: try: - t = time.strptime(dt[1:-1], "%a %b %d %H:%M:%S %Y") + t = time.strptime(dt[1:-1].decode('utf-8'), "%a %b %d %H:%M:%S %Y") except ValueError: # Fail to parser for the first time, try a different output. - t = time.strptime(dt[1:-1], "%a %b %d %H:%M:%S.%f %Y") + t = time.strptime(dt[1:-1].decode('utf-8'), "%a %b %d %H:%M:%S.%f %Y") except ValueError: # Probably it fail to parse if parsing_failure < 3: @@ -65,14 +63,14 @@ class RunPromise(GenericPromise): if maximum_delay and (time.time()-time.mktime(t)) > maximum_delay: # no result in the latest hour break - if level != "[error]": + if level != b"[error]": continue # Classify the types of errors - if "(113)No route to host" in msg: + if b"(113)No route to host" in msg: no_route_error += 1 - elif "(101)Network is unreachable" in msg: + elif b"(101)Network is unreachable" in msg: network_is_unreachable += 1 - elif "(110)Connection timed out" in msg: + elif b"(110)Connection timed out" in msg: timeout += 1 error_amount += 1 if error_amount: diff --git a/slapos/promise/plugin/check_error_on_zope_longrequest_log.py b/slapos/promise/plugin/check_error_on_zope_longrequest_log.py index 146424d..ddc9214 100644 --- a/slapos/promise/plugin/check_error_on_zope_longrequest_log.py +++ b/slapos/promise/plugin/check_error_on_zope_longrequest_log.py @@ -1,4 +1,4 @@ -from zope import interface as zope_interface +from zope.interface import implementer from slapos.grid.promise import interface from slapos.grid.promise.generic import GenericPromise import time @@ -7,12 +7,10 @@ import os import sys import re -r = re.compile("^([0-9]+\-[0-9]+\-[0-9]+ [0-9]+\:[0-9]+\:[0-9]+)(\,[0-9]+) - ([A-z]+) (.*)$") +r = re.compile(br"^([0-9]+\-[0-9]+\-[0-9]+ [0-9]+\:[0-9]+\:[0-9]+)(\,[0-9]+) - ([A-z]+) (.*)$") +@implementer(interface.IPromise) class RunPromise(GenericPromise): - - zope_interface.implements(interface.IPromise) - def __init__(self, config): GenericPromise.__init__(self, config) self.setPeriodicity(minute=10) @@ -27,7 +25,7 @@ class RunPromise(GenericPromise): self.logger.info("log file does not exist: log check skipped") return 0 - with open(log_file) as f: + with open(log_file, "rb") as f: f.seek(0, 2) block_end_byte = f.tell() f.seek(-min(block_end_byte, 4096*10), 1) @@ -38,7 +36,7 @@ class RunPromise(GenericPromise): continue dt, _, level, msg = m.groups() try: - t = time.strptime(dt, "%Y-%m-%d %H:%M:%S") + t = time.strptime(dt.decode('utf-8'), "%Y-%m-%d %H:%M:%S") except ValueError: continue if maximum_delay and (time.time()-time.mktime(t)) > maximum_delay: diff --git a/slapos/promise/plugin/check_file_state.py b/slapos/promise/plugin/check_file_state.py index 7c200ec..e7811eb 100644 --- a/slapos/promise/plugin/check_file_state.py +++ b/slapos/promise/plugin/check_file_state.py @@ -1,12 +1,10 @@ -from zope import interface as zope_interface +from zope.interface import implementer from slapos.grid.promise import interface from slapos.grid.promise.generic import GenericPromise +@implementer(interface.IPromise) class RunPromise(GenericPromise): - - zope_interface.implements(interface.IPromise) - def __init__(self, config): GenericPromise.__init__(self, config) # SR can set custom periodicity diff --git a/slapos/promise/plugin/check_free_disk_space.py b/slapos/promise/plugin/check_free_disk_space.py index 7aa5a6f..ec0bda5 100644 --- a/slapos/promise/plugin/check_free_disk_space.py +++ b/slapos/promise/plugin/check_free_disk_space.py @@ -1,4 +1,4 @@ -from zope import interface as zope_interface +from zope.interface import implementer from slapos.grid.promise import interface from slapos.grid.promise.generic import GenericPromise @@ -12,10 +12,9 @@ import psutil from slapos.collect.db import Database +@implementer(interface.IPromise) class RunPromise(GenericPromise): - zope_interface.implements(interface.IPromise) - def __init__(self, config): GenericPromise.__init__(self, config) # check disk space at least every 3 minutes @@ -29,12 +28,12 @@ class RunPromise(GenericPromise): database.connect() where_query = "time between '%s:00' and '%s:30' and partition='%s'" % (time, time, disk_partition) query_result = database.select("disk", date, "free", where=where_query) - result = zip(*query_result) - if not result or not result[0][0]: + result = query_result.fetchone() + if not result or not result[0]: self.logger.info("No result from collector database: disk check skipped") return 0 - disk_free = result[0][0] - except sqlite3.OperationalError, e: + disk_free = result[0] + except sqlite3.OperationalError as e: # if database is still locked after timeout expiration (another process is using it) # we print warning message and try the promise at next run until max warn count locked_message = "database is locked" diff --git a/slapos/promise/plugin/check_icmp_packet_lost.py b/slapos/promise/plugin/check_icmp_packet_lost.py index f872b79..7941400 100644 --- a/slapos/promise/plugin/check_icmp_packet_lost.py +++ b/slapos/promise/plugin/check_icmp_packet_lost.py @@ -1,14 +1,12 @@ -from zope import interface as zope_interface +from zope.interface import implementer from slapos.grid.promise import interface from slapos.grid.promise.generic import GenericPromise, TestResult import re import time from slapos.networkbench.ping import ping, ping6 +@implementer(interface.IPromise) class RunPromise(GenericPromise): - - zope_interface.implements(interface.IPromise) - def __init__(self, config): GenericPromise.__init__(self, config) # set periodicity to run the promise twice per day diff --git a/slapos/promise/plugin/check_partition_deployment_state.py b/slapos/promise/plugin/check_partition_deployment_state.py index 105a932..52745e4 100644 --- a/slapos/promise/plugin/check_partition_deployment_state.py +++ b/slapos/promise/plugin/check_partition_deployment_state.py @@ -1,13 +1,12 @@ -from zope import interface as zope_interface +from zope.interface import implementer from slapos.grid.promise import interface from slapos.grid.promise.generic import GenericPromise import os from datetime import datetime +@implementer(interface.IPromise) class RunPromise(GenericPromise): - zope_interface.implements(interface.IPromise) - def __init__(self, config): GenericPromise.__init__(self, config) self.setPeriodicity(minute=1) diff --git a/slapos/promise/plugin/check_re6st_optimal_status.py b/slapos/promise/plugin/check_re6st_optimal_status.py index 52ea99c..b8d7e00 100644 --- a/slapos/promise/plugin/check_re6st_optimal_status.py +++ b/slapos/promise/plugin/check_re6st_optimal_status.py @@ -1,14 +1,12 @@ -from zope import interface as zope_interface +from zope.interface import implementer from slapos.grid.promise import interface from slapos.grid.promise.generic import GenericPromise, TestResult import re import time from slapos.networkbench.ping import ping, ping6 +@implementer(interface.IPromise) class RunPromise(GenericPromise): - - zope_interface.implements(interface.IPromise) - def __init__(self, config): GenericPromise.__init__(self, config) # set periodicity to run the promise twice per day diff --git a/slapos/promise/plugin/check_server_cpu_load.py b/slapos/promise/plugin/check_server_cpu_load.py index 60512c2..92afe5c 100644 --- a/slapos/promise/plugin/check_server_cpu_load.py +++ b/slapos/promise/plugin/check_server_cpu_load.py @@ -1,14 +1,13 @@ -from zope import interface as zope_interface +from zope.interface import implementer from slapos.grid.promise import interface from slapos.grid.promise.generic import GenericPromise import subprocess import os +@implementer(interface.IPromise) class RunPromise(GenericPromise): - zope_interface.implements(interface.IPromise) - def __init__(self, config): GenericPromise.__init__(self, config) # test load every 3 minutes @@ -17,7 +16,7 @@ class RunPromise(GenericPromise): def checkCPULoad(self, tolerance=2.2): # tolerance=1.5 => accept CPU load up to 1.5 =150% - uptime_result = subprocess.check_output(['uptime']) + uptime_result = subprocess.check_output(['uptime'], universal_newlines=True) line = uptime_result.strip().split(' ') load, load5, long_load = line[-3:] long_load = float(long_load.replace(',', '.')) @@ -44,7 +43,7 @@ class RunPromise(GenericPromise): if load_threshold is not None: try: threshold = float(load_threshold) - except ValueError, e: + except ValueError as e: self.logger.error("CPU load threshold %r is not valid: %s" % (load_threshold, e)) return diff --git a/slapos/promise/plugin/check_url_available.py b/slapos/promise/plugin/check_url_available.py index d23d16e..86713b1 100644 --- a/slapos/promise/plugin/check_url_available.py +++ b/slapos/promise/plugin/check_url_available.py @@ -1,14 +1,12 @@ -from zope import interface as zope_interface +from zope.interface import implementer from slapos.grid.promise import interface from slapos.grid.promise.generic import GenericPromise import requests +@implementer(interface.IPromise) class RunPromise(GenericPromise): - - zope_interface.implements(interface.IPromise) - def __init__(self, config): GenericPromise.__init__(self, config) # SR can set custom periodicity @@ -47,7 +45,7 @@ class RunPromise(GenericPromise): result = requests.get( url, verify=verify, allow_redirects=True, timeout=timeout, cert=cert) except requests.exceptions.SSLError as e: - if 'certificate verify failed' in str(e.message): + if 'certificate verify failed' in str(e): self.logger.error( "ERROR SSL verify failed while accessing %r" % (url,)) else: @@ -58,7 +56,7 @@ class RunPromise(GenericPromise): self.logger.error( "ERROR connection not possible while accessing %r" % (url, )) return - except Exception, e: + except Exception as e: self.logger.error("ERROR: %s" % (e,)) return diff --git a/slapos/promise/plugin/monitor_bootstrap_status.py b/slapos/promise/plugin/monitor_bootstrap_status.py index e9117fc..787f4a4 100644 --- a/slapos/promise/plugin/monitor_bootstrap_status.py +++ b/slapos/promise/plugin/monitor_bootstrap_status.py @@ -1,4 +1,4 @@ -from zope import interface as zope_interface +from zope.interface import implementer from slapos.grid.promise import interface from slapos.grid.promise.generic import GenericPromise import os @@ -6,10 +6,9 @@ import time import psutil from .util import tail_file +@implementer(interface.IPromise) class RunPromise(GenericPromise): - zope_interface.implements(interface.IPromise) - def __init__(self, config): GenericPromise.__init__(self, config) self.setPeriodicity(minute=2) @@ -23,7 +22,7 @@ class RunPromise(GenericPromise): with open(process_pid_file) as f: try: pid = int(f.read()) - except ValueError, e: + except ValueError as e: raise ValueError("%r is empty or doesn't contain a valid pid number: %s" % ( process_pid_file, str(e))) diff --git a/slapos/pubsub/__init__.py b/slapos/pubsub/__init__.py index a90e5c7..1bf9b6a 100644 --- a/slapos/pubsub/__init__.py +++ b/slapos/pubsub/__init__.py @@ -1,7 +1,7 @@ import argparse import csv import feedparser -import httplib # To avoid magic numbers +from six.moves import http_client as httplib # To avoid magic numbers import io import json import logging diff --git a/slapos/qemuqmpclient/__init__.py b/slapos/qemuqmpclient/__init__.py index 3e9b4c8..9c1ed89 100644 --- a/slapos/qemuqmpclient/__init__.py +++ b/slapos/qemuqmpclient/__init__.py @@ -25,6 +25,8 @@ # ############################################################################## +from __future__ import print_function + import argparse import json import os @@ -71,7 +73,7 @@ def getInitialQemuResourceDict(pid_file): with open(pid_file) as f: try: pid = int(f.read()) - except ValueError, e: + except ValueError as e: raise ValueError("%r is empty or doesn't contain a valid pid number: %s" % ( pid_file, str(e))) @@ -81,7 +83,7 @@ def getInitialQemuResourceDict(pid_file): process = psutil.Process(pid) break except psutil.NoSuchProcess: - print "Qemu process is not started yet..." + print("Qemu process is not started yet...") wait_count -= 1 time.sleep(0.5) else: @@ -130,7 +132,7 @@ class QemuQMPWrapper(object): if not os.path.exists(unix_socket_location): raise Exception('unix socket %s does not exist.' % unix_socket_location) - print 'Connecting to qemu...' + print('Connecting to qemu...') so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) connected = False while not connected: @@ -138,7 +140,7 @@ class QemuQMPWrapper(object): so.connect(unix_socket_location) except socket.error: time.sleep(1) - print 'Could not connect, retrying...' + print('Could not connect, retrying...') else: connected = True so.recv(1024) @@ -159,7 +161,7 @@ class QemuQMPWrapper(object): raise QmpCommandError(response["error"]["desc"]) if 'event' in response: self._event_list.append(response) - print response + print(response) if not only_event: continue @@ -171,7 +173,7 @@ class QemuQMPWrapper(object): for i in range(0, retry): if response is not None: break - print "Retrying send command after %s second(s)..." % sleep + print("Retrying send command after %s second(s)..." % sleep) time.sleep(sleep) self.socket.sendall(json.dumps(message)) response = self._readResponse() @@ -191,14 +193,14 @@ class QemuQMPWrapper(object): if actual_status == wanted_status: return else: - print 'VM in %s status, wanting it to be %s, retrying...' % ( - actual_status, wanted_status) + print('VM in %s status, wanting it to be %s, retrying...' % ( + actual_status, wanted_status)) time.sleep(1) except IOError: - print 'VM not ready, retrying...' + print('VM not ready, retrying...') def capabilities(self): - print 'Asking for capabilities...' + print('Asking for capabilities...') self._send({'execute': 'qmp_capabilities'}) def getEventList(self, timeout=0, cleanup=False): @@ -223,7 +225,7 @@ class QemuQMPWrapper(object): self.socket.setblocking(0) try: self._readResponse(only_event=True) - except socket.error, err: + except socket.error as err: if err[0] == errno.EAGAIN: # No data available pass @@ -240,7 +242,7 @@ class QemuQMPWrapper(object): def setVNCPassword(self, password): # Set VNC password - print 'Setting VNC password...' + print('Setting VNC password...') result = self._send({ "execute": "change", "arguments": { @@ -251,19 +253,19 @@ class QemuQMPWrapper(object): }) if result and result.get('return', None) != {}: raise ValueError(result) - print 'Done.' + print('Done.') def powerdown(self): - print 'Stopping the VM...' + print('Stopping the VM...') self._send({'execute': 'system_powerdown'}) def suspend(self): - print 'Suspending VM...' + print('Suspending VM...') self._send({'execute': 'stop'}) self._waitForVMStatus('paused') def resume(self): - print 'Resuming VM...' + print('Resuming VM...') self._send({'execute': 'cont'}) self._waitForVMStatus('running') @@ -285,7 +287,7 @@ class QemuQMPWrapper(object): return def driveBackup(self, backup_target, source_device='virtio0', sync_type='full'): - print 'Asking Qemu to perform backup to %s' % backup_target + print('Asking Qemu to perform backup to %s' % backup_target) # XXX: check for error self._send({ 'execute': 'drive-backup', @@ -296,17 +298,17 @@ class QemuQMPWrapper(object): } }) while self._getRunningJobList(backup_target): - print 'Job is not finished yet.' + print('Job is not finished yet.') time.sleep(20) def createSnapshot(self, snapshot_file, device='virtio0'): - print self._send({ + print(self._send({ 'execute': 'blockdev-snapshot-sync', 'arguments': { 'device': device, 'snapshot-file': snapshot_file, } - }) + })) def createInternalSnapshot(self, name=None, device='virtio0'): if name is None: @@ -372,9 +374,9 @@ class QemuQMPWrapper(object): try: if resend: result = self._send(command_dict) - except QmpCommandError, e: - print "ERROR: ", str(e) - print "%s\nRetry remove %r in few seconds..." % (result, dev_id) + except QmpCommandError as e: + print("ERROR: ", e) + print("%s\nRetry remove %r in few seconds..." % (result, dev_id)) resend = True else: for event in self.getEventList(timeout=2, cleanup=True): @@ -388,13 +390,13 @@ class QemuQMPWrapper(object): if stop_retry: break elif result is None and max_retry > 0: - print "Retry remove %r in few seconds..." % dev_id + print("Retry remove %r in few seconds..." % dev_id) time.sleep(2) if result is not None: if result.get('return', None) == {} or ('error' in result and \ result['error'].get('class', '') == 'DeviceNotFound'): - print 'Device %s was removed.' % dev_id + print('Device %s was removed.' % dev_id) return # device was not remove after retries @@ -417,7 +419,7 @@ class QemuQMPWrapper(object): if not system_exited: # hard reset the VM - print "Trying hard shutdown of the VM..." + print("Trying hard shutdown of the VM...") self._send({"execute": "quit"}) raise QmpDeviceRemoveError("Stopped Qemu in order to remove the device %r" % dev_id) @@ -459,14 +461,14 @@ class QemuQMPWrapper(object): if cpu_amount == hotplug_amount: # no chanches - print "Hotplug CPU is up to date." + print("Hotplug CPU is up to date.") return if cpu_amount > hotplug_amount: # we will remove CPU cpu_diff = -1 * cpu_diff if cpu_diff >= 1: - print "Request remove %s CPUs..." % cpu_diff + print("Request remove %s CPUs..." % cpu_diff) used_socket_id_list.reverse() for i in range(0, cpu_diff): self._removeDevice(used_socket_id_list[i], { @@ -478,7 +480,7 @@ class QemuQMPWrapper(object): # no hotplugable cpu socket found for Add raise ValueError("Cannot Configure %s CPUs, the maximum amount of " \ "hotplugable CPU is %s!" % (hotplug_amount, max_hotplug_cpu)) - print "Adding %s CPUs..." % cpu_diff + print("Adding %s CPUs..." % cpu_diff) for i in range(0, cpu_diff): self._send({ 'execute': 'device_add', @@ -491,10 +493,10 @@ class QemuQMPWrapper(object): if hotplug_amount != final_cpu_count: raise ValueError("Consistency error: Expected %s hotplugged CPU(s) but" \ " current CPU amount is %s" % (hotplug_amount, final_cpu_count)) - print "Done." + print("Done.") def _removeMemory(self, id_dict, auto_reboot=False): - print "Trying to remove devices %s, %s..." % (id_dict['id'], id_dict['memdev']) + print("Trying to remove devices %s, %s..." % (id_dict['id'], id_dict['memdev'])) self._removeDevice(id_dict['id'] ,{ 'execute': 'device_del', 'arguments': {'id': id_dict['id']} @@ -544,7 +546,7 @@ class QemuQMPWrapper(object): # cleanup memdev that was not removed because of failure for memdev in cleanup_memdev_id_dict.keys(): - print "Cleaning up memdev %s..." % memdev + print("Cleaning up memdev %s..." % memdev) self._removeDevice(memdev, { 'execute': 'object-del', 'arguments': { @@ -559,9 +561,9 @@ class QemuQMPWrapper(object): if (mem_size / slot_size) > slot_amount: raise ValueError("No enough slots available to add %sMB of RAM" % mem_size) - current_size = current_size/(1024 * 1024) + current_size //= (1024 * 1024) if current_size == mem_size: - print "Hotplug Memory size is up to date." + print("Hotplug Memory size is up to date.") return if mem_size < 0: @@ -569,7 +571,7 @@ class QemuQMPWrapper(object): elif current_size > mem_size: # Request to remove memory to_remove_size = current_size - mem_size - print "Removing %s MB of memory..." % to_remove_size + print("Removing %s MB of memory..." % to_remove_size) for i in range(num_slot_used, 0, -1): # remove all slots that won't be used @@ -587,9 +589,9 @@ class QemuQMPWrapper(object): ) elif current_size < mem_size: # ask for increase memory - slot_add = (mem_size - current_size) / slot_size + slot_add = (mem_size - current_size) // slot_size - print "Adding %s memory slot(s) of %s MB..." % (slot_add, slot_size) + print("Adding %s memory slot(s) of %s MB..." % (slot_add, slot_size)) for i in range(0, slot_add): index = num_slot_used + i + 1 self._send({ @@ -618,11 +620,11 @@ class QemuQMPWrapper(object): if mem_size != final_mem_size: raise ValueError("Consistency error: Expected %s MB of hotplugged RAM " \ "but current RAM size is %s MB" % (mem_size, final_mem_size)) - print "Done." + print("Done.") def updateDevice(self, option_dict): argument_dict = {} - if option_dict.has_key('device'): + if 'device' in option_dict: if option_dict['device'] == 'cpu': return self._updateCPU( amount=int(option_dict['amount']), diff --git a/slapos/resilient/runner_exporter.py b/slapos/resilient/runner_exporter.py index 8895239..7a12459 100644 --- a/slapos/resilient/runner_exporter.py +++ b/slapos/resilient/runner_exporter.py @@ -68,7 +68,8 @@ def synchroniseRunnerConfigurationDirectory(config, backup_path): os.makedirs(backup_path) file_list = ['config.json'] - for hidden_file in os.listdir('.'): + # `sorted` is used for Python 2-3 compatibility + for hidden_file in sorted(os.listdir('.')): if hidden_file[0] == '.': file_list.append(hidden_file) rsync(config.rsync_binary, file_list, backup_path, dry=config.dry) @@ -80,7 +81,8 @@ def synchroniseRunnerWorkingDirectory(config, backup_path): if os.path.isdir('instance'): file_list.append('instance') - exclude_list = getExcludePathList(os.getcwd()) + # `sorted` is used for Python 2-3 compatibility + exclude_list = sorted(getExcludePathList(os.getcwd())) # XXX: proxy.db should be properly dumped to leverage its # atomic properties diff --git a/slapos/resilient/runner_utils.py b/slapos/resilient/runner_utils.py index df3fdd3..0d79586 100644 --- a/slapos/resilient/runner_utils.py +++ b/slapos/resilient/runner_utils.py @@ -7,6 +7,9 @@ import sys from contextlib import contextmanager from hashlib import sha256 from zc.buildout.configparser import parse +from slapos.util import bytes2str, str2bytes + +import six @contextmanager @@ -63,7 +66,7 @@ def getExcludePathList(path): if e.errno != errno.ENOENT: raise else: - for section in installed.itervalues(): + for section in six.itervalues(installed): append_relative(section.get( '__buildout_installed__', '').splitlines()) @@ -129,7 +132,7 @@ def writeSignatureFile(slappart_signature_method_dict, runner_working_path, sign if signature_process: (output, error_output) = signature_process.communicate( - '\0'.join(filepath_list) + str2bytes('\0'.join(filepath_list)) ) if signature_process.returncode != 0: @@ -143,7 +146,7 @@ def writeSignatureFile(slappart_signature_method_dict, runner_working_path, sign # We have to rstrip as most programs return an empty line # at the end of their output - signature_list.extend(output.strip('\n').split('\n')) + signature_list.extend(bytes2str(output).strip('\n').split('\n')) else: signature_list.extend( getSha256Sum(filepath_list) diff --git a/slapos/runner/runnertest.py b/slapos/runner/runnertest.py index b2d8ead..0fddd1d 100644 --- a/slapos/runner/runnertest.py +++ b/slapos/runner/runnertest.py @@ -10,20 +10,23 @@ # or it will NOT work ############################################# +from __future__ import print_function + import argparse import base64 -import ConfigParser +from six.moves.configparser import SafeConfigParser import datetime import hashlib import json import os import shutil -import sup_process -from StringIO import StringIO +from . import sup_process +from io import StringIO import ssl import time import unittest -import urllib2 +from six.moves.urllib.request import Request, urlopen +import six from slapos.runner.utils import (getProfilePath, getSession, isInstanceRunning, @@ -125,7 +128,7 @@ class SlaprunnerTestCase(unittest.TestCase): partition_id=cls.partition_id ) cls.parameter_dict = cls.partition.getConnectionParameterDict() - for attribute, value in cls.parameter_dict.iteritems(): + for attribute, value in six.iteritems(cls.parameter_dict): setattr(cls, attribute.replace('-', '_'), value) #create slaprunner configuration @@ -188,7 +191,7 @@ class SlaprunnerTestCase(unittest.TestCase): shutil.rmtree(self.app.config['software_link']) def updateConfigParameter(self, parameter, value): - config_parser = ConfigParser.SafeConfigParser() + config_parser = SafeConfigParser() config_parser.read(os.getenv('RUNNER_CONFIG')) for section in config_parser.sections(): if config_parser.has_option(section, parameter): @@ -256,11 +259,11 @@ setuptools = 33.1.1 open(template, "w").write(content) def assertCanLoginWith(self, username, password): - request = urllib2.Request(self.backend_url) + request = Request(self.backend_url) base64string = base64.encodestring('%s:%s' % (username, password))[:-1] request.add_header("Authorization", "Basic %s" % base64string) ssl_context = ssl._create_unverified_context() - result = urllib2.urlopen(request, context=ssl_context) + result = urlopen(request, context=ssl_context) self.assertEqual(result.getcode(), 200) def test_updateAccount(self): @@ -606,7 +609,7 @@ setuptools = 33.1.1 class PrintStringIO(StringIO): def write(self, data): StringIO.write(self, data) - print data + print(data) def main(): """ diff --git a/slapos/runner/sup_process.py b/slapos/runner/sup_process.py index ce2774a..894a2a0 100644 --- a/slapos/runner/sup_process.py +++ b/slapos/runner/sup_process.py @@ -3,7 +3,7 @@ import os import signal import time -import xmlrpclib +import six.moves.xmlrpc_client as xmlrpclib # This mini-library is used to communicate with supervisord process # It aims to replace the file "process.py" diff --git a/slapos/runner/utils.py b/slapos/runner/utils.py index 82463c2..044a5e3 100644 --- a/slapos/runner/utils.py +++ b/slapos/runner/utils.py @@ -2,20 +2,20 @@ # vim: set et sts=2: # pylint: disable-msg=W0311,C0301,C0103,C0111,W0141,W0142 -import ConfigParser +from six.moves import configparser import datetime import json import logging -import md5 +import hashlib import os -import sup_process +from . import sup_process import re import shutil import stat -import thread +from six.moves import _thread, range import time -import urllib -import xmlrpclib +from six.moves.urllib.request import urlopen +import six.moves.xmlrpc_client as xmlrpclib from xml.dom import minidom import xml_marshaller @@ -43,7 +43,8 @@ html_escape_table = { def getBuildAndRunParams(config): json_file = os.path.join(config['etc_dir'], 'config.json') - json_params = json.load(open(json_file)) + with open(json_file) as f: + json_params = json.load(f) return json_params @@ -52,7 +53,8 @@ def saveBuildAndRunParams(config, params): Works like that because this function do not care about how you got the parameters""" json_file = os.path.join(config['etc_dir'], 'config.json') - open(json_file, "w").write(json.dumps(params)) + with open(json_file, "w") as f: + f.write(json.dumps(params)) def html_escape(text): @@ -92,11 +94,11 @@ def updateUserCredential(config, username, password): def getRcode(config): - parser = ConfigParser.ConfigParser() + parser = configparser.ConfigParser() try: parser.read(config['knowledge0_cfg']) return parser.get('public', 'recovery-code') - except (ConfigParser.NoSectionError, IOError) as e: + except (configparser.NoSectionError, IOError) as e: return None def getUsernameList(config): @@ -188,12 +190,12 @@ def updateProxy(config): 'software_root': config['software_root'] } - for i in xrange(0, int(config['partition_amount'])): + for i in range(int(config['partition_amount'])): partition_reference = '%s%s' % (prefix, i) partition_path = os.path.join(config['instance_root'], partition_reference) if not os.path.exists(partition_path): os.mkdir(partition_path) - os.chmod(partition_path, 0750) + os.chmod(partition_path, 0o750) slap_config['partition_list'].append({ 'address_list': [ { @@ -423,7 +425,7 @@ def getSlapStatus(config): except Exception: pass if partition_list: - for i in xrange(0, int(config['partition_amount'])): + for i in range(int(config['partition_amount'])): slappart_id = '%s%s' % ("slappart", i) if not [x[0] for x in partition_list if slappart_id == x[0]]: partition_list.append((slappart_id, [])) @@ -460,7 +462,7 @@ def removeInstanceRootDirectory(config): fullPath = os.path.join(root, fname) if not os.access(fullPath, os.W_OK): # Some directories may be read-only, preventing to remove files in it - os.chmod(fullPath, 0744) + os.chmod(fullPath, 0o744) shutil.rmtree(instance_directory) def removeCurrentInstance(config): @@ -589,7 +591,7 @@ def newSoftware(folder, config, session): software = "https://lab.nexedi.com/nexedi/slapos/raw/master/software/lamp-template/software.cfg" softwareContent = "" try: - softwareContent = urllib.urlopen(software).read() + softwareContent = urlopen(software).read() except: #Software.cfg and instance.cfg content will be empty pass @@ -777,7 +779,7 @@ def md5sum(file): return False try: fh = open(file, 'rb') - m = md5.md5() + m = hashlib.md5() while True: data = fh.read(8192) if not data: @@ -830,7 +832,7 @@ def readParameters(path): sub_obj[str(subnode.getAttribute('id'))] = subnode.childNodes[0].data # .decode('utf-8').decode('utf-8') obj[str(elt.tagName)] = sub_obj return obj - except Exception, e: + except Exception as e: return str(e) else: return "No such file or directory: %s" % path @@ -901,7 +903,8 @@ def runSlapgridUntilSuccess(config, step): else: return -1 counter_file = os.path.join(config['runner_workdir'], '.turn-left') - open(counter_file, 'w+').write(str(max_tries)) + with open(counter_file, 'w+') as f: + f.write(str(max_tries)) counter = max_tries slapgrid = True # XXX-Nico runSoftwareWithLock can return 0 or False (0==False) @@ -911,9 +914,11 @@ def runSlapgridUntilSuccess(config, step): # slapgrid == 0 because EXIT_SUCCESS == 0 if slapgrid == 0: break - times_left = int(open(counter_file).read()) - 1 + with open(counter_file) as f: + times_left = int(f.read()) - 1 if times_left > 0 : - open(counter_file, 'w+').write(str(times_left)) + with open(counter_file, 'w+') as f: + f.write(str(times_left)) counter = times_left else : counter = 0 @@ -934,7 +939,7 @@ def setupDefaultSR(config): if not os.path.exists(project) and config['default_sr'] != '': configNewSR(config, config['default_sr']) if config['auto_deploy']: - thread.start_new_thread(buildAndRun, (config,)) + _thread.start_new_thread(buildAndRun, (config,)) def setMiniShellHistory(config, command): diff --git a/slapos/runner/views.py b/slapos/runner/views.py index 7602b58..49b5560 100644 --- a/slapos/runner/views.py +++ b/slapos/runner/views.py @@ -7,9 +7,9 @@ import json import os import shutil import subprocess -import sup_process -import thread -import urllib +from . import sup_process +from six.moves import _thread +from six.moves.urllib.parse import unquote from flask import (Flask, request, redirect, url_for, render_template, g, flash, jsonify, session, abort, send_file) @@ -160,7 +160,7 @@ def removeSoftware(): def runSoftwareProfile(): - thread.start_new_thread(runSlapgridUntilSuccess, (app.config, "software")) + _thread.start_new_thread(runSlapgridUntilSuccess, (app.config, "software")) return jsonify(result=True) @@ -233,7 +233,7 @@ def removeInstance(): def runInstanceProfile(): if not os.path.exists(app.config['instance_root']): os.mkdir(app.config['instance_root']) - thread.start_new_thread(runSlapgridUntilSuccess, (app.config, "instance")) + _thread.start_new_thread(runSlapgridUntilSuccess, (app.config, "instance")) return jsonify(result=True) @@ -293,7 +293,7 @@ def cloneRepository(): try: cloneRepo(request.form['repo'], path, request.form['user'], request.form['email']) return jsonify(code=1, result="") - except GitCommandError, e: + except GitCommandError as e: return jsonify(code=0, result=safeResult(str(e))) @@ -324,7 +324,7 @@ def getProjectStatus(): try: result, branch, isdirty = gitStatus(path) return jsonify(code=1, result=result, branch=branch, dirty=isdirty) - except GitCommandError, e: + except GitCommandError as e: return jsonify(code=0, result=safeResult(str(e))) else: return jsonify(code=0, result="Can not read folder: Permission Denied") @@ -414,7 +414,7 @@ def changeBranch(): else: json = "This is already your active branch for this project" return jsonify(code=1, result=json) - except GitCommandError, e: + except GitCommandError as e: return jsonify(code=0, result=safeResult(str(e))) else: return jsonify(code=0, result="Can not read folder: Permission Denied") @@ -432,7 +432,7 @@ def newBranch(): return jsonify(code=1, result="") else: return jsonify(code=0, result="Failed to checkout to branch %s.") - except GitCommandError, e: + except GitCommandError as e: return jsonify(code=0, result=safeResult(str(e))) else: return jsonify(code=0, result="Can not read folder: Permission Denied") @@ -638,7 +638,7 @@ def updateAccount(): try: updateGitConfig(app.config['default_repository_path'], name, email) - except GitCommandError, e: + except GitCommandError as e: return jsonify(code=0, result=str(e)) git_user_file = os.path.join(app.config['etc_dir'], '.git_user') with codecs.open(git_user_file, 'w', encoding='utf-8') as gfile: @@ -684,10 +684,8 @@ def fileBrowser(): dir = request.form['dir'].encode('utf-8') newfilename = request.form.get('newfilename', '').encode('utf-8') files = request.form.get('files', '').encode('utf-8') - if not request.form.has_key('opt') or not request.form['opt']: - opt = 1 - else: - opt = int(request.form['opt']) + opt = request.form.get('opt') + opt = int(opt) if opt else 1 else: opt = int(request.args.get('opt')) @@ -751,9 +749,9 @@ def fileBrowser(): def editFile(): return render_template('editFile.html', workDir='workspace', - profile=urllib.unquote(request.args.get('profile', '')), + profile=unquote(request.args.get('profile', '')), projectList=listFolder(app.config, 'workspace'), - filename=urllib.unquote(request.args.get('filename', ''))) + filename=unquote(request.args.get('filename', ''))) def shell(): return render_template('shell.html') diff --git a/slapos/securedelete.py b/slapos/securedelete.py index c9ad57c..98b02cc 100644 --- a/slapos/securedelete.py +++ b/slapos/securedelete.py @@ -24,6 +24,8 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # ############################################################################## +from __future__ import print_function + import os import argparse import subprocess @@ -96,7 +98,8 @@ def shred(options): arg_list.extend(getFileList(options.file_list, options.check_exist)) pshred = subprocess.Popen(arg_list, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + universal_newlines=True) result, stderr = pshred.communicate() if pshred.returncode is None: pshred.kill() @@ -108,4 +111,4 @@ def shred(options): def main(): arg_parser = getAgumentParser() output = shred(arg_parser.parse_args()) - print output + print(output) diff --git a/slapos/systool.py b/slapos/systool.py index 074f1a0..486f252 100644 --- a/slapos/systool.py +++ b/slapos/systool.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import argparse import sys import os @@ -12,7 +14,7 @@ def killpidfromfile(): if sig is None: raise ValueError('Unknown signal name %s' % sys.argv[2]) pid = int(open(file).read()) - print 'Killing pid %s with signal %s' % (pid, sys.argv[2]) + print('Killing pid %s with signal %s' % (pid, sys.argv[2])) os.kill(pid, sig) def sublist(a, b): @@ -63,7 +65,7 @@ def kill(): cmdline = p.cmdline() if cmdline == args.arg if args.full else sublist(cmdline, args.arg): p.send_signal(s) - print 'killed pid %s with signal %s' % (p.pid, args.signal) + print('killed pid %s with signal %s' % (p.pid, args.signal)) r = 0 except psutil.Error: pass diff --git a/slapos/test/monitor/test_config_document.py b/slapos/test/monitor/test_config_document.py index 3a5de9d..bcef37a 100644 --- a/slapos/test/monitor/test_config_document.py +++ b/slapos/test/monitor/test_config_document.py @@ -78,7 +78,7 @@ echo "htpasswd $@" > %s/monitor-htpasswd self.writeContent(self.monitor_https_cors, '{% set allow_domain = "|".join(domain.replace(".", "\.").split()) -%}\n' 'SetEnvIf Origin "^http(s)?://(.+\.)?({{ allow_domain }})$" ORIGIN_DOMAIN=$0\n' 'Header always set Access-Control-Allow-Origin "%{ORIGIN_DOMAIN}e" env=ORIGIN_DOMAIN') - os.chmod(self.httpd_passwd_bin, 0755) + os.chmod(self.httpd_passwd_bin, 0o755) def tearDown(self): if os.path.exists(self.base_dir): @@ -101,31 +101,37 @@ echo "htpasswd $@" > %s/monitor-htpasswd def check_config(self): config_parameter = os.path.join(self.config_dir, 'config.parameters.json') - config_parameter_json = json.load(open(config_parameter)) - config_json = json.load(open(self.config_path)) + with open(config_parameter) as f: + config_parameter_json = json.load(f) + with open(self.config_path) as f: + config_json = json.load(f) for config in config_json: if config["key"]: - self.assertTrue(config_parameter_json.has_key(config["key"])) + self.assertIn(config["key"], config_parameter_json) parameter = config_parameter_json[config["key"]] else: continue if config["key"] == 'from-file': self.assertTrue(os.path.exists(parameter['file'])) - self.assertEqual(config["value"], open(parameter['file']).read()) + with open(parameter['file']) as f: + self.assertEqual(config["value"], f.read()) elif config["key"] == 'httpd-password': http_passwd = "%s/monitor-htpasswd" % self.base_dir #XXX where \n bellow come from ? command = 'htpasswd -cb %s admin %s%s' % (http_passwd, config["value"], '\n') self.assertTrue(os.path.exists(parameter['file'])) self.assertTrue(os.path.exists(http_passwd)) - self.assertEqual(config["value"], open(parameter['file']).read()) - self.assertEqual(open(http_passwd).read(), command) + with open(parameter['file']) as f: + self.assertEqual(config["value"], f.read()) + with open(http_passwd) as f: + self.assertEqual(f.read(), command) elif config["key"] == 'cors-domain': cors_file = "%s/test-httpd-cors.cfg" % self.base_dir self.assertTrue(os.path.exists(cors_file)) cors_string = self.generate_cors_string(config["value"].split()) - self.assertEqual(cors_string, open(cors_file).read()) + with open(cors_file) as f: + self.assertEqual(cors_string, f.read()) def check_cfg_config(self, config_list): cfg_output = os.path.join(self.config_dir, 'config.cfg') diff --git a/slapos/test/monitor/testbootstrap.py b/slapos/test/monitor/testbootstrap.py index 4cfe3fb..3c7a739 100644 --- a/slapos/test/monitor/testbootstrap.py +++ b/slapos/test/monitor/testbootstrap.py @@ -73,7 +73,6 @@ monitor-url-list = %(url_list)s collector-db = base-url = %(base_url)s title = %(title)s -service-pid-folder = %(base_dir)s/run promise-output-file = %(base_dir)s/monitor-bootstrap-status promise-runner = %(promise_run_script)s randomsleep = /bin/echo sleep @@ -102,11 +101,11 @@ partition-folder = %(base_dir)s for index in range(1, amount+1): promise_file = os.path.join(promise_dir, 'monitor_promise-%s' % index) self.writeContent(promise_file, promse_content) - os.chmod(promise_file, 0755) + os.chmod(promise_file, 0o755) for index in range(1, amount+1): promise_file = os.path.join(plugin_dir, 'monitor_promise-%s.py' % index) self.writeContent(promise_file, promse_content) - os.chmod(promise_file, 0644) + os.chmod(promise_file, 0o644) def checkOPML(self, url_list): opml_title = "%(root_title)s" % self.monitor_config_dict @@ -250,7 +249,8 @@ partition-folder = %(base_dir)s instance_config = os.path.join(instance.config_folder, '.jio_documents', 'config.json') self.assertTrue(os.path.exists(instance_config)) - config_content = json.loads(open(instance_config).read()) + with open(instance_config) as f: + config_content = json.load(f) self.assertEqual(len(config_content), 4) key_list = ['', 'sample', 'monitor-password', 'cors-domain'] for parameter in config_content: diff --git a/slapos/test/monitor/testglobalstate.py b/slapos/test/monitor/testglobalstate.py index 3d29ee6..dab553c 100644 --- a/slapos/test/monitor/testglobalstate.py +++ b/slapos/test/monitor/testglobalstate.py @@ -44,7 +44,7 @@ class MonitorGlobalTest(unittest.TestCase): pkg_resources.resource_string( 'slapos.monitor', 'doc/monitor_instance.schema.json') - self.monitor_instance_schema = json.loads(monitor_schema_string) + self.monitor_instance_schema = json.loads(monitor_schema_string.decode('utf-8')) self.monitor_config_dict = dict( @@ -91,7 +91,6 @@ monitor-url-list = %(url_list)s collector-db = base-url = %(base_url)s title = %(title)s -service-pid-folder = %(base_dir)s/run promise-output-file = %(base_dir)s/monitor-bootstrap-status promise-runner = %(promise_run_script)s randomsleep = /bin/echo sleep @@ -132,7 +131,7 @@ exit %(code)s """ % result_dict promise_path = os.path.join(self.etc_dir, 'promise', name) self.writeContent(promise_path, content) - os.chmod(promise_path, 0755) + os.chmod(promise_path, 0o755) return promise_path def getPromiseParser(self): @@ -230,7 +229,7 @@ exit %(code)s }""" with open(os.path.join(self.private_dir, 'monitor.global.json')) as r: - result = json.loads(r.read().decode("utf-8")) + result = json.load(r) result.pop("date") self.assertEqual(result, json.loads(expected_result)) @@ -248,7 +247,7 @@ exit %(code)s expected_result_dict["state"] = {'error': 0, 'success': 4} instance_result_dict = None with open(os.path.join(self.private_dir, 'monitor.global.json')) as r: - instance_result_dict = json.loads(r.read().decode("utf-8")) + instance_result_dict = json.load(r) result = instance_result_dict.copy() result.pop("date") self.assertEqual(result, diff --git a/slapos/test/monitor/testrunpromise.py b/slapos/test/monitor/testrunpromise.py index 97cb3f7..58775ba 100644 --- a/slapos/test/monitor/testrunpromise.py +++ b/slapos/test/monitor/testrunpromise.py @@ -69,7 +69,7 @@ exit 0 """ promise_path = os.path.join(self.old_promise_dir, name) self.writeContent(promise_path, content) - os.chmod(promise_path, 0755) + os.chmod(promise_path, 0o755) return promise_path def writePromiseNOK(self, name): @@ -80,19 +80,18 @@ exit 2 """ promise_path = os.path.join(self.old_promise_dir, name) self.writeContent(promise_path, content) - os.chmod(promise_path, 0755) + os.chmod(promise_path, 0o755) return promise_path def generatePromiseScript(self, name, success=True, failure_count=1, content="", periodicity=0.03): - promise_content = """from zope import interface as zope_interface + promise_content = """from zope.interface import implementer from slapos.grid.promise import interface from slapos.grid.promise import GenericPromise +@implementer(interface.IPromise) class RunPromise(GenericPromise): - zope_interface.implements(interface.IPromise) - def __init__(self, config): GenericPromise.__init__(self, config) self.setPeriodicity(minute=%(periodicity)s) @@ -151,21 +150,23 @@ class RunPromise(GenericPromise): result_file = os.path.join(self.output_dir, 'my_promise.status.json') os.system('cat %s' % result_file) self.assertTrue(os.path.exists(result_file)) - my_result = json.loads(open(result_file).read().decode("utf-8")) + with open(result_file) as f: + my_result = json.load(f) my_result['result'].pop('date') expected_result = { u'title': u'my_promise', u'name': u'my_promise.py', u'result': { u'failed': False, u'message': u'success', u'type': u'Test Result' }, - u'execution-time': 0.05, u'path': u'%s/my_promise.py' % self.promise_dir, } + self.assertTrue(my_result.pop('execution-time')) self.assertEqual(expected_result, my_result) result_file = os.path.join(self.output_dir, 'my_second_promise.status.json') self.assertTrue(os.path.exists(result_file)) - second_result = json.loads(open(result_file).read().decode("utf-8")) + with open(result_file) as f: + second_result = json.load(f) second_result['result'].pop('date') expected_result = { @@ -173,9 +174,9 @@ class RunPromise(GenericPromise): u'result': { u'failed': False, u'message': u'success', u'type': u'Test Result' }, - u'execution-time': 0.05, u'path': u'%s/my_second_promise.py' % self.promise_dir, } + self.assertTrue(second_result.pop('execution-time')) self.assertEqual(expected_result, second_result) def test_promise_generic_failed(self): @@ -186,7 +187,8 @@ class RunPromise(GenericPromise): result_file = os.path.join(self.output_dir, 'my_promise.status.json') self.assertTrue(os.path.exists(result_file)) - my_result = json.loads(open(result_file).read().decode("utf-8")) + with open(result_file) as f: + my_result = json.load(f) my_result['result'].pop('date') expected_result = { @@ -194,9 +196,9 @@ class RunPromise(GenericPromise): u'result': { u'failed': True, u'message': u'failed', u'type': u'Test Result' }, - u'execution-time': 0.05, u'path': u'%s/my_promise.py' % self.promise_dir, } + self.assertTrue(my_result.pop('execution-time')) self.assertEqual(expected_result, my_result) def test_promise_generic_status_change(self): @@ -207,7 +209,8 @@ class RunPromise(GenericPromise): result_file = os.path.join(self.output_dir, 'my_promise.status.json') self.assertTrue(os.path.exists(result_file)) - my_result = json.loads(open(result_file).read().decode("utf-8")) + with open(result_file) as f: + my_result = json.load(f) my_result['result'].pop('date') expected_result = { @@ -215,9 +218,9 @@ class RunPromise(GenericPromise): u'result': { u'failed': True, u'message': u'failed', u'type': u'Test Result' }, - u'execution-time': 0.05, u'path': u'%s/my_promise.py' % self.promise_dir, } + self.assertTrue(my_result.pop('execution-time')) self.assertEqual(expected_result, my_result) os.system('rm %s/*.pyc' % self.promise_dir) @@ -226,7 +229,8 @@ class RunPromise(GenericPromise): promise_runner2 = MonitorPromiseLauncher(parser) promise_runner2.start() - my_result = json.loads(open(result_file).read().decode("utf-8")) + with open(result_file) as f: + my_result = json.load(f) my_result['result'].pop('date') expected_result = { @@ -234,9 +238,9 @@ class RunPromise(GenericPromise): u'result': { u'failed': False, u'message': u'success', u'type': u'Test Result' }, - u'execution-time': 0.05, u'path': u'%s/my_promise.py' % self.promise_dir, } + self.assertTrue(my_result.pop('execution-time')) self.assertEqual(expected_result, my_result) def test_promise_generic_periodicity(self): @@ -287,7 +291,8 @@ class RunPromise(GenericPromise): result_file = os.path.join(self.output_dir, 'promise_1.status.json') self.assertTrue(os.path.exists(result_file)) - result1 = json.loads(open(result_file).read().decode("utf-8")) + with open(result_file) as f: + result1 = json.load(f) start_date = datetime.strptime(result1['result'].pop('date'), '%Y-%m-%dT%H:%M:%S+0000') expected_result = { @@ -295,7 +300,6 @@ class RunPromise(GenericPromise): u'result': { u'failed': False, u'message': u'success', u'type': u'Test Result' }, - u'execution-time': 0.05, u'path': u'%s' % promise, } @@ -303,8 +307,10 @@ class RunPromise(GenericPromise): parser = self.getPromiseParser(force=True) promise_runner = MonitorPromiseLauncher(parser) promise_runner.start() - result2 = json.loads(open(result_file).read().decode("utf-8")) + with open(result_file) as f: + result2 = json.load(f) start_date2 = datetime.strptime(result2['result'].pop('date'), '%Y-%m-%dT%H:%M:%S+0000') + self.assertTrue(result2.pop('execution-time')) self.assertEqual(expected_result, result2) def test_promise_two_folder(self): @@ -319,7 +325,8 @@ class RunPromise(GenericPromise): result2_file = os.path.join(self.output_dir, 'promise_2.status.json') self.assertTrue(os.path.exists(result_file)) self.assertTrue(os.path.exists(result2_file)) - result1 = json.loads(open(result_file).read().decode("utf-8")) + with open(result_file) as f: + result1 = json.load(f) start_date = datetime.strptime(result1['result'].pop('date'), '%Y-%m-%dT%H:%M:%S+0000') expected_result = { @@ -327,12 +334,13 @@ class RunPromise(GenericPromise): u'result': { u'failed': False, u'message': u'success', u'type': u'Test Result' }, - u'execution-time': 0.05, u'path': u'%s' % promise, } + self.assertTrue(result1.pop('execution-time')) self.assertEqual(expected_result, result1) - result2 = json.loads(open(result2_file).read()) + with open(result2_file) as f: + result2 = json.load(f) start_date2 = datetime.strptime(result2['result'].pop('date'), '%Y-%m-%dT%H:%M:%S+0000') expected_result = { @@ -340,9 +348,9 @@ class RunPromise(GenericPromise): u'result': { u'failed': False, u'message': u'success', u'type': u'Test Result' }, - u'execution-time': 0.05, u'path': u'%s' % promise2, } + self.assertTrue(result2.pop('execution-time')) self.assertEqual(expected_result, result2) def test_promise_NOK(self): @@ -353,23 +361,26 @@ class RunPromise(GenericPromise): result_file = os.path.join(self.output_dir, 'promise_1.status.json') self.assertTrue(os.path.exists(result_file)) - result1 = json.loads(open(result_file).read().decode("utf-8")) + with open(result_file) as f: + result1 = json.load(f) result1['result'].pop('date') expected_result = { u'title': u'promise_1', u'name': u'promise_1', u'result': { u'failed': True, u'message': u'failed', u'type': u'Test Result' }, - u'execution-time': 0.05, u'path': u'%s' % promise, } + self.assertTrue(result1.pop('execution-time')) self.assertEqual(expected_result, result1) # second run promise_runner = MonitorPromiseLauncher(parser) promise_runner.start() - result2 = json.loads(open(result_file).read().decode("utf-8")) + with open(result_file) as f: + result2 = json.load(f) result2['result'].pop('date') + self.assertTrue(result2.pop('execution-time')) self.assertEqual(expected_result, result2) def test_promise_mixed(self): @@ -380,16 +391,17 @@ class RunPromise(GenericPromise): result_file = os.path.join(self.output_dir, 'promise_1.status.json') self.assertTrue(os.path.exists(result_file)) - result1 = json.loads(open(result_file).read().decode("utf-8")) + with open(result_file) as f: + result1 = json.load(f) result1['result'].pop('date') expected_result = { u'title': u'promise_1', u'name': u'promise_1', u'result': { u'failed': False, u'message': u'success', u'type': u'Test Result' }, - u'execution-time': 0.05, u'path': u'%s' % promise, } + self.assertTrue(result1.pop('execution-time')) self.assertEqual(expected_result, result1) # second run with failure @@ -401,7 +413,9 @@ class RunPromise(GenericPromise): promise_runner = MonitorPromiseLauncher(parser) promise_runner.start() - result2 = json.loads(open(result_file).read().decode("utf-8")) + with open(result_file) as f: + result2 = json.load(f) result2['result'].pop('date') + self.assertTrue(result2.pop('execution-time')) self.assertEqual(expected_result, result2) diff --git a/slapos/test/promise/plugin/test_check_file_state.py b/slapos/test/promise/plugin/test_check_file_state.py index fd5ebdf..b1e2310 100644 --- a/slapos/test/promise/plugin/test_check_file_state.py +++ b/slapos/test/promise/plugin/test_check_file_state.py @@ -32,6 +32,7 @@ import tempfile import os import unittest import shutil +import six class TestCheckFileState(TestPromisePluginMixin): @@ -69,8 +70,10 @@ extra_config_dict = { self.assertEqual(result['result']['failed'], True) self.assertEqual( result['result']['message'], - "ERROR IOError(21, 'Is a directory') " - "during opening and reading file %r" % (filename,) + "ERROR %s(21, 'Is a directory') " + "during opening and reading file %r" % ( + "IsADirectoryError" if six.PY3 else "IOError", + filename) ) def test_check_file_not_exists(self): @@ -88,8 +91,10 @@ extra_config_dict = { self.assertEqual(result['result']['failed'], True) self.assertEqual( result['result']['message'], - "ERROR IOError(2, 'No such file or directory') " - "during opening and reading file %r" % (filename,) + "ERROR %s(2, 'No such file or directory') " + "during opening and reading file %r" % ( + "FileNotFoundError" if six.PY3 else "IOError", + filename) ) def test_check_file_empty(self): diff --git a/slapos/test/promise/plugin/test_check_url_available.py b/slapos/test/promise/plugin/test_check_url_available.py index 0b3c60f..9c32188 100644 --- a/slapos/test/promise/plugin/test_check_url_available.py +++ b/slapos/test/promise/plugin/test_check_url_available.py @@ -27,6 +27,7 @@ from slapos.grid.promise import PromiseError from slapos.test.promise.plugin import TestPromisePluginMixin +from slapos.util import str2bytes from cryptography import x509 from cryptography.hazmat.backends import default_backend @@ -34,12 +35,13 @@ from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.x509.oid import NameOID -import BaseHTTPServer +from six.moves import BaseHTTPServer import datetime import ipaddress import json import multiprocessing import os +import six import ssl import tempfile import time @@ -66,10 +68,10 @@ def createCSR(common_name, ip=None): subject_alternative_name_list = [] if ip is not None: subject_alternative_name_list.append( - x509.IPAddress(ipaddress.ip_address(unicode(ip))) + x509.IPAddress(ipaddress.ip_address(ip)) ) csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([ - x509.NameAttribute(NameOID.COMMON_NAME, unicode(common_name)), + x509.NameAttribute(NameOID.COMMON_NAME, common_name), ])) if len(subject_alternative_name_list): @@ -89,10 +91,10 @@ class CertificateAuthority(object): public_key = self.key.public_key() builder = x509.CertificateBuilder() builder = builder.subject_name(x509.Name([ - x509.NameAttribute(NameOID.COMMON_NAME, unicode(common_name)), + x509.NameAttribute(NameOID.COMMON_NAME, common_name), ])) builder = builder.issuer_name(x509.Name([ - x509.NameAttribute(NameOID.COMMON_NAME, unicode(common_name)), + x509.NameAttribute(NameOID.COMMON_NAME, common_name), ])) builder = builder.not_valid_before( datetime.datetime.utcnow() - datetime.timedelta(days=2)) @@ -147,16 +149,19 @@ class TestHandler(BaseHTTPServer.BaseHTTPRequestHandler): response = { 'Path': self.path, } - self.wfile.write(json.dumps(response, indent=2)) + self.wfile.write(str2bytes(json.dumps(response, indent=2))) class CheckUrlAvailableMixin(TestPromisePluginMixin): @classmethod def setUpClass(cls): - cls.another_server_ca = CertificateAuthority("Another Server Root CA") - cls.test_server_ca = CertificateAuthority("Test Server Root CA") + cls.another_server_ca = CertificateAuthority(u"Another Server Root CA") + cls.test_server_ca = CertificateAuthority(u"Test Server Root CA") + ip = SLAPOS_TEST_IPV4.decode('utf-8') \ + if isinstance(SLAPOS_TEST_IPV4, bytes) \ + else SLAPOS_TEST_IPV4 key, key_pem, csr, csr_pem = createCSR( - "testserver.example.com", SLAPOS_TEST_IPV4) + u"testserver.example.com", ip) _, cls.test_server_certificate_pem = cls.test_server_ca.signCSR(csr) cls.test_server_certificate_file = tempfile.NamedTemporaryFile( @@ -175,17 +180,17 @@ class CheckUrlAvailableMixin(TestPromisePluginMixin): cls.test_server_ca.certificate_pem) cls.test_server_ca_certificate_file.close() - server = BaseHTTPServer.HTTPServer( - (SLAPOS_TEST_IPV4, SLAPOS_TEST_IPV4_PORT), - TestHandler) - - server.socket = ssl.wrap_socket( - server.socket, - certfile=cls.test_server_certificate_file.name, - server_side=True) - - cls.server_process = multiprocessing.Process( - target=server.serve_forever) + def server(): + server = BaseHTTPServer.HTTPServer( + (SLAPOS_TEST_IPV4, SLAPOS_TEST_IPV4_PORT), + TestHandler) + server.socket = ssl.wrap_socket( + server.socket, + certfile=cls.test_server_certificate_file.name, + server_side=True) + server.serve_forever() + + cls.server_process = multiprocessing.Process(target=server) cls.server_process.start() @classmethod @@ -269,7 +274,8 @@ class TestCheckUrlAvailable(CheckUrlAvailableMixin): self.assertEqual(result['result']['failed'], True) self.assertEqual( result['result']['message'], - "ERROR: Invalid URL u'https://': No host supplied" + "ERROR: Invalid URL %s'https://': No host supplied" % + ('' if six.PY3 else 'u') ) def test_check_url_malformed(self): diff --git a/slapos/test/promise/test_apache_mpm_watchdog.py b/slapos/test/promise/test_apache_mpm_watchdog.py index cd1f4ac..d36e629 100644 --- a/slapos/test/promise/test_apache_mpm_watchdog.py +++ b/slapos/test/promise/test_apache_mpm_watchdog.py @@ -87,7 +87,7 @@ class TestApacheMPMWatchdog(unittest.TestCase): self.assertEqual(None, getServerStatus("http://localhost/", "user", "password")) - self.assertNotEquals(None, + self.assertNotEqual(None, getServerStatus("https://www.erp5.com/", None, None)) diff --git a/slapos/test/test_agent.py b/slapos/test/test_agent.py index 6892b9e..a21b72b 100644 --- a/slapos/test/test_agent.py +++ b/slapos/test/test_agent.py @@ -29,58 +29,59 @@ import unittest import os.path from slapos.agent.agent import AutoSTemp, TestMap -TESTMAP_DICT = { - "test-wendelin-software-release": { - "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/wendelin/software.cfg", - "supply_computer": "COMP-2", - "group": "COMP-2", - "title": "test-wendelin-software-release"}, - "test-agent-software-release": { - "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/agent/software.cfg", - "supply_computer": "COMP-2", - "group": "COMP-2", - "title": "test-agent-software-release"}, - "test-powerdns-software-release": { - "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/powerdns/software.cfg", - "supply_computer": "COMP-2", - "group": "COMP-2", - "title": "test-powerdns-software-release"}, - "test-monitor-software-release": { - "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/monitor/software.cfg", - "supply_computer": "COMP-2", - "group": "COMP-2", - "title": "test-monitor-software-release"}, - "test-slapos-master-software-release": { - "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/slapos-master/software.cfg", - "supply_computer": "COMP-1", - "group": "COMP-1", - "title": "test-slapos-master-software-release"}, - "test-webrunner-software-release": { - "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/slaprunner/software.cfg", - "supply_computer": "COMP-1", - "group": "COMP-1", - "title": "test-webrunner-software-release"}, - "test-re6stnetmaster-software-release": { - "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/re6stnet/software.cfg", - "supply_computer": "COMP-2", - "group": "COMP-2", - "title": "test-re6stnetmaster-software-release"}, - "test-erp5testnode-software-release": { - "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/erp5testnode/software.cfg", - "supply_computer": "COMP-1", - "group": "COMP-1", - "title": "test-erp5testnode-software-release"}, - "test-apache-frontend-software-release": { - "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/apache-frontend/software.cfg", - "supply_computer": "COMP-1", - "group": "COMP-1", - "title": "test-apache-frontend-software-release"}, - "test-nayuos-software-release": { - "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/nayuos/software.cfg", - "supply_computer": "COMP-1", - "group": "COMP-1", - "title": "test-nayuos-software-release"} - } +from collections import OrderedDict +TESTMAP_DICT = OrderedDict([ + ("test-apache-frontend-software-release", { + "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/apache-frontend/software.cfg", + "supply_computer": "COMP-1", + "group": "COMP-1", + "title": "test-apache-frontend-software-release"}), + ("test-slapos-master-software-release", { + "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/slapos-master/software.cfg", + "supply_computer": "COMP-1", + "group": "COMP-1", + "title": "test-slapos-master-software-release"}), + ("test-erp5testnode-software-release", { + "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/erp5testnode/software.cfg", + "supply_computer": "COMP-1", + "group": "COMP-1", + "title": "test-erp5testnode-software-release"}), + ("test-webrunner-software-release", { + "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/slaprunner/software.cfg", + "supply_computer": "COMP-1", + "group": "COMP-1", + "title": "test-webrunner-software-release"}), + ("test-agent-software-release", { + "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/agent/software.cfg", + "supply_computer": "COMP-2", + "group": "COMP-2", + "title": "test-agent-software-release"}), + ("test-powerdns-software-release", { + "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/powerdns/software.cfg", + "supply_computer": "COMP-2", + "group": "COMP-2", + "title": "test-powerdns-software-release"}), + ("test-nayuos-software-release", { + "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/nayuos/software.cfg", + "supply_computer": "COMP-1", + "group": "COMP-1", + "title": "test-nayuos-software-release"}), + ("test-wendelin-software-release", { + "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/wendelin/software.cfg", + "supply_computer": "COMP-2", + "group": "COMP-2", + "title": "test-wendelin-software-release"}), + ("test-monitor-software-release", { + "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/monitor/software.cfg", + "supply_computer": "COMP-2", + "group": "COMP-2", + "title": "test-monitor-software-release"}), + ("test-re6stnetmaster-software-release", { + "url": "https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/re6stnet/software.cfg", + "supply_computer": "COMP-2", + "group": "COMP-2", + "title": "test-re6stnetmaster-software-release"}), +]) @@ -158,7 +159,8 @@ class TestAutoSTemp(unittest.TestCase): removes it when deleted. """ f = AutoSTemp("foo") - self.assertEqual(open(f.name, "r").read(), "foo") + with open(f.name, "r") as f_: + self.assertEqual(f_.read(), "foo") fname = f.name self.assertTrue(os.path.isfile(fname)) del f diff --git a/slapos/test/test_checkfeedaspromise.py b/slapos/test/test_checkfeedaspromise.py index e0cff83..ce6177c 100644 --- a/slapos/test/test_checkfeedaspromise.py +++ b/slapos/test/test_checkfeedaspromise.py @@ -87,10 +87,10 @@ class TestCheckFeedAsPromise(unittest.TestCase): feed = self.generateKOFeed() option.ko_pattern_list = ['Error'] - self.assertNotEquals(checkFeedAsPromise(feed, option), "") + self.assertNotEqual(checkFeedAsPromise(feed, option), "") option.title, option.description = False, True option.ko_pattern_list = ['FAILURE', 'Error'] - self.assertNotEquals(checkFeedAsPromise(feed, option), "") + self.assertNotEqual(checkFeedAsPromise(feed, option), "") def test_ifNoOKPatternFoundErrorIsRaised(self): @@ -100,7 +100,7 @@ class TestCheckFeedAsPromise(unittest.TestCase): # If no time buffer, then not OK is always wrong option.ok_pattern_list = ['OK'] - self.assertNotEquals(len(checkFeedAsPromise(feed, option)), 0) + self.assertNotEqual(len(checkFeedAsPromise(feed, option)), 0) # if time buffer, then not OK is wrong only after buffer expires extra_item = { @@ -115,7 +115,7 @@ class TestCheckFeedAsPromise(unittest.TestCase): # shorter buffer, we want to raise an error option.time_buffer = 1800 - self.assertNotEquals(len(checkFeedAsPromise(feed, option)), 0) + self.assertNotEqual(len(checkFeedAsPromise(feed, option)), 0) def test_noItemInTheFeedIsNotAnError(self): diff --git a/slapos/test/test_generatefeed.py b/slapos/test/test_generatefeed.py index 655e6a9..8851f47 100644 --- a/slapos/test/test_generatefeed.py +++ b/slapos/test/test_generatefeed.py @@ -7,6 +7,7 @@ import shutil import tempfile import time import unittest +import six from slapos.generatefeed import generateFeed @@ -120,8 +121,8 @@ class TestGenerateFeed(unittest.TestCase): for i in range(5-3, 5): # older items (from 1 to 2) have been deleted expected_remaining_item_list.append('%s.item' % i) - self.assertItemsEqual(remaining_status_item_list, - expected_remaining_item_list) + six.assertCountEqual(self, remaining_status_item_list, + expected_remaining_item_list) if __name__ == '__main__': unittest.main() diff --git a/slapos/test/test_qemuqmpclient.py b/slapos/test/test_qemuqmpclient.py index bd8ac7b..6bfbe4f 100644 --- a/slapos/test/test_qemuqmpclient.py +++ b/slapos/test/test_qemuqmpclient.py @@ -140,7 +140,7 @@ class TestQemuQMPWrapper(unittest.TestCase): elif message['execute'] == 'query-memory-devices': memory_list = [] added_mem = self.readChange('dimm') + self.hotplugged_memory_amount - slot_amount = added_mem / self.memory_slot_size + slot_amount = added_mem // self.memory_slot_size for i in range(slot_amount, 0, -1): memory_list.append({ u'data': { @@ -159,7 +159,7 @@ class TestQemuQMPWrapper(unittest.TestCase): elif message['execute'] == 'query-memdev': memory_list = [] added_mem = self.readChange('dimm') + self.hotplugged_memory_amount - slot_amount = added_mem / self.memory_slot_size + slot_amount = added_mem // self.memory_slot_size for i in range(slot_amount, 0, -1): memory_list.append({ u'dump': True, diff --git a/slapos/test/test_runner.py b/slapos/test/test_runner.py index c0b366e..93943cc 100644 --- a/slapos/test/test_runner.py +++ b/slapos/test/test_runner.py @@ -3,7 +3,6 @@ import os import string import random import supervisor -import thread import unittest @@ -29,14 +28,6 @@ class TestRunnerBackEnd(unittest.TestCase): if os.path.exists(garbage_file): os.remove(garbage_file) - def _startSupervisord(self): - cwd = os.getcwd() - supervisord_config_file = os.path.join(cwd, 'supervisord.conf') - open(supervisord_config_file, 'w').write(""" - """) - supervisord = supervisor.supervisord.Supervisord('-c', supervisord_config_file) - thread.start_new_thread() - def test_UserCanLoginAndUpdateCredentials(self): """ * Create a user with createNewUser diff --git a/slapos/test/test_runner_exporter.py b/slapos/test/test_runner_exporter.py index 350f1e1..b3a27b2 100644 --- a/slapos/test/test_runner_exporter.py +++ b/slapos/test/test_runner_exporter.py @@ -5,7 +5,6 @@ import time import unittest from slapos.resilient import runner_exporter -from StringIO import StringIO tested_instance_cfg = """[buildout] installed_develop_eggs = @@ -75,7 +74,7 @@ class TestRunnerExporter(unittest.TestCase): def _createExecutableFile(self, path, content=''): self._createFile(path, content) - os.chmod(path, 0700) + os.chmod(path, 0o700) def _setUpFakeInstanceFolder(self): self._createFile('proxy.db') @@ -232,9 +231,9 @@ class TestRunnerExporter(unittest.TestCase): self.assertEqual( runner_exporter.getBackupFilesModifiedDuringExportList(config, time.time() - 5), - ['instance/slappart0/srv/backup/data.dat', - 'instance/slappart0/srv/backup/important_logs/this_is_a.log', - 'instance/slappart1/srv/backup/data.dat'] + [b'instance/slappart0/srv/backup/data.dat', + b'instance/slappart0/srv/backup/important_logs/this_is_a.log', + b'instance/slappart1/srv/backup/data.dat'] ) time.sleep(2) self.assertFalse( @@ -243,5 +242,5 @@ class TestRunnerExporter(unittest.TestCase): self._createFile('instance/slappart1/srv/backup/bakckup.data', 'my backup') self.assertEqual( runner_exporter.getBackupFilesModifiedDuringExportList(config, time.time() - 1), - ['instance/slappart1/srv/backup/bakckup.data'] + [b'instance/slappart1/srv/backup/bakckup.data'] ) diff --git a/slapos/test/test_securedelete.py b/slapos/test/test_securedelete.py index 01b0a74..5c3790d 100644 --- a/slapos/test/test_securedelete.py +++ b/slapos/test/test_securedelete.py @@ -58,32 +58,32 @@ class TestSecureDelete(unittest.TestCase): passes = 2 + 1 # Option -z is used, plus one more pass result = shred(options) self.assertFalse(os.path.exists(self.remove_file)) - self.assertTrue("pass %s/%s" % (passes, passes) in result) - self.assertTrue("%s: removed" % os.path.basename(self.remove_file) in result) + self.assertIn("pass %d/%d" % (passes, passes), result) + self.assertIn("%s: removed" % os.path.basename(self.remove_file), result) def test_secure_remove_file_keep_file(self): options = getAgumentParser().parse_args(['-n', '2', '-z', '--file', self.remove_file]) passes = 2 + 1 # Option -z is used, plus one more pass result = shred(options) self.assertTrue(os.path.exists(self.remove_file)) - self.assertTrue("pass %s/%s" % (passes, passes) in result) - self.assertFalse("%s: removed" % os.path.basename(self.remove_file) in result) + self.assertIn("pass %d/%d" % (passes, passes), result) + self.assertNotIn("%s: removed" % os.path.basename(self.remove_file), result) def test_secure_remove_file_non_zero(self): options = getAgumentParser().parse_args(['-n', '2', '-u', '--file', self.remove_file]) passes = 2 result = shred(options) self.assertFalse(os.path.exists(self.remove_file)) - self.assertTrue("pass %s/%s" % (passes, passes) in result) - self.assertTrue("%s: removed" % os.path.basename(self.remove_file) in result) + self.assertIn("pass %d/%d" % (passes, passes), result) + self.assertIn("%s: removed" % os.path.basename(self.remove_file), result) def test_secure_remove_file_check_exist(self): options = getAgumentParser().parse_args(['-n', '2', '-u', '-s', '--file', 'random.txt', self.remove_file]) passes = 2 result = shred(options) self.assertFalse(os.path.exists(self.remove_file)) - self.assertTrue("pass %s/%s" % (passes, passes) in result) - self.assertTrue("%s: removed" % os.path.basename(self.remove_file) in result) + self.assertIn("pass %d/%d" % (passes, passes), result) + self.assertIn("%s: removed" % os.path.basename(self.remove_file), result) def test_secure_remove_file_check_exist_false(self): options = getAgumentParser().parse_args(['-n', '2', '-u', '--file', 'random.txt']) @@ -99,19 +99,19 @@ class TestSecureDelete(unittest.TestCase): # shred removed link and target file self.assertFalse(os.path.exists(self.remove_file)) self.assertFalse(os.path.exists(self.link_name)) - self.assertTrue("pass %s/%s" % (passes, passes) in result) - self.assertTrue("%s: removed" % os.path.basename(self.remove_file) in result) + self.assertIn("pass %d/%d" % (passes, passes), result) + self.assertIn("%s: removed" % os.path.basename(self.remove_file), result) def test_secure_remove_file_multiple_files(self): options = getAgumentParser().parse_args(['-n', '2', '-u', '-z', '--file', self.remove_file, self.remove_file2]) passes = 2 + 1 # Option -z is used, plus one more pass result = shred(options) self.assertFalse(os.path.exists(self.remove_file)) - self.assertTrue("pass %s/%s" % (passes, passes) in result) - self.assertTrue("%s: removed" % os.path.basename(self.remove_file) in result) + self.assertIn("pass %d/%d" % (passes, passes), result) + self.assertIn("%s: removed" % os.path.basename(self.remove_file), result) self.assertFalse(os.path.exists(self.remove_file2)) - self.assertTrue("%s: removed" % os.path.basename(self.remove_file2) in result) + self.assertIn("%s: removed" % os.path.basename(self.remove_file2), result) if __name__ == '__main__': unittest.main() -- 2.30.9