Add Python 3 support

slapos/bang.py

@@ -45,6 +45,6 @@ def do_bang(configp, message):
   slap = slapos.slap.slap()
   slap.initializeConnection(master_url, key_file=key_file, cert_file=cert_file)
   computer = slap.registerComputer(computer_id)
-  print 'Banging to %r' % master_url
+  print('Banging to %r' % master_url)
   computer.bang(message)
-  print 'Bang with message %r' % message
+  print('Bang with message %r' % message)

slapos/cli/boot.py

@@ -27,8 +27,10 @@
 #
 ##############################################################################
 
+from __future__ import print_function
+
 import subprocess
-import urlparse
+from six.moves.urllib.parse import urlparse
 from time import sleep
 import glob
 import os
@@ -46,14 +48,14 @@ def _removeTimestamp(instancehome):
     """
     timestamp_glob_path = "%s/slappart*/.timestamp" % instancehome
     for timestamp_path in glob.glob(timestamp_glob_path):
-       print "Removing %s" % timestamp_path
+       print("Removing %s" % timestamp_path)
        os.remove(timestamp_path)
 
 def _runBang(app):
     """
     Launch slapos node format.
     """
-    print "[BOOT] Invoking slapos node bang..."
+    print("[BOOT] Invoking slapos node bang...")
     result = app.run(['node', 'bang', '-m', 'Reboot'])
     if result == 1:
       return 0
@@ -63,7 +65,7 @@ def _runFormat(app):
     """
     Launch slapos node format.
     """
-    print "[BOOT] Invoking slapos node format..."
+    print("[BOOT] Invoking slapos node format...")
     result = app.run(['node', 'format', '--now', '--verbose'])
     if result == 1:
       return 0
@@ -73,30 +75,30 @@ def _ping(hostname):
     """ 
     Ping a hostname
     """
-    print "[BOOT] Invoking ipv4 ping to %s..." % hostname
+    print("[BOOT] Invoking ipv4 ping to %s..." % hostname)
     p = subprocess.Popen(
       ["ping", "-c", "2", hostname],
        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     stdout, stderr = p.communicate()
     if p.returncode == 0:
-      print "[BOOT] IPv4 network reachable..."
+      print("[BOOT] IPv4 network reachable...")
       return 1
-    print "[BOOT] [ERROR] IPv4 network unreachable..."
+    print("[BOOT] [ERROR] IPv4 network unreachable...")
     return 0
 
 def _ping6(hostname):
     """ 
     Ping an ipv6 address
     """
-    print "[BOOT] Invoking ipv6 ping to %s..." % hostname
+    print("[BOOT] Invoking ipv6 ping to %s..." % hostname)
     p = subprocess.Popen(
         ["ping6", "-c", "2", hostname],
         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     stdout, stderr = p.communicate()
     if p.returncode == 0:
-        print "[BOOT] IPv6 network reachable..."
+        print("[BOOT] IPv6 network reachable...")
         return 1
-    print "[BOOT] [ERROR] IPv6 network unreachable..."
+    print("[BOOT] [ERROR] IPv6 network unreachable...")
     return 0
 
 def _test_ping(hostname):
@@ -126,7 +128,7 @@ def _waitIpv6Ready(ipv6_interface):
     test if ipv6 is ready on ipv6_interface
   """
   ipv6_address = ""
-  print "[BOOT] Checking if %r has IPv6..." % ipv6_interface
+  print("[BOOT] Checking if %r has IPv6..." % ipv6_interface)
   while ipv6_address == "":
     for inet_dict in netifaces.ifaddresses(ipv6_interface)[socket.AF_INET6]:
       ipv6_address = inet_dict['addr'].split('%')[0]
@@ -134,8 +136,8 @@ def _waitIpv6Ready(ipv6_interface):
         break
     else:
       ipv6_address = ""
-      print "[BOOT] [ERROR] No IPv6 found on interface %r, " \
-        "try again in 5 seconds..." % ipv6_interface
+      print("[BOOT] [ERROR] No IPv6 found on interface %r, "
+        "try again in 5 seconds..." % ipv6_interface))
       sleep(5)
 
 class BootCommand(ConfigCommand):
@@ -155,7 +157,7 @@ class BootCommand(ConfigCommand):
     def take_action(self, args):
         configp = self.fetch_config(args)
         instance_root = configp.get('slapos','instance_root')
-        master_url = urlparse.urlparse(configp.get('slapos','master_url'))
+        master_url = urlparse(configp.get('slapos','master_url'))
         master_hostname = master_url.hostname
 
         # Check that we have IPv6 ready
@@ -177,12 +179,12 @@ class BootCommand(ConfigCommand):
         app = SlapOSApp()
         # Make sure slapos node format returns ok
         while not _runFormat(app):
-            print "[BOOT] [ERROR] Fail to format, try again in 15 seconds..."
+            print("[BOOT] [ERROR] Fail to format, try again in 15 seconds...")
             sleep(15)
        
         # Make sure slapos node bang returns ok
         while not _runBang(app):
-            print "[BOOT] [ERROR] Fail to bang, try again in 15 seconds..."
+            print("[BOOT] [ERROR] Fail to bang, try again in 15 seconds...")
             sleep(15)
 
         _removeTimestamp(instance_root)

slapos/cli/config.py

@@ -27,7 +27,7 @@
 #
 ##############################################################################
 
-import ConfigParser
+from six.moves import configparser
 import os
 
 from slapos.cli.command import Command
@@ -77,7 +77,7 @@ class ConfigCommand(Command):
         if not os.path.exists(cfg_path):
             raise ConfigError('Configuration file does not exist: %s' % cfg_path)
 
-        configp = ConfigParser.SafeConfigParser()
+        configp = configparser.SafeConfigParser()
         if configp.read(cfg_path) != [cfg_path]:
             # bad permission, etc.
             raise ConfigError('Cannot parse configuration file: %s' % cfg_path)

slapos/cli/configure_client.py

@@ -31,6 +31,7 @@ import re
 import os
 import sys
 import json
+from six.moves import input
 
 import requests
 
@@ -94,7 +95,7 @@ def fetch_configuration_template():
 
 def do_configure_client(logger, master_url_web, token, config_path, master_url):
     while not token:
-        token = raw_input('Credential security token: ').strip()
+        token = input('Credential security token: ').strip()
 
     # Check for existence of previous configuration, certificate or key files
     # where we expect to create them. If so, ask the use to manually remove them.

slapos/cli/configure_local/__init__.py

@@ -39,6 +39,7 @@ from slapos.cli.config import ConfigCommand
 from slapos.grid.slapgrid import create_slapgrid_object
 from slapos.grid.utils import updateFile, createPrivateDirectory
 from slapos.grid.svcbackend import launchSupervisord
+from slapos.util import bytes2str
 
 DEFAULT_COMPUTER_ID = 'local_computer'
 
@@ -126,8 +127,8 @@ def _replaceParameterValue(original_content, to_replace):
 
 def _generateSlaposNodeConfigurationFile(slapos_node_config_path, args):
     template_arg_list = (__name__, '../../slapos.cfg.example')
-    with pkg_resources.resource_stream(*template_arg_list) as fout:
-      slapos_node_configuration_template = fout.read()
+    slapos_node_configuration_template = \
+      bytes2str(pkg_resources.resource_string(*template_arg_list))
     master_url = 'http://%s:%s' % (args.daemon_listen_ip, args.daemon_listen_port)
     slapos_home = args.slapos_buildout_directory
     to_replace = [
@@ -149,12 +150,12 @@ def _generateSlaposNodeConfigurationFile(slapos_node_config_path, args):
         '(key_file|cert_file|certificate_repository_path).*=.*\n',
         '', slapos_node_configuration_content)
     with open(slapos_node_config_path, 'w') as fout:
-        fout.write(slapos_node_configuration_content.encode('utf8'))
+        fout.write(slapos_node_configuration_content)
 
 def _generateSlaposProxyConfigurationFile(conf):
     template_arg_list = (__name__, '../../slapos-proxy.cfg.example')
-    with pkg_resources.resource_stream(*template_arg_list) as fout:
-      slapos_proxy_configuration_template = fout.read()
+    slapos_proxy_configuration_template = \
+      bytes2str(pkg_resources.resource_string(*template_arg_list))
     slapos_proxy_configuration_path = os.path.join(
       conf.slapos_configuration_directory, 'slapos-proxy.cfg')
     listening_ip, listening_port = \
@@ -172,7 +173,7 @@ def _generateSlaposProxyConfigurationFile(conf):
         slapos_proxy_configuration_template, to_replace)
 
     with open(slapos_proxy_configuration_path, 'w') as fout:
-        fout.write(slapos_proxy_configuration_content.encode('utf8'))
+        fout.write(slapos_proxy_configuration_content)
 
     return slapos_proxy_configuration_path
 

slapos/cli/console.py

@@ -32,6 +32,8 @@ import textwrap
 from slapos.cli.config import ClientConfigCommand
 from slapos.client import init, do_console, ClientConfig
 
+from six import exec_
+
 
 class ShellNotFound(Exception):
     pass
@@ -90,7 +92,9 @@ class ConsoleCommand(ClientConfigCommand):
         local = init(conf, self.app.log)
 
         if args.script_file:
-            return execfile(args.script_file, globals(), local)
+            with open(args.script_file) as f:
+                code = compile(f.read(), args.script_file, 'exec')
+                return exec_(code, globals(), local)
 
         if not any([args.python, args.ipython, args.bpython]):
             args.ipython = True

slapos/cli/entry.py

@@ -247,7 +247,7 @@ class SlapOSApp(App):
           return
 
         if self.options.log_color:
-            import coloredlogs
+            from slapos.cli import coloredlogs
             console = coloredlogs.ColoredStreamHandler(show_name=True,     # logger name (slapos) and PID
                                                        show_severity=True,
                                                        show_timestamps=self.options.log_time,

slapos/cli/list.py

@@ -29,6 +29,7 @@
 
 import logging
 import sys
+import six
 
 from slapos.cli.config import ClientConfigCommand
 from slapos.client import init, ClientConfig
@@ -64,5 +65,5 @@ def do_list(logger, conf, local):
       logger.info('No existing service.')
       return
     logger.info('List of services:')
-    for title, instance in instance_dict.iteritems():
+    for title, instance in six.iteritems(instance_dict):
       logger.info('%s %s', title, instance._software_release_url)

slapos/cli/proxy_show.py

@@ -34,7 +34,6 @@ import logging
 import sys
 import os
 import subprocess
-import StringIO
 
 import lxml.etree
 import prettytable
@@ -43,8 +42,17 @@ import sqlite3
 from slapos.cli.config import ConfigCommand
 from slapos.proxy import ProxyConfig
 from slapos.proxy.db_version import DB_VERSION
-from slapos.util import sqlite_connect
-
+from slapos.util import sqlite_connect, str2bytes
+
+if bytes is str:
+  from io import BytesIO
+  class StringIO(BytesIO):
+    # Something between strict io.BytesIO and laxist/slow StringIO.StringIO
+    # (which starts returning unicode once unicode is written) for logging. 
+    def write(self, b):
+      return BytesIO.write(self, b.encode('utf-8'))
+else:
+  from io import StringIO
 
 class ProxyShowCommand(ConfigCommand):
     """
@@ -139,7 +147,7 @@ def log_params(logger, conn):
         if not row['connection_xml']:
             continue
 
-        xml = str(row['connection_xml'])
+        xml = str2bytes(row['connection_xml'])
         logger.info('%s: %s (type %s)', row['reference'], row['partition_reference'], row['software_type'])
         instance = lxml.etree.fromstring(xml)
         for parameter in list(instance):
@@ -150,9 +158,12 @@ def log_params(logger, conn):
             # _ is usually json encoded - re-format to make it easier to read
             if name == '_':
                 try:
-                    text = json.dumps(json.loads(text), indent=2)
+                    text = json.dumps(json.loads(text),
+                        indent=2, sort_keys=True)
                 except ValueError:
                     pass
+                else: # to avoid differences between Py2 and Py3 in unit tests
+                    text = '\n'.join(map(str.rstrip, text.splitlines()))
             logger.info('    %s = %s', name, text)
 
 
@@ -208,7 +219,7 @@ def do_show(conf):
     # to paginate input, honoring $PAGER.
     output = sys.stdout
     if output.isatty():
-       output = StringIO.StringIO()
+      output = StringIO()
     proxy_show_logger = logging.getLogger(__name__)
     handler = logging.StreamHandler(output)
     handler.setLevel(logging.DEBUG)
@@ -221,7 +232,8 @@ def do_show(conf):
     conn = sqlite_connect(conf.database_uri)
     conn.row_factory = sqlite3.Row
 
-    conn.create_function('md5', 1, lambda s: hashlib.md5(s).hexdigest())
+    conn.create_function('md5', 1,
+                         lambda s: hashlib.md5(str2bytes(s)).hexdigest())
 
     call_table = [
         (conf.computers, log_computer_table),
@@ -248,4 +260,4 @@ def do_show(conf):
             close_fds=True,
             shell=True,
             stdin=subprocess.PIPE,)
-        pager.communicate(output.getvalue().encode('utf-8'))
+        pager.communicate(str2bytes(output.getvalue()))

slapos/cli/register.py

@@ -36,6 +36,7 @@ import sys
 import pkg_resources
 import requests
 import json
+from six.moves import input
 
 from slapos.cli.command import Command, must_be_root
 
@@ -327,7 +328,7 @@ def gen_auth(conf):
         else:
             yield conf.login, getpass.getpass()
     while ask:
-        yield raw_input('SlapOS Master Login: '), getpass.getpass()
+        yield input('SlapOS Master Login: '), getpass.getpass()
 
 
 def do_register(conf):
@@ -348,7 +349,7 @@ def do_register(conf):
                                                     password=password)
     else:
         while not conf.token:
-            conf.token = raw_input('Computer security token: ').strip()
+            conf.token = input('Computer security token: ').strip()
 
         certificate, key = get_certificate_key_pair(conf.logger,
                                                     conf.master_url_web,

slapos/client.py

@@ -28,7 +28,7 @@
 ##############################################################################
 
 import atexit
-import ConfigParser
+from six.moves import configparser
 import os
 import sys
 
@@ -52,7 +52,7 @@ class ClientConfig(object):
     # Merges the arguments and configuration
     try:
       configuration_dict = dict(configp.items('slapconsole'))
-    except ConfigParser.NoSectionError:
+    except configparser.NoSectionError:
       pass
     else:
       for key in configuration_dict:
@@ -119,7 +119,7 @@ def _getSoftwareReleaseFromSoftwareString(logger, software_string, product):
     try:
         return product.__getattr__(software_string[len(SOFTWARE_PRODUCT_NAMESPACE):])
     except AttributeError as e:
-       logger.error('Error: %s Exiting now.' % e.message)
+       logger.error('Error: %s Exiting now.', e)
        sys.exit(1)
 
 def do_console(local):

slapos/collect/__init__.py

@@ -27,6 +27,7 @@
 #
 ##############################################################################
 
+from __future__ import print_function
 from psutil import process_iter, NoSuchProcess, AccessDenied
 from time import strftime
 import shutil
@@ -42,7 +43,7 @@ from slapos.collect.reporter import RawCSVDumper, \
                                     compressLogFolder, \
                                     ConsumptionReport 
 
-from entity import get_user_list, Computer
+from .entity import get_user_list, Computer
 
 def _get_time():
   return strftime("%Y-%m-%d -- %H:%M:%S").split(" -- ")
@@ -157,5 +158,5 @@ def do_collect(conf):
     database.garbageCollect()
 
   except AccessDenied:
-    print "You HAVE TO execute this script with root permission."
+    print("You HAVE TO execute this script with root permission.")
 

slapos/collect/entity.py

@@ -74,19 +74,15 @@ class User(object):
       time_cycle = self.disk_snapshot_params.get('time_cycle', 0)
       database.connect()
       if time_cycle:
-        order = 'date DESC, time DESC'
-        limit = 1
-        query = database.select(table="folder", columns="date, time",
-                                    order=order, limit=limit,
-                                    where="partition='%s'" % self.name)
-        query_result = zip(*query)
-        if len(query_result):
-          date, time = (query_result[0][0], query_result[1][0])
-          latest_date = datetime.strptime('%s %s' % (date, time),
-                                           "%Y-%m-%d %H:%M:%S")
+        for date_time in database.select(table="folder", columns="date, time",
+                                    order='date DESC, time DESC', limit=1,
+                                    where="partition='%s'" % self.name):
+          latest_date = datetime.strptime('%s %s' % date_time,
+                                          "%Y-%m-%d %H:%M:%S")
           if (datetime.now() - latest_date).seconds < time_cycle:
             # wait the time cycle
             return
+          break
       pid_file = self.disk_snapshot_params.get('pid_folder', None)
       if pid_file is not None:
         pid_file = os.path.join(pid_file, '%s_disk_size.pid' % self.name)

slapos/collect/reporter.py

@@ -40,6 +40,8 @@ import tarfile
 import time
 import psutil
 
+import six
+
 log_file = False
 
 class Dumper(object):
@@ -59,10 +61,10 @@ class SystemReporter(Dumper):
     """ Dump data """
     _date = time.strftime("%Y-%m-%d")
     self.db.connect()
-    for item, collected_item_list in self.db.exportSystemAsDict(_date).iteritems():
+    for item, collected_item_list in six.iteritems(self.db.exportSystemAsDict(_date)):
       self.writeFile(item, folder, collected_item_list)
 
-    for partition, collected_item_list in self.db.exportDiskAsDict(_date).iteritems():
+    for partition, collected_item_list in six.iteritems(self.db.exportDiskAsDict(_date)):
       partition_id = "_".join(partition.split("-")[:-1]).replace("/", "_")
       item = "memory_%s" % partition.split("-")[-1]
       self.writeFile("disk_%s_%s" % (item, partition_id), folder, collected_item_list)
@@ -135,62 +137,53 @@ class ConsumptionReportBase(object):
 
   def getPartitionCPULoadAverage(self, partition_id, date_scope):
     self.db.connect()
-    query_result_cursor = self.db.select("user", date_scope,
+    (cpu_percent_sum,), = self.db.select("user", date_scope,
                        columns="SUM(cpu_percent)", 
                        where="partition = '%s'" % partition_id)
 
-    cpu_percent_sum = zip(*query_result_cursor)
-    if len(cpu_percent_sum) and cpu_percent_sum[0][0] is None:
+    if cpu_percent_sum is None:
       return
 
-    query_result_cursor = self.db.select("user", date_scope,
+    (sample_amount,), = self.db.select("user", date_scope,
                        columns="COUNT(DISTINCT time)", 
                        where="partition = '%s'" % partition_id)
 
-    sample_amount = zip(*query_result_cursor)
     self.db.close()
 
-    if len(sample_amount) and len(cpu_percent_sum):
-      return cpu_percent_sum[0][0]/sample_amount[0][0]
+    return cpu_percent_sum/sample_amount
 
   def getPartitionUsedMemoryAverage(self, partition_id, date_scope):
     self.db.connect()
-    query_result_cursor = self.db.select("user", date_scope,
+    (memory_sum,), = self.db.select("user", date_scope,
                        columns="SUM(memory_rss)", 
                        where="partition = '%s'" % partition_id)
 
-    memory_sum = zip(*query_result_cursor)
-    if len(memory_sum) and memory_sum[0][0] is None:
+    if memory_sum is None:
       return
 
-    query_result_cursor = self.db.select("user", date_scope,
+    (sample_amount,), = self.db.select("user", date_scope,
                        columns="COUNT(DISTINCT time)", 
                        where="partition = '%s'" % partition_id)
 
-    sample_amount = zip(*query_result_cursor)
     self.db.close()
 
-    if len(sample_amount) and len(memory_sum):
-      return memory_sum[0][0]/sample_amount[0][0]
+    return memory_sum/sample_amount
 
   def getPartitionDiskUsedAverage(self, partition_id, date_scope):
     self.db.connect()
-    query_result_cursor = self.db.select("folder", date_scope,
+    (disk_used_sum,), = self.db.select("folder", date_scope,
                        columns="SUM(disk_used)", 
                        where="partition = '%s'" % partition_id)
 
-    disk_used_sum = zip(*query_result_cursor)
-    if len(disk_used_sum) and disk_used_sum[0][0] is None:
+    if disk_used_sum is None:
       return
-    query_result_cursor = self.db.select("folder", date_scope,
+    (collect_amount,), = self.db.select("folder", date_scope,
                        columns="COUNT(DISTINCT time)", 
                        where="partition = '%s'" % partition_id)
 
-    collect_amount = zip(*query_result_cursor)
     self.db.close()
 
-    if len(collect_amount) and len(disk_used_sum):
-      return disk_used_sum[0][0]/collect_amount[0][0]
+    return disk_used_sum/collect_amount
 
 class ConsumptionReport(ConsumptionReportBase):
 
@@ -287,7 +280,7 @@ class ConsumptionReport(ConsumptionReportBase):
                            reference=user,
                            category="")
 
-     with open(xml_report_path, 'w') as f:
+     with open(xml_report_path, 'wb') as f:
        f.write(journal.getXML())
        f.close()
 
@@ -298,23 +291,19 @@ class ConsumptionReport(ConsumptionReportBase):
 
   def _getCpuLoadAverageConsumption(self, date_scope):
     self.db.connect()
-    query_result_cursor = self.db.select("system", date_scope, 
+    (cpu_load_percent_list,), = self.db.select("system", date_scope,
                        columns="SUM(cpu_percent)/COUNT(cpu_percent)")
 
-    cpu_load_percent_list = zip(*query_result_cursor)
     self.db.close()
-    if len(cpu_load_percent_list):
-      return cpu_load_percent_list[0][0]
+    return cpu_load_percent_list
 
   def _getMemoryAverageConsumption(self, date_scope):
     self.db.connect()
-    query_result_cursor = self.db.select("system", date_scope, 
+    (memory_used_list,), = self.db.select("system", date_scope,
                        columns="SUM(memory_used)/COUNT(memory_used)")
 
-    memory_used_list = zip(*query_result_cursor)
     self.db.close()
-    if len(memory_used_list):
-      return memory_used_list[0][0]
+    return memory_used_list
 
   def _getZeroEmissionContribution(self):
     self.db.connect()
@@ -329,7 +318,7 @@ class Journal(object):
 
    def getXML(self):
      report = ElementTree.tostring(self.root) 
-     return "<?xml version='1.0' encoding='utf-8'?>%s" % report
+     return b"<?xml version='1.0' encoding='utf-8'?>%s" % report
    
    def newTransaction(self, portal_type="Sale Packing List"):
      transaction = ElementTree.SubElement(self.root, "transaction")

slapos/collect/snapshot.py

@@ -27,13 +27,15 @@
 #
 ##############################################################################
 
+from __future__ import print_function
 import psutil
 import os
 import subprocess
-from temperature import collectComputerTemperature, \
-                        launchTemperatureTest
+from .temperature import collectComputerTemperature, launchTemperatureTest
 
-from temperature.heating import get_contribution_ratio
+from .temperature.heating import get_contribution_ratio
+
+import six
 
 MEASURE_INTERVAL = 5
 
@@ -159,7 +161,7 @@ class HeatingContributionSnapshot(_Snapshot):
     
     result = launchTemperatureTest(sensor_id)
     if result is None:
-      print "Impossible to test sensor: %s " % sensor_id
+      print("Impossible to test sensor: %s " % sensor_id)
       
 
     initial_temperature, final_temperature, duration = result 
@@ -215,8 +217,7 @@ class ComputerSnapshot(_Snapshot):
     #
     self.system_snapshot = SystemSnapshot()
     self.temperature_snapshot_list = self._get_temperature_snapshot_list()
-    self.disk_snapshot_list = []
-    self.partition_list = self._get_physical_disk_info()
+    self._get_physical_disk_info()
 
     if test_heating and model_id is not None \
                     and sensor_id is not None:
@@ -231,16 +232,16 @@ class ComputerSnapshot(_Snapshot):
     return temperature_snapshot_list
 
   def _get_physical_disk_info(self):
-    partition_dict = {}
+    # XXX: merge the following 2 to avoid calling disk_usage() twice
+    self.disk_snapshot_list = []
+    self.partition_list = []
+    partition_set = set()
 
     for partition in psutil.disk_partitions():
-      if partition.device not in partition_dict:
+      dev = partition.device
+      if dev not in partition_set: # XXX: useful ?
+        partition_set.add(dev)
         usage = psutil.disk_usage(partition.mountpoint)
-        partition_dict[partition.device] = usage.total
+        self.partition_list.append((dev, usage.total))
         self.disk_snapshot_list.append(
-          DiskPartitionSnapshot(partition.device, 
-                                partition.mountpoint))
-
-    return [(k, v) for k, v in partition_dict.iteritems()]
-
-  
+          DiskPartitionSnapshot(dev, partition.mountpoint))

slapos/collect/temperature/__init__.py

+from __future__ import print_function
 
 from multiprocessing import Process, active_children, cpu_count, Pipe
 import subprocess
@@ -14,28 +15,21 @@ except NotImplementedError:
     DEFAULT_CPU = 1
 
 def collectComputerTemperature(sensor_bin="sensors"):
-  cmd = ["%s -u" % sensor_bin]
-  
-  sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, 
-                        stderr=subprocess.PIPE, shell=True)
-  
-  stdout, stderr = sp.communicate()
-  
+  stdout = subprocess.check_output((sensor_bin, '-u'), universal_newlines=True)
+
   sensor_output_list = stdout.splitlines()
   adapter_name = ""
   
   sensor_temperature_list = []
-  for line_number in range(len(sensor_output_list)):
-    found_sensor = None
-    stripped_line = sensor_output_list[line_number].strip()
+  for line_number, sensor_output in enumerate(sensor_output_list):
+    stripped_line = sensor_output.strip()
     if stripped_line.startswith("Adapter:"):
       adapter_name = sensor_output_list[line_number-1]
    
     elif stripped_line.startswith("temp") and "_input" in stripped_line:
-      temperature = sensor_output_list[line_number].split()[-1] 
+      temperature = sensor_output.split()[-1]
       found_sensor = ["%s %s" % (adapter_name, sensor_output_list[line_number-1]), float(temperature)]
   
-    if found_sensor is not None:
       critical = '1000'
       maximal = '1000'
       for next_line in sensor_output_list[line_number+1:line_number+3]:
@@ -120,7 +114,7 @@ def launchTemperatureTest(sensor_id, sensor_bin="sensors", timeout=600, interval
 
   for connection in process_connection_list:
     try:
-      print connection.recv()
+      print(connection.recv())
     except EOFError:
       continue
 

slapos/format.py

@@ -28,7 +28,7 @@
 #
 ##############################################################################
 
-import ConfigParser
+from six.moves import configparser
 import errno
 import fcntl
 import grp
@@ -51,12 +51,13 @@ import time
 import traceback
 import zipfile
 import platform
-from urllib2 import urlopen
+from six.moves.urllib.request import urlopen
+import six
 
 import lxml.etree
 import xml_marshaller.xml_marshaller
 
-from slapos.util import mkdir_p, ipv6FromBin, binFromIpv6, lenNetmaskIpv6
+from slapos.util import dumps, mkdir_p, ipv6FromBin, binFromIpv6, lenNetmaskIpv6
 import slapos.slap as slap
 from slapos import version
 from slapos import manager as slapmanager
@@ -87,7 +88,7 @@ class OS(object):
   def _addWrapper(self, name):
     def wrapper(*args, **kw):
       arg_list = [repr(x) for x in args] + [
-          '%s=%r' % (x, y) for x, y in kw.iteritems()
+          '%s=%r' % (x, y) for x, y in six.iteritems(kw)
       ]
       self._logger.debug('%s(%s)' % (name, ', '.join(arg_list)))
       if not self._dry_run:
@@ -232,8 +233,8 @@ def _getDict(obj):
       return obj
 
   return {
-    key: _getDict(value) \
-    for key, value in dikt.iteritems() \
+    key: _getDict(value)
+    for key, value in six.iteritems(dikt)
     # do not attempt to serialize logger: it is both useless and recursive.
     # do not serialize attributes starting with "_", let the classes have some privacy
     if not key.startswith("_")
@@ -336,7 +337,7 @@ class Computer(object):
     if conf.dry_run:
       return
     try:
-      slap_computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps(_getDict(self)))
+      slap_computer.updateConfiguration(dumps(_getDict(self)))
     except slap.NotFoundError as error:
       raise slap.NotFoundError("%s\nERROR: This SlapOS node is not recognised by "
           "SlapOS Master and/or computer_id and certificates don't match. "
@@ -358,7 +359,7 @@ class Computer(object):
       with open(path_to_json, 'wb') as fout:
         fout.write(json.dumps(computer_dict, sort_keys=True, indent=2))
 
-    new_xml = xml_marshaller.xml_marshaller.dumps(computer_dict)
+    new_xml = dumps(computer_dict)
     new_pretty_xml = prettify_xml(new_xml)
 
     path_to_archive = path_to_xml + '.zip'
@@ -1197,7 +1198,7 @@ class Interface(object):
 
 def parse_computer_definition(conf, definition_path):
   conf.logger.info('Using definition file %r' % definition_path)
-  computer_definition = ConfigParser.RawConfigParser({
+  computer_definition = configparser.RawConfigParser({
     'software_user': 'slapsoft',
   })
   computer_definition.read(definition_path)
@@ -1308,7 +1309,7 @@ def parse_computer_xml(conf, xml_path):
 
 
 def write_computer_definition(conf, computer):
-  computer_definition = ConfigParser.RawConfigParser()
+  computer_definition = configparser.RawConfigParser()
   computer_definition.add_section('computer')
   if computer.address is not None and computer.netmask is not None:
     computer_definition.set('computer', 'address', '/'.join(

slapos/grid/SlapObject.py

@@ -38,7 +38,7 @@ import subprocess
 import tarfile
 import tempfile
 import time
-import xmlrpclib
+from six.moves import xmlrpc_client as xmlrpclib, range
 
 from supervisor import xmlrpc
 
@@ -51,6 +51,7 @@ from slapos.grid.exception import (BuildoutFailedError, WrongPermissionError,
                                    PathDoesNotExistError, DiskSpaceError)
 from slapos.grid.networkcache import download_network_cached, upload_network_cached
 from slapos.human import bytes2human
+from slapos.util import bytes2str
 
 
 WATCHDOG_MARK = '-on-watch'
@@ -60,8 +61,8 @@ REQUIRED_COMPUTER_PARTITION_PERMISSION = 0o750
 CP_STORAGE_FOLDER_NAME = 'DATA'
 
 # XXX not very clean. this is changed when testing
-PROGRAM_PARTITION_TEMPLATE = pkg_resources.resource_stream(__name__,
-            'templates/program_partition_supervisord.conf.in').read()
+PROGRAM_PARTITION_TEMPLATE = bytes2str(pkg_resources.resource_string(__name__,
+            'templates/program_partition_supervisord.conf.in'))
 
 
 def free_space(path, fn):
@@ -150,7 +151,7 @@ class Software(object):
     self.software_min_free_space = software_min_free_space
 
   def check_free_space(self):
-    required = self.software_min_free_space
+    required = self.software_min_free_space or 0
     available = free_space_nonroot(self.software_path)
 
     if available < required:
@@ -292,7 +293,7 @@ class Software(object):
         f.close()
 
   def _create_buildout_profile(self, buildout_cfg, url):
-    with open(buildout_cfg, 'wb') as fout:
+    with open(buildout_cfg, 'w') as fout:
       fout.write('[buildout]\nextends = ' + url + '\n')
     self._set_ownership(buildout_cfg)
 
@@ -419,7 +420,7 @@ class Partition(object):
 
 
   def check_free_space(self):
-    required = self.instance_min_free_space
+    required = self.instance_min_free_space or 0
     available = free_space_nonroot(self.instance_path)
 
     if available < required:
@@ -481,8 +482,8 @@ class Partition(object):
       }
 
   def addCustomGroup(self, group_suffix, partition_id, program_list):
-    group_partition_template = pkg_resources.resource_stream(__name__,
-      'templates/group_partition_supervisord.conf.in').read()
+    group_partition_template = bytes2str(pkg_resources.resource_string(__name__,
+      'templates/group_partition_supervisord.conf.in'))
     group_id = '{}-{}'.format(partition_id, group_suffix)
 
     self.supervisor_configuration_group += group_partition_template % {
@@ -568,8 +569,8 @@ class Partition(object):
 
     # fill generated buildout with additional information
     buildout_text = open(config_location).read()
-    buildout_text += '\n\n' + pkg_resources.resource_string(__name__,
-        'templates/buildout-tail.cfg.in') % {
+    buildout_text += '\n\n' + bytes2str(pkg_resources.resource_string(__name__,
+        'templates/buildout-tail.cfg.in')) % {
             'computer_id': self.computer_id,
             'partition_id': self.partition_id,
             'server_url': self.server_url,
@@ -671,8 +672,8 @@ class Partition(object):
         os.unlink(self.supervisord_partition_configuration_path)
     else:
       partition_id = self.computer_partition.getId()
-      group_partition_template = pkg_resources.resource_stream(__name__,
-          'templates/group_partition_supervisord.conf.in').read()
+      group_partition_template = bytes2str(pkg_resources.resource_string(__name__,
+          'templates/group_partition_supervisord.conf.in'))
       self.supervisor_configuration_group = group_partition_template % {
           'instance_id': partition_id,
           'program_list': ','.join(['_'.join([partition_id, runner])
@@ -842,7 +843,7 @@ class Partition(object):
         self.logger.warning('Problem while stopping process %r, will try later' % gname)
       else:
         self.logger.info('Stopped %r' % gname)
-      for i in xrange(0, 10):
+      for i in range(0, 10):
         # Some process may be still running, be nice and wait for them to be stopped.
         try:
           supervisor.removeProcessGroup(gname)

slapos/grid/networkcache.py

@@ -11,6 +11,7 @@
 # FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
+from __future__ import print_function
 
 import ast
 import json
@@ -29,10 +30,10 @@ try:
     else:
         LIBNETWORKCACHE_ENABLED = True
 except:
-    print 'There was problem while trying to import slapos.libnetworkcache:'\
-        '\n%s' % traceback.format_exc()
+    print('There was problem while trying to import slapos.libnetworkcache:\n%s'
+          % traceback.format_exc())
     LIBNETWORKCACHE_ENABLED = False
-    print 'Networkcache forced to be disabled.'
+    print('Networkcache forced to be disabled.')
 
 
 
@@ -45,8 +46,8 @@ def fallback_call(function):
         try:
             return function(self, *args, **kwd)
         except: # indeed, *any* exception is swallowed
-            print 'There was problem while calling method %r:\n%s' % (
-                function.__name__, traceback.format_exc())
+            print('There was problem while calling method %r:\n%s' % (
+                function.__name__, traceback.format_exc()))
             return False
     wrapper.__doc__ = function.__doc__
     return wrapper
@@ -107,7 +108,7 @@ def download_network_cached(cache_url, dir_url, software_url, software_root,
                 f.close()
                 file_descriptor.close()
             return True
-    except (IOError, DirectoryNotFound), e:
+    except (IOError, DirectoryNotFound) as e:
         logger.info('Failed to download from network cache %s: %s' % \
                                                        (software_url, str(e)))
     return False
@@ -169,7 +170,7 @@ def upload_network_cached(software_root, software_url, cached_key,
 
     try:
         return nc.upload_generic(f, cached_key, **kw)
-    except (IOError, UploadError), e:
+    except (IOError, UploadError) as e:
         logger.info('Failed to upload file. %s' % (str(e)))
         return False
     finally:

slapos/grid/promise/__init__.py

@@ -38,7 +38,7 @@ import importlib
 import traceback
 import psutil
 from multiprocessing import Process, Queue as MQueue
-import Queue
+from six.moves import queue, reload_module
 from slapos.util import mkdir_p, chownDirectory
 from slapos.grid.utils import dropPrivileges, killProcessTree
 from slapos.grid.promise import interface
@@ -168,7 +168,7 @@ class PromiseProcess(Process):
     if not os.path.exists(init_file):
       with open(init_file, 'w') as f:
         f.write("")
-      os.chmod(init_file, 0644)
+      os.chmod(init_file, 0o644)
     # add promise folder to sys.path so we can import promise script
     if sys.path[0] != promise_folder:
       sys.path[0:0] = [promise_folder]
@@ -184,9 +184,8 @@ class PromiseProcess(Process):
       raise AttributeError("Class RunPromise not found in promise" \
         "%s" % self.name)
     if not interface.IPromise.implementedBy(promise_module.RunPromise):
-      raise RuntimeError("RunPromise class in %s must implements 'IPromise'" \
-        " interface. zope_interface.implements(interface.IPromise) is" \
-        " missing ?" % self.name)
+      raise RuntimeError("RunPromise class in %s must implement 'IPromise'"
+        " interface. @implementer(interface.IPromise) is missing ?" % self.name)
 
     from slapos.grid.promise.generic import GenericPromise
     if not issubclass(promise_module.RunPromise, GenericPromise):
@@ -195,7 +194,7 @@ class PromiseProcess(Process):
 
     if promise_module.__file__ != self.promise_path:
       # cached module need to be updated
-      promise_module = reload(promise_module)
+      promise_module = reload_module(promise_module)
     # load extra parameters
     self._loadPromiseParameterDict(promise_module)
 
@@ -208,7 +207,7 @@ class PromiseProcess(Process):
       if not isinstance(extra_dict, dict):
         raise ValueError("Extra parameter is not a dict")
       for key in extra_dict:
-        if self.argument_dict.has_key(key):
+        if key in self.argument_dict:
           raise ValueError("Extra parameter name %r cannot be used.\n%s" % (
                            key, extra_dict))
         self.argument_dict[key] = extra_dict[key]
@@ -362,7 +361,7 @@ class PromiseLauncher(object):
         try:
           result = PromiseQueueResult()
           result.load(json.loads(f.read()))
-        except ValueError, e:
+        except ValueError as e:
           result = None
           self.logger.warn('Bad promise JSON result at %r: %s' % (
             promise_output_file,
@@ -375,7 +374,7 @@ class PromiseLauncher(object):
     while True:
       try:
         self.queue_result.get_nowait()
-      except Queue.Empty:
+      except queue.Empty:
         return
 
   def _updateFolderOwner(self, folder_path=None):
@@ -443,7 +442,7 @@ class PromiseLauncher(object):
       if not promise_process.is_alive():
         try:
           queue_item = self.queue_result.get(True, 1)
-        except Queue.Empty:
+        except queue.Empty:
           # no result found in process result Queue
           pass
         else:

slapos/grid/promise/generic.py

@@ -35,8 +35,10 @@ import time
 import random
 import traceback
 import slapos.slap
-from slapos.util import mkdir_p
+from slapos.util import bytes2str, mkdir_p
 from abc import ABCMeta, abstractmethod
+import six
+from six import PY3, with_metaclass
 from datetime import datetime, timedelta
 
 PROMISE_STATE_FOLDER_NAME = '.slapgrid/promise'
@@ -46,6 +48,10 @@ PROMISE_LOG_FOLDER_NAME = '.slapgrid/promise/log'
 PROMISE_PARAMETER_NAME = 'extra_config_dict'
 PROMISE_PERIOD_FILE_NAME = '%s.periodicity'
 
+LOGLINE_RE = r"(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})\s+\-?\s*(\w{4,7})\s+\-?\s+(\d+\-\d{3})\s+\-?\s*(.*)"
+matchLogStr = re.compile(LOGLINE_RE).match
+matchLogBytes = re.compile(LOGLINE_RE.encode()).match if PY3 else matchLogStr
+
 class BaseResult(object):
   def __init__(self, problem=False, message=None, date=None):
     self.__problem = problem
@@ -129,10 +135,7 @@ class PromiseQueueResult(object):
     self.path = data['path']
     self.execution_time = data['execution-time']
 
-class GenericPromise(object):
-
-  # Abstract class
-  __metaclass__ = ABCMeta
+class GenericPromise(with_metaclass(ABCMeta, object)):
 
   def __init__(self, config):
     self.__config = config
@@ -160,9 +163,7 @@ class GenericPromise(object):
     for handler in self.logger.handlers:
       self.logger.removeHandler(handler)
     if self.__log_folder is None:
-      # configure logger with StringIO
-      import cStringIO
-      self.__logger_buffer = cStringIO.StringIO()
+      self.__logger_buffer = six.StringIO()
       logger_handler = logging.StreamHandler(self.__logger_buffer)
       self.__log_file = None
     else:
@@ -230,9 +231,9 @@ class GenericPromise(object):
     """
       Call bang if requested
     """
-    if self.__config.has_key('master-url') and \
-        self.__config.has_key('partition-id') and \
-        self.__config.has_key('computer-id'):
+    if 'master-url' in self.__config and \
+       'partition-in' in self.__config and \
+       'computer-id' in self.__config:
 
       slap = slapos.slap.slap()
       slap.initializeConnection(
@@ -247,18 +248,14 @@ class GenericPromise(object):
       computer_partition.bang(message)
       self.logger.info("Bang with message %r." % message)
 
-  def __getLogRegex(self):
-    return re.compile(r"(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})\s+\-?\s*(\w{4,7})\s+\-?\s+(\d+\-\d{3})\s+\-?\s*(.*)")
-
   def __getResultFromString(self, result_string, only_failure=False):
     line_list = result_string.split('\n')
     result_list = []
     line_part = ""
-    regex = self.__getLogRegex()
     for line in line_list:
       if not line:
         continue
-      match = regex.match(line)
+      match = matchLogStr(line)
       if match is not None:
         if not only_failure or (only_failure and match.groups()[1] == 'ERROR'):
           result_list.append({
@@ -297,37 +294,36 @@ class GenericPromise(object):
     if not os.path.exists(self.__log_file):
       return []
 
-    regex = self.__getLogRegex()
-    max_date_string = ""
     if latest_minute > 0:
       date = datetime.now() - timedelta(minutes=latest_minute)
       max_date_string = date.strftime('%Y-%m-%d %H:%M:%S')
+    else:
+      max_date_string = ""
 
     line_list = []
     result_list = []
     transaction_id = None
     transaction_count = 0
-    with open(self.__log_file, 'r') as f:
-      offset = 0
+    with open(self.__log_file, 'rb') as f:
       f.seek(0, 2)
-      size = f.tell() * -1
-      line = line_part = ""
-      while offset > size:
+      offset = f.tell()
+      line = b""
+      line_part = ""
+      while offset:
         offset -= 1
-        f.seek(offset, 2)
+        f.seek(offset)
         char = f.read(1)
-        if char != '\n':
+        if char != b'\n':
           line = char + line
-        if char == '\n' or offset == size:
-          # Add new line
-          if offset == -1:
+          if offset:
             continue
-          if line != "":
-            result = regex.match(line)
+        if line:
+            result = matchLogBytes(line)
             if result is not None:
-              if max_date_string and result.groups()[0] <= max_date_string:
+              date, level, tid, msg = map(bytes2str, result.groups())
+              if max_date_string and date <= max_date_string:
                 break
-              if transaction_id != result.groups()[2]:
+              if transaction_id != tid:
                 if transaction_id is not None:
                   # append new result
                   result_list.append(line_list)
@@ -335,20 +331,18 @@ class GenericPromise(object):
                 transaction_count += 1
                 if transaction_count > result_count:
                   break
-                transaction_id = result.groups()[2]
-              if not only_failure or \
-                  (only_failure and result.groups()[1] == 'ERROR'):
+                transaction_id = tid
+              if not only_failure or level == 'ERROR':
                 line_list.insert(0, {
-                  'date': datetime.strptime(result.groups()[0],
+                  'date': datetime.strptime(date,
                                             '%Y-%m-%d %H:%M:%S'),
-                  'status': result.groups()[1],
-                  'message': (result.groups()[3] + line_part).strip(),
+                  'status': level,
+                  'message': (msg + line_part).strip(),
                 })
+              line_part = ""
             else:
-              line_part = '\n' + line + line_part
-              line = ""
-              continue
-            line = line_part = ""
+              line_part = '\n' + bytes2str(line) + line_part
+            line = b""
 
     if len(line_list):
       result_list.append(line_list)
@@ -410,7 +404,7 @@ class GenericPromise(object):
       try:
         self.__queue.put_nowait(result_item)
         break
-      except Queue.Full, e:
+      except Queue.Full as e:
         error = e
         time.sleep(0.5)
     if error:
@@ -459,7 +453,7 @@ class GenericPromise(object):
     """
     try:
       self.sense()
-    except Exception, e:
+    except Exception as e:
       # log the result
       self.logger.error(str(e))
     if check_anomaly:
@@ -468,7 +462,7 @@ class GenericPromise(object):
         result = self.anomaly()
         if result is None:
           raise ValueError("Promise anomaly method returned 'None'")
-      except Exception, e:
+      except Exception as e:
         result = AnomalyResult(problem=True, message=str(e))
       else:
         if isinstance(result, AnomalyResult) and result.hasFailed() and can_bang:
@@ -482,7 +476,7 @@ class GenericPromise(object):
         result = self.test()
         if result is None:
           raise ValueError("Promise test method returned 'None'")
-      except Exception, e:
+      except Exception as e:
         result = TestResult(problem=True, message=str(e))
 
     if self.__logger_buffer is not None:

slapos/grid/promise/wrapper.py

@@ -32,17 +32,16 @@ import subprocess
 import functools
 import signal
 import traceback
-from zope import interface as zope_interface
+from zope.interface import implementer
 from slapos.grid.promise import interface
 from slapos.grid.promise.generic import GenericPromise
 
+@implementer(interface.IPromise)
 class WrapPromise(GenericPromise):
   """
     A wrapper promise used to run old promises style and bash promises
   """
 
-  zope_interface.implements(interface.IPromise)
-
   def __init__(self, config):
     GenericPromise.__init__(self, config)
     self.setPeriodicity(minute=2)
@@ -62,21 +61,19 @@ class WrapPromise(GenericPromise):
         [self.getPromiseFile()],
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
-        cwd=self.getPartitionFolder()
+        cwd=self.getPartitionFolder(),
+        universal_newlines=True,
     )
     handler = functools.partial(self.terminate, self.getName(), self.logger,
                                 promise_process)
     signal.signal(signal.SIGINT, handler)
     signal.signal(signal.SIGTERM, handler)
 
-    output, error = promise_process.communicate()
-    message = output or ""
-    if error:
-      message += "\n" + error
-    if promise_process.returncode != 0:
-      self.logger.error(message.strip())
+    message = promise_process.communicate()[0].strip()
+    if promise_process.returncode:
+      self.logger.error(message)
     else:
-      self.logger.info(message.strip())
+      self.logger.info(message)
 
   def test(self):
     # Fail if the latest promise result failed

slapos/grid/slapgrid.py

@@ -32,7 +32,7 @@ import os
 import pkg_resources
 import random
 import socket
-import StringIO
+from io import BytesIO
 import subprocess
 import sys
 import tempfile
@@ -42,6 +42,7 @@ import warnings
 import logging
 import json
 import shutil
+import six
 
 if sys.version_info < (2, 6):
   warnings.warn('Used python version (%s) is old and has problems with'
@@ -167,7 +168,7 @@ def merged_options(args, configp):
 
   if configp.has_section('networkcache'):
     options.update(dict(configp.items('networkcache')))
-  for key, value in vars(args).iteritems():
+  for key, value in six.iteritems(vars(args)):
     if value is not None:
       options[key] = value
 
@@ -672,7 +673,7 @@ stderr_logfile_backups=1
           computer_partition.setComputerPartitionRelatedInstanceList(
             [reference for reference in tf.read().split('\n') if reference]
           )
-        except NotFoundError, e:
+        except NotFoundError as e:
           # Master doesn't implement this feature ?
           self.logger.warning("NotFoundError: %s. \nCannot send requested instance "\
                             "list to master. Please check if this feature is"\
@@ -682,12 +683,12 @@ stderr_logfile_backups=1
     """
     """
     query_cmd = rule_command.replace('--add-rule', '--query-rule')
-    process = FPopen(query_cmd)
+    process = FPopen(query_cmd, universal_newlines=True)
     result, stderr = process.communicate()
     if result.strip() == 'no':
       # rule doesn't exist add to firewall
       self.logger.debug(rule_command)
-      process = FPopen(rule_command)
+      process = FPopen(rule_command, universal_newlines=True)
       rule_result, stderr = process.communicate()
       if process.returncode == 0:
         if rule_result.strip() != 'success':
@@ -705,13 +706,13 @@ stderr_logfile_backups=1
     """
     """
     query_cmd = rule_command.replace('--add-rule', '--query-rule')
-    process = FPopen(query_cmd)
+    process = FPopen(query_cmd, universal_newlines=True)
     result, stderr = process.communicate()
     if result.strip() == 'yes':
       # The rule really exist, remove it
       remove_command = rule_command.replace('--add-rule', '--remove-rule')
       self.logger.debug(remove_command)
-      process = FPopen(remove_command)
+      process = FPopen(remove_command, universal_newlines=True)
       rule_result, stderr = process.communicate()
       if process.returncode == 0:
         if rule_result.strip() != 'success':
@@ -764,7 +765,7 @@ stderr_logfile_backups=1
       # XXX - need to check firewalld reload instead of restart
       self.logger.info("Reloading firewall configuration...")
       reload_cmd = self.firewall_conf['reload_config_cmd']
-      reload_process = FPopen(reload_cmd)
+      reload_process = FPopen(reload_cmd, universal_newlines=True)
       stdout, stderr = reload_process.communicate()
       if reload_process.returncode != 0:
         raise Exception("Failed to load firewalld rules with command %s.\n%" % (
@@ -922,7 +923,7 @@ stderr_logfile_backups=1
       self._checkPromiseList(local_partition,
                              check_anomaly=True,
                              force=False)
-    except PromiseError, e:
+    except PromiseError as e:
       self.logger.error(e)
       if partition_access_status is None or not status_error:
         computer_partition.error(e, logger=self.logger)
@@ -1140,7 +1141,7 @@ stderr_logfile_backups=1
           (computer_partition_id, computer_partition_state)
         computer_partition.error(error_string, logger=self.logger)
         raise NotImplementedError(error_string)
-    except Exception, e:
+    except Exception as e:
       if not isinstance(e, PromiseError):
         with open(error_output_file, 'w') as error_file:
           # Write error message in a log file assible to computer partition user
@@ -1309,7 +1310,7 @@ stderr_logfile_backups=1
   def validateXML(self, to_be_validated, xsd_model):
     """Validates a given xml file"""
     #We retrieve the xsd model
-    xsd_model = StringIO.StringIO(xsd_model)
+    xsd_model = BytesIO(xsd_model)
     xmlschema_doc = etree.parse(xsd_model)
     xmlschema = etree.XMLSchema(xmlschema_doc)
 

slapos/grid/svcbackend.py

@@ -35,9 +35,10 @@ import subprocess
 import stat
 import sys
 import time
-import xmlrpclib
+from six.moves import xmlrpc_client as xmlrpclib
 
 from slapos.grid.utils import (createPrivateDirectory, SlapPopen, updateFile)
+from slapos.util import bytes2str
 
 from supervisor import xmlrpc, states
 
@@ -89,8 +90,8 @@ def createSupervisordConfiguration(instance_root, watchdog_command=''):
 
   # Creates supervisord configuration
   updateFile(supervisord_configuration_file_path,
-    pkg_resources.resource_stream(__name__,
-      'templates/supervisord.conf.in').read() % {
+    bytes2str(pkg_resources.resource_string(__name__,
+      'templates/supervisord.conf.in')) % {
           'supervisord_configuration_directory': supervisord_configuration_directory,
           'supervisord_socket': os.path.abspath(supervisord_socket),
           'supervisord_loglevel': 'info',

slapos/grid/utils.py

@@ -40,6 +40,8 @@ import logging
 import psutil
 import time
 
+import six
+
 from slapos.grid.exception import BuildoutFailedError, WrongPermissionError
 
 # Such umask by default will create paths with full permission
@@ -123,20 +125,18 @@ class SlapPopen(subprocess.Popen):
     self.stdin.close()
     self.stdin = None
 
-    # XXX-Cedric: this algorithm looks overkill for simple logging.
     output_lines = []
-    while True:
-      line = self.stdout.readline()
-      if line == '' and self.poll() is not None:
-        break
-      if line:
-        output_lines.append(line)
-        logger.info(line.rstrip('\n'))
+    for line in self.stdout:
+      if type(line) is not str:
+        line = line.decode(errors='replace')
+      output_lines.append(line)
+      logger.info(line.rstrip('\n'))
+    self.wait()
     self.output = ''.join(output_lines)
 
 
 def md5digest(url):
-  return hashlib.md5(url).hexdigest()
+  return hashlib.md5(url.encode('utf-8')).hexdigest()
 
 
 def getCleanEnvironment(logger, home_path='/tmp'):
@@ -150,7 +150,7 @@ def getCleanEnvironment(logger, home_path='/tmp'):
     if old is not None:
       removed_env.append(k)
   changed_env['HOME'] = env['HOME'] = home_path
-  for k in sorted(changed_env.iterkeys()):
+  for k in sorted(six.iterkeys(changed_env)):
     logger.debug('Overridden %s = %r' % (k, changed_env[k]))
   if removed_env:
     logger.debug('Removed from environment: %s' % ', '.join(sorted(removed_env)))
@@ -351,18 +351,20 @@ def launchBuildout(path, buildout_binary, logger,
 
 def updateFile(file_path, content, mode=0o600):
   """Creates or updates a file with "content" as content."""
-  altered = False
-  if not (os.path.isfile(file_path)) or \
-     not (hashlib.md5(open(file_path).read()).digest() ==
-          hashlib.md5(content).digest()):
-    with open(file_path, 'w') as fout:
-      fout.write(content)
-    altered = True
-  os.chmod(file_path, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
-  if stat.S_IMODE(os.stat(file_path).st_mode) != mode:
-    os.chmod(file_path, mode)
-    altered = True
-  return altered
+  content = content.encode('utf-8')
+  try:
+    with open(file_path, 'rb') as f:
+      if f.read(len(content) + 1) == content:
+        if stat.S_IMODE(os.fstat(f.fileno()).st_mode) == mode:
+          return False
+        os.fchmod(f.fileno(), mode)
+        return True
+  except IOError:
+    pass
+  with open(file_path, 'wb') as f:
+    os.fchmod(f.fileno(), mode)
+    f.write(content)
+  return True
 
 
 def updateExecutable(executable_path, content):
@@ -399,7 +401,7 @@ def killProcessTree(pid, logger):
     for child in running_process_list:
       try:
         child.suspend()
-      except psutil.Error, e:
+      except psutil.Error as e:
         logger.debug(str(e))
 
     time.sleep(0.2)
@@ -408,5 +410,5 @@ def killProcessTree(pid, logger):
   for process in process_list:
     try:
       process.kill()
-    except psutil.Error, e:
+    except psutil.Error as e:
       logger.debug("Process kill: %s" % e)

slapos/grid/watchdog.py

@@ -30,6 +30,7 @@
 import argparse
 import os.path
 import sys
+import six
 
 import slapos.slap.slap
 from slapos.grid.slapgrid import COMPUTER_PARTITION_TIMESTAMP_FILENAME, \
@@ -56,7 +57,7 @@ def parseArgumentTuple():
 
   # Build option_dict
   option_dict = {}
-  for argument_key, argument_value in vars(option).iteritems():
+  for argument_key, argument_value in six.iteritems(vars(option)):
     option_dict.update({argument_key: argument_value})
 
   return option_dict

slapos/manager/cpuset.py

@@ -5,12 +5,12 @@ import os.path
 import pwd
 import time
 
-from zope import interface as zope_interface
+from zope.interface import implementer
 from slapos.manager import interface
 
 logger = logging.getLogger(__name__)
 
-
+@implementer(interface.IManager)
 class Manager(object):
   """Manage cgroup's cpuset in terms on initializing and runtime operations.
 
@@ -21,8 +21,6 @@ class Manager(object):
 
   TODO: there is no limit on number of reserved cores per user.
   """
-  zope_interface.implements(interface.IManager)
-
   cpu_exclusive_file = ".slapos-cpu-exclusive"
   cpuset_path = "/sys/fs/cgroup/cpuset/"
   task_write_mode = "wt"

slapos/manager/devperm.py

@@ -5,7 +5,6 @@ import os
 import pwd
 import grp
 from .interface import IManager
-from itertools import ifilter
 from zope import interface
 
 logger = logging.getLogger(__name__)

slapos/manager/portredir.py

@@ -5,8 +5,8 @@ import netaddr
 import os
 
 from .interface import IManager
-from itertools import ifilter
-from zope import interface
+from six.moves import filter
+from zope.interface import implementer
 
 logger = logging.getLogger(__name__)
 
@@ -23,9 +23,8 @@ def which(exename):
       return full_path
   return None
 
+@implementer(IManager)
 class Manager(object):
-  interface.implements(IManager)
-
   port_redirect_filename = '.slapos-port-redirect'
 
   def __init__(self, config):
@@ -89,7 +88,7 @@ class Manager(object):
       'full_ip_list', [])
     partition_ip_list = [tup[1] for tup in partition_ip_list]
 
-    partition_ipv6 = next(ifilter(lambda ip_addr: ':' in ip_addr,
+    partition_ipv6 = next(filter(lambda ip_addr: ':' in ip_addr,
                                  partition_ip_list),
                           None)
 

slapos/manager/prerm.py

@@ -4,17 +4,16 @@ import os
 import sys
 import subprocess
 
-from zope import interface as zope_interface
+from zope.interface import implementer
 from slapos.manager import interface
 from slapos.grid.slapgrid import COMPUTER_PARTITION_WAIT_LIST_FILENAME
 
 logger = logging.getLogger(__name__)
 
+@implementer(interface.IManager)
 class Manager(object):
   """Manager is called in every step of preparation of the computer."""
 
-  zope_interface.implements(interface.IManager)
-
   def __init__(self, config):
     """Manager needs to know config for its functioning.
     """

slapos/proxy/__init__.py

@@ -33,6 +33,8 @@ import logging
 from slapos.proxy.views import app
 from slapos.util import sqlite_connect
 
+import six
+
 def _generateSoftwareProductListFromString(software_product_list_string):
   """
   Take a string as argument (which usually comes from the software_product_list
@@ -72,7 +74,7 @@ class ProxyConfig(object):
       elif section.startswith('multimaster/'):
         # Merge multimaster configuration if any
         # XXX: check for duplicate SR entries
-        for key, value in configuration_dict.iteritems():
+        for key, value in six.iteritems(configuration_dict):
           if key == 'software_release_list':
             # Split multi-lines values
             configuration_dict[key] = [line.strip() for line in value.strip().split('\n')]

slapos/proxy/db_version.py

 # -*- coding: utf-8 -*-
 
 import pkg_resources
+from slapos.util import bytes2str
 
-DB_VERSION = pkg_resources.resource_stream('slapos.proxy', 'schema.sql').readline().strip().split(':')[1]
+DB_VERSION = bytes2str(pkg_resources.resource_stream('slapos.proxy', 'schema.sql').readline()).strip().split(':')[1]
 

slapos/proxy/views.py

@@ -28,6 +28,8 @@
 #
 ##############################################################################
 
+from __future__ import print_function
+
 from lxml import etree
 import random
 import string
@@ -35,12 +37,13 @@ from slapos.slap.slap import Computer, ComputerPartition, \
     SoftwareRelease, SoftwareInstance, NotFoundError
 from slapos.proxy.db_version import DB_VERSION
 import slapos.slap
-from slapos.util import sqlite_connect
+from slapos.util import bytes2str, str2bytes, unicode2str, sqlite_connect
 
 from flask import g, Flask, request, abort
-import xml_marshaller
-from xml_marshaller.xml_marshaller import loads
-from xml_marshaller.xml_marshaller import dumps
+from slapos.util import loads, dumps
+
+import six
+from six.moves import range
 
 app = Flask(__name__)
 
@@ -50,19 +53,10 @@ class UnauthorizedError(Exception):
   pass
 
 
-# cast everything to string, utf-8 encoded
-def to_str(v):
-  if isinstance(v, str):
-    return v
-  if not isinstance(v, unicode):
-    v = unicode(v)
-  return v.encode('utf-8')
-
-
 def xml2dict(xml):
   result_dict = {}
-  if xml is not None and xml != '':
-    tree = etree.fromstring(to_str(xml))
+  if xml:
+    tree = etree.fromstring(str2bytes(xml))
     for element in tree.iter(tag=etree.Element):
       if element.tag == 'parameter':
         key = element.get('id')
@@ -77,21 +71,16 @@ def xml2dict(xml):
 
 def dict2xml(dictionary):
   instance = etree.Element('instance')
-  for parameter_id, parameter_value in dictionary.iteritems():
-    # cast everything to string
-    parameter_value = unicode(parameter_value)
+  for parameter_id, parameter_value in six.iteritems(dictionary):
     etree.SubElement(instance, "parameter",
                      attrib={'id': parameter_id}).text = parameter_value
-  return etree.tostring(instance,
+  return bytes2str(etree.tostring(instance,
                         pretty_print=True,
                         xml_declaration=True,
-                        encoding='utf-8')
+                        encoding='utf-8'))
 
 
 def partitiondict2partition(partition):
-  for key, value in partition.iteritems():
-    if type(value) is unicode:
-      partition[key] = value.encode()
   slap_partition = ComputerPartition(partition['computer_reference'],
       partition['reference'])
   slap_partition._software_release_document = None
@@ -121,7 +110,7 @@ def partitiondict2partition(partition):
         root_partition['partition_reference']
     if partition['slave_instance_list'] is not None:
       slap_partition._parameter_dict['slave_instance_list'] = \
-          xml_marshaller.xml_marshaller.loads(partition['slave_instance_list'])
+          loads(partition['slave_instance_list'].encode('utf-8'))
     else:
       slap_partition._parameter_dict['slave_instance_list'] = []
     slap_partition._connection_dict = xml2dict(partition['connection_xml'])
@@ -139,7 +128,7 @@ def execute_db(table, query, args=(), one=False, db_version=None, log=False, db=
     db_version = DB_VERSION
   query = query % (table + db_version,)
   if log:
-    print query
+    print(query)
   try:
     cur = db.execute(query, args)
   except:
@@ -184,8 +173,8 @@ def _upgradeDatabaseIfNeeded():
   if current_schema_version == DB_VERSION:
     return
 
-  schema = app.open_resource('schema.sql')
-  schema = schema.read() % dict(version=DB_VERSION, computer=app.config['computer_id'])
+  with app.open_resource('schema.sql', 'r') as f:
+    schema = f.read() % dict(version=DB_VERSION, computer=app.config['computer_id'])
   g.db.cursor().executescript(schema)
   g.db.commit()
 
@@ -242,24 +231,20 @@ def getFullComputerInformation():
   for partition in execute_db('partition', 'SELECT * FROM %s WHERE computer_reference=?', [computer_id]):
     slap_computer._computer_partition_list.append(partitiondict2partition(
       partition))
-  return xml_marshaller.xml_marshaller.dumps(slap_computer)
+  return dumps(slap_computer)
 
 @app.route('/setComputerPartitionConnectionXml', methods=['POST'])
 def setComputerPartitionConnectionXml():
   slave_reference = request.form.get('slave_reference', None)
-  computer_partition_id = request.form['computer_partition_id'].encode()
-  computer_id = request.form['computer_id'].encode()
-  connection_xml = request.form['connection_xml'].encode()
-  connection_dict = xml_marshaller.xml_marshaller.loads(
-                                            connection_xml)
-  connection_xml = dict2xml(connection_dict)
+  computer_partition_id = unicode2str(request.form['computer_partition_id'])
+  computer_id = unicode2str(request.form['computer_id'])
+  connection_xml = dict2xml(loads(request.form['connection_xml'].encode('utf-8')))
   if not slave_reference or slave_reference == 'None':
     query = 'UPDATE %s SET connection_xml=? WHERE reference=? AND computer_reference=?'
     argument_list = [connection_xml, computer_partition_id, computer_id]
     execute_db('partition', query, argument_list)
     return 'done'
   else:
-    slave_reference = slave_reference.encode()
     query = 'UPDATE %s SET connection_xml=? , hosted_by=? WHERE reference=?'
     argument_list = [connection_xml, computer_partition_id, slave_reference]
     execute_db('slave', query, argument_list)
@@ -304,7 +289,7 @@ def useComputer():
 @app.route('/loadComputerConfigurationFromXML', methods=['POST'])
 def loadComputerConfigurationFromXML():
   xml = request.form['xml']
-  computer_dict = xml_marshaller.xml_marshaller.loads(str(xml))
+  computer_dict = loads(xml.encode('utf-8'))
   execute_db('computer', 'INSERT OR REPLACE INTO %s values(:reference, :address, :netmask)',
              computer_dict)
   for partition in computer_dict['partition_list']:
@@ -322,14 +307,13 @@ def loadComputerConfigurationFromXML():
 
 @app.route('/registerComputerPartition', methods=['GET'])
 def registerComputerPartition():
-  computer_reference = request.args['computer_reference'].encode()
-  computer_partition_reference = request.args['computer_partition_reference'].encode()
+  computer_reference = unicode2str(request.args['computer_reference'])
+  computer_partition_reference = unicode2str(request.args['computer_partition_reference'])
   partition = execute_db('partition', 'SELECT * FROM %s WHERE reference=? and computer_reference=?',
       [computer_partition_reference, computer_reference], one=True)
   if partition is None:
     raise UnauthorizedError
-  return xml_marshaller.xml_marshaller.dumps(
-      partitiondict2partition(partition))
+  return dumps(partitiondict2partition(partition))
 
 @app.route('/supplySupply', methods=['POST'])
 def supplySupply():
@@ -348,7 +332,7 @@ def requestComputerPartition():
   parsed_request_dict = parseRequestComputerPartitionForm(request.form)
 
   # Is it a slave instance?
-  slave = loads(request.form.get('shared_xml', EMPTY_DICT_XML).encode())
+  slave = loads(request.form.get('shared_xml', EMPTY_DICT_XML).encode('utf-8'))
 
   # Check first if instance is already allocated
   if slave:
@@ -391,20 +375,21 @@ def parseRequestComputerPartitionForm(form):
   """
   Parse without intelligence a form from a request(), return it.
   """
-  parsed_dict = {}
-  parsed_dict['software_release'] = form['software_release'].encode()
-  parsed_dict['software_type'] = form.get('software_type').encode()
-  parsed_dict['partition_reference'] = form.get('partition_reference', '').encode()
-  parsed_dict['partition_id'] = form.get('computer_partition_id', '').encode()
-  parsed_dict['partition_parameter_kw'] = loads(form.get('partition_parameter_xml', EMPTY_DICT_XML).encode())
-  parsed_dict['filter_kw'] = loads(form.get('filter_xml', EMPTY_DICT_XML).encode())
-  # Note: currently ignored for slave instance (slave instances
-  # are always started).
-  parsed_dict['requested_state'] = loads(form.get('state').encode())
+  parsed_dict = {
+    'software_release': unicode2str(form['software_release']),
+    'software_type': unicode2str(form.get('software_type')),
+    'partition_reference': unicode2str(form.get('partition_reference', '')),
+    'partition_id': unicode2str(form.get('computer_partition_id', '')),
+    'partition_parameter_kw': loads(form.get('partition_parameter_xml', EMPTY_DICT_XML).encode('utf-8')),
+    'filter_kw': loads(form.get('filter_xml', EMPTY_DICT_XML).encode('utf-8')),
+    # Note: currently ignored for slave instance (slave instances
+    # are always started).
+    'requested_state': loads(form.get('state').encode('utf-8')),
+  }
 
   return parsed_dict
 
-run_id = ''.join([random.choice(string.ascii_letters + string.digits) for n in xrange(32)])
+run_id = ''.join([random.choice(string.ascii_letters + string.digits) for n in range(32)])
 def checkIfMasterIsCurrentMaster(master_url):
   """
   Because there are several ways to contact this server, we can't easily check
@@ -421,12 +406,9 @@ def checkIfMasterIsCurrentMaster(master_url):
   slap = slapos.slap.slap()
   slap.initializeConnection(master_url)
   try:
-    master_run_id = slap._connection_helper.GET('/getRunId')
+    return run_id == bytes2str(slap._connection_helper.GET('/getRunId'))
   except:
     return False
-  if master_run_id == run_id:
-    return True
-  return False
 
 @app.route('/getRunId', methods=['GET'])
 def getRunId():
@@ -469,7 +451,7 @@ def isRequestToBeForwardedToExternalMaster(parsed_request_dict):
       return master_url
 
     software_release = parsed_request_dict['software_release']
-    for mutimaster_url, mutimaster_entry in app.config.get('multimaster').iteritems():
+    for mutimaster_url, mutimaster_entry in six.iteritems(app.config.get('multimaster')):
       if software_release in mutimaster_entry['software_release_list']:
         # Don't allocate the instance locally, but forward to specified master
         return mutimaster_url
@@ -495,20 +477,17 @@ def forwardRequestToExternalMaster(master_url, request_form):
   else:
     slap.initializeConnection(master_url)
 
-  partition_reference = request_form['partition_reference'].encode()
+  partition_reference = unicode2str(request_form['partition_reference'])
   # Store in database
   execute_db('forwarded_partition_request', 'INSERT OR REPLACE INTO %s values(:partition_reference, :master_url)',
              {'partition_reference':partition_reference, 'master_url': master_url})
 
   new_request_form = request_form.copy()
-  filter_kw = loads(new_request_form['filter_xml'].encode())
+  filter_kw = loads(new_request_form['filter_xml'].encode('utf-8'))
   filter_kw['source_instance_id'] = partition_reference
   new_request_form['filter_xml'] = dumps(filter_kw)
 
   xml = slap._connection_helper.POST('/requestComputerPartition', data=new_request_form)
-  if type(xml) is unicode:
-    xml = str(xml)
-    xml.encode('utf-8')
   partition = loads(xml)
 
   # XXX move to other end
@@ -568,7 +547,6 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
   instance_xml = dict2xml(partition_parameter_kw)
   requested_computer_id = filter_kw['computer_guid']
 
-  instance_xml = dict2xml(partition_parameter_kw)
   args = []
   a = args.append
   q = 'SELECT * FROM %s WHERE partition_reference=?'
@@ -599,7 +577,7 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
       software_type = 'RootSoftwareInstance'
   else:
     # XXX Check if software_release should be updated
-    if partition['software_release'].encode() != software_release:
+    if partition['software_release'] != software_release:
       q += ' ,software_release=?'
       a(software_release)
     if partition['requested_by']:
@@ -625,13 +603,13 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
     q += ' ,xml=?'
     a(instance_xml)
   q += ' WHERE reference=? AND computer_reference=?'
-  a(partition['reference'].encode())
-  a(partition['computer_reference'].encode())
+  a(partition['reference'])
+  a(partition['computer_reference'])
 
   execute_db('partition', q, args)
   args = []
   partition = execute_db('partition', 'SELECT * FROM %s WHERE reference=? and computer_reference=?',
-      [partition['reference'].encode(), partition['computer_reference'].encode()], one=True)
+      [partition['reference'], partition['computer_reference']], one=True)
   address_list = []
   for address in execute_db('partition_network', 'SELECT * FROM %s WHERE partition_reference=?', [partition['reference']]):
     address_list.append((address['reference'], address['address']))
@@ -642,12 +620,12 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
   software_instance = SoftwareInstance(_connection_dict=xml2dict(partition['connection_xml']),
                                        _parameter_dict=xml2dict(partition['xml']),
                                        connection_xml=partition['connection_xml'],
-                                       slap_computer_id=partition['computer_reference'].encode(),
+                                       slap_computer_id=partition['computer_reference'],
                                        slap_computer_partition_id=partition['reference'],
                                        slap_software_release_url=partition['software_release'],
                                        slap_server_url='slap_server_url',
                                        slap_software_type=partition['software_type'],
-                                       _instance_guid='%s-%s' % (partition['computer_reference'].encode(), partition['reference']),
+                                       _instance_guid='%s-%s' % (partition['computer_reference'], partition['reference']),
                                        _requested_state=requested_state,
                                        ip_list=address_list)
   return software_instance
@@ -704,28 +682,29 @@ def requestSlave(software_release, software_type, partition_reference, partition
 
   # Add slave to partition slave_list if not present else replace information
   slave_instance_list = partition['slave_instance_list']
-  if slave_instance_list is None:
-    slave_instance_list = []
-  else:
-    slave_instance_list = xml_marshaller.xml_marshaller.loads(slave_instance_list.encode())
-    for x in slave_instance_list:
+  if slave_instance_list:
+    slave_instance_list = loads(slave_instance_list.encode('utf-8'))
+    for i, x in enumerate(slave_instance_list):
       if x['slave_reference'] == slave_reference:
-        slave_instance_list.remove(x)
-
-  slave_instance_list.append(new_slave)
+        slave_instance_list[i] = new_slave
+        break
+    else:
+      slave_instance_list.append(new_slave)
+  else:
+    slave_instance_list = [new_slave]
 
   # Update slave_instance_list in database
   args = []
   a = args.append
   q = 'UPDATE %s SET slave_instance_list=?'
-  a(xml_marshaller.xml_marshaller.dumps(slave_instance_list))
+  a(bytes2str(dumps(slave_instance_list)))
   q += ' WHERE reference=? and computer_reference=?'
-  a(partition['reference'].encode())
+  a(partition['reference'])
   a(requested_computer_id)
   execute_db('partition', q, args)
   args = []
   partition = execute_db('partition', 'SELECT * FROM %s WHERE reference=? and computer_reference=?',
-      [partition['reference'].encode(), requested_computer_id], one=True)
+      [partition['reference'], requested_computer_id], one=True)
 
   # Add slave to slave table if not there
   slave = execute_db('slave', 'SELECT * FROM %s WHERE reference=? and computer_reference=?',
@@ -757,9 +736,9 @@ def requestSlave(software_release, software_type, partition_reference, partition
 
 @app.route('/softwareInstanceRename', methods=['POST'])
 def softwareInstanceRename():
-  new_name = request.form['new_name'].encode()
-  computer_partition_id = request.form['computer_partition_id'].encode()
-  computer_id = request.form['computer_id'].encode()
+  new_name = unicode2str(request.form['new_name'])
+  computer_partition_id = unicode2str(request.form['computer_partition_id'])
+  computer_id = unicode2str(request.form['computer_id'])
 
   q = 'UPDATE %s SET partition_reference = ? WHERE reference = ? AND computer_reference = ?'
   execute_db('partition', q, [new_name, computer_partition_id, computer_id])
@@ -767,16 +746,16 @@ def softwareInstanceRename():
 
 @app.route('/getComputerPartitionStatus', methods=['GET'])
 def getComputerPartitionStatus():
-  return xml_marshaller.xml_marshaller.dumps('Not implemented.')
+  return dumps('Not implemented.')
 
 @app.route('/computerBang', methods=['POST'])
 def computerBang():
-  return xml_marshaller.xml_marshaller.dumps('')
+  return dumps('')
 
 @app.route('/getComputerPartitionCertificate', methods=['GET'])
 def getComputerPartitionCertificate():
   # proxy does not use partition certificate, but client calls this.
-  return xml_marshaller.xml_marshaller.dumps({'certificate': '', 'key': ''})
+  return dumps({'certificate': '', 'key': ''})
 
 @app.route('/getSoftwareReleaseListFromSoftwareProduct', methods=['GET'])
 def getSoftwareReleaseListFromSoftwareProduct():
@@ -788,10 +767,10 @@ def getSoftwareReleaseListFromSoftwareProduct():
     raise NotImplementedError('software_release_url parameter is not supported yet.')
   else:
     assert(software_product_reference is not None)
-    if app.config['software_product_list'].has_key(software_product_reference):
+    if software_product_reference in app.config['software_product_list']:
       software_release_url_list =\
           [app.config['software_product_list'][software_product_reference]]
     else:
       software_release_url_list = []
-    return xml_marshaller.xml_marshaller.dumps(software_release_url_list)
+    return dumps(software_release_url_list)
 

slapos/slap/__init__.py

@@ -30,4 +30,4 @@ if sys.version_info < (2, 6):
   import warnings
   warnings.warn('Used python version (%s) is old and has problems with'
       ' IPv6 connections' % '.'.join([str(q) for q in sys.version_info[:3]]))
-from slap import *
+from .slap import *

slapos/slap/slap.py

@@ -40,15 +40,19 @@ import os
 import json
 import logging
 import re
-import urlparse
 import hashlib
-from util import xml2dict
+from functools import wraps
+
+import six
+from six.moves.urllib import parse
+
+from .util import xml2dict
+from slapos.util import loads, dumps, bytes2str
 
 import netaddr
 from xml.sax import saxutils
-import zope.interface
-from interface import slap as interface
-from xml_marshaller import xml_marshaller
+from zope.interface import implementer
+from .interface import slap as interface
 
 from uritemplate import expand
 
@@ -92,33 +96,29 @@ class SlapRequester(SlapDocument):
         request_dict=request_dict,
         connection_helper=self._connection_helper,
       )
-    if type(xml) is unicode:
-      xml = str(xml)
-      xml.encode('utf-8')
-    software_instance = xml_marshaller.loads(xml)
+    software_instance = loads(xml)
     computer_partition = ComputerPartition(
-      software_instance.slap_computer_id.encode('UTF-8'),
-      software_instance.slap_computer_partition_id.encode('UTF-8'),
+      software_instance.slap_computer_id,
+      software_instance.slap_computer_partition_id,
       connection_helper=self._connection_helper,
     )
     # Hack to give all object attributes to the ComputerPartition instance
     # XXX Should be removed by correctly specifying difference between
     # ComputerPartition and SoftwareInstance
-    computer_partition.__dict__ = dict(computer_partition.__dict__.items() +
-                                       software_instance.__dict__.items())
+    computer_partition.__dict__.update(software_instance.__dict__)
     # XXX not generic enough.
-    if xml_marshaller.loads(request_dict['shared_xml']):
+    if loads(request_dict['shared_xml']):
       computer_partition._synced = True
       computer_partition._connection_dict = software_instance._connection_dict
       computer_partition._parameter_dict = software_instance._parameter_dict
     return computer_partition
 
 
+@implementer(interface.ISoftwareRelease)
 class SoftwareRelease(SlapDocument):
   """
   Contains Software Release information
   """
-  zope.interface.implements(interface.ISoftwareRelease)
 
   def __init__(self, software_release=None, computer_guid=None, **kw):
     """
@@ -129,8 +129,6 @@ class SoftwareRelease(SlapDocument):
     SlapDocument.__init__(self, kw.pop('connection_helper', None),
                                 kw.pop('hateoas_navigator', None))
     self._software_instance_list = []
-    if software_release is not None:
-      software_release = software_release.encode('UTF-8')
     self._software_release = software_release
     self._computer_guid = computer_guid
 
@@ -178,8 +176,8 @@ class SoftwareRelease(SlapDocument):
     return getattr(self, '_requested_state', 'available')
 
 
+@implementer(interface.ISoftwareProductCollection)
 class SoftwareProductCollection(object):
-  zope.interface.implements(interface.ISoftwareProductCollection)
 
   def __init__(self, logger, slap):
     self.logger = logger
@@ -201,40 +199,41 @@ class SoftwareProductCollection(object):
 
 
 # XXX What is this SoftwareInstance class?
+@implementer(interface.ISoftwareInstance)
 class SoftwareInstance(SlapDocument):
   """
   Contains Software Instance information
   """
-  zope.interface.implements(interface.ISoftwareInstance)
 
-  def __init__(self, **kwargs):
+  def __init__(self, **kw):
     """
     Makes easy initialisation of class parameters
     """
-    for k, v in kwargs.iteritems():
-      setattr(self, k, v)
+    self.__dict__.update(kw)
 
 
 """Exposed exceptions"""
+@implementer(interface.IResourceNotReady)
 class ResourceNotReady(Exception):
-  zope.interface.implements(interface.IResourceNotReady)
+  pass
 
+@implementer(interface.IServerError)
 class ServerError(Exception):
-  zope.interface.implements(interface.IServerError)
+  pass
 
+@implementer(interface.INotFoundError)
 class NotFoundError(Exception):
-  zope.interface.implements(interface.INotFoundError)
+  pass
 
 class AuthenticationError(Exception):
   pass
 
-
+@implementer(interface.IConnectionError)
 class ConnectionError(Exception):
-  zope.interface.implements(interface.IConnectionError)
-
+  pass
 
+@implementer(interface.ISupply)
 class Supply(SlapDocument):
-  zope.interface.implements(interface.ISupply)
 
   def supply(self, software_release, computer_guid=None, state='available'):
     try:
@@ -247,8 +246,8 @@ class Supply(SlapDocument):
           % computer_guid)
 
 
+@implementer(interface.IOpenOrder)
 class OpenOrder(SlapRequester):
-  zope.interface.implements(interface.IOpenOrder)
 
   def request(self, software_release, partition_reference,
               partition_parameter_kw=None, software_type=None,
@@ -260,12 +259,12 @@ class OpenOrder(SlapRequester):
     request_dict = {
         'software_release': software_release,
         'partition_reference': partition_reference,
-        'partition_parameter_xml': xml_marshaller.dumps(partition_parameter_kw),
-        'filter_xml': xml_marshaller.dumps(filter_kw),
+        'partition_parameter_xml': dumps(partition_parameter_kw),
+        'filter_xml': dumps(filter_kw),
         # XXX Cedric: Why state and shared are marshalled? First is a string
         #             And second is a boolean.
-        'state': xml_marshaller.dumps(state),
-        'shared_xml': xml_marshaller.dumps(shared),
+        'state': dumps(state),
+        'shared_xml': dumps(shared),
     }
     if software_type is not None:
       request_dict['software_type'] = software_type
@@ -280,7 +279,7 @@ class OpenOrder(SlapRequester):
     raw_information = self._hateoas_navigator.getHostingSubscriptionRootSoftwareInstanceInformation(partition_reference)
     software_instance = SoftwareInstance()
     # XXX redefine SoftwareInstance to be more consistent
-    for key, value in raw_information.iteritems():
+    for key, value in six.iteritems(raw_information):
       if key in ['_links']:
         continue
       setattr(software_instance, '_%s' % key, value)
@@ -292,7 +291,7 @@ class OpenOrder(SlapRequester):
     Requests a computer.
     """
     xml = self._connection_helper.POST('requestComputer', data={'computer_title': computer_reference})
-    computer = xml_marshaller.loads(xml)
+    computer = loads(xml)
     computer._connection_helper = self._connection_helper
     computer._hateoas_navigator = self._hateoas_navigator
     return computer
@@ -303,24 +302,19 @@ def _syncComputerInformation(func):
   Synchronize computer object with server information
   """
   def decorated(self, *args, **kw):
-    if getattr(self, '_synced', 0):
-      return func(self, *args, **kw)
-    computer = self._connection_helper.getFullComputerInformation(self._computer_id)
-    for key, value in computer.__dict__.items():
-      if isinstance(value, unicode):
-        # convert unicode to utf-8
-        setattr(self, key, value.encode('utf-8'))
-      else:
-        setattr(self, key, value)
-    setattr(self, '_synced', True)
-    for computer_partition in self.getComputerPartitionList():
-      setattr(computer_partition, '_synced', True)
+    if not getattr(self, '_synced', 0):
+      computer = self._connection_helper.getFullComputerInformation(
+        self._computer_id)
+      self.__dict__.update(computer.__dict__)
+      self._synced = True
+      for computer_partition in self.getComputerPartitionList():
+        computer_partition._synced = True
     return func(self, *args, **kw)
-  return decorated
+  return wraps(func)(decorated)
 
 
+@implementer(interface.IComputer)
 class Computer(SlapDocument):
-  zope.interface.implements(interface.IComputer)
 
   def __init__(self, computer_id, connection_helper=None, hateoas_navigator=None):
     SlapDocument.__init__(self, connection_helper, hateoas_navigator)
@@ -366,7 +360,7 @@ class Computer(SlapDocument):
 
   def getStatus(self):
     xml = self._connection_helper.GET('getComputerStatus', params={'computer_id': self._computer_id})
-    return xml_marshaller.loads(xml)
+    return loads(xml)
 
   def revokeCertificate(self):
     self._connection_helper.POST('revokeComputerCertificate', data={
@@ -375,7 +369,7 @@ class Computer(SlapDocument):
   def generateCertificate(self):
     xml = self._connection_helper.POST('generateComputerCertificate', data={
       'computer_id': self._computer_id})
-    return xml_marshaller.loads(xml)
+    return loads(xml)
 
 
 def parsed_error_message(status, body, path):
@@ -391,8 +385,8 @@ def parsed_error_message(status, body, path):
     return 'Server responded with wrong code %s with %s' % (status, path)
 
 
+@implementer(interface.IComputerPartition)
 class ComputerPartition(SlapRequester):
-  zope.interface.implements(interface.IComputerPartition)
 
   def __init__(self, computer_id=None, partition_id=None,
                request_dict=None, connection_helper=None,
@@ -466,11 +460,10 @@ class ComputerPartition(SlapRequester):
         'software_release': software_release,
         'software_type': software_type,
         'partition_reference': partition_reference,
-        'shared_xml': xml_marshaller.dumps(shared),
-        'partition_parameter_xml': xml_marshaller.dumps(
-                                        partition_parameter_kw),
-        'filter_xml': xml_marshaller.dumps(filter_kw),
-        'state': xml_marshaller.dumps(state),
+        'shared_xml': dumps(shared),
+        'partition_parameter_xml': dumps(partition_parameter_kw),
+        'filter_xml': dumps(filter_kw),
+        'state': dumps(state),
     }
     self._updateTransactionFile(partition_reference)
     return self._requestComputerPartition(request_dict)
@@ -529,7 +522,7 @@ class ComputerPartition(SlapRequester):
     raw_information = self._hateoas_navigator.getRelatedInstanceInformation(partition_reference)
     software_instance = SoftwareInstance()
     # XXX redefine SoftwareInstance to be more consistent
-    for key, value in raw_information.iteritems():
+    for key, value in six.iteritems(raw_information):
       if key in ['_links']:
         continue
       setattr(software_instance, '_%s' % key, value)
@@ -615,7 +608,7 @@ class ComputerPartition(SlapRequester):
     self._connection_helper.POST('setComputerPartitionConnectionXml', data={
           'computer_id': self._computer_id,
           'computer_partition_id': self._partition_id,
-          'connection_xml': xml_marshaller.dumps(connection_dict),
+          'connection_xml': dumps(connection_dict),
           'slave_reference': slave_reference})
 
   def getInstanceParameter(self, key):
@@ -643,7 +636,7 @@ class ComputerPartition(SlapRequester):
                 'computer_partition_id': self._partition_id,
                 }
             )
-    return xml_marshaller.loads(xml)
+    return loads(xml)
 
   def getStatus(self):
     xml = self._connection_helper.GET('getComputerPartitionStatus',
@@ -652,7 +645,7 @@ class ComputerPartition(SlapRequester):
                 'computer_partition_id': self._partition_id,
                 }
             )
-    return xml_marshaller.loads(xml)
+    return loads(xml)
   
   def getFullHostingIpAddressList(self):
     xml = self._connection_helper.GET('getHostingSubscriptionIpList',
@@ -661,14 +654,14 @@ class ComputerPartition(SlapRequester):
                 'computer_partition_id': self._partition_id,
                 }
             )
-    return xml_marshaller.loads(xml)
+    return loads(xml)
 
   def setComputerPartitionRelatedInstanceList(self, instance_reference_list):
     self._connection_helper.POST('updateComputerPartitionRelatedInstanceList',
         data={
           'computer_id': self._computer_id,
           'computer_partition_id': self._partition_id,
-          'instance_reference_xml': xml_marshaller.dumps(instance_reference_list)
+          'instance_reference_xml': dumps(instance_reference_list)
           }
         )
 
@@ -676,7 +669,7 @@ def _addIpv6Brackets(url):
   # if master_url contains an ipv6 without bracket, add it
   # Note that this is mostly to limit specific issues with
   # backward compatiblity, not to ensure generic detection.
-  api_scheme, api_netloc, api_path, api_query, api_fragment = urlparse.urlsplit(url)
+  api_scheme, api_netloc, api_path, api_query, api_fragment = parse.urlsplit(url)
   try:
     ip = netaddr.IPAddress(api_netloc)
     port = None
@@ -690,7 +683,7 @@ def _addIpv6Brackets(url):
     api_netloc = '[%s]' % ip
     if port:
       api_netloc = '%s:%s' % (api_netloc, port)
-    url = urlparse.urlunsplit((api_scheme, api_netloc, api_path, api_query, api_fragment))
+    url = parse.urlunsplit((api_scheme, api_netloc, api_path, api_query, api_fragment))
   return url
 
 class ConnectionHelper:
@@ -709,7 +702,7 @@ class ConnectionHelper:
 
   def getComputerInformation(self, computer_id):
     xml = self.GET('getComputerInformation', params={'computer_id': computer_id})
-    return xml_marshaller.loads(xml)
+    return loads(xml)
 
   def getFullComputerInformation(self, computer_id):
     """
@@ -728,13 +721,10 @@ class ConnectionHelper:
       # We should stablise slap library soon.
       xml = self.GET('getComputerInformation', params=params)
 
-    if type(xml) is unicode:
-      xml = str(xml)
-      xml.encode('utf-8')
-    return xml_marshaller.loads(xml)
+    return loads(xml)
 
   def do_request(self, method, path, params=None, data=None, headers=None):
-    url = urlparse.urljoin(self.slapgrid_uri, path)
+    url = parse.urljoin(self.slapgrid_uri, path)
     if headers is None:
       headers = {}
     headers.setdefault('Accept', '*/*')
@@ -754,7 +744,7 @@ class ConnectionHelper:
       # Behavior kept for compatibility with old slapproxies (< v1.3.3).
       # Can be removed when old slapproxies are no longer in use.
       if data:
-        for k, v in data.iteritems():
+        for k, v in six.iteritems(data):
           if v is None:
             data[k] = 'None'
 
@@ -765,7 +755,12 @@ class ConnectionHelper:
                    data=data,
                    headers=headers,
                    timeout=self.timeout)
-      req.raise_for_status()
+      try:
+        req.raise_for_status()
+      except TypeError:
+        # In Py3, a comparison between NoneType and int can occur if req has no
+        # status_code (= None).
+        pass
 
     except (requests.Timeout, requests.ConnectionError) as exc:
       raise ConnectionError("Couldn't connect to the server. Please "
@@ -816,8 +811,8 @@ class ConnectionHelper:
 
 
 getHateoasUrl_cache = {}
+@implementer(interface.slap)
 class slap:
-  zope.interface.implements(interface.slap)
 
   def initializeConnection(self, slapgrid_uri,
                            key_file=None, cert_file=None,
@@ -837,7 +832,8 @@ class slap:
         pass
     if not slapgrid_rest_uri:
       try:
-        slapgrid_rest_uri = getHateoasUrl_cache[getHateoasUrl_cache_key] = self._connection_helper.GET('getHateoasUrl')
+        slapgrid_rest_uri = getHateoasUrl_cache[getHateoasUrl_cache_key] = \
+          bytes2str(self._connection_helper.GET('getHateoasUrl'))
       except:
         pass
     if slapgrid_rest_uri:
@@ -885,10 +881,7 @@ class slap:
                 'computer_partition_reference': partition_id,
                 }
             )
-    if type(xml) is unicode:
-      xml = str(xml)
-      xml.encode('utf-8')
-    result = xml_marshaller.loads(xml)
+    result = loads(xml)
     # XXX: dirty hack to make computer partition usable. xml_marshaller is too
     # low-level for our needs here.
     result._connection_helper = self._connection_helper
@@ -923,10 +916,7 @@ class slap:
       params['software_release_url'] = software_release_url
 
     xml = self._connection_helper.GET(url, params=params)
-    if type(xml) is unicode:
-      xml = str(xml)
-      xml.encode('utf-8')
-    result = xml_marshaller.loads(xml)
+    result = loads(xml)
     assert(type(result) == list)
     return result
 
@@ -1046,7 +1036,7 @@ class SlapHateoasNavigator(HateoasNavigator):
       raw_information = self.getHostingSubscriptionRootSoftwareInstanceInformation(hosting_subscription_link['title'])
       software_instance = SoftwareInstance()
       # XXX redefine SoftwareInstance to be more consistent
-      for key, value in raw_information.iteritems():
+      for key, value in six.iteritems(raw_information):
         if key in ['_links']:
           continue
         setattr(software_instance, '_%s' % key, value)

slapos/tests/cli.py

@@ -29,23 +29,21 @@ import logging
 import pprint
 import unittest
 import tempfile
-import StringIO
 import sys
 import os
-import sqlite3
 import pkg_resources
 
-
+from contextlib import contextmanager
 from mock import patch, create_autospec
 import mock
-from slapos.util import sqlite_connect
+from slapos.util import sqlite_connect, bytes2str
 
 import slapos.cli.console
 import slapos.cli.entry
 import slapos.cli.info
 import slapos.cli.list
 import slapos.cli.supervisorctl
-import slapos.cli.proxy_show
+from slapos.cli.proxy_show import do_show, StringIO
 from slapos.client import ClientConfig
 import slapos.grid.svcbackend
 import slapos.proxy
@@ -99,9 +97,9 @@ class TestCliProxyShow(CliMixin):
     self.conf.logger = self.logger
 
     # load database
-    schema = pkg_resources.resource_stream('slapos.tests.slapproxy', 'database_dump_version_11.sql')
+    schema = bytes2str(pkg_resources.resource_string('slapos.tests.slapproxy', 'database_dump_version_11.sql'))
     db = sqlite_connect(self.db_file.name)
-    db.cursor().executescript(schema.read())
+    db.cursor().executescript(schema)
     db.commit()
 
     # by default we simulate being invoked with "show all" arguments
@@ -121,7 +119,7 @@ class TestCliProxyShow(CliMixin):
     with mock.patch(
             'slapos.cli.proxy_show.logging.getLogger',
             return_value=logger):
-        slapos.cli.proxy_show.do_show(self.conf)
+        do_show(self.conf)
 
     # installed softwares are listed
     logger.info.assert_any_call(
@@ -132,7 +130,7 @@ class TestCliProxyShow(CliMixin):
     logger.info.assert_any_call(
         '    %s = %s',
         '_',
-        '{\n  "url": "memcached://10.0.30.235:2003/", \n  "monitor-base-url": ""\n}')
+        '{\n  "monitor-base-url": "",\n  "url": "memcached://10.0.30.235:2003/"\n}')
 
     # other parameters are displayed as simple string
     logger.info.assert_any_call(
@@ -152,10 +150,10 @@ class TestCliProxyShow(CliMixin):
   def test_proxy_show_displays_on_stdout(self):
     saved_stderr = sys.stderr
     saved_stdout = sys.stdout
-    sys.stderr = stderr = StringIO.StringIO()
-    sys.stdout = stdout = StringIO.StringIO()
+    sys.stderr = stderr = StringIO()
+    sys.stdout = stdout = StringIO()
     try:
-      slapos.cli.proxy_show.do_show(self.conf)
+      do_show(self.conf)
     finally:
       sys.stderr = saved_stderr
       sys.stdout = saved_stdout
@@ -169,8 +167,8 @@ class TestCliProxyShow(CliMixin):
   def test_proxy_show_use_pager(self):
     saved_stderr = sys.stderr
     saved_stdout = sys.stdout
-    sys.stderr = stderr = StringIO.StringIO()
-    sys.stdout = stdout = StringIO.StringIO()
+    sys.stderr = stderr = StringIO()
+    sys.stdout = stdout = StringIO()
     stdout.isatty = lambda *args: True
 
     # use a pager that just output to a file.
@@ -179,7 +177,7 @@ class TestCliProxyShow(CliMixin):
     os.environ['PAGER'] = 'cat > {}'.format(tmp.name)
 
     try:
-      slapos.cli.proxy_show.do_show(self.conf)
+      do_show(self.conf)
     finally:
       sys.stderr = saved_stderr
       sys.stdout = saved_stdout
@@ -327,53 +325,34 @@ class TestCliSupervisorctl(CliMixin):
 
 
 class TestCliConsole(unittest.TestCase):
-  def setUp(self):
-    cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
-    cp._parameter_dict = {'parameter_name': 'parameter_value'}
 
-    request_patch = patch.object(slapos.slap.OpenOrder, 'request', return_value = cp)
-    self.mock_request = request_patch.start()
+  script = """\
+print(request('software_release', 'instance').getInstanceParameterDict()['parameter_name'])
+"""
 
-    self.config_file = tempfile.NamedTemporaryFile()
-    self.config_file.write('''[slapos]
-master_url=null
-''')
-    self.config_file.flush()
-
-  def tearDown(self):
-    self.mock_request.stop()
-    self.config_file.close()
+  @contextmanager
+  def _test_console(self):
+    cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
+    cp._parameter_dict = {'parameter_name': 'parameter_value'}
+    with patch.object(slapos.slap.OpenOrder, 'request',
+                      return_value = cp) as mock_request, \
+         patch.object(sys, 'stdout', StringIO()) as app_stdout, \
+         tempfile.NamedTemporaryFile() as config_file:
+      config_file.write(b'[slapos]\nmaster_url=null\n')
+      config_file.flush()
+      yield slapos.cli.entry.SlapOSApp(), config_file.name
+      mock_request.assert_called_once_with(
+          'software_release', 'instance')
+      self.assertIn('parameter_value', app_stdout.getvalue())
 
   def test_console_interactive(self):
-      app = slapos.cli.entry.SlapOSApp()
-      saved_stdin = sys.stdin
-      saved_stdout = sys.stdout
-      try:
-        sys.stdin = app_stdin = StringIO.StringIO(
-            """print request('software_release', 'instance').getInstanceParameterDict()['parameter_name']\n""")
-        sys.stdout = app_stdout = StringIO.StringIO()
-        app.run(('console', '--cfg', self.config_file.name))
-      finally:
-        sys.stdin = saved_stdin
-        sys.stdout = saved_stdout
-
-      self.mock_request.assert_called_once_with('software_release', 'instance')
-      self.assertIn('parameter_value', app_stdout.getvalue())
+    with self._test_console() as (app, config_file), \
+         patch.object(sys, 'stdin', StringIO(self.script)):
+      app.run(('console', '--cfg', config_file))
 
   def test_console_script(self):
-    with tempfile.NamedTemporaryFile() as script:
-      script.write(
-        """print request('software_release', 'instance').getInstanceParameterDict()['parameter_name']\n""")
+    with self._test_console() as (app, config_file), \
+         tempfile.NamedTemporaryFile('w') as script:
+      script.write(self.script)
       script.flush()
-
-      app = slapos.cli.entry.SlapOSApp()
-      saved_stdout = sys.stdout
-      try:
-        sys.stdout = app_stdout = StringIO.StringIO()
-        app.run(('console', '--cfg', self.config_file.name, script.name))
-      finally:
-        sys.stdout = saved_stdout
-
-      self.mock_request.assert_called_once_with('software_release', 'instance')
-      self.assertIn('parameter_value', app_stdout.getvalue())
-
+      app.run(('console', '--cfg', config_file, script.name))

slapos/tests/collect.py

@@ -36,7 +36,7 @@ import psutil
 from time import strftime
 from slapos.collect import entity, snapshot, db, reporter
 from slapos.cli.entry import SlapOSApp
-from ConfigParser import ConfigParser
+from six.moves.configparser import ConfigParser
 
 class FakeDatabase(object):
     def __init__(self):
@@ -364,7 +364,7 @@ class TestCollectReport(unittest.TestCase):
         with tarfile.open("%s.tar.gz" % dump_folder) as tf:
             self.assertEqual(tf.getmembers()[0].name, "1990-01-01")
             self.assertEqual(tf.getmembers()[1].name, "1990-01-01/test.txt")
-            self.assertEqual(tf.extractfile(tf.getmembers()[1]).read(), 'hi')
+            self.assertEqual(tf.extractfile(tf.getmembers()[1]).read(), b'hi')
 
 class TestCollectSnapshot(unittest.TestCase):
 
@@ -482,10 +482,10 @@ class TestCollectEntity(unittest.TestCase):
         config.set('slapos', 'instance_root', self.instance_root)
  
         user_dict = entity.get_user_list(config)
-        username_list = ['slapuser0', 'slapuser1', 'slapuser2'] 
-        self.assertEqual(username_list, user_dict.keys())
+        username_set = {'slapuser0', 'slapuser1', 'slapuser2'} 
+        self.assertEquals(username_set, set(user_dict))
        
-        for name in username_list:
+        for name in username_set:
           self.assertEqual(user_dict[name].name, name)
           self.assertEqual(user_dict[name].snapshot_list, [])
           expected_path = "%s/slappart%s" % (self.instance_root, name.strip("slapuser")) 
@@ -508,11 +508,11 @@ class TestCollectEntity(unittest.TestCase):
 
         self.assertEqual(database.invoked_method_list[1][0], "insertUserSnapshot")
         self.assertEqual(database.invoked_method_list[1][1][0], ("fakeuser0",))
-        self.assertEqual(database.invoked_method_list[1][1][1].keys(), 
-                   ['cpu_time', 'cpu_percent', 'process',
+        self.assertEqual(set(database.invoked_method_list[1][1][1]), 
+                   {'cpu_time', 'cpu_percent', 'process',
                     'memory_rss', 'pid', 'memory_percent',
                     'io_rw_counter', 'insertion_date', 'insertion_time',
-                    'io_cycles_counter', 'cpu_num_threads'])
+                    'io_cycles_counter', 'cpu_num_threads'})
         self.assertEqual(database.invoked_method_list[2], ("commit", ""))
         self.assertEqual(database.invoked_method_list[3], ("close", ""))
 
@@ -527,19 +527,19 @@ class TestCollectEntity(unittest.TestCase):
 
         self.assertEqual(database.invoked_method_list[1][0], "insertUserSnapshot")
         self.assertEqual(database.invoked_method_list[1][1][0], ("fakeuser0",))
-        self.assertEqual(database.invoked_method_list[1][1][1].keys(), 
-                   ['cpu_time', 'cpu_percent', 'process',
+        self.assertEqual(set(database.invoked_method_list[1][1][1]), 
+                   {'cpu_time', 'cpu_percent', 'process',
                     'memory_rss', 'pid', 'memory_percent',
                     'io_rw_counter', 'insertion_date', 'insertion_time',
-                    'io_cycles_counter', 'cpu_num_threads'])
-        self.assertEqual(database.invoked_method_list[2], ("commit", ""))
-        self.assertEqual(database.invoked_method_list[3], ("close", ""))
+                    'io_cycles_counter', 'cpu_num_threads'})
+        self.assertEquals(database.invoked_method_list[2], ("commit", ""))
+        self.assertEquals(database.invoked_method_list[3], ("close", ""))
 
         self.assertEqual(database.invoked_method_list[4], ("connect", ""))
         self.assertEqual(database.invoked_method_list[5][0], "inserFolderSnapshot")
         self.assertEqual(database.invoked_method_list[5][1][0], ("fakeuser0",))
-        self.assertEqual(database.invoked_method_list[5][1][1].keys(), 
-                   ['insertion_date', 'disk_usage', 'insertion_time'])
+        self.assertEqual(set(database.invoked_method_list[5][1][1]), 
+                   {'insertion_date', 'disk_usage', 'insertion_time'})
         self.assertEqual(database.invoked_method_list[6], ("commit", ""))
         self.assertEqual(database.invoked_method_list[7], ("close", ""))
 
@@ -554,23 +554,23 @@ class TestCollectEntity(unittest.TestCase):
 
         self.assertEqual(database.invoked_method_list[1][0], "insertUserSnapshot")
         self.assertEqual(database.invoked_method_list[1][1][0], ("fakeuser0",))
-        self.assertEqual(database.invoked_method_list[1][1][1].keys(), 
-                   ['cpu_time', 'cpu_percent', 'process',
+        self.assertEqual(set(database.invoked_method_list[1][1][1]), 
+                   {'cpu_time', 'cpu_percent', 'process',
                     'memory_rss', 'pid', 'memory_percent',
                     'io_rw_counter', 'insertion_date', 'insertion_time',
-                    'io_cycles_counter', 'cpu_num_threads'])
+                    'io_cycles_counter', 'cpu_num_threads'})
         self.assertEqual(database.invoked_method_list[2], ("commit", ""))
         self.assertEqual(database.invoked_method_list[3], ("close", ""))
 
         self.assertEqual(database.invoked_method_list[4], ("connect", ""))
         self.assertEqual(database.invoked_method_list[5][0], "select")
         self.assertEqual(database.invoked_method_list[5][1][0], ())
-        self.assertEqual(database.invoked_method_list[5][1][1].keys(),
-                                ['table', 'where', 'limit', 'order', 'columns'])
+        self.assertEqual(set(database.invoked_method_list[5][1][1]),
+                                {'table', 'where', 'limit', 'order', 'columns'})
         self.assertEqual(database.invoked_method_list[6][0], "inserFolderSnapshot")
         self.assertEqual(database.invoked_method_list[6][1][0], ("fakeuser0",))
-        self.assertEqual(database.invoked_method_list[6][1][1].keys(), 
-                   ['insertion_date', 'disk_usage', 'insertion_time'])
+        self.assertEqual(set(database.invoked_method_list[6][1][1]), 
+                   {'insertion_date', 'disk_usage', 'insertion_time'})
         self.assertEqual(database.invoked_method_list[7], ("commit", ""))
         self.assertEqual(database.invoked_method_list[8], ("close", ""))
 
@@ -583,14 +583,14 @@ class TestCollectEntity(unittest.TestCase):
 
         self.assertEqual(database.invoked_method_list[1][0], "insertComputerSnapshot")
         self.assertEqual(database.invoked_method_list[1][1][0], ())
-        self.assertEqual(database.invoked_method_list[1][1][1].keys(), 
-                 ['insertion_time', 'insertion_date', 'cpu_num_core',
+        self.assertEqual(set(database.invoked_method_list[1][1][1]), 
+                 {'insertion_time', 'insertion_date', 'cpu_num_core',
                   'partition_list', 'cpu_frequency', 'memory_size', 
-                  'cpu_type', 'memory_type'])
+                  'cpu_type', 'memory_type'})
  
         self.assertEqual(database.invoked_method_list[2][0], "insertSystemSnapshot")
         self.assertEqual(database.invoked_method_list[2][1][0], ())
-        self.assertEqual(set(database.invoked_method_list[2][1][1].keys()), 
+        self.assertEqual(set(database.invoked_method_list[2][1][1]), 
           set([ 'memory_used', 'cpu_percent', 'insertion_date', 'insertion_time',
                 'loadavg', 'memory_free', 'net_in_bytes', 'net_in_dropped', 
                 'net_in_errors', 'net_out_bytes', 'net_out_dropped', 
@@ -598,7 +598,7 @@ class TestCollectEntity(unittest.TestCase):
 
         self.assertEqual(database.invoked_method_list[3][0], "insertDiskPartitionSnapshot")
         self.assertEqual(database.invoked_method_list[3][1][0], ())
-        self.assertEqual(set(database.invoked_method_list[3][1][1].keys()), 
+        self.assertEqual(set(database.invoked_method_list[3][1][1]), 
           set([ 'used', 'insertion_date', 'partition', 'free', 
                 'mountpoint', 'insertion_time' ]))
 

slapos/tests/configure_local.py

@@ -34,7 +34,7 @@ import slapos.cli.configure_local
 from slapos.cli.configure_local import ConfigureLocalCommand, _createConfigurationDirectory 
 from slapos.cli.entry import SlapOSApp
 from argparse import Namespace
-from ConfigParser import ConfigParser
+from six.moves.configparser import ConfigParser
 
 # Disable any command to launch slapformat and supervisor
 slapos.cli.configure_local._runFormat = lambda x: "Do nothing"

slapos/tests/grid_utils.py

@@ -47,7 +47,7 @@ class SlapPopenTestCase(unittest.TestCase):
   def test_exec(self):
     """Test command execution with SlapPopen.
     """
-    self.script.write('#!/bin/sh\necho "hello"\nexit 123')
+    self.script.write(b'#!/bin/sh\necho "hello"\nexit 123')
     self.script.close()
 
     logger = mock.MagicMock()
@@ -65,7 +65,7 @@ class SlapPopenTestCase(unittest.TestCase):
   def test_debug(self):
     """Test debug=True, which keeps interactive.
     """
-    self.script.write('#!/bin/sh\necho "exit code?"\nread rc\nexit $rc')
+    self.script.write(b'#!/bin/sh\necho "exit code?"\nread rc\nexit $rc')
     self.script.close()
 
     # keep a reference to stdin and stdout to restore them later
@@ -74,7 +74,7 @@ class SlapPopenTestCase(unittest.TestCase):
 
     # replace stdin with a pipe that will write 123
     child_stdin_r, child_stdin_w = os.pipe()
-    os.write(child_stdin_w, "123")
+    os.write(child_stdin_w, b"123")
     os.close(child_stdin_w)
     os.dup2(child_stdin_r, sys.stdin.fileno())
 
@@ -88,7 +88,7 @@ class SlapPopenTestCase(unittest.TestCase):
           debug=True,
           logger=logging.getLogger())
       # program output
-      self.assertEqual('exit code?\n', os.read(child_stdout_r, 1024))
+      self.assertEqual(b'exit code?\n', os.read(child_stdout_r, 1024))
 
       self.assertEqual(123, program.returncode)
       self.assertEqual('(output not captured in debug mode)', program.output)

slapos/tests/interface.py

@@ -28,17 +28,15 @@ import unittest
 
 from zope.interface.verify import verifyClass
 import zope.interface
-import types
+from six import class_types
 from slapos import slap
 
 def getOnlyImplementationAssertionMethod(klass, method_list):
   """Returns method which verifies if a klass only implements its interfaces"""
   def testMethod(self):
-    implemented_method_list = [x for x in dir(klass) \
-        if ((not x.startswith('_')) and callable(getattr(klass, x)))]
-    for interface_method in method_list:
-      if interface_method in implemented_method_list:
-        implemented_method_list.remove(interface_method)
+    implemented_method_list = {x for x in dir(klass)
+        if not x.startswith('_') and callable(getattr(klass, x))}
+    implemented_method_list.difference_update(method_list)
 
     if implemented_method_list:
       raise AssertionError("Unexpected methods %s" % implemented_method_list)
@@ -61,7 +59,7 @@ def generateTestMethodListOnClass(klass, module):
   """Generate test method on klass"""
   for class_id in dir(module):
     implementing_class = getattr(module, class_id)
-    if type(implementing_class) not in (types.ClassType, types.TypeType):
+    if not isinstance(implementing_class, class_types):
       continue
     # add methods to assert that publicly available classes are defining
     # interfaces
@@ -69,7 +67,7 @@ def generateTestMethodListOnClass(klass, module):
     setattr(klass, method_name, getDeclarationAssertionMethod(
       implementing_class))
 
-    implemented_method_list = []
+    implemented_method_list = ['with_traceback']
     for interface in list(zope.interface.implementedBy(implementing_class)):
       # for each interface which class declares add a method which verify
       # implementation

slapos/tests/promise.py

@@ -34,8 +34,8 @@ import json
 import random
 import logging
 from datetime import datetime, timedelta
-import Queue
-from zope import interface as zope_interface
+import six
+from six.moves import queue
 from slapos.grid.promise import interface, PromiseLauncher, PromiseProcess, PromiseError
 from slapos.grid.promise.generic import (GenericPromise, TestResult, AnomalyResult,
                                          PromiseQueueResult, PROMISE_STATE_FOLDER_NAME,
@@ -59,7 +59,7 @@ class TestSlapOSPromiseMixin(unittest.TestCase):
   def writeInit(self):
     with open(os.path.join(self.plugin_dir, '__init__'), 'w') as f:
       f.write('')
-    os.chmod(os.path.join(self.plugin_dir, '__init__'), 0644)
+    os.chmod(os.path.join(self.plugin_dir, '__init__'), 0o644)
     if sys.path[0] != self.plugin_dir:
       sys.path[0:0] = [self.plugin_dir]
 
@@ -111,7 +111,7 @@ class TestSlapOSPromiseMixin(unittest.TestCase):
       'debug': False,
       'name': promise_name,
       'path': os.path.join(self.plugin_dir, promise_name),
-      'queue': Queue.Queue(),
+      'queue': queue.Queue(),
     }
 
   def createPromiseProcess(self, promise_name, check_anomaly=False, wrap=False):
@@ -128,21 +128,20 @@ class TestSlapOSPromiseMixin(unittest.TestCase):
       wrap=wrap,
     )
 
-  def writeFile(self, path, content, mode=0644):
+  def writeFile(self, path, content, mode=0o644):
     with open(path, 'w') as f:
       f.write(content)
     os.chmod(path, mode)
 
   def generatePromiseScript(self, name, success=True, failure_count=1, content="",
     periodicity=0.03):
-    promise_content = """from zope import interface as zope_interface
+    promise_content = """from zope.interface import implementer
 from slapos.grid.promise import interface
 from slapos.grid.promise import GenericPromise
 
+@implementer(interface.IPromise)
 class RunPromise(GenericPromise):
 
-  zope_interface.implements(interface.IPromise)
-
   def __init__(self, config):
     GenericPromise.__init__(self, config)
     self.setPeriodicity(minute=%(periodicity)s)
@@ -186,7 +185,9 @@ class TestSlapOSPromiseLauncher(TestSlapOSPromiseMixin):
 
     with self.assertRaises(ImportError) as exc:
       promise_module = promise_process._loadPromiseModule()
-    self.assertEqual(exc.exception.message, 'No module named my_promise_no_py')
+    self.assertEqual(str(exc.exception), 'No module named %s' %
+                                          ("'%s'" % promise_name if six.PY3 else
+                                           promise_name))
 
   def test_promise_match_interface_no_implement(self):
     promise_name = 'my_promise_noimplement.py'
@@ -208,19 +209,18 @@ class RunPromise(GenericPromise):
 
     with self.assertRaises(RuntimeError) as exc:
       promise_module = promise_process._loadPromiseModule()
-    message = "RunPromise class in my_promise_noimplement.py must implements" \
-      " 'IPromise' interface. zope_interface.implements(interface.IPromise) is missing ?"
-    self.assertEqual(exc.exception.message, message)
+    message = "RunPromise class in my_promise_noimplement.py must implement" \
+      " 'IPromise' interface. @implementer(interface.IPromise) is missing ?"
+    self.assertEqual(str(exc.exception), message)
 
   def test_promise_match_interface_no_generic(self):
     promise_name = 'my_promise_nogeneric.py'
-    promise_content = """from zope import interface as zope_interface
+    promise_content = """from zope.interface import implementer
 from slapos.grid.promise import interface
 
+@implementer(interface.IPromise)
 class RunPromise(object):
 
-  zope_interface.implements(interface.IPromise)
-
   def __init__(self, config):
     pass
 
@@ -236,18 +236,17 @@ class RunPromise(object):
     with self.assertRaises(RuntimeError) as exc:
       promise_module = promise_process._loadPromiseModule()
 
-    self.assertEqual(exc.exception.message, 'RunPromise class is not a subclass of GenericPromise class.')
+    self.assertEqual(str(exc.exception), 'RunPromise class is not a subclass of GenericPromise class.')
 
   def test_promise_match_interface_no_sense(self):
     promise_name = 'my_promise_nosense.py'
-    promise_content = """from zope import interface as zope_interface
+    promise_content = """from zope.interface import implementer
 from slapos.grid.promise import interface
 from slapos.grid.promise import GenericPromise
 
+@implementer(interface.IPromise)
 class RunPromise(GenericPromise):
 
-  zope_interface.implements(interface.IPromise)
-
   def __init__(self, config):
     pass
 
@@ -263,23 +262,22 @@ class RunPromise(GenericPromise):
     with self.assertRaises(TypeError) as exc:
       promise_module = promise_process._loadPromiseModule()
       promise = promise_module.RunPromise({})
-    self.assertEqual(exc.exception.message,
+    self.assertEqual(str(exc.exception),
       "Can't instantiate abstract class RunPromise with abstract methods sense")
 
   def test_promise_extra_config(self):
     promise_name = 'my_promise_extra.py'
     config_dict = {'foo': 'bar', 'my-config': 4522111,
                    'text': 'developers \ninformation, sample\n\ner'}
-    promise_content = """from zope import interface as zope_interface
+    promise_content = """from zope.interface import implementer
 from slapos.grid.promise import interface
 from slapos.grid.promise import GenericPromise
 
 %(config_name)s = %(config_content)s
 
+@implementer(interface.IPromise)
 class RunPromise(GenericPromise):
 
-  zope_interface.implements(interface.IPromise)
-
   def sense(self):
     pass
     
@@ -298,17 +296,19 @@ class RunPromise(GenericPromise):
 
   def test_promise_extra_config_reserved_name(self):
     promise_name = 'my_promise_extra.py'
-    config_dict = {'name': 'bar', 'my-config': 4522111}
-    promise_content = """from zope import interface as zope_interface
+    from collections import OrderedDict
+    config_dict = OrderedDict([('name', 'bar'), ('my-config', 4522111)])
+    promise_content = """from collections import OrderedDict
+
+from zope.interface import implementer
 from slapos.grid.promise import interface
 from slapos.grid.promise import GenericPromise
 
 %(config_name)s = %(config_content)s
 
+@implementer(interface.IPromise)
 class RunPromise(GenericPromise):
 
-  zope_interface.implements(interface.IPromise)
-
   def sense(self):
     pass
     
@@ -321,7 +321,7 @@ class RunPromise(GenericPromise):
 
     with self.assertRaises(ValueError) as exc:
       promise_module = promise_process._loadPromiseModule()
-    self.assertEqual(exc.exception.message, "Extra parameter name 'name' cannot be used.\n%s" % config_dict)
+    self.assertEqual(str(exc.exception), "Extra parameter name 'name' cannot be used.\n%s" % config_dict)
 
   def test_runpromise(self):
     promise_name = 'my_promise.py'
@@ -540,7 +540,7 @@ class RunPromise(GenericPromise):
     # run promise will fail when promise fail (usefull for slapgrid)
     with self.assertRaises(PromiseError) as exc:
       self.launcher.run()
-    self.assertEqual(exc.exception.message, 'Promise %r failed.' % second_promise)
+    self.assertEqual(str(exc.exception), 'Promise %r failed.' % second_promise)
 
     # force to reload the module without rerun python
     os.system('rm %s/*.pyc' % self.plugin_dir)
@@ -643,7 +643,7 @@ class RunPromise(GenericPromise):
 
     with self.assertRaises(PromiseError) as exc:
       self.launcher.run()
-    self.assertEqual(exc.exception.message, 'Promise %r failed.' % second_promise)
+    self.assertEqual(str(exc.exception), 'Promise %r failed.' % second_promise)
 
     self.assertTrue(os.path.exists(first_state_file))
     self.assertTrue(os.path.exists(second_state_file))
@@ -658,7 +658,7 @@ class RunPromise(GenericPromise):
     time.sleep(2)
     with self.assertRaises(PromiseError) as exc:
       self.launcher.run() # only my_first_promise will run but second_promise still failing
-    self.assertEqual(exc.exception.message, 'Promise %r failed.' % second_promise)
+    self.assertEqual(str(exc.exception), 'Promise %r failed.' % second_promise)
 
     first_result = json.load(open(first_state_file))
     second_result = json.load(open(second_state_file))
@@ -670,7 +670,7 @@ class RunPromise(GenericPromise):
     self.configureLauncher()
     with self.assertRaises(PromiseError) as exc:
       self.launcher.run()
-    self.assertEqual(exc.exception.message, 'Promise %r failed.' % second_promise)
+    self.assertEqual(str(exc.exception), 'Promise %r failed.' % second_promise)
 
     first_result = json.load(open(first_state_file))
     second_result = json.load(open(second_state_file))
@@ -693,7 +693,7 @@ class RunPromise(GenericPromise):
 
     with self.assertRaises(PromiseError) as exc:
       self.launcher.run()
-    self.assertEqual(exc.exception.message, 'Promise %r failed.' % second_promise)
+    self.assertEqual(str(exc.exception), 'Promise %r failed.' % second_promise)
 
     self.assertTrue(os.path.exists(first_state_file))
     self.assertTrue(os.path.exists(second_state_file))
@@ -708,7 +708,7 @@ class RunPromise(GenericPromise):
     time.sleep(2)
     with self.assertRaises(PromiseError) as exc:
       self.launcher.run() # only my_first_promise will run but second_promise still failing
-    self.assertEqual(exc.exception.message, 'Promise %r failed.' % second_promise)
+    self.assertEqual(str(exc.exception), 'Promise %r failed.' % second_promise)
 
     first_result = json.load(open(first_state_file))
     second_result = json.load(open(second_state_file))
@@ -781,7 +781,7 @@ class RunPromise(GenericPromise):
       f.write("""#!/bin/bash
 echo "success"
 """)
-    os.chmod(promise_path, 0744)
+    os.chmod(promise_path, 0o744)
 
     def test_method(result):
       self.called = True
@@ -810,13 +810,13 @@ echo "success"
 echo "This promise failed"
 exit 1
 """)
-    os.chmod(promise_path, 0744)
+    os.chmod(promise_path, 0o744)
 
     self.configureLauncher()
     state_file = os.path.join(self.partition_dir, PROMISE_STATE_FOLDER_NAME)
     with self.assertRaises(PromiseError) as exc:
       self.launcher.run()
-    self.assertEqual(exc.exception.message, 'Promise %r failed.' % promise_name)
+    self.assertEqual(str(exc.exception), 'Promise %r failed.' % promise_name)
 
   def test_runpromise_wrapped_mixed(self):
     self.called = 0
@@ -841,13 +841,13 @@ exit 1
 echo "promise 1 succeeded"
 exit 0
 """)
-    os.chmod(promise_path, 0744)
+    os.chmod(promise_path, 0o744)
     with open(promise_path2, 'w') as f:
       f.write("""#!/bin/bash
 echo "promise 2 succeeded"
 exit 0
 """)
-    os.chmod(promise_path2, 0744)
+    os.chmod(promise_path2, 0o744)
     self.generatePromiseScript("first_promise.py", success=True)
     self.generatePromiseScript("second_promise.py", success=True)
 
@@ -989,7 +989,7 @@ exit 0
 sleep 20
 echo "success"
 """)
-    os.chmod(promise_path, 0744)
+    os.chmod(promise_path, 0o744)
 
     def test_method(result):
       self.called = True
@@ -1025,7 +1025,7 @@ sleep 20
 echo "success"
 exit 1
 """)
-      os.chmod(promise_path, 0744)
+      os.chmod(promise_path, 0o744)
 
     createPromise(first_promise_path)
     createPromise(second_promise_path)
@@ -1073,7 +1073,7 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
     self.configureLauncher()
     self.generatePromiseScript(self.promise_name, periodicity=1, content=promise_content, success=success)
     self.writeInit()
-    self.queue = Queue.Queue()
+    self.queue = queue.Queue()
     self.promise_config = {
       'log-folder': self.log_dir,
       'partition-folder': self.partition_dir,

slapos/tests/slap.py

@@ -25,16 +25,18 @@
 #
 ##############################################################################
 
+from __future__ import print_function
+
 import logging
 import os
 import unittest
-import urlparse
+from six.moves.urllib import parse
 import tempfile
 
 import httmock
 
 import slapos.slap
-import xml_marshaller
+from slapos.util import dumps
 
 
 class UndefinedYetException(Exception):
@@ -51,11 +53,10 @@ class SlapMixin(unittest.TestCase):
       self.server_url = 'http://localhost/'
     else:
       self.server_url = self._server_url
-    print 'Testing against SLAP server %r' % self.server_url
+    print('Testing against SLAP server %r' % self.server_url)
     self.slap = slapos.slap.slap()
     self.partition_id = 'PARTITION_01'
-    if os.environ.has_key('SLAPGRID_INSTANCE_ROOT'):
-      del os.environ['SLAPGRID_INSTANCE_ROOT']
+    os.environ.pop('SLAPGRID_INSTANCE_ROOT', None)
 
   def tearDown(self):
     pass
@@ -185,7 +186,7 @@ class TestSlap(SlapMixin):
     self.slap.registerComputer(computer_guid)
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition'
             and qs == {
                 'computer_reference': [computer_guid],
@@ -194,7 +195,7 @@ class TestSlap(SlapMixin):
         partition = slapos.slap.ComputerPartition(computer_guid, partition_id)
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(partition)
+                'content': dumps(partition)
                 }
       else:
         return {'status_code': 400}
@@ -226,7 +227,7 @@ class TestSlap(SlapMixin):
     partition_id = 'PARTITION_01'
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition'
             and qs == {
                 'computer_reference': [computer_guid],
@@ -318,12 +319,12 @@ class TestSlap(SlapMixin):
     software_release_url_list = ['1', '2']
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/getSoftwareReleaseListFromSoftwareProduct'
             and qs == {'software_product_reference': [software_product_reference]}):
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(software_release_url_list)
+                'content': dumps(software_release_url_list)
                 }
 
     with httmock.HTTMock(handler):
@@ -344,12 +345,12 @@ class TestSlap(SlapMixin):
     software_release_url_list = ['1', '2']
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/getSoftwareReleaseListFromSoftwareProduct'
          and qs == {'software_release_url': [software_release_url]}):
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(software_release_url_list)
+                'content': dumps(software_release_url_list)
                 }
 
     with httmock.HTTMock(handler):
@@ -385,7 +386,7 @@ class TestSlap(SlapMixin):
     """
     hateoas_url = 'foo'
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/getHateoasUrl'):
         return {
                 'status_code': 200,
@@ -406,7 +407,7 @@ class TestSlap(SlapMixin):
     """
     hateoas_url = 'foo'
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/getHateoasUrl'):
         self.fail('slap should not have contacted master to get Hateoas URL.')
 
@@ -424,7 +425,7 @@ class TestSlap(SlapMixin):
     """
     hateoas_url = 'foo'
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/getHateoasUrl'):
         return {
                 'status_code': 404,
@@ -450,7 +451,7 @@ class TestComputer(SlapMixin):
     slap.initializeConnection(self.server_url)
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition'
               and 'computer_reference' in qs
               and 'computer_partition_reference' in qs):
@@ -459,7 +460,7 @@ class TestComputer(SlapMixin):
             qs['computer_partition_reference'][0])
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(slap_partition)
+                'content': dumps(slap_partition)
                 }
       elif (url.path == '/getFullComputerInformation'
               and 'computer_id' in qs):
@@ -468,7 +469,7 @@ class TestComputer(SlapMixin):
         slap_computer._computer_partition_list = []
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(slap_computer)
+                'content': dumps(slap_computer)
                 }
       elif url.path == '/requestComputerPartition':
         return {'status_code': 408}
@@ -520,7 +521,7 @@ class TestComputer(SlapMixin):
     self.slap.initializeConnection(self.server_url)
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition'
             and qs == {
                 'computer_reference': [self.computer_guid],
@@ -529,7 +530,7 @@ class TestComputer(SlapMixin):
         partition = slapos.slap.ComputerPartition(self.computer_guid, partition_id)
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(partition)
+                'content': dumps(partition)
                 }
       elif (url.path == '/getFullComputerInformation'
               and 'computer_id' in qs):
@@ -537,7 +538,7 @@ class TestComputer(SlapMixin):
         slap_computer._computer_partition_list = []
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(slap_computer)
+                'content': dumps(slap_computer)
                 }
       else:
         return {'status_code': 400}
@@ -602,7 +603,7 @@ class TestComputerPartition(SlapMixin):
     partition_id = 'PARTITION_01'
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition'
               and 'computer_reference' in qs
               and 'computer_partition_reference' in qs):
@@ -611,7 +612,7 @@ class TestComputerPartition(SlapMixin):
             qs['computer_partition_reference'][0])
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(slap_partition)
+                'content': dumps(slap_partition)
                 }
       elif (url.path == '/getComputerInformation'
               and 'computer_id' in qs):
@@ -623,7 +624,7 @@ class TestComputerPartition(SlapMixin):
         slap_computer._computer_partition_list = [slap_partition]
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(slap_computer)
+                'content': dumps(slap_computer)
                 }
       elif url.path == '/requestComputerPartition':
         raise RequestWasCalled
@@ -647,7 +648,7 @@ class TestComputerPartition(SlapMixin):
     partition_id = 'PARTITION_01'
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition'
               and 'computer_reference' in qs
               and 'computer_partition_reference' in qs):
@@ -656,7 +657,7 @@ class TestComputerPartition(SlapMixin):
             qs['computer_partition_reference'][0])
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(slap_partition)
+                'content': dumps(slap_partition)
                 }
       elif (url.path == '/getComputerInformation'
               and 'computer_id' in qs):
@@ -668,7 +669,7 @@ class TestComputerPartition(SlapMixin):
         slap_computer._computer_partition_list = [slap_partition]
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(slap_computer)
+                'content': dumps(slap_computer)
                 }
       elif url.path == '/requestComputerPartition':
         return {'status_code': 408}
@@ -691,7 +692,7 @@ class TestComputerPartition(SlapMixin):
     partition_id = 'PARTITION_01'
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition' and
               'computer_reference' in qs and
               'computer_partition_reference' in qs):
@@ -700,7 +701,7 @@ class TestComputerPartition(SlapMixin):
             qs['computer_partition_reference'][0])
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(slap_partition)
+                'content': dumps(slap_partition)
                 }
       elif (url.path == '/getComputerInformation'
               and 'computer_id' in qs):
@@ -712,7 +713,7 @@ class TestComputerPartition(SlapMixin):
         slap_computer._computer_partition_list = [slap_partition]
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(slap_computer)
+                'content': dumps(slap_computer)
                 }
       elif url.path == '/requestComputerPartition':
         return {'status_code': 408}
@@ -740,7 +741,7 @@ class TestComputerPartition(SlapMixin):
     computer_guid = self._getTestComputerId()
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition' and
               'computer_reference' in qs and
               'computer_partition_reference' in qs):
@@ -749,7 +750,7 @@ class TestComputerPartition(SlapMixin):
             qs['computer_partition_reference'][0])
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(slap_partition)
+                'content': dumps(slap_partition)
                 }
       elif (url.path == '/getComputerInformation' and 'computer_id' in qs):
         slap_computer = slapos.slap.Computer(qs['computer_id'][0])
@@ -760,7 +761,7 @@ class TestComputerPartition(SlapMixin):
         slap_computer._computer_partition_list = [slap_partition]
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(slap_computer)
+                'content': dumps(slap_computer)
                 }
       elif url.path == '/requestComputerPartition':
         from slapos.slap.slap import SoftwareInstance
@@ -769,7 +770,7 @@ class TestComputerPartition(SlapMixin):
             slap_computer_partition_id=requested_partition_id)
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(slap_partition)
+                'content': dumps(slap_partition)
                 }
       else:
         return {'status_code': 404}
@@ -801,7 +802,7 @@ class TestComputerPartition(SlapMixin):
     transaction_file_path = os.path.join(partition_root, transaction_file_name)
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition'
               and 'computer_reference' in qs
               and 'computer_partition_reference' in qs):
@@ -810,7 +811,7 @@ class TestComputerPartition(SlapMixin):
             qs['computer_partition_reference'][0])
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(slap_partition)
+                'content': dumps(slap_partition)
                 }
       elif (url.path == '/getComputerInformation'
               and 'computer_id' in qs):
@@ -822,7 +823,7 @@ class TestComputerPartition(SlapMixin):
         slap_computer._computer_partition_list = [slap_partition]
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(slap_computer)
+                'content': dumps(slap_computer)
                 }
       elif url.path == '/requestComputerPartition':
         raise RequestWasCalled
@@ -848,7 +849,7 @@ class TestComputerPartition(SlapMixin):
                         'software_type', 'myref')
       self.assertTrue(os.path.exists(transaction_file_path))
       with open(transaction_file_path, 'r') as f:
-        content_list = f.read().strip().split('\n')
+        content_list = f.read().splitlines()
         self.assertEqual(content_list, ['myref'])
 
       # Not override
@@ -856,7 +857,7 @@ class TestComputerPartition(SlapMixin):
           self.computer_guid, partition_id)
       self.assertTrue(os.path.exists(transaction_file_path))
       with open(transaction_file_path, 'r') as f:
-        content_list = f.read().strip().split('\n')
+        content_list = f.read().splitlines()
         self.assertEqual(content_list, ['myref'])
 
       # Request a second instance
@@ -865,8 +866,8 @@ class TestComputerPartition(SlapMixin):
                         'http://server/new/' + self._getTestComputerId(),
                         'software_type', 'mysecondref')
       with open(transaction_file_path, 'r') as f:
-        content_list = f.read().strip().split('\n')
-        self.assertEqual(list(set(content_list)), ['myref', 'mysecondref'])
+        content_list = f.read().splitlines()
+        self.assertEqual(sorted(content_list), ['myref', 'mysecondref'])
 
   def _test_new_computer_partition_state(self, state):
     """
@@ -879,7 +880,7 @@ class TestComputerPartition(SlapMixin):
     slap.initializeConnection(self.server_url)
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition' and
               qs['computer_reference'][0] == computer_guid and
               qs['computer_partition_reference'][0] == partition_id):
@@ -887,7 +888,7 @@ class TestComputerPartition(SlapMixin):
             computer_guid, partition_id)
         return {
                 'status_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(partition)
+                'content': dumps(partition)
                 }
       else:
         return {'status_code': 404}
@@ -923,7 +924,7 @@ class TestComputerPartition(SlapMixin):
     slap.initializeConnection(self.server_url)
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition' and
               qs['computer_reference'][0] == computer_guid and
               qs['computer_partition_reference'][0] == partition_id):
@@ -931,10 +932,10 @@ class TestComputerPartition(SlapMixin):
             computer_guid, partition_id)
         return {
                 'statu_code': 200,
-                'content': xml_marshaller.xml_marshaller.dumps(partition)
+                'content': dumps(partition)
                 }
       elif url.path == '/softwareInstanceError':
-        parsed_qs_body = urlparse.parse_qs(req.body)
+        parsed_qs_body = parse.parse_qs(req.body)
         # XXX: why do we have computer_id and not computer_reference?
         # XXX: why do we have computer_partition_id and not
         # computer_partition_reference?
@@ -995,7 +996,7 @@ class TestSoftwareRelease(SlapMixin):
     slap.initializeConnection(self.server_url)
 
     def handler(url, req):
-      qs = urlparse.parse_qs(req.body)
+      qs = parse.parse_qs(req.body)
       if (url.path == '/softwareReleaseError' and
               qs['computer_id'][0] == computer_guid and
               qs['url'][0] == software_release_uri and
@@ -1079,7 +1080,7 @@ class TestOpenOrder(SlapMixin):
           slap_computer_partition_id=requested_partition_id)
       return {
               'status_code': 200,
-              'content': xml_marshaller.xml_marshaller.dumps(slap_partition)
+              'content': dumps(slap_partition)
               }
 
     with httmock.HTTMock(handler):
@@ -1106,7 +1107,7 @@ class TestOpenOrder(SlapMixin):
           slap_computer_partition_id=requested_partition_id)
       return {
               'status_code': 200,
-              'content': xml_marshaller.xml_marshaller.dumps(slap_partition)
+              'content': dumps(slap_partition)
               }
 
 
@@ -1139,7 +1140,7 @@ class TestOpenOrder(SlapMixin):
           slap_computer_partition_id=requested_partition_id)
       return {
               'status_code': 200,
-              'content': xml_marshaller.xml_marshaller.dumps(slap_partition)
+              'content': dumps(slap_partition)
               }
 
     with httmock.HTTMock(handler):
@@ -1205,7 +1206,7 @@ class TestSoftwareProductCollection(SlapMixin):
     self.assertEqual(self.product_collection.foo, '0')
 
 if __name__ == '__main__':
-  print 'You can point to any SLAP server by setting TEST_SLAP_SERVER_URL '\
-      'environment variable'
+  print('You can point to any SLAP server by setting TEST_SLAP_SERVER_URL'
+        ' environment variable')
   unittest.main()
 

slapos/tests/slapformat.py

@@ -26,6 +26,8 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import print_function
+
 import glob
 import logging
 import slapos.format
@@ -49,6 +51,8 @@ import mock
 
 from .slapgrid import DummyManager
 
+import six
+
 USER_LIST = []
 GROUP_LIST = []
 INTERFACE_DICT = {}
@@ -89,7 +93,7 @@ class FakeCallAndRead:
     retval = 0, 'UP'
     global INTERFACE_DICT
     if 'useradd' in argument_list:
-      print argument_list
+      print(argument_list)
       global USER_LIST
       username = argument_list[-1]
       if username == '-r':
@@ -130,7 +134,7 @@ class LoggableWrapper:
 
   def __call__(self, *args, **kwargs):
     arg_list = [repr(x) for x in args] + [
-      '%s=%r' % (x, y) for x, y in kwargs.iteritems()]
+      '%s=%r' % (x, y) for x, y in six.iteritems(kwargs)]
     self.__logger.debug('%s(%s)' % (self.__name, ', '.join(arg_list)))
 
 
@@ -197,6 +201,8 @@ class SlapformatMixin(unittest.TestCase):
     self.netifaces = NetifacesMock()
     self.saved_netifaces = {}
     for fake in vars(NetifacesMock):
+      if fake.startswith("__"):
+        continue
       self.saved_netifaces[fake] = getattr(netifaces, fake, None)
       setattr(netifaces, fake, getattr(self.netifaces, fake))
 
@@ -208,6 +214,8 @@ class SlapformatMixin(unittest.TestCase):
   def patchPwd(self):
     self.saved_pwd = {}
     for fake in vars(PwdMock):
+      if fake.startswith("__"):
+        continue
       self.saved_pwd[fake] = getattr(pwd, fake, None)
       setattr(pwd, fake, getattr(PwdMock, fake))
 
@@ -219,6 +227,8 @@ class SlapformatMixin(unittest.TestCase):
   def patchTime(self):
     self.saved_time = {}
     for fake in vars(TimeMock):
+      if fake.startswith("__"):
+        continue
       self.saved_time[fake] = getattr(time, fake, None)
       setattr(time, fake, getattr(TimeMock, fake))
 
@@ -230,6 +240,8 @@ class SlapformatMixin(unittest.TestCase):
   def patchGrp(self):
     self.saved_grp = {}
     for fake in vars(GrpMock):
+      if fake.startswith("__"):
+        continue
       self.saved_grp[fake] = getattr(grp, fake, None)
       setattr(grp, fake, getattr(GrpMock, fake))
 

slapos/tests/slapgrid.py

@@ -38,13 +38,13 @@ import tempfile
 import textwrap
 import time
 import unittest
-import urlparse
+import six
+from six.moves.urllib import parse
 import json
 import re
 
-import xml_marshaller
 from mock import patch
-from zope import interface
+from zope.interface import implementer
 
 import slapos.slap.slap
 import slapos.grid.utils
@@ -57,6 +57,7 @@ from slapos.manager.interface import IManager
 from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
 import slapos.grid.SlapObject
 from slapos import manager as slapmanager
+from slapos.util import dumps
 
 import httmock
 
@@ -113,7 +114,7 @@ class BasicMixin(object):
     self.manager_list = []
     self.software_root = os.path.join(self._tempdir, 'software')
     self.instance_root = os.path.join(self._tempdir, 'instance')
-    if os.environ.has_key('SLAPGRID_INSTANCE_ROOT'):
+    if 'SLAPGRID_INSTANCE_ROOT' in os.environ:
       del os.environ['SLAPGRID_INSTANCE_ROOT']
     logging.basicConfig(level=logging.DEBUG)
     self.setSlapgrid()
@@ -197,7 +198,7 @@ class BasicMixin(object):
     instance_list.append('etc')
     instance_list.append('var')
     instance_list.append('supervisord.socket')
-    self.assertItemsEqual(os.listdir(self.instance_root), instance_list)
+    six.assertCountEqual(self, os.listdir(self.instance_root), instance_list)
 
   def tearDown(self):
     # XXX: Hardcoded pid, as it is not configurable in slapos
@@ -314,21 +315,21 @@ class ComputerForTest(object):
     """
     self.sequence.append(url.path)
     if req.method == 'GET':
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
     else:
-      qs = urlparse.parse_qs(req.body)
+      qs = parse.parse_qs(req.body)
     if (url.path == '/getFullComputerInformation'
             and 'computer_id' in qs):
       slap_computer = self.getComputer(qs['computer_id'][0])
       return {
               'status_code': 200,
-              'content': xml_marshaller.xml_marshaller.dumps(slap_computer)
+              'content': dumps(slap_computer)
               }
     elif url.path == '/getHostingSubscriptionIpList':
       ip_address_list = self.ip_address_list
       return {
               'status_code': 200,
-              'content': xml_marshaller.xml_marshaller.dumps(ip_address_list)
+              'content': dumps(ip_address_list)
               }
     if req.method == 'POST' and 'computer_partition_id' in qs:
       instance = self.instance_list[int(qs['computer_partition_id'][0])]
@@ -549,9 +550,8 @@ touch worked"""):
       fout.write(str(periodicity))
 
 
+@implementer(IManager)
 class DummyManager(object):
-  interface.implements(IManager)
-
   def __init__(self):
     self.sequence = []
 
@@ -584,7 +584,7 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
     with httmock.HTTMock(computer.request_handler):
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual([])
-      self.assertItemsEqual(os.listdir(self.software_root), [])
+      six.assertCountEqual(self, os.listdir(self.software_root), [])
       st = os.stat(os.path.join(self.instance_root, 'var'))
       self.assertEqual(stat.S_IMODE(st.st_mode), 0o755)
 
@@ -595,9 +595,9 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
-      self.assertItemsEqual(os.listdir(partition), ['.slapgrid', 'buildout.cfg',
+      six.assertCountEqual(self, os.listdir(partition), ['.slapgrid', 'buildout.cfg',
                                                     'software_release', 'worked', '.slapos-retention-lock-delay'])
-      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
+      six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       self.assertEqual(computer.sequence,
                        ['/getFullComputerInformation', 
                         '/stoppedComputerPartition'])
@@ -613,9 +613,9 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
-      self.assertItemsEqual(os.listdir(partition), ['.slapgrid', 'buildout.cfg',
+      six.assertCountEqual(self, os.listdir(partition), ['.slapgrid', 'buildout.cfg',
                                                     'software_release', 'worked', '.slapos-retention-lock-delay'])
-      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
+      six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       self.assertEqual(computer.sequence,
                        ['/getFullComputerInformation', 
                         '/stoppedComputerPartition'])
@@ -632,8 +632,8 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
       partition.requested_state = 'destroyed'
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(partition.partition_path), [])
-      self.assertItemsEqual(os.listdir(self.software_root), [])
+      six.assertCountEqual(self, os.listdir(partition.partition_path), [])
+      six.assertCountEqual(self, os.listdir(self.software_root), [])
       self.assertEqual(partition.sequence, [])
 
   def test_one_partition_started(self):
@@ -644,12 +644,12 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
       partition.software.setBuildout(WRAPPER_CONTENT)
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(partition.partition_path),
+      six.assertCountEqual(self, os.listdir(partition.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
       wrapper_log = os.path.join(partition.partition_path, '.0_wrapper.log')
       self.assertLogContent(wrapper_log, 'Working')
-      self.assertItemsEqual(os.listdir(self.software_root), [partition.software.software_hash])
+      six.assertCountEqual(self, os.listdir(self.software_root), [partition.software.software_hash])
       self.assertEqual(computer.sequence,
                        ['/getFullComputerInformation', 
                         '/startedComputerPartition'])
@@ -663,12 +663,12 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
       partition.software.setBuildout(WRAPPER_CONTENT)
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(partition.partition_path),
+      six.assertCountEqual(self, os.listdir(partition.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
       wrapper_log = os.path.join(partition.partition_path, '.0_wrapper.log')
       self.assertLogContent(wrapper_log, 'Working')
-      self.assertItemsEqual(os.listdir(self.software_root), [partition.software.software_hash])
+      six.assertCountEqual(self, os.listdir(self.software_root), [partition.software.software_hash])
       self.assertEqual(computer.sequence,
                        ['/getFullComputerInformation', 
                         '/startedComputerPartition'])
@@ -680,7 +680,7 @@ exit 1
 """)
       self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_FAIL)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(instance.partition_path),
+      six.assertCountEqual(self, os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked',
                              '.slapos-retention-lock-delay', '.slapgrid-0-error.log'])
@@ -702,27 +702,28 @@ mkdir -p etc/run &&
 (
 cat <<'HEREDOC'
 #!%(python)s
+from __future__ import print_function
 import signal
 def handler(signum, frame):
   for i in range(30):
-    print 'Signal handler called with signal', signum
+    print('Signal handler called with signal', signum)
   raise SystemExit
 signal.signal(signal.SIGTERM, handler)
 
 while True:
-  print "Working"
+  print("Working")
 HEREDOC
 )> etc/run/wrapper &&
 chmod 755 etc/run/wrapper
 """ % {'python': sys.executable})
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(instance.partition_path),
+      six.assertCountEqual(self, os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
       wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
       self.assertLogContent(wrapper_log, 'Working')
-      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
+      six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       self.assertEqual(computer.sequence,
                        ['/getFullComputerInformation', 
                         '/startedComputerPartition'])
@@ -732,7 +733,7 @@ chmod 755 etc/run/wrapper
       instance.requested_state = 'stopped'
       self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(instance.partition_path),
+      six.assertCountEqual(self, os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
       self.assertLogContent(wrapper_log, 'Signal handler called with signal 15')
@@ -758,27 +759,28 @@ mkdir -p etc/run &&
 (
 cat <<'HEREDOC'
 #!%(python)s
+from __future__ import print_function
 import signal
 def handler(signum, frame):
   for i in range(30):
-    print 'Signal handler called with signal', signum
+    print('Signal handler called with signal', signum)
   raise SystemExit
 signal.signal(signal.SIGTERM, handler)
 
 while True:
-  print "Working"
+  print("Working")
 HEREDOC
 )> etc/run/wrapper &&
 chmod 755 etc/run/wrapper
 """ % {'python': sys.executable})
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(instance.partition_path),
+      six.assertCountEqual(self, os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
       wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
       self.assertLogContent(wrapper_log, 'Working')
-      self.assertItemsEqual(os.listdir(self.software_root),
+      six.assertCountEqual(self, os.listdir(self.software_root),
                             [instance.software.software_hash])
       self.assertEqual(computer.sequence,
                        ['/getFullComputerInformation', 
@@ -792,7 +794,7 @@ exit 1
 """)
       self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_FAIL)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(instance.partition_path),
+      six.assertCountEqual(self, os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked',
                              '.slapos-retention-lock-delay', '.slapgrid-0-error.log'])
@@ -812,9 +814,9 @@ exit 1
 
       self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
-      self.assertItemsEqual(os.listdir(partition),
+      six.assertCountEqual(self, os.listdir(partition),
                             ['.slapgrid', 'buildout.cfg', 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
-      self.assertItemsEqual(os.listdir(self.software_root),
+      six.assertCountEqual(self, os.listdir(self.software_root),
                             [instance.software.software_hash])
       self.assertEqual(computer.sequence,
                        ['/getFullComputerInformation', 
@@ -826,10 +828,10 @@ exit 1
       self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
-      self.assertItemsEqual(os.listdir(partition),
+      six.assertCountEqual(self, os.listdir(partition),
                             ['.slapgrid', '.0_wrapper.log', 'etc',
                              'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
-      self.assertItemsEqual(os.listdir(self.software_root),
+      six.assertCountEqual(self, os.listdir(self.software_root),
                             [instance.software.software_hash])
       wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
       self.assertLogContent(wrapper_log, 'Working')
@@ -856,8 +858,8 @@ exit 1
 
       self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
-      self.assertItemsEqual(os.listdir(partition), ['.slapgrid', dummy_file_name])
-      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
+      six.assertCountEqual(self, os.listdir(partition), ['.slapgrid', dummy_file_name])
+      six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       self.assertEqual(computer.sequence,
                        ['/getFullComputerInformation',
                         '/stoppedComputerPartition'])
@@ -900,7 +902,7 @@ class TestSlapgridCPWithMasterWatchdog(MasterMixin, unittest.TestCase):
 
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(partition.partition_path),
+      six.assertCountEqual(self, os.listdir(partition.partition_path),
                             ['.slapgrid', '.0_daemon.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
       daemon_log = os.path.join(partition.partition_path, '.0_daemon.log')
@@ -946,7 +948,7 @@ class TestSlapgridCPWithMasterWatchdog(MasterMixin, unittest.TestCase):
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       time.sleep(1)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(partition.partition_path),
+      six.assertCountEqual(self, os.listdir(partition.partition_path),
                             ['.slapgrid', '.0_daemon.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay',
                              'launched', 'crashed'])
@@ -1229,9 +1231,9 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
-      self.assertItemsEqual(os.listdir(partition),
+      six.assertCountEqual(self, os.listdir(partition),
                             ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
-      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
+      six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       timestamp_path = os.path.join(instance.partition_path, '.timestamp')
       self.setSlapgrid()
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
@@ -1249,10 +1251,10 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
-      self.assertItemsEqual(os.listdir(partition),
+      six.assertCountEqual(self, os.listdir(partition),
                             ['.slapgrid', '.timestamp', 'buildout.cfg',
                              'software_release', 'worked', '.slapos-retention-lock-delay'])
-      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
+      six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
 
       self.assertEqual(self.launchSlapgrid(develop=True),
                        slapgrid.SLAPGRID_SUCCESS)
@@ -1272,9 +1274,9 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
-      self.assertItemsEqual(os.listdir(partition),
+      six.assertCountEqual(self, os.listdir(partition),
                             ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
-      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
+      six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       instance.timestamp = str(int(timestamp) - 1)
       self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
       self.assertEqual(instance.sequence,
@@ -1290,9 +1292,9 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
-      self.assertItemsEqual(os.listdir(partition),
+      six.assertCountEqual(self, os.listdir(partition),
                             ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
-      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
+      six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       instance.timestamp = str(int(timestamp) + 1)
       self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
       self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
@@ -1315,9 +1317,9 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       self.launchSlapgrid()
       self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
-      self.assertItemsEqual(os.listdir(partition),
+      six.assertCountEqual(self, os.listdir(partition),
                             ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
-      self.assertItemsEqual(os.listdir(self.software_root),
+      six.assertCountEqual(self, os.listdir(self.software_root),
                             [instance.software.software_hash])
       instance.timestamp = None
       self.launchSlapgrid()
@@ -1344,7 +1346,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
 
       self.launchSlapgrid()
       partition = os.path.join(self.instance_root, '0')
-      self.assertItemsEqual(os.listdir(partition),
+      six.assertCountEqual(self, os.listdir(partition),
                             ['.slapgrid', '.timestamp', 'buildout.cfg',
                              'software_release', 'worked', '.slapos-retention-lock-delay'])
 
@@ -1354,7 +1356,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       instance.install = lambda: None
 
       self.launchSlapgrid()
-      self.assertItemsEqual(os.listdir(partition),
+      six.assertCountEqual(self, os.listdir(partition),
                             ['.slapgrid', '.timestamp', 'buildout.cfg',
                              'software_release', 'worked', '.slapos-retention-lock-delay'])
 
@@ -1638,12 +1640,12 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
       instance.software.setBuildout(WRAPPER_CONTENT)
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(instance.partition_path),
+      six.assertCountEqual(self, os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
       wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
       self.assertLogContent(wrapper_log, 'Working')
-      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
+      six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       self.assertEqual(computer.sequence,
                        ['/getFullComputerInformation',
                         
@@ -1656,8 +1658,8 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is empty
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(instance.partition_path), [])
-      self.assertItemsEqual(os.listdir(self.software_root),
+      six.assertCountEqual(self, os.listdir(instance.partition_path), [])
+      six.assertCountEqual(self, os.listdir(self.software_root),
                             [instance.software.software_hash])
       # Assert supervisor stopped process
       wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
@@ -1687,8 +1689,8 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is empty
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(instance.partition_path), [])
-      self.assertItemsEqual(os.listdir(self.software_root),
+      six.assertCountEqual(self, os.listdir(instance.partition_path), [])
+      six.assertCountEqual(self, os.listdir(self.software_root),
                             [instance.software.software_hash])
       # Assert supervisor stopped process
       wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
@@ -1709,12 +1711,12 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
       instance.software.setBuildout(WRAPPER_CONTENT)
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(instance.partition_path),
+      six.assertCountEqual(self, os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
       wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
       self.assertLogContent(wrapper_log, 'Working')
-      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
+      six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       self.assertEqual(computer.sequence,
                        ['/getFullComputerInformation',
                         
@@ -1728,14 +1730,14 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
       from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
       request_list_file = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % instance.name
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(instance.partition_path),
+      six.assertCountEqual(self, os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked',
                              '.slapos-retention-lock-delay', request_list_file])
       wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
       self.assertLogContent(wrapper_log, 'Working')
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(instance.partition_path),
+      six.assertCountEqual(self, os.listdir(instance.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked',
                              '.slapos-retention-lock-delay', request_list_file])
@@ -1760,8 +1762,8 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is empty
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(instance.partition_path), [])
-      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
+      six.assertCountEqual(self, os.listdir(instance.partition_path), [])
+      six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       self.assertEqual(computer.sequence, ['/getFullComputerInformation'])
 
   def test_slapgrid_report_ignore_free_instance(self):
@@ -1779,8 +1781,8 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is empty
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(instance.partition_path), [])
-      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
+      six.assertCountEqual(self, os.listdir(instance.partition_path), [])
+      six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       self.assertEqual(computer.sequence, ['/getFullComputerInformation'])
 
 
@@ -2033,7 +2035,7 @@ exit 1
       self.assertEqual(self.grid.processComputerPartitionList(),
                        slapos.grid.slapgrid.SLAPGRID_FAIL)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(instance.partition_path),
+      six.assertCountEqual(self, os.listdir(instance.partition_path),
                             ['.slapgrid', 'buildout.cfg', 'software_release',
                              '.slapgrid-0-error.log'])
 
@@ -2059,7 +2061,7 @@ exit 1
       self.assertEqual(self.grid.processComputerPartitionList(),
                        slapos.grid.slapgrid.SLAPGRID_FAIL)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(instance.partition_path),
+      six.assertCountEqual(self, os.listdir(instance.partition_path),
                             ['.slapgrid', 'buildout.cfg', 'software_release',
                              '.slapgrid-0-error.log'])
 
@@ -2151,8 +2153,8 @@ echo "ERROR: $var"
 exit 1
 """)
 
-    os.chmod(self.firewall_cmd_add, 0755)
-    os.chmod(self.firewall_cmd_remove, 0755)
+    os.chmod(self.firewall_cmd_add, 0o755)
+    os.chmod(self.firewall_cmd_remove, 0o755)
 
     firewall_conf= dict(
       authorized_sources=source_ip,
@@ -2543,9 +2545,9 @@ exit 0
       os.makedirs(pre_delete_dir, 0o700)
       with open(pre_delete_script, 'w') as f:
         f.write(self.prerm_script_content)
-      os.chmod(pre_delete_script, 0754)
+      os.chmod(pre_delete_script, 0o754)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(partition.partition_path),
+      six.assertCountEqual(self, os.listdir(partition.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
       self.assertEqual(computer.sequence,
@@ -2559,12 +2561,12 @@ exit 0
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is not destroyed (pre-delete is running)
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(partition.partition_path),
+      six.assertCountEqual(self, os.listdir(partition.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay',
                              '.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list',
                              '.slapos-request-transaction-0'])
-      self.assertItemsEqual(os.listdir(self.software_root),
+      six.assertCountEqual(self, os.listdir(self.software_root),
                             [partition.software.software_hash])
 
       # wait until the pre-delete script is finished
@@ -2573,7 +2575,7 @@ exit 0
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is empty
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(partition.partition_path), [])
+      six.assertCountEqual(self, os.listdir(partition.partition_path), [])
 
   def test_partition_destroy_pre_remove_with_retention_lock(self):
     computer = ComputerForTest(self.software_root, self.instance_root)
@@ -2593,7 +2595,7 @@ exit 0
       os.makedirs(pre_delete_dir, 0o700)
       with open(pre_delete_script, 'w') as f:
         f.write(self.prerm_script_content)
-      os.chmod(pre_delete_script, 0754)
+      os.chmod(pre_delete_script, 0o754)
       self.assertTrue(os.path.exists(pre_delete_script))
 
       manager_list = slapmanager.from_config({'manager_list': 'prerm'})
@@ -2602,7 +2604,7 @@ exit 0
       partition.requested_state = 'destroyed'
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is not destroyed (retention-delay-lock)
-      self.assertItemsEqual(os.listdir(partition.partition_path),
+      six.assertCountEqual(self, os.listdir(partition.partition_path),
                             ['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
                              'worked', '.slapos-retention-lock-delay',
                              '.slapos-retention-lock-date', '.slapos-request-transaction-0'])
@@ -2616,7 +2618,7 @@ exit 0
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
 
       # Assert partition directory is not destroyed (pre-delete is running)
-      self.assertItemsEqual(os.listdir(partition.partition_path),
+      six.assertCountEqual(self, os.listdir(partition.partition_path),
                             ['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
                              'worked', '.slapos-retention-lock-delay', '.slapos-retention-lock-date',
                              '.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list',
@@ -2627,7 +2629,7 @@ exit 0
 
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is empty
-      self.assertItemsEqual(os.listdir(partition.partition_path), [])
+      six.assertCountEqual(self, os.listdir(partition.partition_path), [])
 
   def test_partition_destroy_pre_remove_script_not_stopped(self):
     computer = ComputerForTest(self.software_root, self.instance_root)
@@ -2640,7 +2642,7 @@ exit 0
       os.makedirs(pre_delete_dir, 0o700)
       with open(pre_delete_script, 'w') as f:
         f.write(self.prerm_script_content)
-      os.chmod(pre_delete_script, 0754)
+      os.chmod(pre_delete_script, 0o754)
       self.assertEqual(partition.state, 'started')
       manager_list = slapmanager.from_config({'manager_list': 'prerm'})
       self.grid._manager_list = manager_list
@@ -2648,7 +2650,7 @@ exit 0
       partition.requested_state = 'destroyed'
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is not destroyed (pre-delete is running)
-      self.assertItemsEqual(os.listdir(partition.partition_path),
+      six.assertCountEqual(self, os.listdir(partition.partition_path),
                             ['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
                              'worked', '.slapos-retention-lock-delay', '.slapos-request-transaction-0',
                              '.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list'])
@@ -2662,7 +2664,7 @@ exit 0
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is empty
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(partition.partition_path), [])
+      six.assertCountEqual(self, os.listdir(partition.partition_path), [])
 
   def test_partition_destroy_pre_remove_script_run_as_partition_user(self):
     computer = ComputerForTest(self.software_root, self.instance_root)
@@ -2675,7 +2677,7 @@ exit 0
       os.makedirs(pre_delete_dir, 0o700)
       with open(pre_delete_script, 'w') as f:
         f.write(self.prerm_script_content)
-      os.chmod(pre_delete_script, 0754)
+      os.chmod(pre_delete_script, 0o754)
 
       manager_list = slapmanager.from_config({'manager_list': 'prerm'})
       self.grid._manager_list = manager_list
@@ -2683,7 +2685,7 @@ exit 0
       partition.requested_state = 'destroyed'
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is not destroyed (pre-delete is running)
-      self.assertItemsEqual(os.listdir(partition.partition_path),
+      six.assertCountEqual(self, os.listdir(partition.partition_path),
                             ['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
                              'worked', '.slapos-retention-lock-delay', '.slapos-request-transaction-0',
                              '.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list'])
@@ -2714,7 +2716,7 @@ exit 0
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is empty
       self.assertInstanceDirectoryListEqual(['0'])
-      self.assertItemsEqual(os.listdir(partition.partition_path), [])
+      six.assertCountEqual(self, os.listdir(partition.partition_path), [])
 
 
 # test that slapgrid commands, like e.g. 'slapos node software' do not leak

slapos/tests/slapproxy/__init__.py

@@ -28,7 +28,8 @@
 #
 ##############################################################################
 
-import ConfigParser
+import six
+from six.moves import configparser
 import os
 import logging
 import shutil
@@ -38,14 +39,12 @@ import sys
 import tempfile
 import time
 import unittest
-import xml_marshaller
-from xml_marshaller.xml_marshaller import loads, dumps
 
 import slapos.proxy
 import slapos.proxy.views as views
 import slapos.slap
 import slapos.slap.slap
-from slapos.util import sqlite_connect
+from slapos.util import loads, dumps, sqlite_connect, bytes2str
 
 import sqlite3
 import pkg_resources
@@ -77,7 +76,8 @@ class BasicMixin(object):
     self.startProxy()
 
   def createSlapOSConfigurationFile(self):
-    open(self.slapos_cfg, 'w').write("""[slapos]
+    with open(self.slapos_cfg, 'w') as f:
+      f.write("""[slapos]
 software_root = %(tempdir)s/opt/slapgrid
 instance_root = %(tempdir)s/srv/slapgrid
 master_url = %(proxyaddr)s
@@ -106,7 +106,7 @@ database_uri = %(tempdir)s/lib/proxy.db
     Set config for slapproxy and start it
     """
     conf = slapos.proxy.ProxyConfig(logger=logging.getLogger())
-    configp = ConfigParser.SafeConfigParser()
+    configp = configparser.SafeConfigParser()
     configp.read(self.slapos_cfg)
     conf.mergeConfig(ProxyOption(self.proxy_db), configp)
     conf.setConfig()
@@ -142,7 +142,7 @@ database_uri = %(tempdir)s/lib/proxy.db
 
     request_dict = {
         'computer_id': self.computer_id,
-        'xml': xml_marshaller.xml_marshaller.dumps(computer_dict),
+        'xml': dumps(computer_dict),
     }
     rv = self.app.post('/loadComputerConfigurationFromXML',
                   data=request_dict)
@@ -168,7 +168,7 @@ class TestInformation(BasicMixin, unittest.TestCase):
     """
     rv = self.app.get('/getComputerInformation?computer_id=%s' % self.computer_id)
     self.assertIsInstance(
-        xml_marshaller.xml_marshaller.loads(rv.data),
+        loads(rv.data),
         slapos.slap.Computer)
     self.assertTrue(os.path.exists(self.proxy_db))
 
@@ -179,7 +179,7 @@ class TestInformation(BasicMixin, unittest.TestCase):
     """
     rv = self.app.get('/getFullComputerInformation?computer_id=%s' % self.computer_id)
     self.assertIsInstance(
-        xml_marshaller.xml_marshaller.loads(rv.data),
+        loads(rv.data),
         slapos.slap.Computer)
     self.assertTrue(os.path.exists(self.proxy_db))
 
@@ -197,7 +197,7 @@ class TestInformation(BasicMixin, unittest.TestCase):
     """
     self.add_free_partition(10)
     rv = self.app.get('/getFullComputerInformation?computer_id=%s' % self.computer_id)
-    computer = xml_marshaller.xml_marshaller.loads(rv.data)
+    computer = loads(rv.data)
     for slap_partition in computer._computer_partition_list:
         self.assertIsNone(slap_partition._software_release_document)
         self.assertEqual(slap_partition._requested_state, 'destroyed')
@@ -216,8 +216,7 @@ class TestInformation(BasicMixin, unittest.TestCase):
     response = self.app.get('/getSoftwareReleaseListFromSoftwareProduct'
                             '?software_product_reference=%s' %\
                             software_product_reference)
-    software_release_url_list = xml_marshaller.xml_marshaller.loads(
-        response.data)
+    software_release_url_list = loads(response.data)
     self.assertEqual(
         software_release_url_list,
         [software_release_url]
@@ -232,8 +231,7 @@ class TestInformation(BasicMixin, unittest.TestCase):
     self.app_config['software_product_list'] = {'random': 'random'}
     response = self.app.get('/getSoftwareReleaseListFromSoftwareProduct'
                             '?software_product_reference=idonotexist')
-    software_release_url_list = xml_marshaller.xml_marshaller.loads(
-        response.data)
+    software_release_url_list = loads(response.data)
     self.assertEqual(
         software_release_url_list,
         []
@@ -269,7 +267,7 @@ class TestInformation(BasicMixin, unittest.TestCase):
     rv = self.app.get(
       '/getComputerPartitionCertificate?computer_id=%s&computer_partition_id=%s' % (
       self.computer_id, 'slappart0'))
-    response = xml_marshaller.xml_marshaller.loads(rv.data)
+    response = loads(rv.data)
     self.assertEqual({'certificate': '', 'key': ''}, response)
 
   def test_computerBang(self):
@@ -277,7 +275,7 @@ class TestInformation(BasicMixin, unittest.TestCase):
     Tests that computerBang method is implemented in slapproxy.
     """
     rv = self.app.post( '/computerBang?computer_id=%s' % ( self.computer_id))
-    response = xml_marshaller.xml_marshaller.loads(rv.data)
+    response = loads(rv.data)
     self.assertEqual('', response)
 
 class MasterMixin(BasicMixin, unittest.TestCase):
@@ -305,11 +303,10 @@ class MasterMixin(BasicMixin, unittest.TestCase):
         'software_release': software_release,
         'software_type': software_type,
         'partition_reference': partition_reference,
-        'shared_xml': xml_marshaller.xml_marshaller.dumps(shared),
-        'partition_parameter_xml': xml_marshaller.xml_marshaller.dumps(
-            partition_parameter_kw),
-        'filter_xml': xml_marshaller.xml_marshaller.dumps(filter_kw),
-        'state': xml_marshaller.xml_marshaller.dumps(state),
+        'shared_xml': dumps(shared),
+        'partition_parameter_xml': dumps(partition_parameter_kw),
+        'filter_xml': dumps(filter_kw),
+        'state': dumps(state),
     }
     return self.app.post('/requestComputerPartition', data=request_dict)
 
@@ -321,7 +318,7 @@ class MasterMixin(BasicMixin, unittest.TestCase):
     rv = self._requestComputerPartition(*args, **kwargs)
     self.assertEqual(rv._status_code, 200)
     xml = rv.data
-    software_instance = xml_marshaller.xml_marshaller.loads(xml)
+    software_instance = loads(xml)
 
     computer_partition = slapos.slap.ComputerPartition(
         software_instance.slap_computer_id,
@@ -343,7 +340,7 @@ class MasterMixin(BasicMixin, unittest.TestCase):
     self.app.post('/setComputerPartitionConnectionXml', data={
         'computer_id': self.computer_id,
         'computer_partition_id': partition_id,
-        'connection_xml': xml_marshaller.xml_marshaller.dumps(connection_dict),
+        'connection_xml': dumps(connection_dict),
         'slave_reference': slave_reference})
 
   def getPartitionInformation(self, computer_partition_id):
@@ -351,7 +348,7 @@ class MasterMixin(BasicMixin, unittest.TestCase):
     Return computer information as stored in proxy for corresponding id
     """
     rv = self.app.get('/getFullComputerInformation?computer_id=%s' % self.computer_id)
-    computer = xml_marshaller.xml_marshaller.loads(rv.data)
+    computer = loads(rv.data)
     for instance in computer._computer_partition_list:
       if instance._partition_id == computer_partition_id:
         return instance
@@ -1006,7 +1003,8 @@ class TestMultiMasterSupport(MasterMixin):
     super(TestMultiMasterSupport, self).tearDown()
 
   def createExternalProxyConfigurationFile(self):
-    open(self.external_slapproxy_configuration_file_location, 'w').write("""[slapos]
+    with open(self.external_slapproxy_configuration_file_location, 'w') as f:
+      f.write("""[slapos]
 computer_id = %(external_computer_id)s
 [slapproxy]
 host = %(host)s
@@ -1040,7 +1038,7 @@ database_uri = %(tempdir)s/lib/external_proxy.db
         self.external_proxy_slap._connection_helper.GET('/')
       except slapos.slap.NotFoundError:
         break
-      except slapos.slap.ConnectionError, socket.error:
+      except (slapos.slap.ConnectionError, socket.error):
         attempts = attempts + 1
         time.sleep(0.1)
     else:
@@ -1054,9 +1052,9 @@ database_uri = %(tempdir)s/lib/external_proxy.db
     Overwrite default slapos configuration file to enable specific multimaster
     behaviours.
     """
-    configuration = pkg_resources.resource_stream(
+    configuration = bytes2str(pkg_resources.resource_string(
         'slapos.tests.slapproxy', 'slapos_multimaster.cfg.in'
-    ).read() % {
+    )) % {
         'tempdir': self._tempdir, 'proxyaddr': self.proxyaddr,
         'external_proxy_host': self.external_proxy_host,
         'external_proxy_port': self.external_proxy_port
@@ -1089,7 +1087,7 @@ database_uri = %(tempdir)s/lib/external_proxy.db
 
     request_dict = {
         'computer_id': self.computer_id,
-        'xml': xml_marshaller.xml_marshaller.dumps(computer_dict),
+        'xml': dumps(computer_dict),
     }
     self.external_proxy_slap._connection_helper.POST('/loadComputerConfigurationFromXML',
                                                      data=request_dict)
@@ -1121,7 +1119,7 @@ database_uri = %(tempdir)s/lib/external_proxy.db
     external_slap.initializeConnection(self.external_master_url)
     external_computer = external_slap.registerComputer(self.external_computer_id)
     external_partition = external_computer.getComputerPartitionList()[0]
-    for k, v in partition_parameter_kw.iteritems():
+    for k, v in six.iteritems(partition_parameter_kw):
       self.assertEqual(
           external_partition.getInstanceParameter(k),
           v
@@ -1146,7 +1144,7 @@ database_uri = %(tempdir)s/lib/external_proxy.db
         '/getFullComputerInformation?computer_id=%s' % self.computer_id
     ).data)
     partition = computer._computer_partition_list[0]
-    for k, v in partition_parameter_kw.iteritems():
+    for k, v in six.iteritems(partition_parameter_kw):
       self.assertEqual(
           partition.getInstanceParameter(k),
           v
@@ -1251,7 +1249,7 @@ database_uri = %(tempdir)s/lib/external_proxy.db
         '/getFullComputerInformation?computer_id=%s' % self.computer_id
     ).data)
     partition = computer._computer_partition_list[0]
-    for k, v in dummy_parameter_dict.iteritems():
+    for k, v in six.iteritems(dummy_parameter_dict):
       self.assertEqual(
           partition.getInstanceParameter(k),
           v
@@ -1271,8 +1269,10 @@ class TestMigrateVersion10To11(TestInformation, TestRequest, TestSlaveRequest, T
   """
   def setUp(self):
     super(TestMigrateVersion10To11, self).setUp()
-    schema = pkg_resources.resource_stream('slapos.tests.slapproxy', 'database_dump_version_10.sql')
-    schema = schema.read() % dict(version='11')
+    schema = bytes2str(pkg_resources.resource_string(
+      'slapos.tests.slapproxy',
+      'database_dump_version_10.sql'
+    )) % dict(version='11')
     self.db = sqlite_connect(self.proxy_db)
     self.db.cursor().executescript(schema)
     self.db.commit()

slapos/tests/util.py

@@ -65,9 +65,9 @@ class TestUtil(unittest.TestCase):
     wanted_directory0 = os.path.join(root_slaptest, 'slap-write0')
     wanted_directory1 = os.path.join(root_slaptest, 'slap-write0', 'write-slap1')
     wanted_directory2 = os.path.join(root_slaptest, 'slap-write0', 'write-slap1', 'write-teste2')
-    wanted_directory_mkdir0 = os.makedirs(wanted_directory0, mode=0777)
-    wanted_directory_mkdir1 = os.makedirs(wanted_directory1, mode=0777)
-    wanted_directory_mkdir2 = os.makedirs(wanted_directory2, mode=0777)
+    wanted_directory_mkdir0 = os.makedirs(wanted_directory0, mode=0o777)
+    wanted_directory_mkdir1 = os.makedirs(wanted_directory1, mode=0o777)
+    wanted_directory_mkdir2 = os.makedirs(wanted_directory2, mode=0o777)
     create_file_txt = tempfile.mkstemp(suffix='.txt', prefix='tmp', dir=wanted_directory2, text=True)
     user = 'nobody'
     try:
@@ -109,23 +109,14 @@ class TestUtil(unittest.TestCase):
     shutil.rmtree(root_slaptest)
 
   def test_string_to_boolean_with_true_values(self):
-    """
-    Check that mkdir_p doesn't raise if directory already exist.
-    """
     for value in ['true', 'True', 'TRUE']:
       self.assertTrue(string_to_boolean(value))
 
   def test_string_to_boolean_with_false_values(self):
-    """
-    Check that mkdir_p doesn't raise if directory already exist.
-    """
     for value in ['false', 'False', 'False']:
       self.assertFalse(string_to_boolean(value))
 
   def test_string_to_boolean_with_incorrect_values(self):
-    """
-    Check that mkdir_p doesn't raise if directory already exist.
-    """
     for value in [True, False, 1, '1', 't', 'tru', 'truelle', 'f', 'fals', 'falsey']:
       self.assertRaises(ValueError, string_to_boolean, value)
 

slapos/util.py

@@ -33,6 +33,7 @@ import socket
 import struct
 import subprocess
 import sqlite3
+from xml_marshaller.xml_marshaller import dumps, loads
 
 
 def mkdir_p(path, mode=0o700):
@@ -86,16 +87,9 @@ def string_to_boolean(string):
 
   The parser is completely arbitrary, see code for actual implementation.
   """
-  if not isinstance(string, str) and not isinstance(string, unicode):
-    raise ValueError('Given value is not a string.')
-  acceptable_true_values = ['true']
-  acceptable_false_values = ['false']
-  string = string.lower()
-  if string in acceptable_true_values:
-    return True
-  if string in acceptable_false_values:
-    return False
-  else:
+  try:
+    return ('false', 'true').index(string.lower())
+  except Exception:
     raise ValueError('%s is neither True nor False.' % string)
 
 
@@ -138,3 +132,15 @@ def ipv6FromBin(ip, suffix=''):
 def lenNetmaskIpv6(netmask):
   return len(binFromIpv6(netmask).rstrip('0'))
 
+# Used for Python 2-3 compatibility
+if str is bytes:
+  bytes2str = str2bytes = lambda s: s
+  def unicode2str(s):
+    return s.encode('utf-8')
+else:
+  def bytes2str(s):
+    return s.decode()
+  def str2bytes(s):
+    return s.encode()
+  def unicode2str(s):
+    return s

[Jérome Perrin]

This seems to be a change in behavior, before this change, if sensors command is not available, the test fail. Which I believe is a good thing, it was hiding errors.

Isn't the missing part that we need to include component/lmsensors/buildout.cfg in the PATH of the test here ? similar to what we do for socat for example ?

/cc @rafael @tomo

[Jérome Perrin]

Apparently this is also something we can get from psutil https://psutil.readthedocs.io/en/latest/index.html#psutil.sensors_temperatures

[Jérome Perrin]

Test can fail because command is not available (like on my development environment), but also some tests fail because command exit with non 0, for example:

test_computer_snapshot (slapos.tests.collect.TestCollectSnapshot) ... No sensors found!
Make sure you loaded all the kernel drivers you need.
Try sensors-detect to find out which these are.
ERROR

I noticed this on rapidspace-testnode-003-3Nodes-ERP5PROJECT3 , also on RAPIDCLOUD-VM-COMP-2867-3Nodes-ERP5PROJECT6

[Rafael Monnerat]

it should fail if sensor is not available.... and it should get sensors from the component like you said.

[Jérome Perrin]

Thanks. So if sensors command is not available it should fail. OK, let's change the software profile to add the command from component in $PATH -> slapos!479 (merged)

But what to do when sensors command is available and it does no find sensors (the No sensors found! error message from above) ? slapos!479 (merged) won't solve this case.

Also according to https://github.com/lm-sensors/lm-sensors , this is linux only, psutil version seems a bit more portable but it's not something that's universally available, so slapos should maybe consider this as something optional - only required on linux for example

[Jérome Perrin]

From my understanding of the discussions in slapos!479 (comment 71803) , here we should not use check_output but an approach which would tolerate a non 0 exit code here.

In !73 (merged) we discuss adding subprocess32, so we can maybe switch to subprocess.run here ?

[Jérome Perrin]

for the subprocess.run approach -> !81 (merged)

[Jérome Perrin]

Isn't it better to let the TypeError propagate in this case ? Without knowing under which conditions these response without status code occur, it sounds like a error case, wasn't it ?

The concern is about the try: except: pass that might catch too much error.