wip

slapos/cli/configure_local/__init__.py

@@ -149,8 +149,8 @@ def _generateSlaposNodeConfigurationFile(slapos_node_config_path, args):
     slapos_node_configuration_content = re.sub(
         '(key_file|cert_file|certificate_repository_path).*=.*\n',
         '', slapos_node_configuration_content)
-    with open(slapos_node_config_path, 'wb') as fout:
-        fout.write(slapos_node_configuration_content.encode('utf8'))
+    with open(slapos_node_config_path, 'w') as fout:
+        fout.write(slapos_node_configuration_content)
 
 def _generateSlaposProxyConfigurationFile(conf):
     template_arg_list = (__name__, '../../slapos-proxy.cfg.example')
@@ -172,8 +172,8 @@ def _generateSlaposProxyConfigurationFile(conf):
     slapos_proxy_configuration_content = _replaceParameterValue(
         slapos_proxy_configuration_template, to_replace)
 
-    with open(slapos_proxy_configuration_path, 'wb') as fout:
-        fout.write(slapos_proxy_configuration_content.encode('utf8'))
+    with open(slapos_proxy_configuration_path, 'w') as fout:
+        fout.write(slapos_proxy_configuration_content)
 
     return slapos_proxy_configuration_path
 

slapos/client.py

@@ -28,7 +28,7 @@
 ##############################################################################
 
 import atexit
-import six.moves.configparser
+from six.moves import configparser
 import os
 import sys
 
@@ -52,7 +52,7 @@ class ClientConfig(object):
     # Merges the arguments and configuration
     try:
       configuration_dict = dict(configp.items('slapconsole'))
-    except ConfigParser.NoSectionError:
+    except configparser.NoSectionError:
       pass
     else:
       for key in configuration_dict:

slapos/collect/entity.py

@@ -74,19 +74,15 @@ class User(object):
       time_cycle = self.disk_snapshot_params.get('time_cycle', 0)
       database.connect()
       if time_cycle:
-        order = 'date DESC, time DESC'
-        limit = 1
-        query = database.select(table="folder", columns="date, time",
-                                    order=order, limit=limit,
-                                    where="partition='%s'" % self.name)
-        query_result = zip(*query)
-        if len(query_result):
-          date, time = (query_result[0][0], query_result[1][0])
-          latest_date = datetime.strptime('%s %s' % (date, time),
-                                           "%Y-%m-%d %H:%M:%S")
+        for date_time in database.select(table="folder", columns="date, time",
+                                    order='date DESC, time DESC', limit=1,
+                                    where="partition='%s'" % self.name):
+          latest_date = datetime.strptime('%s %s' % date_time,
+                                          "%Y-%m-%d %H:%M:%S")
           if (datetime.now() - latest_date).seconds < time_cycle:
             # wait the time cycle
             return
+          break
       pid_file = self.disk_snapshot_params.get('pid_folder', None)
       if pid_file is not None:
         pid_file = os.path.join(pid_file, '%s_disk_size.pid' % self.name)

slapos/collect/reporter.py

@@ -40,6 +40,8 @@ import tarfile
 import time
 import psutil
 
+import six
+
 log_file = False
 
 class Dumper(object):
@@ -59,10 +61,10 @@ class SystemReporter(Dumper):
     """ Dump data """
     _date = time.strftime("%Y-%m-%d")
     self.db.connect()
-    for item, collected_item_list in self.db.exportSystemAsDict(_date).iteritems():
+    for item, collected_item_list in six.iteritems(self.db.exportSystemAsDict(_date)):
       self.writeFile(item, folder, collected_item_list)
 
-    for partition, collected_item_list in self.db.exportDiskAsDict(_date).iteritems():
+    for partition, collected_item_list in six.iteritems(self.db.exportDiskAsDict(_date)):
       partition_id = "_".join(partition.split("-")[:-1]).replace("/", "_")
       item = "memory_%s" % partition.split("-")[-1]
       self.writeFile("disk_%s_%s" % (item, partition_id), folder, collected_item_list)

slapos/collect/snapshot.py

@@ -36,6 +36,8 @@ from .temperature import collectComputerTemperature, \
 
 from .temperature.heating import get_contribution_ratio
 
+import six
+
 MEASURE_INTERVAL = 5
 
 class _Snapshot(object):
@@ -242,6 +244,6 @@ class ComputerSnapshot(_Snapshot):
           DiskPartitionSnapshot(partition.device, 
                                 partition.mountpoint))
 
-    return [(k, v) for k, v in partition_dict.iteritems()]
+    return [(k, v) for k, v in six.iteritems(partition_dict)]
 
   

slapos/collect/temperature/__init__.py

@@ -18,7 +18,8 @@ def collectComputerTemperature(sensor_bin="sensors"):
   cmd = ["%s -u" % sensor_bin]
   
   sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, 
-                        stderr=subprocess.PIPE, shell=True)
+                        stderr=subprocess.PIPE, shell=True,
+                        universal_newlines=True)
   
   stdout, stderr = sp.communicate()
   

slapos/grid/promise/generic.py

@@ -161,8 +161,8 @@ class GenericPromise(object):
       self.logger.removeHandler(handler)
     if self.__log_folder is None:
       # configure logger with StringIO
-      import cStringIO
-      self.__logger_buffer = cStringIO.StringIO()
+      from six.moves import cStringIO
+      self.__logger_buffer = cStringIO()
       logger_handler = logging.StreamHandler(self.__logger_buffer)
       self.__log_file = None
     else:

slapos/manager/portredir.py

@@ -6,7 +6,7 @@ import os
 
 from .interface import IManager
 from six.moves import filter
-from zope import interface
+from zope.interface import implementer
 
 logger = logging.getLogger(__name__)
 
@@ -22,9 +22,8 @@ def which(exename):
       return full_path
   return None
 
+@implementer(IManager)
 class Manager(object):
-  interface.implements(IManager)
-
   port_redirect_filename = '.slapos-port-redirect'
 
   def __init__(self, config):

slapos/proxy/db_version.py

@@ -3,5 +3,5 @@
 import pkg_resources
 from slapos.util import bytes2str
 
-DB_VERSION = bytes2str(pkg_resources.resource_stream('slapos.proxy', 'schema.sql').readline().strip().split(b':')[1])
+DB_VERSION = bytes2str(pkg_resources.resource_stream('slapos.proxy', 'schema.sql').readline()).strip().split(':')[1]
 

slapos/proxy/views.py

@@ -298,7 +298,7 @@ def useComputer():
 @app.route('/loadComputerConfigurationFromXML', methods=['POST'])
 def loadComputerConfigurationFromXML():
   xml = request.form['xml']
-  computer_dict = xml_marshaller.xml_marshaller.loads(str(xml).encode('utf-8'))
+  computer_dict = xml_marshaller.xml_marshaller.loads(xml.encode('utf-8'))
   execute_db('computer', 'INSERT OR REPLACE INTO %s values(:reference, :address, :netmask)',
              computer_dict)
   for partition in computer_dict['partition_list']:
@@ -622,7 +622,7 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
   execute_db('partition', q, args)
   args = []
   partition = execute_db('partition', 'SELECT * FROM %s WHERE reference=? and computer_reference=?',
-      [partition['reference'].encode(), partition['computer_reference'].encode()], one=True)
+      [partition['reference'], partition['computer_reference']], one=True)
   address_list = []
   for address in execute_db('partition_network', 'SELECT * FROM %s WHERE partition_reference=?', [partition['reference']]):
     address_list.append((address['reference'], address['address']))

slapos/slap/slap.py

@@ -873,12 +873,12 @@ class slap:
       # XXX-Cedric: should raise something smarter than NotFound
       raise NotFoundError
 
-    xml = bytes(self._connection_helper.GET('registerComputerPartition',
+    xml = self._connection_helper.GET('registerComputerPartition',
             params = {
                 'computer_reference': computer_guid,
                 'computer_partition_reference': partition_id,
                 }
-            ))
+            )
     result = xml_marshaller.loads(xml)
     # XXX: dirty hack to make computer partition usable. xml_marshaller is too
     # low-level for our needs here.
@@ -913,7 +913,7 @@ class slap:
                              'software_release_url parameters are specified.')
       params['software_release_url'] = software_release_url
 
-    xml = bytes(self._connection_helper.GET(url, params=params))
+    xml = self._connection_helper.GET(url, params=params)
     result = xml_marshaller.loads(xml)
     assert(type(result) == list)
     return result

slapos/tests/collect.py

@@ -364,7 +364,7 @@ class TestCollectReport(unittest.TestCase):
         with tarfile.open("%s.tar.gz" % dump_folder) as tf:
             self.assertEquals(tf.getmembers()[0].name, "1990-01-01")
             self.assertEquals(tf.getmembers()[1].name, "1990-01-01/test.txt")
-            self.assertEquals(tf.extractfile(tf.getmembers()[1]).read(), 'hi')
+            self.assertEquals(tf.extractfile(tf.getmembers()[1]).read(), b'hi')
 
 class TestCollectSnapshot(unittest.TestCase):
 
@@ -482,10 +482,10 @@ class TestCollectEntity(unittest.TestCase):
         config.set('slapos', 'instance_root', self.instance_root)
  
         user_dict = entity.get_user_list(config)
-        username_list = ['slapuser0', 'slapuser1', 'slapuser2'] 
-        self.assertEquals(username_list, user_dict.keys())
+        username_set = {'slapuser0', 'slapuser1', 'slapuser2'} 
+        self.assertEquals(username_set, set(user_dict))
        
-        for name in username_list:
+        for name in username_set:
           self.assertEquals(user_dict[name].name, name)
           self.assertEquals(user_dict[name].snapshot_list, [])
           expected_path = "%s/slappart%s" % (self.instance_root, name.strip("slapuser")) 
@@ -508,11 +508,11 @@ class TestCollectEntity(unittest.TestCase):
 
         self.assertEquals(database.invoked_method_list[1][0], "insertUserSnapshot")
         self.assertEquals(database.invoked_method_list[1][1][0], ("fakeuser0",))
-        self.assertEquals(database.invoked_method_list[1][1][1].keys(), 
-                   ['cpu_time', 'cpu_percent', 'process',
+        self.assertEquals(set(database.invoked_method_list[1][1][1]), 
+                   {'cpu_time', 'cpu_percent', 'process',
                     'memory_rss', 'pid', 'memory_percent',
                     'io_rw_counter', 'insertion_date', 'insertion_time',
-                    'io_cycles_counter', 'cpu_num_threads'])
+                    'io_cycles_counter', 'cpu_num_threads'})
         self.assertEquals(database.invoked_method_list[2], ("commit", ""))
         self.assertEquals(database.invoked_method_list[3], ("close", ""))
 
@@ -527,19 +527,19 @@ class TestCollectEntity(unittest.TestCase):
 
         self.assertEquals(database.invoked_method_list[1][0], "insertUserSnapshot")
         self.assertEquals(database.invoked_method_list[1][1][0], ("fakeuser0",))
-        self.assertEquals(database.invoked_method_list[1][1][1].keys(), 
-                   ['cpu_time', 'cpu_percent', 'process',
+        self.assertEquals(set(database.invoked_method_list[1][1][1]), 
+                   {'cpu_time', 'cpu_percent', 'process',
                     'memory_rss', 'pid', 'memory_percent',
                     'io_rw_counter', 'insertion_date', 'insertion_time',
-                    'io_cycles_counter', 'cpu_num_threads'])
+                    'io_cycles_counter', 'cpu_num_threads'})
         self.assertEquals(database.invoked_method_list[2], ("commit", ""))
         self.assertEquals(database.invoked_method_list[3], ("close", ""))
 
         self.assertEquals(database.invoked_method_list[4], ("connect", ""))
         self.assertEquals(database.invoked_method_list[5][0], "inserFolderSnapshot")
         self.assertEquals(database.invoked_method_list[5][1][0], ("fakeuser0",))
-        self.assertEquals(database.invoked_method_list[5][1][1].keys(), 
-                   ['insertion_date', 'disk_usage', 'insertion_time'])
+        self.assertEquals(set(database.invoked_method_list[5][1][1]), 
+                   {'insertion_date', 'disk_usage', 'insertion_time'})
         self.assertEquals(database.invoked_method_list[6], ("commit", ""))
         self.assertEquals(database.invoked_method_list[7], ("close", ""))
 
@@ -554,23 +554,23 @@ class TestCollectEntity(unittest.TestCase):
 
         self.assertEquals(database.invoked_method_list[1][0], "insertUserSnapshot")
         self.assertEquals(database.invoked_method_list[1][1][0], ("fakeuser0",))
-        self.assertEquals(database.invoked_method_list[1][1][1].keys(), 
-                   ['cpu_time', 'cpu_percent', 'process',
+        self.assertEquals(set(database.invoked_method_list[1][1][1]), 
+                   {'cpu_time', 'cpu_percent', 'process',
                     'memory_rss', 'pid', 'memory_percent',
                     'io_rw_counter', 'insertion_date', 'insertion_time',
-                    'io_cycles_counter', 'cpu_num_threads'])
+                    'io_cycles_counter', 'cpu_num_threads'})
         self.assertEquals(database.invoked_method_list[2], ("commit", ""))
         self.assertEquals(database.invoked_method_list[3], ("close", ""))
 
         self.assertEquals(database.invoked_method_list[4], ("connect", ""))
         self.assertEquals(database.invoked_method_list[5][0], "select")
         self.assertEquals(database.invoked_method_list[5][1][0], ())
-        self.assertEquals(database.invoked_method_list[5][1][1].keys(),
-                                ['table', 'where', 'limit', 'order', 'columns'])
+        self.assertEquals(set(database.invoked_method_list[5][1][1]),
+                                {'table', 'where', 'limit', 'order', 'columns'})
         self.assertEquals(database.invoked_method_list[6][0], "inserFolderSnapshot")
         self.assertEquals(database.invoked_method_list[6][1][0], ("fakeuser0",))
-        self.assertEquals(database.invoked_method_list[6][1][1].keys(), 
-                   ['insertion_date', 'disk_usage', 'insertion_time'])
+        self.assertEquals(set(database.invoked_method_list[6][1][1]), 
+                   {'insertion_date', 'disk_usage', 'insertion_time'})
         self.assertEquals(database.invoked_method_list[7], ("commit", ""))
         self.assertEquals(database.invoked_method_list[8], ("close", ""))
 
@@ -583,14 +583,14 @@ class TestCollectEntity(unittest.TestCase):
 
         self.assertEquals(database.invoked_method_list[1][0], "insertComputerSnapshot")
         self.assertEquals(database.invoked_method_list[1][1][0], ())
-        self.assertEquals(database.invoked_method_list[1][1][1].keys(), 
-                 ['insertion_time', 'insertion_date', 'cpu_num_core',
+        self.assertEquals(set(database.invoked_method_list[1][1][1]), 
+                 {'insertion_time', 'insertion_date', 'cpu_num_core',
                   'partition_list', 'cpu_frequency', 'memory_size', 
-                  'cpu_type', 'memory_type'])
+                  'cpu_type', 'memory_type'})
  
         self.assertEquals(database.invoked_method_list[2][0], "insertSystemSnapshot")
         self.assertEquals(database.invoked_method_list[2][1][0], ())
-        self.assertEquals(set(database.invoked_method_list[2][1][1].keys()), 
+        self.assertEquals(set(database.invoked_method_list[2][1][1]), 
           set([ 'memory_used', 'cpu_percent', 'insertion_date', 'insertion_time',
                 'loadavg', 'memory_free', 'net_in_bytes', 'net_in_dropped', 
                 'net_in_errors', 'net_out_bytes', 'net_out_dropped', 
@@ -598,7 +598,7 @@ class TestCollectEntity(unittest.TestCase):
 
         self.assertEquals(database.invoked_method_list[3][0], "insertDiskPartitionSnapshot")
         self.assertEquals(database.invoked_method_list[3][1][0], ())
-        self.assertEquals(set(database.invoked_method_list[3][1][1].keys()), 
+        self.assertEquals(set(database.invoked_method_list[3][1][1]), 
           set([ 'used', 'insertion_date', 'partition', 'free', 
                 'mountpoint', 'insertion_time' ]))
 

slapos/util.py

@@ -84,8 +84,6 @@ def string_to_boolean(string):
 
   The parser is completely arbitrary, see code for actual implementation.
   """
-  if isinstance(string, bytes):
-    string = string.decode('utf-8')
   try:
     return ('false', 'true').index(string.lower())
   except Exception: