wip

slapos/cli/configure_local/__init__.py

@@ -126,7 +126,8 @@ def _replaceParameterValue(original_content, to_replace):
 
 def _generateSlaposNodeConfigurationFile(slapos_node_config_path, args):
     template_arg_list = (__name__, '../../slapos.cfg.example')
-    slapos_node_configuration_template = pkg_resources.resource_string(*template_arg_list).decode('utf-8')
+    slapos_node_configuration_template = \
+      pkg_resources.resource_string(*template_arg_list).decode('utf-8')
     master_url = 'http://%s:%s' % (args.daemon_listen_ip, args.daemon_listen_port)
     slapos_home = args.slapos_buildout_directory
     to_replace = [
@@ -152,7 +153,8 @@ def _generateSlaposNodeConfigurationFile(slapos_node_config_path, args):
 
 def _generateSlaposProxyConfigurationFile(conf):
     template_arg_list = (__name__, '../../slapos-proxy.cfg.example')
-    slapos_proxy_configuration_template = pkg_resources.resource_string(*template_arg_list).decode('utf-8')
+    slapos_proxy_configuration_template = \
+      pkg_resources.resource_string(*template_arg_list).decode('utf-8')
     slapos_proxy_configuration_path = os.path.join(
       conf.slapos_configuration_directory, 'slapos-proxy.cfg')
     listening_ip, listening_port = \

slapos/client.py

@@ -119,7 +119,7 @@ def _getSoftwareReleaseFromSoftwareString(logger, software_string, product):
     try:
         return product.__getattr__(software_string[len(SOFTWARE_PRODUCT_NAMESPACE):])
     except AttributeError as e:
-       logger.error('Error: %s Exiting now.' % e)
+       logger.error('Error: %s Exiting now.', e)
        sys.exit(1)
 
 def do_console(local):

slapos/collect/reporter.py

@@ -27,8 +27,6 @@
 #
 ##############################################################################
 
-from six.moves import zip
-
 from lxml import etree as ElementTree
 from slapos.util import mkdir_p
 
@@ -137,19 +135,17 @@ class ConsumptionReportBase(object):
 
   def getPartitionCPULoadAverage(self, partition_id, date_scope):
     self.db.connect()
-    query_result_cursor = self.db.select("user", date_scope,
+    cpu_percent_sum = self.db.select("user", date_scope,
                        columns="SUM(cpu_percent)", 
                        where="partition = '%s'" % partition_id)
 
-    cpu_percent_sum = list(zip(*query_result_cursor))
     if len(cpu_percent_sum) and cpu_percent_sum[0][0] is None:
       return
 
-    query_result_cursor = self.db.select("user", date_scope,
+    sample_amount = self.db.select("user", date_scope,
                        columns="COUNT(DISTINCT time)", 
                        where="partition = '%s'" % partition_id)
 
-    sample_amount = list(zip(*query_result_cursor))
     self.db.close()
 
     if len(sample_amount) and len(cpu_percent_sum):
@@ -157,19 +153,17 @@ class ConsumptionReportBase(object):
 
   def getPartitionUsedMemoryAverage(self, partition_id, date_scope):
     self.db.connect()
-    query_result_cursor = self.db.select("user", date_scope,
+    memory_sum = self.db.select("user", date_scope,
                        columns="SUM(memory_rss)", 
                        where="partition = '%s'" % partition_id)
 
-    memory_sum = list(zip(*query_result_cursor))
     if len(memory_sum) and memory_sum[0][0] is None:
       return
 
-    query_result_cursor = self.db.select("user", date_scope,
+    sample_amount = self.db.select("user", date_scope,
                        columns="COUNT(DISTINCT time)", 
                        where="partition = '%s'" % partition_id)
 
-    sample_amount = list(zip(*query_result_cursor))
     self.db.close()
 
     if len(sample_amount) and len(memory_sum):
@@ -177,18 +171,16 @@ class ConsumptionReportBase(object):
 
   def getPartitionDiskUsedAverage(self, partition_id, date_scope):
     self.db.connect()
-    query_result_cursor = self.db.select("folder", date_scope,
+    disk_used_sum = self.db.select("folder", date_scope,
                        columns="SUM(disk_used)", 
                        where="partition = '%s'" % partition_id)
 
-    disk_used_sum = list(zip(*query_result_cursor))
     if len(disk_used_sum) and disk_used_sum[0][0] is None:
       return
-    query_result_cursor = self.db.select("folder", date_scope,
+    collect_amount = self.db.select("folder", date_scope,
                        columns="COUNT(DISTINCT time)", 
                        where="partition = '%s'" % partition_id)
 
-    collect_amount = list(zip(*query_result_cursor))
     self.db.close()
 
     if len(collect_amount) and len(disk_used_sum):
@@ -289,7 +281,7 @@ class ConsumptionReport(ConsumptionReportBase):
                            reference=user,
                            category="")
 
-     with open(xml_report_path, 'w') as f:
+     with open(xml_report_path, 'wb') as f:
        f.write(journal.getXML())
        f.close()
 
@@ -300,20 +292,18 @@ class ConsumptionReport(ConsumptionReportBase):
 
   def _getCpuLoadAverageConsumption(self, date_scope):
     self.db.connect()
-    query_result_cursor = self.db.select("system", date_scope, 
+    cpu_load_percent_list = self.db.select("system", date_scope, 
                        columns="SUM(cpu_percent)/COUNT(cpu_percent)")
 
-    cpu_load_percent_list = list(zip(*query_result_cursor))
     self.db.close()
     if len(cpu_load_percent_list):
       return cpu_load_percent_list[0][0]
 
   def _getMemoryAverageConsumption(self, date_scope):
     self.db.connect()
-    query_result_cursor = self.db.select("system", date_scope, 
+    memory_used_list = self.db.select("system", date_scope, 
                        columns="SUM(memory_used)/COUNT(memory_used)")
 
-    memory_used_list = list(zip(*query_result_cursor))
     self.db.close()
     if len(memory_used_list):
       return memory_used_list[0][0]
@@ -331,7 +321,7 @@ class Journal(object):
 
    def getXML(self):
      report = ElementTree.tostring(self.root) 
-     return "<?xml version='1.0' encoding='utf-8'?>%s" % report
+     return b"<?xml version='1.0' encoding='utf-8'?>%s" % report
    
    def newTransaction(self, portal_type="Sale Packing List"):
      transaction = ElementTree.SubElement(self.root, "transaction")

slapos/grid/SlapObject.py

@@ -292,8 +292,8 @@ class Software(object):
         f.close()
 
   def _create_buildout_profile(self, buildout_cfg, url):
-    with open(buildout_cfg, 'wb') as fout:
-      fout.write(('[buildout]\nextends = ' + url + '\n').encode('utf-8'))
+    with open(buildout_cfg, 'w') as fout:
+      fout.write('[buildout]\nextends = ' + url + '\n')
     self._set_ownership(buildout_cfg)
 
   def uploadSoftwareRelease(self, tarpath):

slapos/grid/networkcache.py

@@ -30,8 +30,8 @@ try:
     else:
         LIBNETWORKCACHE_ENABLED = True
 except:
-    print('There was problem while trying to import slapos.libnetworkcache:'\
-        '\n%s' % traceback.format_exc())
+    print('There was problem while trying to import slapos.libnetworkcache:\n%s'
+          % traceback.format_exc())
     LIBNETWORKCACHE_ENABLED = False
     print('Networkcache forced to be disabled.')
 

slapos/grid/promise/__init__.py

@@ -38,7 +38,7 @@ import importlib
 import traceback
 import psutil
 from multiprocessing import Process, Queue as MQueue
-from six.moves import queue
+from six.moves import queue, reload_module
 from slapos.util import mkdir_p, chownDirectory
 from slapos.grid.utils import dropPrivileges, killProcessTree
 from slapos.grid.promise import interface
@@ -195,7 +195,7 @@ class PromiseProcess(Process):
 
     if promise_module.__file__ != self.promise_path:
       # cached module need to be updated
-      promise_module = reload(promise_module)
+      promise_module = reload_module(promise_module)
     # load extra parameters
     self._loadPromiseParameterDict(promise_module)
 
@@ -208,7 +208,7 @@ class PromiseProcess(Process):
       if not isinstance(extra_dict, dict):
         raise ValueError("Extra parameter is not a dict")
       for key in extra_dict:
-        if self.argument_dict.has_key(key):
+        if key in self.argument_dict:
           raise ValueError("Extra parameter name %r cannot be used.\n%s" % (
                            key, extra_dict))
         self.argument_dict[key] = extra_dict[key]

slapos/grid/slapgrid.py

@@ -32,7 +32,7 @@ import os
 import pkg_resources
 import random
 import socket
-from six import StringIO
+from io import BytesIO
 import subprocess
 import sys
 import tempfile
@@ -1309,7 +1309,7 @@ stderr_logfile_backups=1
   def validateXML(self, to_be_validated, xsd_model):
     """Validates a given xml file"""
     #We retrieve the xsd model
-    xsd_model = StringIO.StringIO(xsd_model)
+    xsd_model = BytesIO(xsd_model)
     xmlschema_doc = etree.parse(xsd_model)
     xmlschema = etree.XMLSchema(xmlschema_doc)
 

slapos/grid/utils.py

@@ -352,10 +352,12 @@ def launchBuildout(path, buildout_binary, logger,
 def updateFile(file_path, content, mode=0o600):
   """Creates or updates a file with "content" as content."""
   altered = False
+  if isinstance(content, six.text_type):
+    content = content.encode('utf-8')
   if not (os.path.isfile(file_path)) or \
      not (hashlib.md5(open(file_path, 'rb').read()).digest() ==
           hashlib.md5(content).digest()):
-    with open(file_path, 'w') as fout:
+    with open(file_path, 'wb') as fout:
       fout.write(content)
     altered = True
   os.chmod(file_path, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)

slapos/proxy/views.py

@@ -55,19 +55,10 @@ class UnauthorizedError(Exception):
   pass
 
 
-# cast everything to string, utf-8 encoded
-def to_str(v):
-  if isinstance(v, six.binary_type):
-    return v
-  if not isinstance(v, six.text_type):
-    v = six.text_type(v)
-  return v.encode('utf-8')
-
-
 def xml2dict(xml):
   result_dict = {}
-  if xml is not None and xml != '':
-    tree = etree.fromstring(to_str(xml))
+  if xml:
+    tree = etree.fromstring(xml)
     for element in tree.iter(tag=etree.Element):
       if element.tag == 'parameter':
         key = element.get('id')
@@ -83,8 +74,6 @@ def xml2dict(xml):
 def dict2xml(dictionary):
   instance = etree.Element('instance')
   for parameter_id, parameter_value in six.iteritems(dictionary):
-    # cast everything to string
-    parameter_value = six.text_type(parameter_value)
     etree.SubElement(instance, "parameter",
                      attrib={'id': parameter_id}).text = parameter_value
   return etree.tostring(instance,
@@ -511,8 +500,6 @@ def forwardRequestToExternalMaster(master_url, request_form):
   new_request_form['filter_xml'] = dumps(filter_kw)
 
   xml = slap._connection_helper.POST('/requestComputerPartition', data=new_request_form)
-  if type(xml) is six.text_type:
-    xml.encode('utf-8')
   partition = loads(xml)
 
   # XXX move to other end

slapos/slap/slap.py

@@ -94,8 +94,6 @@ class SlapRequester(SlapDocument):
         request_dict=request_dict,
         connection_helper=self._connection_helper,
       )
-    if type(xml) is six.text_type:
-      xml.encode('utf-8')
     software_instance = xml_marshaller.loads(xml)
     computer_partition = ComputerPartition(
       software_instance.slap_computer_id.encode('UTF-8'),
@@ -206,12 +204,11 @@ class SoftwareInstance(SlapDocument):
   Contains Software Instance information
   """
 
-  def __init__(self, **kwargs):
+  def __init__(self, **kw):
     """
     Makes easy initialisation of class parameters
     """
-    for k, v in six.iteritems(kwargs):
-      setattr(self, k, v)
+    self.__dict__.update(kw)
 
 
 """Exposed exceptions"""
@@ -728,8 +725,6 @@ class ConnectionHelper:
       # We should stablise slap library soon.
       xml = self.GET('getComputerInformation', params=params)
 
-    if type(xml) is six.text_type:
-      xml.encode('utf-8')
     return xml_marshaller.loads(xml)
 
   def do_request(self, method, path, params=None, data=None, headers=None):

slapos/tests/slap.py

@@ -30,7 +30,7 @@ from __future__ import print_function
 import logging
 import os
 import unittest
-import urlparse
+from six.moves.urllib import parse
 import tempfile
 
 import httmock
@@ -187,7 +187,7 @@ class TestSlap(SlapMixin):
     self.slap.registerComputer(computer_guid)
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition'
             and qs == {
                 'computer_reference': [computer_guid],
@@ -228,7 +228,7 @@ class TestSlap(SlapMixin):
     partition_id = 'PARTITION_01'
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition'
             and qs == {
                 'computer_reference': [computer_guid],
@@ -320,7 +320,7 @@ class TestSlap(SlapMixin):
     software_release_url_list = ['1', '2']
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/getSoftwareReleaseListFromSoftwareProduct'
             and qs == {'software_product_reference': [software_product_reference]}):
         return {
@@ -346,7 +346,7 @@ class TestSlap(SlapMixin):
     software_release_url_list = ['1', '2']
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/getSoftwareReleaseListFromSoftwareProduct'
          and qs == {'software_release_url': [software_release_url]}):
         return {
@@ -387,7 +387,7 @@ class TestSlap(SlapMixin):
     """
     hateoas_url = 'foo'
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/getHateoasUrl'):
         return {
                 'status_code': 200,
@@ -408,7 +408,7 @@ class TestSlap(SlapMixin):
     """
     hateoas_url = 'foo'
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/getHateoasUrl'):
         self.fail('slap should not have contacted master to get Hateoas URL.')
 
@@ -426,7 +426,7 @@ class TestSlap(SlapMixin):
     """
     hateoas_url = 'foo'
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/getHateoasUrl'):
         return {
                 'status_code': 404,
@@ -452,7 +452,7 @@ class TestComputer(SlapMixin):
     slap.initializeConnection(self.server_url)
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition'
               and 'computer_reference' in qs
               and 'computer_partition_reference' in qs):
@@ -522,7 +522,7 @@ class TestComputer(SlapMixin):
     self.slap.initializeConnection(self.server_url)
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition'
             and qs == {
                 'computer_reference': [self.computer_guid],
@@ -604,7 +604,7 @@ class TestComputerPartition(SlapMixin):
     partition_id = 'PARTITION_01'
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition'
               and 'computer_reference' in qs
               and 'computer_partition_reference' in qs):
@@ -649,7 +649,7 @@ class TestComputerPartition(SlapMixin):
     partition_id = 'PARTITION_01'
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition'
               and 'computer_reference' in qs
               and 'computer_partition_reference' in qs):
@@ -693,7 +693,7 @@ class TestComputerPartition(SlapMixin):
     partition_id = 'PARTITION_01'
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition' and
               'computer_reference' in qs and
               'computer_partition_reference' in qs):
@@ -742,7 +742,7 @@ class TestComputerPartition(SlapMixin):
     computer_guid = self._getTestComputerId()
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition' and
               'computer_reference' in qs and
               'computer_partition_reference' in qs):
@@ -803,7 +803,7 @@ class TestComputerPartition(SlapMixin):
     transaction_file_path = os.path.join(partition_root, transaction_file_name)
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition'
               and 'computer_reference' in qs
               and 'computer_partition_reference' in qs):
@@ -881,7 +881,7 @@ class TestComputerPartition(SlapMixin):
     slap.initializeConnection(self.server_url)
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition' and
               qs['computer_reference'][0] == computer_guid and
               qs['computer_partition_reference'][0] == partition_id):
@@ -925,7 +925,7 @@ class TestComputerPartition(SlapMixin):
     slap.initializeConnection(self.server_url)
 
     def handler(url, req):
-      qs = urlparse.parse_qs(url.query)
+      qs = parse.parse_qs(url.query)
       if (url.path == '/registerComputerPartition' and
               qs['computer_reference'][0] == computer_guid and
               qs['computer_partition_reference'][0] == partition_id):
@@ -936,7 +936,7 @@ class TestComputerPartition(SlapMixin):
                 'content': xml_marshaller.xml_marshaller.dumps(partition)
                 }
       elif url.path == '/softwareInstanceError':
-        parsed_qs_body = urlparse.parse_qs(req.body)
+        parsed_qs_body = parse.parse_qs(req.body)
         # XXX: why do we have computer_id and not computer_reference?
         # XXX: why do we have computer_partition_id and not
         # computer_partition_reference?
@@ -997,7 +997,7 @@ class TestSoftwareRelease(SlapMixin):
     slap.initializeConnection(self.server_url)
 
     def handler(url, req):
-      qs = urlparse.parse_qs(req.body)
+      qs = parse.parse_qs(req.body)
       if (url.path == '/softwareReleaseError' and
               qs['computer_id'][0] == computer_guid and
               qs['url'][0] == software_release_uri and

slapos/tests/util.py

@@ -109,23 +109,14 @@ class TestUtil(unittest.TestCase):
     shutil.rmtree(root_slaptest)
 
   def test_string_to_boolean_with_true_values(self):
-    """
-    Check that mkdir_p doesn't raise if directory already exist.
-    """
     for value in ['true', 'True', 'TRUE']:
       self.assertTrue(string_to_boolean(value))
 
   def test_string_to_boolean_with_false_values(self):
-    """
-    Check that mkdir_p doesn't raise if directory already exist.
-    """
     for value in ['false', 'False', 'False']:
       self.assertFalse(string_to_boolean(value))
 
   def test_string_to_boolean_with_incorrect_values(self):
-    """
-    Check that mkdir_p doesn't raise if directory already exist.
-    """
     for value in [True, False, 1, '1', 't', 'tru', 'truelle', 'f', 'fals', 'falsey']:
       self.assertRaises(ValueError, string_to_boolean, value)
 

slapos/util.py

@@ -31,7 +31,6 @@ import errno
 import os
 import subprocess
 import sqlite3
-import six
 
 
 def mkdir_p(path, mode=0o700):
@@ -85,16 +84,11 @@ def string_to_boolean(string):
 
   The parser is completely arbitrary, see code for actual implementation.
   """
-  if not isinstance(string, six.binary_type) and not isinstance(string, six.text_type):
-    raise ValueError('Given value is not a string.')
-  acceptable_true_values = ['true']
-  acceptable_false_values = ['false']
-  string = string.lower()
-  if string in acceptable_true_values:
-    return True
-  if string in acceptable_false_values:
-    return False
-  else:
+  if isinstance(string, bytes):
+    string = string.decode('utf-8')
+  try:
+    return ('false', 'true').index(string.lower())
+  except Exception:
     raise ValueError('%s is neither True nor False.' % string)