Add support for Python 3

slapos/crypto.py

@@ -11,7 +11,7 @@ FILETYPE_PEM = 1
 
 def load_privatekey(type, buffer):
   r = _tmpfile()
-  r.write(buffer)
+  r.write(buffer.encode())
   r.flush()
   return r
 
@@ -20,7 +20,7 @@ def load_certificate(type, buffer):
   r = _tmpfile()
   p = Popen(("openssl", "x509", "-pubkey", "-noout"),
             stdin=PIPE, stdout=r, stderr=PIPE)
-  err = p.communicate(buffer)[1]
+  err = p.communicate(buffer.encode())[1]
   if p.poll():
     raise Error(err)
   return r

slapos/libnetworkcache.py

@@ -13,10 +13,7 @@
 ##############################################################################
 
 import argparse
-import ConfigParser
 import hashlib
-import httplib
-import inspect
 import json
 import logging
 import os
@@ -26,8 +23,19 @@ import sys
 import tarfile
 import tempfile
 import traceback
-import urllib2
-import urlparse
+from base64 import b64encode, decodestring, encodestring
+try:
+  import configparser
+  from http.client import HTTPConnection, HTTPSConnection
+  from urllib.error import HTTPError
+  from urllib.parse import urlsplit
+  from urllib.request import urlopen
+  basestring = bytes, str
+except ImportError:
+  import ConfigParser as configparser
+  from httplib import HTTPConnection, HTTPSConnection
+  from urllib2 import HTTPError, urlopen
+  from urlparse import urlsplit
 try:
   from OpenSSL import crypto
 except ImportError:
@@ -52,17 +60,21 @@ class short_exc_info(tuple):
     l += traceback.format_exception_only(t, v)
     return ''.join(l).rstrip()
 
-def byteify(input):
+def strify(input):
   '''Transform every unicode string inside a list or dict into normal strings. '''
   if isinstance(input, dict):
-    return dict([(byteify(key), byteify(value)) for key, value in input.iteritems()])
+    return dict((strify(key), strify(value)) for key, value in input.items())
   elif isinstance(input, list):
-    return [byteify(element) for element in input]
+    return map(strify, input)
   elif isinstance(input, unicode):
     return input.encode('utf-8')
   else:
     return input
-
+try:
+  unicode
+except NameError:
+  def strify(input):
+    return input
 
 class NetworkcacheClient(object):
   '''
@@ -100,7 +112,7 @@ class NetworkcacheClient(object):
 
   def __new_init(self, config, signature_certificate_list=None):
     if not hasattr(config, 'get'):
-      parser = ConfigParser.SafeConfigParser()
+      parser = configparser.SafeConfigParser()
       parser.readfp(config)
       config = dict(parser.items('networkcache'))
     self.config = config
@@ -147,12 +159,12 @@ class NetworkcacheClient(object):
       method = 'PUT' if name else 'POST'
       url = self.config['upload-%s-url' % where]
       timeout = UPLOAD_TIMEOUT
-    parsed_url = urlparse.urlsplit(url.rstrip('/') + ('/' + name if name else ''))
+    parsed_url = urlsplit(url.rstrip('/') + ('/' + name if name else ''))
     if not headers:
       headers = {}
     if parsed_url.username:
-      headers['Authorization'] = 'Basic %s' % ('%s:%s' % (
-        parsed_url.username, parsed_url.password)).encode('base64').strip()
+      headers['Authorization'] = 'Basic ' + b64encode('%s:%s' % (
+        parsed_url.username, parsed_url.password))
     headers["Connection"] = "close"
     connection_kw = {
       'host': parsed_url.hostname,
@@ -162,23 +174,21 @@ class NetworkcacheClient(object):
     if parsed_url.scheme == 'https':
       connection_kw['cert_file'] = self.config['sha%s-cert-file' % where]
       connection_kw['key_file'] = self.config['sha%s-key-file' % where]
-      if 'context' in inspect.getargspec(
-          httplib.HTTPSConnection.__init__).args:
+      if hasattr(ssl, 'create_default_context'):
         connection_kw['context'] = ssl.create_default_context(
           cafile=self.config.get('sha%s-ca-file' % where)
         )
-      connection = httplib.HTTPSConnection(**connection_kw)
+      connection = HTTPSConnection(**connection_kw)
     else:
-      connection = httplib.HTTPConnection(**connection_kw)
+      connection = HTTPConnection(**connection_kw)
     try:
       connection.request(method, parsed_url.path, data, headers)
       r = connection.getresponse()
       if 200 <= r.status < 300:
         return r
-    except:
+    finally:
       connection.close()
-      raise
-    raise urllib2.HTTPError(url, r.status, r.reason, r.msg, r.fp)
+    raise HTTPError(url, r.status, r.reason, r.msg, r.fp)
 
   @staticmethod
   def archive(path):
@@ -216,7 +226,7 @@ class NetworkcacheClient(object):
     try:
       try:
         file_descriptor.seek(0)
-      except StandardError:
+      except Exception:
         f = tempfile.TemporaryFile()
         while 1:
             data = file_descriptor.read(65536)
@@ -234,14 +244,14 @@ class NetworkcacheClient(object):
       sha512sum = sha512sum.hexdigest()
       try:
         self._request('cache', sha512sum).close()
-      except urllib2.HTTPError:
+      except HTTPError:
         size = file_descriptor.tell()
         file_descriptor.seek(0)
         result = self._request('cache', data=file_descriptor, headers={
           'Content-Length': str(size),
           'Content-Type': 'application/octet-stream'})
         data = result.read()
-        if result.status != 201 or data != sha512sum:
+        if result.status != 201 or data != sha512sum.encode():
           raise NetworkcacheException(
             'Failed to upload data to SHACACHE Server.'
             ' Response code: %s. Response data: %s' % (result.status, data))
@@ -266,7 +276,7 @@ class NetworkcacheClient(object):
 
   def index(self, key, **kw):
     data = json.dumps(kw)
-    data = [data, self._getSignatureString(data)]
+    data = [data, self._getSignatureString(data.encode())]
     result = self._request('dir', key, json.dumps(data), {
       'Content-Type': 'application/json'})
     if result.status != 201:
@@ -287,19 +297,19 @@ class NetworkcacheClient(object):
     data_list = self.select_generic(key, self.signature_certificate_list)
     for information_json, signature in data_list:
       try:
-        information_dict = byteify(json.loads(information_json))
+        information_dict = strify(json.loads(information_json))
       except Exception:
         logger.info('Failed to parse json-in-json response (%r)',
                     information_json)
         continue
       try:
         len(information_dict['sha512'])
-      except StandardError:
+      except Exception:
         logger.info('Bad or missing sha512 in directory response (%r)',
                     information_dict)
         continue
       if required_key_test(information_dict):
-        for k, v in wanted_metadata_dict.iteritems():
+        for k, v in wanted_metadata_dict.items():
           if information_dict.get(k) != v:
             break
         else:
@@ -310,7 +320,7 @@ class NetworkcacheClient(object):
     '''
     data = self._request('dir', key).read()
     try:
-      data_list = json.loads(data)
+      data_list = json.loads(data.decode())
     except Exception:
       raise NetworkcacheException('Failed to parse json response (%r)' % data)
     if filter:
@@ -323,7 +333,9 @@ class NetworkcacheClient(object):
       Return the signature based on certification file.
     """
     k = self.signature_private_key
-    return '' if k is None else crypto.sign(k, content, 'sha1').encode('base64')
+    if k is None:
+        return ''
+    return encodestring(crypto.sign(k, content, 'sha1')).decode()
 
   def _verifySignatureInCertificateList(self, content, signature_string):
     """
@@ -331,8 +343,8 @@ class NetworkcacheClient(object):
       find any.
     """
     if signature_string:
-      signature = signature_string.decode('base64')
-      content = str(content)
+      signature = decodestring(signature_string.encode())
+      content = content.encode()
       for certificate in self.signature_certificate_list:
         try:
           crypto.verify(certificate, signature, content, 'sha1')
@@ -371,12 +383,12 @@ def cmd_upload(*args):
   f = None
   try:
     if args.file:
-      f = open(args.file)
+      f = open(args.file, 'rb')
     elif os.path.isdir(args.url):
       f = nc.archive(args.url)
     else:
-      f = urllib2.urlopen(args.url)
-    urlmd5 = hashlib.md5(args.url).hexdigest()
+      f = urlopen(args.url)
+    urlmd5 = str(hashlib.md5(args.url.encode()).hexdigest())
     nc.upload(f, args.prefix_key + urlmd5 + args.suffix_key, urlmd5=urlmd5,
               file_name=os.path.basename(args.url) or "file",
               **dict(x.split('=', 1) for x in args.meta))
@@ -387,5 +399,8 @@ def cmd_download(*args):
   parser = _newArgumentParser("URL of data to download.")
   args = parser.parse_args(args or sys.argv[1:])
   nc = NetworkcacheClient(args.config)
-  key = args.prefix_key + hashlib.md5(args.url).hexdigest() + args.suffix_key
-  shutil.copyfileobj(nc.download(nc.select(key).next()['sha512']), sys.stdout)
+  urlmd5 = str(hashlib.md5(args.url.encode()).hexdigest())
+  key = args.prefix_key + urlmd5 + args.suffix_key
+  f = sys.stdout
+  shutil.copyfileobj(nc.download(next(nc.select(key))['sha512']),
+                     getattr(f, 'buffer', f))

slapos/networkcachehelper.py

@@ -51,9 +51,6 @@ def __upload_network_cached(dir_url, cache_url,
       ) as nc:
     return nc.upload(file_descriptor, directory_key, **metadata_dict)
 
-# BBB: slapos.buildout (1.6.0-dev-SlapOS-011) imports it without using it
-helper_upload_network_cached_from_file = None
-
 def helper_upload_network_cached_from_directory(dir_url, cache_url,
     path, directory_key, metadata_dict,
     signature_private_key_file, shacache_ca_file, shacache_cert_file,

slapos/signature.py

@@ -12,11 +12,14 @@
 #
 ##############################################################################
 
-import ConfigParser
 import argparse
 import os
 import subprocess
 import sys
+try:
+  import configparser
+except ImportError:
+  import ConfigParser as configparser
 
 def generateCertificate(certificate_file, key_file, common_name):
   if os.path.lexists(certificate_file):
@@ -26,8 +29,8 @@ def generateCertificate(certificate_file, key_file, common_name):
     raise ValueError("Key %r exists, will not overwrite." %
       key_file)
 
-  print 'Generating certificate for %r (key: %r, certficate: %r)' % (
-    common_name, key_file, certificate_file)
+  print('Generating certificate for %r (key: %r, certficate: %r)' % (
+    common_name, key_file, certificate_file))
   subj = '/CN=%s' % common_name
   subprocess.check_call(["openssl", "req", "-x509", "-nodes", "-days", "36500",
     "-subj", subj, "-newkey", "rsa:1024", "-keyout", key_file, "-out",
@@ -39,7 +42,7 @@ def run(*args):
   parser.add_argument('slapos_config', type=argparse.FileType('r'),
     help='SlapOS configuration file.')
 
-  config = ConfigParser.SafeConfigParser()
+  config = configparser.SafeConfigParser()
   option = parser.parse_args(list(args) or sys.argv[1:])
   config.readfp(option.slapos_config)
   generateCertificate(config.get('networkcache', 'signature-certificate-file'),