Add support for Python 3

slapos/crypto.py

@@ -11,7 +11,7 @@ FILETYPE_PEM = 1
 
 def load_privatekey(type, buffer):
   r = _tmpfile()
-  r.write(buffer)
+  r.write(buffer.encode())
   r.flush()
   return r
 
@@ -20,7 +20,7 @@ def load_certificate(type, buffer):
   r = _tmpfile()
   p = Popen(("openssl", "x509", "-pubkey", "-noout"),
             stdin=PIPE, stdout=r, stderr=PIPE)
-  err = p.communicate(buffer)[1]
+  err = p.communicate(buffer.encode())[1]
   if p.poll():
     raise Error(err)
   return r

slapos/libnetworkcache.py

@@ -13,10 +13,7 @@
 ##############################################################################
 
 import argparse
-import ConfigParser
 import hashlib
-import httplib
-import inspect
 import json
 import logging
 import os
@@ -26,8 +23,19 @@ import sys
 import tarfile
 import tempfile
 import traceback
-import urllib2
-import urlparse
+from base64 import b64encode, decodestring, encodestring
+try:
+  import configparser
+  from http.client import HTTPConnection, HTTPSConnection
+  from urllib.error import HTTPError
+  from urllib.parse import urlsplit
+  from urllib.request import urlopen
+  basestring = bytes, str
+except ImportError:
+  import ConfigParser as configparser
+  from httplib import HTTPConnection, HTTPSConnection
+  from urllib2 import HTTPError, urlopen
+  from urlparse import urlsplit
 try:
   from OpenSSL import crypto
 except ImportError:
@@ -52,17 +60,21 @@ class short_exc_info(tuple):
     l += traceback.format_exception_only(t, v)
     return ''.join(l).rstrip()
 
-def byteify(input):
+def strify(input):
   '''Transform every unicode string inside a list or dict into normal strings. '''
   if isinstance(input, dict):
-    return dict([(byteify(key), byteify(value)) for key, value in input.iteritems()])
+    return dict((strify(key), strify(value)) for key, value in input.items())
   elif isinstance(input, list):
-    return [byteify(element) for element in input]
+    return map(strify, input)
   elif isinstance(input, unicode):
     return input.encode('utf-8')
   else:
     return input
-
+try:
+  unicode
+except NameError:
+  def strify(input):
+    return input
 
 class NetworkcacheClient(object):
   '''
@@ -100,7 +112,7 @@ class NetworkcacheClient(object):
 
   def __new_init(self, config, signature_certificate_list=None):
     if not hasattr(config, 'get'):
-      parser = ConfigParser.SafeConfigParser()
+      parser = configparser.SafeConfigParser()
       parser.readfp(config)
       config = dict(parser.items('networkcache'))
     self.config = config
@@ -147,12 +159,12 @@ class NetworkcacheClient(object):
       method = 'PUT' if name else 'POST'
       url = self.config['upload-%s-url' % where]
       timeout = UPLOAD_TIMEOUT
-    parsed_url = urlparse.urlsplit(url.rstrip('/') + ('/' + name if name else ''))
+    parsed_url = urlsplit(url.rstrip('/') + ('/' + name if name else ''))
     if not headers:
       headers = {}
     if parsed_url.username:
-      headers['Authorization'] = 'Basic %s' % ('%s:%s' % (
-        parsed_url.username, parsed_url.password)).encode('base64').strip()
+      headers['Authorization'] = 'Basic ' + b64encode('%s:%s' % (
+        parsed_url.username, parsed_url.password))
     headers["Connection"] = "close"
     connection_kw = {
       'host': parsed_url.hostname,
@@ -162,23 +174,21 @@ class NetworkcacheClient(object):
     if parsed_url.scheme == 'https':
       connection_kw['cert_file'] = self.config['sha%s-cert-file' % where]
       connection_kw['key_file'] = self.config['sha%s-key-file' % where]
-      if 'context' in inspect.getargspec(
-          httplib.HTTPSConnection.__init__).args:
+      if hasattr(ssl, 'create_default_context'):
         connection_kw['context'] = ssl.create_default_context(
           cafile=self.config.get('sha%s-ca-file' % where)
         )
-      connection = httplib.HTTPSConnection(**connection_kw)
+      connection = HTTPSConnection(**connection_kw)
     else:
-      connection = httplib.HTTPConnection(**connection_kw)
+      connection = HTTPConnection(**connection_kw)
     try:
       connection.request(method, parsed_url.path, data, headers)
       r = connection.getresponse()
       if 200 <= r.status < 300:
         return r
-    except:
+    finally:
       connection.close()
-      raise
-    raise urllib2.HTTPError(url, r.status, r.reason, r.msg, r.fp)
+    raise HTTPError(url, r.status, r.reason, r.msg, r.fp)
 
   @staticmethod
   def archive(path):
@@ -216,7 +226,7 @@ class NetworkcacheClient(object):
     try:
       try:
         file_descriptor.seek(0)
-      except StandardError:
+      except Exception:
         f = tempfile.TemporaryFile()
         while 1:
             data = file_descriptor.read(65536)
@@ -234,14 +244,14 @@ class NetworkcacheClient(object):
       sha512sum = sha512sum.hexdigest()
       try:
         self._request('cache', sha512sum).close()
-      except urllib2.HTTPError:
+      except HTTPError:
         size = file_descriptor.tell()
         file_descriptor.seek(0)
         result = self._request('cache', data=file_descriptor, headers={
           'Content-Length': str(size),
           'Content-Type': 'application/octet-stream'})
         data = result.read()
-        if result.status != 201 or data != sha512sum:
+        if result.status != 201 or data != sha512sum.encode():
           raise NetworkcacheException(
             'Failed to upload data to SHACACHE Server.'
             ' Response code: %s. Response data: %s' % (result.status, data))
@@ -266,7 +276,7 @@ class NetworkcacheClient(object):
 
   def index(self, key, **kw):
     data = json.dumps(kw)
-    data = [data, self._getSignatureString(data)]
+    data = [data, self._getSignatureString(data.encode())]
     result = self._request('dir', key, json.dumps(data), {
       'Content-Type': 'application/json'})
     if result.status != 201:
@@ -287,19 +297,19 @@ class NetworkcacheClient(object):
     data_list = self.select_generic(key, self.signature_certificate_list)
     for information_json, signature in data_list:
       try:
-        information_dict = byteify(json.loads(information_json))
+        information_dict = strify(json.loads(information_json))
       except Exception:
         logger.info('Failed to parse json-in-json response (%r)',
                     information_json)
         continue
       try:
         len(information_dict['sha512'])
-      except StandardError:
+      except Exception:
         logger.info('Bad or missing sha512 in directory response (%r)',
                     information_dict)
         continue
       if required_key_test(information_dict):
-        for k, v in wanted_metadata_dict.iteritems():
+        for k, v in wanted_metadata_dict.items():
           if information_dict.get(k) != v:
             break
         else:
@@ -310,7 +320,7 @@ class NetworkcacheClient(object):
     '''
     data = self._request('dir', key).read()
     try:
-      data_list = json.loads(data)
+      data_list = json.loads(data.decode())
     except Exception:
       raise NetworkcacheException('Failed to parse json response (%r)' % data)
     if filter:
@@ -323,7 +333,9 @@ class NetworkcacheClient(object):
       Return the signature based on certification file.
     """
     k = self.signature_private_key
-    return '' if k is None else crypto.sign(k, content, 'sha1').encode('base64')
+    if k is None:
+        return ''
+    return encodestring(crypto.sign(k, content, 'sha1')).decode()
 
   def _verifySignatureInCertificateList(self, content, signature_string):
     """
@@ -331,8 +343,8 @@ class NetworkcacheClient(object):
       find any.
     """
     if signature_string:
-      signature = signature_string.decode('base64')
-      content = str(content)
+      signature = decodestring(signature_string.encode())
+      content = content.encode()
       for certificate in self.signature_certificate_list:
         try:
           crypto.verify(certificate, signature, content, 'sha1')
@@ -371,12 +383,12 @@ def cmd_upload(*args):
   f = None
   try:
     if args.file:
-      f = open(args.file)
+      f = open(args.file, 'rb')
     elif os.path.isdir(args.url):
       f = nc.archive(args.url)
     else:
-      f = urllib2.urlopen(args.url)
-    urlmd5 = hashlib.md5(args.url).hexdigest()
+      f = urlopen(args.url)
+    urlmd5 = str(hashlib.md5(args.url.encode()).hexdigest())
     nc.upload(f, args.prefix_key + urlmd5 + args.suffix_key, urlmd5=urlmd5,
               file_name=os.path.basename(args.url) or "file",
               **dict(x.split('=', 1) for x in args.meta))
@@ -387,5 +399,8 @@ def cmd_download(*args):
   parser = _newArgumentParser("URL of data to download.")
   args = parser.parse_args(args or sys.argv[1:])
   nc = NetworkcacheClient(args.config)
-  key = args.prefix_key + hashlib.md5(args.url).hexdigest() + args.suffix_key
-  shutil.copyfileobj(nc.download(nc.select(key).next()['sha512']), sys.stdout)
+  urlmd5 = str(hashlib.md5(args.url.encode()).hexdigest())
+  key = args.prefix_key + urlmd5 + args.suffix_key
+  f = sys.stdout
+  shutil.copyfileobj(nc.download(next(nc.select(key))['sha512']),
+                     getattr(f, 'buffer', f))

slapos/libnetworkcachetests.py

-import BaseHTTPServer
 import datetime
 import errno
 import hashlib
-import httplib
 import json
 import logging
 import logging.handlers
 import os
-import urllib2
 import random
 import shutil
 import ssl
@@ -20,21 +17,29 @@ import unittest
 import slapos.libnetworkcache
 import slapos.signature
 import sys
-from cStringIO import StringIO
+from io import BytesIO
+try:
+  from http.server import BaseHTTPRequestHandler, HTTPServer
+  from http.client import HTTPConnection, NOT_FOUND
+  from urllib.error import HTTPError
+except ImportError:
+  from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+  from httplib import HTTPConnection, NOT_FOUND
+  from urllib2 import HTTPError
 
 logging.basicConfig()
 
 
-class NCHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+class NCHandler(BaseHTTPRequestHandler):
   def __init__(self, request, address, server):
     self.__server = server
     self.tree = server.tree
-    BaseHTTPServer.BaseHTTPRequestHandler.__init__(
+    BaseHTTPRequestHandler.__init__(
       self, request, address, server)
 
   def handle(self):
     try:
-      BaseHTTPServer.BaseHTTPRequestHandler.handle(self)
+      BaseHTTPRequestHandler.handle(self)
     except Exception:
       traceback.print_exc()
       raise
@@ -49,80 +54,79 @@ class NCHandler(BaseHTTPServer.BaseHTTPRequestHandler):
       and
       os.path.exists(path)
       ):
-      self.send_response(404, 'Not Found')
+      self.send_error(404)
       return
     self.send_response(200)
-    out = open(path, 'rb').read()
+    with open(path, 'rb') as f:
+      out = f.read()
     self.send_header('Content-Length', len(out))
     self.end_headers()
     self.wfile.write(out)
 
   def do_PUT(self):
     assert 'shadir' in self.path
-    assert self.headers.getheader('content-type') == 'application/json'
+    assert self.headers.get('content-type') == 'application/json'
     path = os.path.abspath(os.path.join(self.tree, *self.path.split('/')))
     if not os.path.exists(os.path.dirname(path)):
       os.makedirs(os.path.dirname(path))
-    data = self.rfile.read(int(self.headers.getheader('content-length')))
+    data = self.rfile.read(int(self.headers.get('content-length')))
     cksum = hashlib.sha512(data).hexdigest()
     if os.path.exists(path):
-      with open(path, 'rb') as f:
-        json_data_list = json.loads(f.read().strip())
+      with open(path) as f:
+        json_data_list = json.load(f)
     else:
       json_data_list = []
-    json_data_list.append(json.loads(data))
-    data = json.dumps(json_data_list)
+    json_data_list.append(json.loads(data.decode()))
 
-    open(path, 'wb').write(data)
+    with open(path, 'w') as f:
+      json.dump(json_data_list, f)
     self.send_response(201)
     self.send_header('Content-Length', str(len(cksum)))
     self.send_header('Content-Type', 'text/html')
     self.end_headers()
-    self.wfile.write(cksum)
-    return
+    self.wfile.write(cksum.encode())
 
   def do_POST(self):
     assert 'shacache' in self.path
-    assert self.headers.getheader('content-type') == 'application/octet-stream'
+    assert self.headers.get('content-type') == 'application/octet-stream'
     path = os.path.abspath(os.path.join(self.tree, *self.path.split('/')))
     if not os.path.exists(path):
       os.makedirs(path)
-    data = self.rfile.read(int(self.headers.getheader('content-length')))
+    data = self.rfile.read(int(self.headers.get('content-length')))
     cksum = hashlib.sha512(data).hexdigest()
     path = os.path.join(path, cksum)
 
     # Although real server would accept the request,
     # clients should avoid uploading same content twice.
     assert not os.path.exists(path)
-    open(path, 'wb').write(data)
+    with open(path, 'wb') as f:
+      f.write(data)
     self.send_response(201)
     self.send_header('Content-Length', str(len(cksum)))
     self.send_header('Content-Type', 'text/html')
     self.end_headers()
-    self.wfile.write(cksum)
-    return
+    self.wfile.write(cksum.encode())
 
 
 class NCHandlerPOST200(NCHandler):
   def do_POST(self):
     self.send_response(200)
-    return
+    self.end_headers()
 
 
 class NCHandlerReturnWrong(NCHandler):
   def do_POST(self):
-    cksum = 'incorrect'
+    cksum = b'incorrect'
     self.send_response(201)
     self.send_header('Content-Length', str(len(cksum)))
     self.send_header('Content-Type', 'text/html')
     self.end_headers()
     self.wfile.write(cksum)
-    return
 
 
-class Server(BaseHTTPServer.HTTPServer):
+class Server(HTTPServer):
   def __init__(self, tree, *args):
-    BaseHTTPServer.HTTPServer.__init__(self, *args)
+    HTTPServer.__init__(self, *args)
     self.tree = os.path.abspath(tree)
 
   __run = True
@@ -158,7 +162,7 @@ class OfflineTest(unittest.TestCase):
   def test_upload_offline(self):
     nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache_url,
       self.shadir_url)
-    self.assertRaises(IOError, nc.upload, StringIO())
+    self.assertRaises(IOError, nc.upload, BytesIO())
 
 
 class OnlineMixin:
@@ -174,18 +178,21 @@ class OnlineMixin:
     if not 'TEST_SHA_CACHE' in os.environ and not 'TEST_SHA_DIR' in os.environ:
       self.tree = tempfile.mkdtemp()
       self.thread = Server.run(self.tree, (self.host, self.port), self.handler)
-    self.test_string = str(random.random())
-    self.test_data = StringIO(self.test_string)
+    self.test_string = str(random.random()).encode()
+    self.test_data = BytesIO(self.test_string)
     self.test_shasum = hashlib.sha512(self.test_string).hexdigest()
     self.handler = logging.handlers.BufferingHandler(float('inf'))
     slapos.libnetworkcache.logger.addHandler(self.handler)
 
   def tearDown(self):
     if not 'TEST_SHA_CACHE' in os.environ and not 'TEST_SHA_DIR' in os.environ:
+      conn = HTTPConnection(self.host, self.port)
       try:
-        httplib.HTTPConnection(self.host, self.port).request('KILL', '/')
+        conn.request('KILL', '/')
       except Exception:
         pass
+      finally:
+        conn.close()
 
       if self.thread is not None:
         self.thread.join()
@@ -204,7 +211,7 @@ class OnlineMixin:
 
   def select(self, nc, key, *args):
     try:
-      return nc.download(nc.select(key).next()['sha512'])
+      return nc.download(next(nc.select(key))['sha512'])
     except StopIteration:
       for msg in args:
         self.assertLog(msg)
@@ -369,8 +376,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     key = 'somekey' + str(random.random())
     nc.upload(self.test_data, key, urlmd5=urlmd5, file_name='my file')
     with nc:
-      nc.select('key_another_key' + str(random.random())).next()
-    self.assertRaised(urllib2.HTTPError)
+      next(nc.select('key_another_key' + str(random.random())))
+    self.assertRaised(HTTPError)
 
   def test_upload_shadir_no_filename(self):
     """Check scenario with shadir used, but not filename passed"""
@@ -403,14 +410,14 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     try:
       nc.download(self.test_shasum)
       self.fail("HTTPError not raised")
-    except urllib2.HTTPError, error:
-      self.assertEqual(error.code, httplib.NOT_FOUND)
+    except HTTPError as error:
+      self.assertEqual(error.code, NOT_FOUND)
 
   def test_select_signed_content(self):
     key = 'somekey' + str(random.random())
     urlmd5 = str(random.random())
     file_name = 'my file'
-    key_file = tempfile.NamedTemporaryFile()
+    key_file = tempfile.NamedTemporaryFile('w+')
     key_file.write(self.key)
     key_file.flush()
     signed_nc = slapos.libnetworkcache.NetworkcacheClient(
@@ -423,7 +430,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     key = 'somekey' + str(random.random())
     urlmd5 = str(random.random())
     file_name = 'my file'
-    key_file = tempfile.NamedTemporaryFile()
+    key_file = tempfile.NamedTemporaryFile('w+')
     key_file.write(self.key)
     key_file.flush()
     signed_nc = slapos.libnetworkcache.NetworkcacheClient(
@@ -437,7 +444,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     key = 'somekey' + str(random.random())
     urlmd5 = str(random.random())
     file_name = 'my file'
-    key_file = tempfile.NamedTemporaryFile()
+    key_file = tempfile.NamedTemporaryFile('w+')
     key_file.write(self.key)
     key_file.flush()
     signed_nc = slapos.libnetworkcache.NetworkcacheClient(
@@ -455,9 +462,10 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
       self.shacache, self.shadir)
     self.assertEqual(self.test_shasum, nc.upload(self.test_data,
         key, urlmd5=urlmd5, file_name=file_name))
-    f = os.path.join(self.tree, 'shadir', key)
+    path = os.path.join(self.tree, 'shadir', key)
     # now remove the entry from shacache
-    open(f, 'w').write(json.dumps([]))
+    with open(path, 'w') as f:
+      json.dump([], f)
     self.assertEqual(self.select(nc, key), None)
 
   def test_select_no_json_response(self):
@@ -472,7 +480,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
       # now remove the entry from shacache
       f.write('This is not a json.')
     with nc:
-      nc.select(key).next()
+      next(nc.select(key))
     self.assertLog('Failed to parse json response')
 
   def test_select_json_no_in_json_response(self):
@@ -505,7 +513,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     key = 'somekey' + str(random.random())
     urlmd5 = str(random.random())
     file_name = 'my file'
-    key_file = tempfile.NamedTemporaryFile()
+    key_file = tempfile.NamedTemporaryFile('w+')
     key_file.write(self.key)
     key_file.flush()
     signed_nc = slapos.libnetworkcache.NetworkcacheClient(
@@ -513,9 +521,9 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
 
     signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
     # Hacker entered the server...
-    f = os.path.join(self.tree, 'shadir', key)
-    original_data = open(f).read()
-    original_json = json.loads(original_data)
+    path = os.path.join(self.tree, 'shadir', key)
+    with open(path) as f:
+      original_json = json.load(f)
     hacked_entry_json = json.loads(original_json[0][0])
     # ...he modifies something...
     hacked_entry_json['file'] = 'hacked'
@@ -524,11 +532,12 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     hacked_json[0][0] = json.dumps(hacked_entry_json)
     # ...but as he has no access to key, no way to sign data..
     # hacked_json[0][1] is still a good key
-    open(f, 'w').write(json.dumps(hacked_json))
+    with open(path, 'w') as f:
+      json.dump(hacked_json, f)
     self.assertEqual(self.select(signed_nc, key), None)
 
   def test_DirectoryNotFound_non_trustable_entry(self):
-    key_file = tempfile.NamedTemporaryFile()
+    key_file = tempfile.NamedTemporaryFile('w+')
     key_file.write(self.key)
     key_file.flush()
 
@@ -573,7 +582,7 @@ class OnlineTestSSLServer(OnlineMixin, unittest.TestCase):
     nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
     try:
       nc.upload(self.test_data)
-    except ssl.SSLError, e:
+    except ssl.SSLError as e:
       self.assertTrue('alert handshake failure' in str(e))
 
   def test_upload_to_ssl_auth(self):
@@ -589,7 +598,7 @@ class OnlineTestSSLServer(OnlineMixin, unittest.TestCase):
     try:
       nc.upload(self.test_data, key=str(random.random()),
           file_name=str(random.random()), urlmd5=str(random.random()))
-    except ssl.SSLError, e:
+    except ssl.SSLError as e:
       self.assertTrue('alert handshake failure' in str(e))
 
   def test_upload_with_key_with_ssl_auth(self):
@@ -641,11 +650,11 @@ class GenerateSignatureScriptTest(unittest.TestCase):
       self.common_name)
     today = datetime.date.today()
     result = subprocess.check_output(['openssl', 'x509', '-noout', '-subject',
-      '-in', self.certificate])
+      '-in', self.certificate], universal_newlines=True)
     self.assertEqual('subject= /CN=%s' % self.common_name, result.strip())
     result = subprocess.check_output(['openssl', 'x509', '-noout', '-enddate',
-      '-in', self.certificate])
-    self.assertTrue(' %s ' % (today.year + 100) in result)
+      '-in', self.certificate], universal_newlines=True)
+    self.assertIn(' %s ' % (today.year + 100), result)
 
   def test_generate_key_exists(self):
     with tempfile.NamedTemporaryFile() as key:

slapos/networkcachehelper.py

@@ -51,9 +51,6 @@ def __upload_network_cached(dir_url, cache_url,
       ) as nc:
     return nc.upload(file_descriptor, directory_key, **metadata_dict)
 
-# BBB: slapos.buildout (1.6.0-dev-SlapOS-011) imports it without using it
-helper_upload_network_cached_from_file = None
-
 def helper_upload_network_cached_from_directory(dir_url, cache_url,
     path, directory_key, metadata_dict,
     signature_private_key_file, shacache_ca_file, shacache_cert_file,

slapos/signature.py

@@ -12,11 +12,14 @@
 #
 ##############################################################################
 
-import ConfigParser
 import argparse
 import os
 import subprocess
 import sys
+try:
+  import configparser
+except ImportError:
+  import ConfigParser as configparser
 
 def generateCertificate(certificate_file, key_file, common_name):
   if os.path.lexists(certificate_file):
@@ -26,8 +29,8 @@ def generateCertificate(certificate_file, key_file, common_name):
     raise ValueError("Key %r exists, will not overwrite." %
       key_file)
 
-  print 'Generating certificate for %r (key: %r, certficate: %r)' % (
-    common_name, key_file, certificate_file)
+  print('Generating certificate for %r (key: %r, certficate: %r)' % (
+    common_name, key_file, certificate_file))
   subj = '/CN=%s' % common_name
   subprocess.check_call(["openssl", "req", "-x509", "-nodes", "-days", "36500",
     "-subj", subj, "-newkey", "rsa:1024", "-keyout", key_file, "-out",
@@ -39,7 +42,7 @@ def run(*args):
   parser.add_argument('slapos_config', type=argparse.FileType('r'),
     help='SlapOS configuration file.')
 
-  config = ConfigParser.SafeConfigParser()
+  config = configparser.SafeConfigParser()
   option = parser.parse_args(list(args) or sys.argv[1:])
   config.readfp(option.slapos_config)
   generateCertificate(config.get('networkcache', 'signature-certificate-file'),