Commit eb0246c2 by Julien Muchembled

Add support for Python 3

1 parent 4dee0c16
......@@ -11,7 +11,7 @@ FILETYPE_PEM = 1
def load_privatekey(type, buffer):
r = _tmpfile()
r.write(buffer)
r.write(buffer.encode())
r.flush()
return r
......@@ -20,7 +20,7 @@ def load_certificate(type, buffer):
r = _tmpfile()
p = Popen(("openssl", "x509", "-pubkey", "-noout"),
stdin=PIPE, stdout=r, stderr=PIPE)
err = p.communicate(buffer)[1]
err = p.communicate(buffer.encode())[1]
if p.poll():
raise Error(err)
return r
......
......@@ -13,10 +13,7 @@
##############################################################################
import argparse
import ConfigParser
import hashlib
import httplib
import inspect
import json
import logging
import os
......@@ -26,8 +23,19 @@ import sys
import tarfile
import tempfile
import traceback
import urllib2
import urlparse
from base64 import b64encode, decodestring, encodestring
try:
import configparser
from http.client import HTTPConnection, HTTPSConnection
from urllib.error import HTTPError
from urllib.parse import urlsplit
from urllib.request import urlopen
basestring = bytes, str
except ImportError:
import ConfigParser as configparser
from httplib import HTTPConnection, HTTPSConnection
from urllib2 import HTTPError, urlopen
from urlparse import urlsplit
try:
from OpenSSL import crypto
except ImportError:
......@@ -52,17 +60,21 @@ class short_exc_info(tuple):
l += traceback.format_exception_only(t, v)
return ''.join(l).rstrip()
def byteify(input):
def strify(input):
'''Transform every unicode string inside a list or dict into normal strings. '''
if isinstance(input, dict):
return dict([(byteify(key), byteify(value)) for key, value in input.iteritems()])
return dict((strify(key), strify(value)) for key, value in input.items())
elif isinstance(input, list):
return [byteify(element) for element in input]
return map(strify, input)
elif isinstance(input, unicode):
return input.encode('utf-8')
else:
return input
try:
unicode
except NameError:
def strify(input):
return input
class NetworkcacheClient(object):
'''
......@@ -100,7 +112,7 @@ class NetworkcacheClient(object):
def __new_init(self, config, signature_certificate_list=None):
if not hasattr(config, 'get'):
parser = ConfigParser.SafeConfigParser()
parser = configparser.SafeConfigParser()
parser.readfp(config)
config = dict(parser.items('networkcache'))
self.config = config
......@@ -147,12 +159,12 @@ class NetworkcacheClient(object):
method = 'PUT' if name else 'POST'
url = self.config['upload-%s-url' % where]
timeout = UPLOAD_TIMEOUT
parsed_url = urlparse.urlsplit(url.rstrip('/') + ('/' + name if name else ''))
parsed_url = urlsplit(url.rstrip('/') + ('/' + name if name else ''))
if not headers:
headers = {}
if parsed_url.username:
headers['Authorization'] = 'Basic %s' % ('%s:%s' % (
parsed_url.username, parsed_url.password)).encode('base64').strip()
headers['Authorization'] = 'Basic ' + b64encode('%s:%s' % (
parsed_url.username, parsed_url.password))
headers["Connection"] = "close"
connection_kw = {
'host': parsed_url.hostname,
......@@ -162,23 +174,21 @@ class NetworkcacheClient(object):
if parsed_url.scheme == 'https':
connection_kw['cert_file'] = self.config['sha%s-cert-file' % where]
connection_kw['key_file'] = self.config['sha%s-key-file' % where]
if 'context' in inspect.getargspec(
httplib.HTTPSConnection.__init__).args:
if hasattr(ssl, 'create_default_context'):
connection_kw['context'] = ssl.create_default_context(
cafile=self.config.get('sha%s-ca-file' % where)
)
connection = httplib.HTTPSConnection(**connection_kw)
connection = HTTPSConnection(**connection_kw)
else:
connection = httplib.HTTPConnection(**connection_kw)
connection = HTTPConnection(**connection_kw)
try:
connection.request(method, parsed_url.path, data, headers)
r = connection.getresponse()
if 200 <= r.status < 300:
return r
except:
finally:
connection.close()
raise
raise urllib2.HTTPError(url, r.status, r.reason, r.msg, r.fp)
raise HTTPError(url, r.status, r.reason, r.msg, r.fp)
@staticmethod
def archive(path):
......@@ -216,7 +226,7 @@ class NetworkcacheClient(object):
try:
try:
file_descriptor.seek(0)
except StandardError:
except Exception:
f = tempfile.TemporaryFile()
while 1:
data = file_descriptor.read(65536)
......@@ -234,14 +244,14 @@ class NetworkcacheClient(object):
sha512sum = sha512sum.hexdigest()
try:
self._request('cache', sha512sum).close()
except urllib2.HTTPError:
except HTTPError:
size = file_descriptor.tell()
file_descriptor.seek(0)
result = self._request('cache', data=file_descriptor, headers={
'Content-Length': str(size),
'Content-Type': 'application/octet-stream'})
data = result.read()
if result.status != 201 or data != sha512sum:
if result.status != 201 or data != sha512sum.encode():
raise NetworkcacheException(
'Failed to upload data to SHACACHE Server.'
' Response code: %s. Response data: %s' % (result.status, data))
......@@ -266,7 +276,7 @@ class NetworkcacheClient(object):
def index(self, key, **kw):
data = json.dumps(kw)
data = [data, self._getSignatureString(data)]
data = [data, self._getSignatureString(data.encode())]
result = self._request('dir', key, json.dumps(data), {
'Content-Type': 'application/json'})
if result.status != 201:
......@@ -287,19 +297,19 @@ class NetworkcacheClient(object):
data_list = self.select_generic(key, self.signature_certificate_list)
for information_json, signature in data_list:
try:
information_dict = byteify(json.loads(information_json))
information_dict = strify(json.loads(information_json))
except Exception:
logger.info('Failed to parse json-in-json response (%r)',
information_json)
continue
try:
len(information_dict['sha512'])
except StandardError:
except Exception:
logger.info('Bad or missing sha512 in directory response (%r)',
information_dict)
continue
if required_key_test(information_dict):
for k, v in wanted_metadata_dict.iteritems():
for k, v in wanted_metadata_dict.items():
if information_dict.get(k) != v:
break
else:
......@@ -310,7 +320,7 @@ class NetworkcacheClient(object):
'''
data = self._request('dir', key).read()
try:
data_list = json.loads(data)
data_list = json.loads(data.decode())
except Exception:
raise NetworkcacheException('Failed to parse json response (%r)' % data)
if filter:
......@@ -323,7 +333,9 @@ class NetworkcacheClient(object):
Return the signature based on certification file.
"""
k = self.signature_private_key
return '' if k is None else crypto.sign(k, content, 'sha1').encode('base64')
if k is None:
return ''
return encodestring(crypto.sign(k, content, 'sha1')).decode()
def _verifySignatureInCertificateList(self, content, signature_string):
"""
......@@ -331,8 +343,8 @@ class NetworkcacheClient(object):
find any.
"""
if signature_string:
signature = signature_string.decode('base64')
content = str(content)
signature = decodestring(signature_string.encode())
content = content.encode()
for certificate in self.signature_certificate_list:
try:
crypto.verify(certificate, signature, content, 'sha1')
......@@ -371,12 +383,12 @@ def cmd_upload(*args):
f = None
try:
if args.file:
f = open(args.file)
f = open(args.file, 'rb')
elif os.path.isdir(args.url):
f = nc.archive(args.url)
else:
f = urllib2.urlopen(args.url)
urlmd5 = hashlib.md5(args.url).hexdigest()
f = urlopen(args.url)
urlmd5 = str(hashlib.md5(args.url.encode()).hexdigest())
nc.upload(f, args.prefix_key + urlmd5 + args.suffix_key, urlmd5=urlmd5,
file_name=os.path.basename(args.url) or "file",
**dict(x.split('=', 1) for x in args.meta))
......@@ -387,5 +399,8 @@ def cmd_download(*args):
parser = _newArgumentParser("URL of data to download.")
args = parser.parse_args(args or sys.argv[1:])
nc = NetworkcacheClient(args.config)
key = args.prefix_key + hashlib.md5(args.url).hexdigest() + args.suffix_key
shutil.copyfileobj(nc.download(nc.select(key).next()['sha512']), sys.stdout)
urlmd5 = str(hashlib.md5(args.url.encode()).hexdigest())
key = args.prefix_key + urlmd5 + args.suffix_key
f = sys.stdout
shutil.copyfileobj(nc.download(next(nc.select(key))['sha512']),
getattr(f, 'buffer', f))
......@@ -51,9 +51,6 @@ def __upload_network_cached(dir_url, cache_url,
) as nc:
return nc.upload(file_descriptor, directory_key, **metadata_dict)
# BBB: slapos.buildout (1.6.0-dev-SlapOS-011) imports it without using it
helper_upload_network_cached_from_file = None
def helper_upload_network_cached_from_directory(dir_url, cache_url,
path, directory_key, metadata_dict,
signature_private_key_file, shacache_ca_file, shacache_cert_file,
......
......@@ -12,11 +12,14 @@
#
##############################################################################
import ConfigParser
import argparse
import os
import subprocess
import sys
try:
import configparser
except ImportError:
import ConfigParser as configparser
def generateCertificate(certificate_file, key_file, common_name):
if os.path.lexists(certificate_file):
......@@ -26,8 +29,8 @@ def generateCertificate(certificate_file, key_file, common_name):
raise ValueError("Key %r exists, will not overwrite." %
key_file)
print 'Generating certificate for %r (key: %r, certficate: %r)' % (
common_name, key_file, certificate_file)
print('Generating certificate for %r (key: %r, certficate: %r)' % (
common_name, key_file, certificate_file))
subj = '/CN=%s' % common_name
subprocess.check_call(["openssl", "req", "-x509", "-nodes", "-days", "36500",
"-subj", subj, "-newkey", "rsa:1024", "-keyout", key_file, "-out",
......@@ -39,7 +42,7 @@ def run(*args):
parser.add_argument('slapos_config', type=argparse.FileType('r'),
help='SlapOS configuration file.')
config = ConfigParser.SafeConfigParser()
config = configparser.SafeConfigParser()
option = parser.parse_args(list(args) or sys.argv[1:])
config.readfp(option.slapos_config)
generateCertificate(config.get('networkcache', 'signature-certificate-file'),
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!