Commit 1cc13345 authored by Julien Muchembled's avatar Julien Muchembled

download: add support for slapos.libnetworkcache

This is a rewrite of commit 6e4b8efe
("Support network cache in Download.download()").
parent c685c3c3
......@@ -245,7 +245,7 @@ _buildout_default_options = _annotate_section({
'use-dependency-links': 'true',
}, 'DEFAULT_VALUE')
network_cache_parameter_dict = {}
networkcache_client = None
class Buildout(DictMixin):
......@@ -516,27 +516,16 @@ class Buildout(DictMixin):
networkcache_section_name = options.get('networkcache-section')
if networkcache_section_name:
networkcache_section = self[networkcache_section_name]
for k in (
'download-cache-url',
'download-dir-url',
'upload-cache-url',
'upload-dir-url',
'signature-certificate-list',
'signature-private-key-file',
'shacache-ca-file',
'shacache-cert-file',
'shacache-key-file',
'shadir-ca-file',
'shadir-cert-file',
'shadir-key-file',
):
network_cache_parameter_dict[k] = networkcache_section.get(k, '')
# parse signature list
cert_marker = '-----BEGIN CERTIFICATE-----'
network_cache_parameter_dict['signature-certificate-list'] = \
[cert_marker + '\n' + q.strip() \
for q in network_cache_parameter_dict['signature-certificate-list'].split(cert_marker) \
if q.strip()]
try:
from slapos.libnetworkcache import NetworkcacheClient
global networkcache_client
networkcache_client = NetworkcacheClient(networkcache_section)
except ImportError:
pass
except Exception:
self._logger.exception("There was problem while trying to"
" import slapos.libnetworkcache."
" Networkcache forced to be disabled.")
def _buildout_path(self, name):
if '${' in name:
......
......@@ -20,13 +20,14 @@ except ImportError:
try:
# Python 3
from urllib.error import HTTPError
from urllib.request import Request, splitport, splituser, urlopen
from urllib.parse import urlparse, urlunparse
except ImportError:
# Python 2
from urlparse import urlparse
from urlparse import urlunparse
from urllib2 import Request, splitport, splituser, urlopen
from urllib2 import HTTPError, Request, splitport, splituser, urlopen
from zc.buildout.easy_install import realpath
from base64 import b64encode
......@@ -216,46 +217,31 @@ class Download(object):
if not path:
handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
os.close(handle)
from .buildout import network_cache_parameter_dict as nc
if not download_network_cached(
nc.get('download-dir-url'),
nc.get('download-cache-url'),
tmp_path, url, self.logger,
nc.get('signature-certificate-list'), md5sum):
# Download from original url if not cached or md5sum doesn't match.
self._download(url, tmp_path, md5sum, alternate_url)
cleanup = False
finally:
if cleanup and tmp_path:
remove(tmp_path)
return tmp_path, not path
def _download(self, url, path, md5sum=None, alternate_url=None):
download_url = url
try:
try:
tmp_path, headers = self.urlretrieve(url, tmp_path)
self.urlretrieve(url, path)
except HTTPError:
if not alternate_url:
raise
self.logger.info('using alternate URL: %s', alternate_url)
download_url = alternate_url
tmp_path, headers = self.urlretrieve(
alternate_url, tmp_path)
if not check_md5sum(tmp_path, md5sum):
raise ChecksumError(
'MD5 checksum mismatch downloading %r' % download_url)
# Upload the file to network cache.
if nc.get('upload-cache-url') and nc.get('upload-dir-url'):
upload_network_cached(
nc.get('upload-dir-url'),
nc.get('upload-cache-url'), url, tmp_path, self.logger,
nc.get('signature-private-key-file'),
nc.get('shacache-ca-file'),
nc.get('shacache-cert-file'),
nc.get('shacache-key-file'),
nc.get('shadir-ca-file'),
nc.get('shadir-cert-file'),
nc.get('shadir-key-file'))
cleanup = False
self.urlretrieve(alternate_url, path)
if not check_md5sum(path, md5sum):
raise ChecksumError('MD5 checksum mismatch downloading %r'
% download_url)
except IOError as e:
raise zc.buildout.UserError("Error downloading %s: %s"
% (download_url, e))
finally:
if cleanup and tmp_path:
remove(tmp_path)
return tmp_path, not path
def filename(self, url):
"""Determine a file name from a URL according to the configuration.
......@@ -306,6 +292,32 @@ class Download(object):
return tmp_path, src.info()
class Download(Download):
def _download(self, url, path, md5sum=None, alternate_url=None):
from .buildout import networkcache_client as nc
key = 'slapos-buildout-' + md5(url.encode()).hexdigest()
if nc and nc.tryDownload(key):
with nc:
# BBB: To avoid collision, wanted_metadata_dict should be
# {'url': url}
entry = next(nc.select(key), None)
if entry is None:
err = 'no matching entry'
else:
with closing(nc.download(entry['sha512'])) as src, \
open(path, 'wb') as dst:
shutil.copyfileobj(src, dst)
if check_md5sum(path, md5sum):
return
err = 'MD5 checksum mismatch'
self.logger.info('Cannot download from network cache: %s', err)
super(Download, self)._download(url, path, md5sum, alternate_url)
if nc and nc.tryUpload(key):
with nc, open(path, 'rb') as f:
nc.upload(f, key, url=url)
def check_md5sum(path, md5sum):
"""Tell whether the MD5 checksum of the file at path matches.
......@@ -331,8 +343,6 @@ def remove(path):
if os.path.exists(path):
os.remove(path)
from zc.buildout.networkcache import \
download_network_cached, upload_network_cached, HTTPError
def locate_at(source, dest):
if dest is None or realpath(dest) == realpath(source):
......
##############################################################################
#
# Copyright (c) 2010 ViFiB SARL and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
#XXX factor with slapos/grid/networkcache.py and use libnetworkcache helpers
from __future__ import absolute_import, print_function, division
import hashlib
import posixpath
import re
import traceback
try:
# Python 3
from urllib.error import HTTPError
from urllib.parse import urlparse
strify = bytes.decode
except ImportError:
# Python 2
from urllib2 import HTTPError
from urlparse import urlparse
strify = str
try:
try:
from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
DirectoryNotFound
from slapos.networkcachehelper import \
helper_download_network_cached, \
helper_download_network_cached_to_file
except ImportError:
LIBNETWORKCACHE_ENABLED = False
else:
LIBNETWORKCACHE_ENABLED = True
except:
print('There was problem while trying to import slapos.libnetworkcache:\n'
+ traceback.format_exc())
LIBNETWORKCACHE_ENABLED = False
print('Networkcache forced to be disabled.')
_md5_re = re.compile(r'md5=([a-f0-9]+)')
def _get_md5_from_url(url):
match = _md5_re.search(url)
if match:
return match.group(1)
def fallback_call(function):
"""Decorator which disallow to have any problem while calling method"""
def wrapper(self, *args, **kwd):
"""
Log the call, and the result of the call
"""
try:
return function(self, *args, **kwd)
except: # indeed, *any* exception is swallowed
print('There was problem while calling method %r:\n%s' % (
function.__name__, traceback.format_exc()))
return False
wrapper.__doc__ = function.__doc__
return wrapper
@fallback_call
def get_directory_key(url):
"""Returns directory hash based on url.
Basically check if the url belongs to pypi:
- if yes, the directory key will be pypi-buildout-urlmd5
- if not, the directory key will be slapos-buildout-urlmd5
# XXX why is that?
"""
urlmd5 = hashlib.md5(url.encode()).hexdigest()
if 'pypi' in url:
return 'pypi-buildout-%s' % urlmd5
return 'slapos-buildout-%s' % urlmd5
@fallback_call
def get_index_directory_key(url, requirement):
"""Returns directory hash based on egg requirement.
"""
urlmd5 = hashlib.md5(url.encode()).hexdigest()
return 'pypi-index-%s-%s' % (urlmd5, requirement)
@fallback_call
def download_network_cached(dir_url, cache_url, path, url, logger,
signature_certificate_list, md5sum=None):
"""Downloads from a network cache provider
If something fail (providor be offline, or hash_string fail), we ignore
network cached files.
return True if download succeeded.
"""
if not LIBNETWORKCACHE_ENABLED:
return False
if md5sum is None:
md5sum = _get_md5_from_url(url)
directory_key = get_directory_key(url)
logger.debug('Trying to download %s from network cache...', url)
if helper_download_network_cached_to_file(
dir_url=dir_url,
cache_url=cache_url,
signature_certificate_list=signature_certificate_list,
directory_key=directory_key,
path=path):
logger.info('Downloaded %s from network cache.', url)
if check_md5sum(path, md5sum):
return True
logger.info('MD5 checksum mismatch downloading %s', url)
else:
logger.info('Cannot download %s from network cache.', url)
return False
@fallback_call
def download_index_network_cached(dir_url, cache_url, url, requirement, logger,
signature_certificate_list):
"""
XXX description
Downloads pypi index from a network cache provider
If something fail (providor be offline, or hash_string fail), we ignore
network cached index.
return index if succeeded, False otherwise.
"""
if not LIBNETWORKCACHE_ENABLED:
return False
directory_key = get_index_directory_key(url, requirement)
wanted_metadata_dict = {
'urlmd5': hashlib.md5(url.encode()).hexdigest(),
'requirement': requirement,
}
required_key_list = ['base']
result = helper_download_network_cached(dir_url, cache_url,
signature_certificate_list,
directory_key, wanted_metadata_dict, required_key_list)
if result:
file_descriptor, metadata = result
try:
content = strify(file_descriptor.read())
logger.info('Downloaded %s from network cache.', url)
return content, metadata['base']
except (IOError, DirectoryNotFound) as e:
if isinstance(e, HTTPError) and e.code == 404:
logger.debug('%s does not exist in network cache.', url)
else:
logger.debug('Failed to download from network cache %s: %s',
url, e)
return False
@fallback_call
def upload_network_cached(dir_url, cache_url, external_url, path, logger,
signature_private_key_file, shacache_ca_file, shacache_cert_file,
shacache_key_file, shadir_ca_file, shadir_cert_file, shadir_key_file):
"""Upload file to a network cache server"""
# XXX use helper and FACTOR code
if not LIBNETWORKCACHE_ENABLED:
return False
if not (dir_url and cache_url):
return False
logger.info('Uploading %s into network cache.', external_url)
file_name = get_filename_from_url(external_url)
directory_key = get_directory_key(external_url)
kw = dict(file_name=file_name,
urlmd5=hashlib.md5(external_url.encode()).hexdigest())
# convert '' into None in order to call nc nicely
if not signature_private_key_file:
signature_private_key_file = None
if not shacache_ca_file:
shacache_ca_file = None
if not shacache_cert_file:
shacache_cert_file = None
if not shacache_key_file:
shacache_key_file = None
if not shadir_ca_file:
shadir_ca_file = None
if not shadir_cert_file:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_private_key_file=signature_private_key_file,
shacache_ca_file=shacache_ca_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_ca_file=shadir_ca_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
try:
with open(path, 'rb') as f:
return nc.upload(f, directory_key, **kw)
except (IOError, UploadError) as e:
logger.info('Fail to upload file. %s', e)
return False
@fallback_call
def upload_index_network_cached(dir_url, cache_url, external_url, base, requirement, content, logger,
signature_private_key_file, shacache_ca_file, shacache_cert_file,
shacache_key_file, shadir_ca_file, shadir_cert_file, shadir_key_file):
# XXX use helper and FACTOR code
"""Upload content of a web page to a network cache server"""
if not LIBNETWORKCACHE_ENABLED:
return False
if not (dir_url and cache_url):
return False
logger.info('Uploading %s content into network cache.', external_url)
directory_key = get_index_directory_key(external_url, requirement)
kw = dict(file="file",
base=base,
urlmd5=hashlib.md5(external_url.encode()).hexdigest(),
requirement=requirement)
import tempfile
f = tempfile.TemporaryFile()
f.write(content.encode())
# convert '' into None in order to call nc nicely
if not signature_private_key_file:
signature_private_key_file = None
if not shacache_ca_file:
shacache_ca_file = None
if not shacache_cert_file:
shacache_cert_file = None
if not shacache_key_file:
shacache_key_file = None
if not shadir_ca_file:
shadir_ca_file = None
if not shadir_cert_file:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_private_key_file=signature_private_key_file,
shacache_ca_file=shacache_ca_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_ca_file=shadir_ca_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
try:
return nc.upload_generic(f, directory_key, **kw)
except (IOError, UploadError) as e:
logger.info('Fail to upload file. %s', e)
return False
finally:
f.close()
return True
@fallback_call
def get_filename_from_url(url):
"""Inspired how pip get filename from url.
"""
parsed_url = urlparse(url)
if parsed_url.query and parsed_url.path.endswith('/'):
name = parsed_url.query.split('?', 1)[0]
elif parsed_url.path.endswith('/') and not parsed_url.query:
name = parsed_url.path.split('/')[-2]
else:
name = posixpath.basename(parsed_url.path)
name = name.split('#', 1)[0]
assert name, (
'URL %r produced no filename' % url)
return name
from .download import check_md5sum
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment