Skip to content

S
slapos.buildout
Commit 1cc13345 authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

download: add support for slapos.libnetworkcache 

This is a rewrite of commit 6e4b8efe
("Support network cache in Download.download()").
parent c685c3c3
Hide whitespace changes
Inline Side-by-side
Showing with 58 additions and 367 deletions
src/zc/buildout/buildout.py
View file @ 1cc13345
...@@ -245,7 +245,7 @@ _buildout_default_options = _annotate_section({ ...@@ -245,7 +245,7 @@ _buildout_default_options = _annotate_section({
'use-dependency-links': 'true', 'use-dependency-links': 'true',
}, 'DEFAULT_VALUE') }, 'DEFAULT_VALUE')
network_cache_parameter_dict = {} networkcache_client = None
class Buildout(DictMixin): class Buildout(DictMixin):
...@@ -516,27 +516,16 @@ class Buildout(DictMixin): ...@@ -516,27 +516,16 @@ class Buildout(DictMixin):
networkcache_section_name = options.get('networkcache-section') networkcache_section_name = options.get('networkcache-section')
if networkcache_section_name: if networkcache_section_name:
networkcache_section = self[networkcache_section_name] networkcache_section = self[networkcache_section_name]
for k in ( try:
'download-cache-url', from slapos.libnetworkcache import NetworkcacheClient
'download-dir-url', global networkcache_client
'upload-cache-url', networkcache_client = NetworkcacheClient(networkcache_section)
'upload-dir-url', except ImportError:
'signature-certificate-list', pass
'signature-private-key-file', except Exception:
'shacache-ca-file', self._logger.exception("There was problem while trying to"
'shacache-cert-file', " import slapos.libnetworkcache."
'shacache-key-file', " Networkcache forced to be disabled.")
'shadir-ca-file',
'shadir-cert-file',
'shadir-key-file',
):
network_cache_parameter_dict[k] = networkcache_section.get(k, '')
# parse signature list
cert_marker = '-----BEGIN CERTIFICATE-----'
network_cache_parameter_dict['signature-certificate-list'] = \
[cert_marker + '\n' + q.strip() \
for q in network_cache_parameter_dict['signature-certificate-list'].split(cert_marker) \
if q.strip()]
def _buildout_path(self, name): def _buildout_path(self, name):
if '${' in name: if '${' in name:
......
src/zc/buildout/download.py
View file @ 1cc13345
...@@ -20,13 +20,14 @@ except ImportError: ...@@ -20,13 +20,14 @@ except ImportError:
try: try:
# Python 3 # Python 3
from urllib.error import HTTPError
from urllib.request import Request, splitport, splituser, urlopen from urllib.request import Request, splitport, splituser, urlopen
from urllib.parse import urlparse, urlunparse from urllib.parse import urlparse, urlunparse
except ImportError: except ImportError:
# Python 2 # Python 2
from urlparse import urlparse from urlparse import urlparse
from urlparse import urlunparse from urlparse import urlunparse
from urllib2 import Request, splitport, splituser, urlopen from urllib2 import HTTPError, Request, splitport, splituser, urlopen
from zc.buildout.easy_install import realpath from zc.buildout.easy_install import realpath
from base64 import b64encode from base64 import b64encode
...@@ -216,47 +217,32 @@ class Download(object): ...@@ -216,47 +217,32 @@ class Download(object):
if not path: if not path:
handle, tmp_path = tempfile.mkstemp(prefix='buildout-') handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
os.close(handle) os.close(handle)
from .buildout import network_cache_parameter_dict as nc self._download(url, tmp_path, md5sum, alternate_url)
if not download_network_cached(
nc.get('download-dir-url'),
nc.get('download-cache-url'),
tmp_path, url, self.logger,
nc.get('signature-certificate-list'), md5sum):
# Download from original url if not cached or md5sum doesn't match.
try:
tmp_path, headers = self.urlretrieve(url, tmp_path)
except HTTPError:
if not alternate_url:
raise
self.logger.info('using alternate URL: %s', alternate_url)
download_url = alternate_url
tmp_path, headers = self.urlretrieve(
alternate_url, tmp_path)
if not check_md5sum(tmp_path, md5sum):
raise ChecksumError(
'MD5 checksum mismatch downloading %r' % download_url)
# Upload the file to network cache.
if nc.get('upload-cache-url') and nc.get('upload-dir-url'):
upload_network_cached(
nc.get('upload-dir-url'),
nc.get('upload-cache-url'), url, tmp_path, self.logger,
nc.get('signature-private-key-file'),
nc.get('shacache-ca-file'),
nc.get('shacache-cert-file'),
nc.get('shacache-key-file'),
nc.get('shadir-ca-file'),
nc.get('shadir-cert-file'),
nc.get('shadir-key-file'))
cleanup = False cleanup = False
except IOError as e:
raise zc.buildout.UserError("Error downloading %s: %s"
% (download_url, e))
finally: finally:
if cleanup and tmp_path: if cleanup and tmp_path:
remove(tmp_path) remove(tmp_path)
return tmp_path, not path return tmp_path, not path
def _download(self, url, path, md5sum=None, alternate_url=None):
download_url = url
try:
try:
self.urlretrieve(url, path)
except HTTPError:
if not alternate_url:
raise
self.logger.info('using alternate URL: %s', alternate_url)
download_url = alternate_url
self.urlretrieve(alternate_url, path)
if not check_md5sum(path, md5sum):
raise ChecksumError('MD5 checksum mismatch downloading %r'
% download_url)
except IOError as e:
raise zc.buildout.UserError("Error downloading %s: %s"
% (download_url, e))
def filename(self, url): def filename(self, url):
"""Determine a file name from a URL according to the configuration. """Determine a file name from a URL according to the configuration.
...@@ -306,6 +292,32 @@ class Download(object): ...@@ -306,6 +292,32 @@ class Download(object):
return tmp_path, src.info() return tmp_path, src.info()
class Download(Download):
def _download(self, url, path, md5sum=None, alternate_url=None):
from .buildout import networkcache_client as nc
key = 'slapos-buildout-' + md5(url.encode()).hexdigest()
if nc and nc.tryDownload(key):
with nc:
# BBB: To avoid collision, wanted_metadata_dict should be
# {'url': url}
entry = next(nc.select(key), None)
if entry is None:
err = 'no matching entry'
else:
with closing(nc.download(entry['sha512'])) as src, \
open(path, 'wb') as dst:
shutil.copyfileobj(src, dst)
if check_md5sum(path, md5sum):
return
err = 'MD5 checksum mismatch'
self.logger.info('Cannot download from network cache: %s', err)
super(Download, self)._download(url, path, md5sum, alternate_url)
if nc and nc.tryUpload(key):
with nc, open(path, 'rb') as f:
nc.upload(f, key, url=url)
def check_md5sum(path, md5sum): def check_md5sum(path, md5sum):
"""Tell whether the MD5 checksum of the file at path matches. """Tell whether the MD5 checksum of the file at path matches.
...@@ -331,8 +343,6 @@ def remove(path): ...@@ -331,8 +343,6 @@ def remove(path):
if os.path.exists(path): if os.path.exists(path):
os.remove(path) os.remove(path)
from zc.buildout.networkcache import \
download_network_cached, upload_network_cached, HTTPError
def locate_at(source, dest): def locate_at(source, dest):
if dest is None or realpath(dest) == realpath(source): if dest is None or realpath(dest) == realpath(source):
......
src/zc/buildout/networkcache.py deleted 100644 → 0
View file @ c685c3c3
##############################################################################
#
# Copyright (c) 2010 ViFiB SARL and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
#XXX factor with slapos/grid/networkcache.py and use libnetworkcache helpers
from __future__ import absolute_import, print_function, division
import hashlib
import posixpath
import re
import traceback
try:
# Python 3
from urllib.error import HTTPError
from urllib.parse import urlparse
strify = bytes.decode
except ImportError:
# Python 2
from urllib2 import HTTPError
from urlparse import urlparse
strify = str
try:
try:
from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
DirectoryNotFound
from slapos.networkcachehelper import \
helper_download_network_cached, \
helper_download_network_cached_to_file
except ImportError:
LIBNETWORKCACHE_ENABLED = False
else:
LIBNETWORKCACHE_ENABLED = True
except:
print('There was problem while trying to import slapos.libnetworkcache:\n'
+ traceback.format_exc())
LIBNETWORKCACHE_ENABLED = False
print('Networkcache forced to be disabled.')
_md5_re = re.compile(r'md5=([a-f0-9]+)')
def _get_md5_from_url(url):
match = _md5_re.search(url)
if match:
return match.group(1)
def fallback_call(function):
"""Decorator which disallow to have any problem while calling method"""
def wrapper(self, *args, **kwd):
"""
Log the call, and the result of the call
"""
try:
return function(self, *args, **kwd)
except: # indeed, *any* exception is swallowed
print('There was problem while calling method %r:\n%s' % (
function.__name__, traceback.format_exc()))
return False
wrapper.__doc__ = function.__doc__
return wrapper
@fallback_call
def get_directory_key(url):
"""Returns directory hash based on url.
Basically check if the url belongs to pypi:
- if yes, the directory key will be pypi-buildout-urlmd5
- if not, the directory key will be slapos-buildout-urlmd5
# XXX why is that?
"""
urlmd5 = hashlib.md5(url.encode()).hexdigest()
if 'pypi' in url:
return 'pypi-buildout-%s' % urlmd5
return 'slapos-buildout-%s' % urlmd5
@fallback_call
def get_index_directory_key(url, requirement):
"""Returns directory hash based on egg requirement.
"""
urlmd5 = hashlib.md5(url.encode()).hexdigest()
return 'pypi-index-%s-%s' % (urlmd5, requirement)
@fallback_call
def download_network_cached(dir_url, cache_url, path, url, logger,
signature_certificate_list, md5sum=None):
"""Downloads from a network cache provider
If something fail (providor be offline, or hash_string fail), we ignore
network cached files.
return True if download succeeded.
"""
if not LIBNETWORKCACHE_ENABLED:
return False
if md5sum is None:
md5sum = _get_md5_from_url(url)
directory_key = get_directory_key(url)
logger.debug('Trying to download %s from network cache...', url)
if helper_download_network_cached_to_file(
dir_url=dir_url,
cache_url=cache_url,
signature_certificate_list=signature_certificate_list,
directory_key=directory_key,
path=path):
logger.info('Downloaded %s from network cache.', url)
if check_md5sum(path, md5sum):
return True
logger.info('MD5 checksum mismatch downloading %s', url)
else:
logger.info('Cannot download %s from network cache.', url)
return False
@fallback_call
def download_index_network_cached(dir_url, cache_url, url, requirement, logger,
signature_certificate_list):
"""
XXX description
Downloads pypi index from a network cache provider
If something fail (providor be offline, or hash_string fail), we ignore
network cached index.
return index if succeeded, False otherwise.
"""
if not LIBNETWORKCACHE_ENABLED:
return False
directory_key = get_index_directory_key(url, requirement)
wanted_metadata_dict = {
'urlmd5': hashlib.md5(url.encode()).hexdigest(),
'requirement': requirement,
}
required_key_list = ['base']
result = helper_download_network_cached(dir_url, cache_url,
signature_certificate_list,
directory_key, wanted_metadata_dict, required_key_list)
if result:
file_descriptor, metadata = result
try:
content = strify(file_descriptor.read())
logger.info('Downloaded %s from network cache.', url)
return content, metadata['base']
except (IOError, DirectoryNotFound) as e:
if isinstance(e, HTTPError) and e.code == 404:
logger.debug('%s does not exist in network cache.', url)
else:
logger.debug('Failed to download from network cache %s: %s',
url, e)
return False
@fallback_call
def upload_network_cached(dir_url, cache_url, external_url, path, logger,
signature_private_key_file, shacache_ca_file, shacache_cert_file,
shacache_key_file, shadir_ca_file, shadir_cert_file, shadir_key_file):
"""Upload file to a network cache server"""
# XXX use helper and FACTOR code
if not LIBNETWORKCACHE_ENABLED:
return False
if not (dir_url and cache_url):
return False
logger.info('Uploading %s into network cache.', external_url)
file_name = get_filename_from_url(external_url)
directory_key = get_directory_key(external_url)
kw = dict(file_name=file_name,
urlmd5=hashlib.md5(external_url.encode()).hexdigest())
# convert '' into None in order to call nc nicely
if not signature_private_key_file:
signature_private_key_file = None
if not shacache_ca_file:
shacache_ca_file = None
if not shacache_cert_file:
shacache_cert_file = None
if not shacache_key_file:
shacache_key_file = None
if not shadir_ca_file:
shadir_ca_file = None
if not shadir_cert_file:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_private_key_file=signature_private_key_file,
shacache_ca_file=shacache_ca_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_ca_file=shadir_ca_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
try:
with open(path, 'rb') as f:
return nc.upload(f, directory_key, **kw)
except (IOError, UploadError) as e:
logger.info('Fail to upload file. %s', e)
return False
@fallback_call
def upload_index_network_cached(dir_url, cache_url, external_url, base, requirement, content, logger,
signature_private_key_file, shacache_ca_file, shacache_cert_file,
shacache_key_file, shadir_ca_file, shadir_cert_file, shadir_key_file):
# XXX use helper and FACTOR code
"""Upload content of a web page to a network cache server"""
if not LIBNETWORKCACHE_ENABLED:
return False
if not (dir_url and cache_url):
return False
logger.info('Uploading %s content into network cache.', external_url)
directory_key = get_index_directory_key(external_url, requirement)
kw = dict(file="file",
base=base,
urlmd5=hashlib.md5(external_url.encode()).hexdigest(),
requirement=requirement)
import tempfile
f = tempfile.TemporaryFile()
f.write(content.encode())
# convert '' into None in order to call nc nicely
if not signature_private_key_file:
signature_private_key_file = None
if not shacache_ca_file:
shacache_ca_file = None
if not shacache_cert_file:
shacache_cert_file = None
if not shacache_key_file:
shacache_key_file = None
if not shadir_ca_file:
shadir_ca_file = None
if not shadir_cert_file:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_private_key_file=signature_private_key_file,
shacache_ca_file=shacache_ca_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_ca_file=shadir_ca_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
try:
return nc.upload_generic(f, directory_key, **kw)
except (IOError, UploadError) as e:
logger.info('Fail to upload file. %s', e)
return False
finally:
f.close()
return True
@fallback_call
def get_filename_from_url(url):
"""Inspired how pip get filename from url.
"""
parsed_url = urlparse(url)
if parsed_url.query and parsed_url.path.endswith('/'):
name = parsed_url.query.split('?', 1)[0]
elif parsed_url.path.endswith('/') and not parsed_url.query:
name = parsed_url.path.split('/')[-2]
else:
name = posixpath.basename(parsed_url.path)
name = name.split('#', 1)[0]
assert name, (
'URL %r produced no filename' % url)
return name
from .download import check_md5sum
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7