Commit e3638903 authored by Julien Muchembled's avatar Julien Muchembled Committed by Xavier Thompson

[feat] download: add support for slapos.libnetworkcache

This is a rewrite of commit 6e4b8efe
("Support network cache in Download.download()").
parent fcb184ea
...@@ -400,7 +400,7 @@ def _get_user_config(): ...@@ -400,7 +400,7 @@ def _get_user_config():
return os.path.join(buildout_home, 'default.cfg') return os.path.join(buildout_home, 'default.cfg')
network_cache_parameter_dict = {} networkcache_client = None
@commands @commands
...@@ -668,27 +668,16 @@ class Buildout(DictMixin): ...@@ -668,27 +668,16 @@ class Buildout(DictMixin):
networkcache_section_name = options.get('networkcache-section') networkcache_section_name = options.get('networkcache-section')
if networkcache_section_name: if networkcache_section_name:
networkcache_section = self[networkcache_section_name] networkcache_section = self[networkcache_section_name]
for k in ( try:
'download-cache-url', from slapos.libnetworkcache import NetworkcacheClient
'download-dir-url', global networkcache_client
'upload-cache-url', networkcache_client = NetworkcacheClient(networkcache_section)
'upload-dir-url', except ImportError:
'signature-certificate-list', pass
'signature-private-key-file', except Exception:
'shacache-ca-file', self._logger.exception("There was problem while trying to"
'shacache-cert-file', " import slapos.libnetworkcache."
'shacache-key-file', " Networkcache forced to be disabled.")
'shadir-ca-file',
'shadir-cert-file',
'shadir-key-file',
):
network_cache_parameter_dict[k] = networkcache_section.get(k, '')
# parse signature list
cert_marker = '-----BEGIN CERTIFICATE-----'
network_cache_parameter_dict['signature-certificate-list'] = \
[cert_marker + '\n' + q.strip() \
for q in network_cache_parameter_dict['signature-certificate-list'].split(cert_marker) \
if q.strip()]
def _buildout_path(self, name): def _buildout_path(self, name):
if '${' in name: if '${' in name:
......
...@@ -20,13 +20,14 @@ except ImportError: ...@@ -20,13 +20,14 @@ except ImportError:
try: try:
# Python 3 # Python 3
from urllib.error import HTTPError
from urllib.request import Request, urlopen from urllib.request import Request, urlopen
from urllib.parse import urlparse, urlunparse from urllib.parse import urlparse, urlunparse
except ImportError: except ImportError:
# Python 2 # Python 2
from urlparse import urlparse from urlparse import urlparse
from urlparse import urlunparse from urlparse import urlunparse
from urllib2 import Request, urlopen from urllib2 import HTTPError, Request, urlopen
from zc.buildout.easy_install import realpath from zc.buildout.easy_install import realpath
from base64 import b64encode from base64 import b64encode
...@@ -219,47 +220,32 @@ class Download(object): ...@@ -219,47 +220,32 @@ class Download(object):
if not path: if not path:
handle, tmp_path = tempfile.mkstemp(prefix='buildout-') handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
os.close(handle) os.close(handle)
from .buildout import network_cache_parameter_dict as nc self._download(url, tmp_path, md5sum, alternate_url)
if not download_network_cached(
nc.get('download-dir-url'),
nc.get('download-cache-url'),
tmp_path, url, self.logger,
nc.get('signature-certificate-list'), md5sum):
# Download from original url if not cached or md5sum doesn't match.
try:
tmp_path, headers = self.urlretrieve(url, tmp_path)
except HTTPError:
if not alternate_url:
raise
self.logger.info('using alternate URL: %s', alternate_url)
download_url = alternate_url
tmp_path, headers = self.urlretrieve(
alternate_url, tmp_path)
if not check_md5sum(tmp_path, md5sum):
raise ChecksumError(
'MD5 checksum mismatch downloading %r' % download_url)
# Upload the file to network cache.
if nc.get('upload-cache-url') and nc.get('upload-dir-url'):
upload_network_cached(
nc.get('upload-dir-url'),
nc.get('upload-cache-url'), url, tmp_path, self.logger,
nc.get('signature-private-key-file'),
nc.get('shacache-ca-file'),
nc.get('shacache-cert-file'),
nc.get('shacache-key-file'),
nc.get('shadir-ca-file'),
nc.get('shadir-cert-file'),
nc.get('shadir-key-file'))
cleanup = False cleanup = False
except IOError as e:
raise zc.buildout.UserError("Error downloading %s: %s"
% (download_url, e))
finally: finally:
if cleanup and tmp_path: if cleanup and tmp_path:
remove(tmp_path) remove(tmp_path)
return tmp_path, not path return tmp_path, not path
def _download(self, url, path, md5sum=None, alternate_url=None):
download_url = url
try:
try:
self.urlretrieve(url, path)
except HTTPError:
if not alternate_url:
raise
self.logger.info('using alternate URL: %s', alternate_url)
download_url = alternate_url
self.urlretrieve(alternate_url, path)
if not check_md5sum(path, md5sum):
raise ChecksumError('MD5 checksum mismatch downloading %r'
% download_url)
except IOError as e:
raise zc.buildout.UserError("Error downloading %s: %s"
% (download_url, e))
def filename(self, url): def filename(self, url):
"""Determine a file name from a URL according to the configuration. """Determine a file name from a URL according to the configuration.
...@@ -310,6 +296,30 @@ class Download(object): ...@@ -310,6 +296,30 @@ class Download(object):
return tmp_path, src.info() return tmp_path, src.info()
class Download(Download):
def _download(self, url, path, md5sum=None, alternate_url=None):
from .buildout import networkcache_client as nc
key = 'file-urlmd5:' + md5(url.encode()).hexdigest()
if nc and nc.tryDownload(key):
with nc:
entry = next(nc.select(key, {'url': url}), None)
if entry is None:
err = 'no matching entry'
else:
with closing(nc.download(entry['sha512'])) as src, \
open(path, 'wb') as dst:
shutil.copyfileobj(src, dst)
if check_md5sum(path, md5sum):
return
err = 'MD5 checksum mismatch'
self.logger.info('Cannot download from network cache: %s', err)
super(Download, self)._download(url, path, md5sum, alternate_url)
if nc and nc.tryUpload(key):
with nc, open(path, 'rb') as f:
nc.upload(f, key, url=url)
def check_md5sum(path, md5sum): def check_md5sum(path, md5sum):
"""Tell whether the MD5 checksum of the file at path matches. """Tell whether the MD5 checksum of the file at path matches.
...@@ -335,8 +345,6 @@ def remove(path): ...@@ -335,8 +345,6 @@ def remove(path):
if os.path.exists(path): if os.path.exists(path):
os.remove(path) os.remove(path)
from zc.buildout.networkcache import \
download_network_cached, upload_network_cached, HTTPError
def locate_at(source, dest): def locate_at(source, dest):
if dest is None or realpath(dest) == realpath(source): if dest is None or realpath(dest) == realpath(source):
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment