Commit 1cc13345 authored by Julien Muchembled's avatar Julien Muchembled

download: add support for slapos.libnetworkcache

This is a rewrite of commit 6e4b8efe
("Support network cache in Download.download()").
parent c685c3c3
...@@ -245,7 +245,7 @@ _buildout_default_options = _annotate_section({ ...@@ -245,7 +245,7 @@ _buildout_default_options = _annotate_section({
'use-dependency-links': 'true', 'use-dependency-links': 'true',
}, 'DEFAULT_VALUE') }, 'DEFAULT_VALUE')
network_cache_parameter_dict = {} networkcache_client = None
class Buildout(DictMixin): class Buildout(DictMixin):
...@@ -516,27 +516,16 @@ class Buildout(DictMixin): ...@@ -516,27 +516,16 @@ class Buildout(DictMixin):
networkcache_section_name = options.get('networkcache-section') networkcache_section_name = options.get('networkcache-section')
if networkcache_section_name: if networkcache_section_name:
networkcache_section = self[networkcache_section_name] networkcache_section = self[networkcache_section_name]
for k in ( try:
'download-cache-url', from slapos.libnetworkcache import NetworkcacheClient
'download-dir-url', global networkcache_client
'upload-cache-url', networkcache_client = NetworkcacheClient(networkcache_section)
'upload-dir-url', except ImportError:
'signature-certificate-list', pass
'signature-private-key-file', except Exception:
'shacache-ca-file', self._logger.exception("There was problem while trying to"
'shacache-cert-file', " import slapos.libnetworkcache."
'shacache-key-file', " Networkcache forced to be disabled.")
'shadir-ca-file',
'shadir-cert-file',
'shadir-key-file',
):
network_cache_parameter_dict[k] = networkcache_section.get(k, '')
# parse signature list
cert_marker = '-----BEGIN CERTIFICATE-----'
network_cache_parameter_dict['signature-certificate-list'] = \
[cert_marker + '\n' + q.strip() \
for q in network_cache_parameter_dict['signature-certificate-list'].split(cert_marker) \
if q.strip()]
def _buildout_path(self, name): def _buildout_path(self, name):
if '${' in name: if '${' in name:
......
...@@ -20,13 +20,14 @@ except ImportError: ...@@ -20,13 +20,14 @@ except ImportError:
try: try:
# Python 3 # Python 3
from urllib.error import HTTPError
from urllib.request import Request, splitport, splituser, urlopen from urllib.request import Request, splitport, splituser, urlopen
from urllib.parse import urlparse, urlunparse from urllib.parse import urlparse, urlunparse
except ImportError: except ImportError:
# Python 2 # Python 2
from urlparse import urlparse from urlparse import urlparse
from urlparse import urlunparse from urlparse import urlunparse
from urllib2 import Request, splitport, splituser, urlopen from urllib2 import HTTPError, Request, splitport, splituser, urlopen
from zc.buildout.easy_install import realpath from zc.buildout.easy_install import realpath
from base64 import b64encode from base64 import b64encode
...@@ -216,46 +217,31 @@ class Download(object): ...@@ -216,46 +217,31 @@ class Download(object):
if not path: if not path:
handle, tmp_path = tempfile.mkstemp(prefix='buildout-') handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
os.close(handle) os.close(handle)
from .buildout import network_cache_parameter_dict as nc self._download(url, tmp_path, md5sum, alternate_url)
if not download_network_cached( cleanup = False
nc.get('download-dir-url'), finally:
nc.get('download-cache-url'), if cleanup and tmp_path:
tmp_path, url, self.logger, remove(tmp_path)
nc.get('signature-certificate-list'), md5sum):
# Download from original url if not cached or md5sum doesn't match. return tmp_path, not path
def _download(self, url, path, md5sum=None, alternate_url=None):
download_url = url
try:
try: try:
tmp_path, headers = self.urlretrieve(url, tmp_path) self.urlretrieve(url, path)
except HTTPError: except HTTPError:
if not alternate_url: if not alternate_url:
raise raise
self.logger.info('using alternate URL: %s', alternate_url) self.logger.info('using alternate URL: %s', alternate_url)
download_url = alternate_url download_url = alternate_url
tmp_path, headers = self.urlretrieve( self.urlretrieve(alternate_url, path)
alternate_url, tmp_path) if not check_md5sum(path, md5sum):
if not check_md5sum(tmp_path, md5sum): raise ChecksumError('MD5 checksum mismatch downloading %r'
raise ChecksumError( % download_url)
'MD5 checksum mismatch downloading %r' % download_url)
# Upload the file to network cache.
if nc.get('upload-cache-url') and nc.get('upload-dir-url'):
upload_network_cached(
nc.get('upload-dir-url'),
nc.get('upload-cache-url'), url, tmp_path, self.logger,
nc.get('signature-private-key-file'),
nc.get('shacache-ca-file'),
nc.get('shacache-cert-file'),
nc.get('shacache-key-file'),
nc.get('shadir-ca-file'),
nc.get('shadir-cert-file'),
nc.get('shadir-key-file'))
cleanup = False
except IOError as e: except IOError as e:
raise zc.buildout.UserError("Error downloading %s: %s" raise zc.buildout.UserError("Error downloading %s: %s"
% (download_url, e)) % (download_url, e))
finally:
if cleanup and tmp_path:
remove(tmp_path)
return tmp_path, not path
def filename(self, url): def filename(self, url):
"""Determine a file name from a URL according to the configuration. """Determine a file name from a URL according to the configuration.
...@@ -306,6 +292,32 @@ class Download(object): ...@@ -306,6 +292,32 @@ class Download(object):
return tmp_path, src.info() return tmp_path, src.info()
class Download(Download):
def _download(self, url, path, md5sum=None, alternate_url=None):
from .buildout import networkcache_client as nc
key = 'slapos-buildout-' + md5(url.encode()).hexdigest()
if nc and nc.tryDownload(key):
with nc:
# BBB: To avoid collision, wanted_metadata_dict should be
# {'url': url}
entry = next(nc.select(key), None)
if entry is None:
err = 'no matching entry'
else:
with closing(nc.download(entry['sha512'])) as src, \
open(path, 'wb') as dst:
shutil.copyfileobj(src, dst)
if check_md5sum(path, md5sum):
return
err = 'MD5 checksum mismatch'
self.logger.info('Cannot download from network cache: %s', err)
super(Download, self)._download(url, path, md5sum, alternate_url)
if nc and nc.tryUpload(key):
with nc, open(path, 'rb') as f:
nc.upload(f, key, url=url)
def check_md5sum(path, md5sum): def check_md5sum(path, md5sum):
"""Tell whether the MD5 checksum of the file at path matches. """Tell whether the MD5 checksum of the file at path matches.
...@@ -331,8 +343,6 @@ def remove(path): ...@@ -331,8 +343,6 @@ def remove(path):
if os.path.exists(path): if os.path.exists(path):
os.remove(path) os.remove(path)
from zc.buildout.networkcache import \
download_network_cached, upload_network_cached, HTTPError
def locate_at(source, dest): def locate_at(source, dest):
if dest is None or realpath(dest) == realpath(source): if dest is None or realpath(dest) == realpath(source):
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment