Commit 6e4b8efe authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki Committed by Julien Muchembled

Support network cache in Download.download().

parent 970ffc9e
...@@ -164,6 +164,8 @@ _buildout_default_options = _annotate_section({ ...@@ -164,6 +164,8 @@ _buildout_default_options = _annotate_section({
'use-dependency-links': 'true', 'use-dependency-links': 'true',
}, 'DEFAULT_VALUE') }, 'DEFAULT_VALUE')
network_cache_parameter_dict = {}
class Buildout(DictMixin): class Buildout(DictMixin):
def __init__(self, config_file, cloptions, def __init__(self, config_file, cloptions,
...@@ -421,6 +423,31 @@ class Buildout(DictMixin): ...@@ -421,6 +423,31 @@ class Buildout(DictMixin):
os.chdir(options['directory']) os.chdir(options['directory'])
networkcache_section_name = options.get('networkcache-section')
if networkcache_section_name:
networkcache_section = self[networkcache_section_name]
for k in (
'download-cache-url',
'download-dir-url',
'upload-cache-url',
'upload-dir-url',
'signature-certificate-list',
'signature-private-key-file',
'shacache-ca-file',
'shacache-cert-file',
'shacache-key-file',
'shadir-ca-file',
'shadir-cert-file',
'shadir-key-file',
):
network_cache_parameter_dict[k] = networkcache_section.get(k, '')
# parse signature list
cert_marker = '-----BEGIN CERTIFICATE-----'
network_cache_parameter_dict['signature-certificate-list'] = \
[cert_marker + '\n' + q.strip() \
for q in network_cache_parameter_dict['signature-certificate-list'].split(cert_marker) \
if q.strip()]
def _buildout_path(self, name): def _buildout_path(self, name):
if '${' in name: if '${' in name:
return name return name
......
...@@ -204,10 +204,29 @@ class Download(object): ...@@ -204,10 +204,29 @@ class Download(object):
handle, tmp_path = tempfile.mkstemp(prefix='buildout-') handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
os.close(handle) os.close(handle)
try: try:
tmp_path, headers = urlretrieve(url, tmp_path) from .buildout import network_cache_parameter_dict as nc
if not check_md5sum(tmp_path, md5sum): if not download_network_cached(
raise ChecksumError( nc.get('download-dir-url'),
'MD5 checksum mismatch downloading %r' % url) nc.get('download-cache-url'),
tmp_path, url, self.logger,
nc.get('signature-certificate-list'), md5sum):
# Download from original url if not cached or md5sum doesn't match.
tmp_path, headers = urlretrieve(url, tmp_path)
if not check_md5sum(tmp_path, md5sum):
raise ChecksumError(
'MD5 checksum mismatch downloading %r' % url)
# Upload the file to network cache.
if nc.get('upload-cache-url') and nc.get('upload-dir-url'):
upload_network_cached(
nc.get('upload-dir-url'),
nc.get('upload-cache-url'), url, tmp_path, self.logger,
nc.get('signature-private-key-file'),
nc.get('shacache-ca-file'),
nc.get('shacache-cert-file'),
nc.get('shacache-key-file'),
nc.get('shadir-ca-file'),
nc.get('shadir-cert-file'),
nc.get('shadir-key-file'))
except IOError: except IOError:
e = sys.exc_info()[1] e = sys.exc_info()[1]
os.remove(tmp_path) os.remove(tmp_path)
...@@ -276,6 +295,8 @@ def remove(path): ...@@ -276,6 +295,8 @@ def remove(path):
if os.path.exists(path): if os.path.exists(path):
os.remove(path) os.remove(path)
from zc.buildout.networkcache import download_network_cached, \
upload_network_cached
def locate_at(source, dest): def locate_at(source, dest):
if dest is None or realpath(dest) == realpath(source): if dest is None or realpath(dest) == realpath(source):
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment