Cache pypi index, extensions and recipes using buildout internals.

parent b0bcdf03
......@@ -1193,7 +1193,17 @@ class Buildout(UserDict.DictMixin):
links = self['buildout'].get('find-links', '').split(),
index = self['buildout'].get('index'),
newest=self.newest, allow_hosts=self._allow_hosts,
prefer_final=not self.accept_buildout_test_releases)
prefer_final=not self.accept_buildout_test_releases,
download_cache_url=self.download_cache_url,
download_dir_url=self.download_dir_url,
upload_cache_url=self.upload_cache_url,
upload_dir_url=self.upload_dir_url,
signature_private_key_file=self.signature_private_key_file,
signature_certificate_list=self.signature_certificate_list,
shacache_cert_file=self.shacache_cert_file,
shacache_key_file=self.shacache_key_file,
shadir_cert_file=self.shadir_cert_file,
shadir_key_file=self.shadir_key_file,)
# Clear cache because extensions might now let us read pages we
# couldn't read before.
......
This diff is collapsed.
......@@ -12,12 +12,12 @@
#
##############################################################################
#XXX factor with slapos/grid/networkcache.py and use libnetworkcache helpers
import hashlib
import os
import posixpath
import re
import shutil
import urllib2
import urlparse
import traceback
......@@ -25,7 +25,11 @@ import traceback
try:
try:
from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
DirectoryNotFound
DirectoryNotFound
from slapos.networkcachehelper import \
helper_download_network_cached, \
helper_upload_network_cached_from_file, \
helper_download_network_cached_to_file
except ImportError:
LIBNETWORKCACHE_ENABLED = False
else:
......@@ -68,12 +72,19 @@ def get_directory_key(url):
Basically check if the url belongs to pypi:
- if yes, the directory key will be pypi-buildout-urlmd5
- if not, the directory key will be slapos-buildout-urlmd5
# XXX why is that?
"""
urlmd5 = hashlib.md5(url).hexdigest()
if 'pypi' in url:
return 'pypi-buildout-%s' % urlmd5
return 'slapos-buildout-%s' % urlmd5
@fallback_call
def get_index_directory_key(url, requirement):
"""Returns directory hash based on egg requirement.
"""
return 'pypi-index-%s-%s' % (hashlib.md5(url).hexdigest(), requirement)
@fallback_call
def download_network_cached(dir_url, cache_url, path, url, logger,
......@@ -88,56 +99,74 @@ def download_network_cached(dir_url, cache_url, path, url, logger,
if not LIBNETWORKCACHE_ENABLED:
return False
if not(dir_url and cache_url):
return False
if md5sum is None:
md5sum = _get_md5_from_url(url)
directory_key = get_directory_key(url)
url = os.path.basename(url)
if len(signature_certificate_list) == 0:
# convert [] into None in order to call nc nicely
signature_certificate_list = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_certificate_list=signature_certificate_list)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
logger.debug('Trying to download %s from network cache...' % url)
try:
file_descriptor = nc.select(directory_key)
f = open(path, 'w+b')
try:
shutil.copyfileobj(file_descriptor, f)
finally:
f.close()
file_descriptor.close()
logger.info('Downloaded %s from network cache.' % url)
if helper_download_network_cached_to_file(
dir_url=dir_url,
cache_url=cache_url,
signature_certificate_list=signature_certificate_list,
directory_key=directory_key,
path=path):
logger.info('Downloaded %s from network cache.' % url)
if not check_md5sum(path, md5sum):
logger.info('MD5 checksum mismatch downloading %s' % url)
return False
return True
return False
except (IOError, DirectoryNotFound), e:
if isinstance(e, urllib2.HTTPError) and e.code == 404:
logger.debug('%s does not exist in network cache.' % url)
else:
logger.debug('Failed to download from network cache %s: %s' % \
(url, str(e)))
@fallback_call
def download_index_network_cached(dir_url, cache_url, url, requirement, logger,
signature_certificate_list):
"""
XXX description
Downloads pypi index from a network cache provider
If something fail (providor be offline, or hash_string fail), we ignore
network cached index.
return index if succeeded, False otherwise.
"""
if not LIBNETWORKCACHE_ENABLED:
return False
return True
directory_key = get_index_directory_key(url, requirement)
wanted_metadata_dict = {
'urlmd5':hashlib.md5(url).hexdigest(),
'requirement':requirement
}
required_key_list = ['base']
result = helper_download_network_cached(dir_url, cache_url,
signature_certificate_list,
directory_key, wanted_metadata_dict, required_key_list)
if result:
file_descriptor, metadata = result
try:
content = file_descriptor.read()
logger.info('Downloaded %s from network cache.' % url)
return content, metadata['base']
except (IOError, DirectoryNotFound), e:
if isinstance(e, urllib2.HTTPError) and e.code == 404:
logger.debug('%s does not exist in network cache.' % url)
else:
logger.debug('Failed to download from network cache %s: %s' % \
(url, str(e)))
return False
@fallback_call
def upload_network_cached(dir_url, cache_url, external_url, path, logger,
signature_private_key_file, shacache_cert_file, shacache_key_file,
shadir_cert_file, shadir_key_file):
"""Upload file to a network cache server"""
# XXX use helper and FACTOR code
if not LIBNETWORKCACHE_ENABLED:
return False
......@@ -187,6 +216,64 @@ def upload_network_cached(dir_url, cache_url, external_url, path, logger,
return True
@fallback_call
def upload_index_network_cached(dir_url, cache_url, external_url, base, requirement, content, logger,
signature_private_key_file, shacache_cert_file, shacache_key_file,
shadir_cert_file, shadir_key_file):
# XXX use helper and FACTOR code
"""Upload content of a web page to a network cache server"""
if not LIBNETWORKCACHE_ENABLED:
return False
if not (dir_url and cache_url):
return False
logger.info('Uploading %s content into network cache.' % external_url)
directory_key = get_index_directory_key(external_url, requirement)
kw = dict(file="file",
base=base,
urlmd5=hashlib.md5(external_url).hexdigest(),
requirement=requirement)
import tempfile
f = tempfile.TemporaryFile()
f.write(content)
# convert '' into None in order to call nc nicely
if not signature_private_key_file:
signature_private_key_file = None
if not shacache_cert_file:
shacache_cert_file = None
if not shacache_key_file:
shacache_key_file = None
if not shadir_cert_file:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_private_key_file=signature_private_key_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
try:
return nc.upload_generic(f, directory_key, **kw)
except (IOError, UploadError), e:
logger.info('Fail to upload file. %s' % \
(str(e)))
return False
finally:
f.close()
return True
@fallback_call
def get_filename_from_url(url):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment