Commit 54a9d788 authored by Lucas Carvalho's avatar Lucas Carvalho

First version of network cache integration with zc.buildout.

parent f0d0ef63
...@@ -138,6 +138,7 @@ _buildout_default_options = _annotate_section({ ...@@ -138,6 +138,7 @@ _buildout_default_options = _annotate_section({
'socket-timeout': '', 'socket-timeout': '',
'unzip': 'false', 'unzip': 'false',
'use-dependency-links': 'true', 'use-dependency-links': 'true',
'network-cache': 'http://127.0.0.1:5001/',
}, 'DEFAULT_VALUE') }, 'DEFAULT_VALUE')
...@@ -362,6 +363,7 @@ class Buildout(UserDict.DictMixin): ...@@ -362,6 +363,7 @@ class Buildout(UserDict.DictMixin):
newest=self.newest, newest=self.newest,
allow_hosts=self._allow_hosts, allow_hosts=self._allow_hosts,
prefer_final=not self.accept_buildout_test_releases, prefer_final=not self.accept_buildout_test_releases,
network_cache=options['network-cache'],
) )
# Now copy buildout and setuptools eggs, and record destination eggs: # Now copy buildout and setuptools eggs, and record destination eggs:
...@@ -860,6 +862,7 @@ class Buildout(UserDict.DictMixin): ...@@ -860,6 +862,7 @@ class Buildout(UserDict.DictMixin):
path = [options['develop-eggs-directory']], path = [options['develop-eggs-directory']],
allow_hosts = self._allow_hosts, allow_hosts = self._allow_hosts,
prefer_final=not self.accept_buildout_test_releases, prefer_final=not self.accept_buildout_test_releases,
network_cache=options.get('network-cache'),
) )
upgraded = [] upgraded = []
...@@ -1087,7 +1090,8 @@ def _install_and_load(spec, group, entry, buildout): ...@@ -1087,7 +1090,8 @@ def _install_and_load(spec, group, entry, buildout):
working_set=pkg_resources.working_set, working_set=pkg_resources.working_set,
newest=buildout.newest, newest=buildout.newest,
allow_hosts=buildout._allow_hosts, allow_hosts=buildout._allow_hosts,
prefer_final=not buildout.accept_buildout_test_releases) prefer_final=not buildout.accept_buildout_test_releases,
network_cache=buildout_options.get('network-cache'))
__doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry __doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
return pkg_resources.load_entry_point( return pkg_resources.load_entry_point(
......
...@@ -40,6 +40,11 @@ class ChecksumError(zc.buildout.UserError): ...@@ -40,6 +40,11 @@ class ChecksumError(zc.buildout.UserError):
url_opener = URLOpener() url_opener = URLOpener()
from zc.buildout.networkcache import get_sha256sum_from_networkcached, \
download_network_cached, \
upload_network_cached
class Download(object): class Download(object):
"""Configurable download utility. """Configurable download utility.
...@@ -72,6 +77,7 @@ class Download(object): ...@@ -72,6 +77,7 @@ class Download(object):
self.fallback = fallback self.fallback = fallback
self.hash_name = hash_name self.hash_name = hash_name
self.logger = logger or logging.getLogger('zc.buildout') self.logger = logger or logging.getLogger('zc.buildout')
self.network_cache = options.get('network-cache')
@property @property
def download_cache(self): def download_cache(self):
...@@ -174,20 +180,28 @@ class Download(object): ...@@ -174,20 +180,28 @@ class Download(object):
self.logger.info('Downloading %s' % url) self.logger.info('Downloading %s' % url)
urllib._urlopener = url_opener urllib._urlopener = url_opener
handle, tmp_path = tempfile.mkstemp(prefix='buildout-') handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
sha256sum = get_sha256sum_from_networkcached(self.network_cache,
url,
self.logger)
try: try:
try:
if not download_network_cached(self.network_cache,
tmp_path, sha256sum, self.logger):
# Download from original url
tmp_path, headers = urllib.urlretrieve(url, tmp_path) tmp_path, headers = urllib.urlretrieve(url, tmp_path)
if not check_md5sum(tmp_path, md5sum): if not check_md5sum(tmp_path, md5sum):
raise ChecksumError( raise ChecksumError(
'MD5 checksum mismatch downloading %r' % url) 'MD5 checksum mismatch downloading %r' % url)
except IOError, e: # Upload the file to networkcached.
os.remove(tmp_path) upload_network_cached(self.network_cache, url,
raise zc.buildout.UserError("Error downloading extends for URL " tmp_path, self.logger)
"%s: %r" % (url, e[1:3]))
except Exception, e:
os.remove(tmp_path)
raise
finally: finally:
os.close(handle) os.close(handle)
except:
os.remove(tmp_path)
raise
if path: if path:
shutil.move(tmp_path, path) shutil.move(tmp_path, path)
......
...@@ -18,6 +18,8 @@ It doesn't install scripts. It uses setuptools and requires it to be ...@@ -18,6 +18,8 @@ It doesn't install scripts. It uses setuptools and requires it to be
installed. installed.
""" """
from zc.buildout.networkcache import fetch_from_network_cache, \
upload_network_cached
import distutils.errors import distutils.errors
import fnmatch import fnmatch
import glob import glob
...@@ -335,6 +337,7 @@ class Installer: ...@@ -335,6 +337,7 @@ class Installer:
include_site_packages=None, include_site_packages=None,
allowed_eggs_from_site_packages=None, allowed_eggs_from_site_packages=None,
prefer_final=None, prefer_final=None,
network_cache=None,
): ):
self._dest = dest self._dest = dest
self._allow_hosts = allow_hosts self._allow_hosts = allow_hosts
...@@ -403,6 +406,8 @@ class Installer: ...@@ -403,6 +406,8 @@ class Installer:
if versions is not None: if versions is not None:
self._versions = versions self._versions = versions
self._network_cache = network_cache
_allowed_eggs_from_site_packages_regex = None _allowed_eggs_from_site_packages_regex = None
def allow_site_package_egg(self, name): def allow_site_package_egg(self, name):
if (not self._include_site_packages or if (not self._include_site_packages or
...@@ -701,8 +706,13 @@ class Installer: ...@@ -701,8 +706,13 @@ class Installer:
and (realpath(os.path.dirname(dist.location)) == download_cache) and (realpath(os.path.dirname(dist.location)) == download_cache)
): ):
return dist return dist
new_location = fetch_from_network_cache(self._network_cache,
dist.location, tmp, logger)
if new_location is None:
new_location = self._index.download(dist.location, tmp) new_location = self._index.download(dist.location, tmp)
upload_network_cached(self._network_cache,
dist.location, new_location, logger)
if (download_cache if (download_cache
and (realpath(new_location) == realpath(dist.location)) and (realpath(new_location) == realpath(dist.location))
and os.path.isfile(new_location) and os.path.isfile(new_location)
...@@ -1080,13 +1090,13 @@ def install(specs, dest, ...@@ -1080,13 +1090,13 @@ def install(specs, dest,
path=None, working_set=None, newest=True, versions=None, path=None, working_set=None, newest=True, versions=None,
use_dependency_links=None, allow_hosts=('*',), use_dependency_links=None, allow_hosts=('*',),
include_site_packages=None, allowed_eggs_from_site_packages=None, include_site_packages=None, allowed_eggs_from_site_packages=None,
prefer_final=None): prefer_final=None, network_cache=None):
installer = Installer( installer = Installer(
dest, links, index, executable, always_unzip, path, newest, dest, links, index, executable, always_unzip, path, newest,
versions, use_dependency_links, allow_hosts=allow_hosts, versions, use_dependency_links, allow_hosts=allow_hosts,
include_site_packages=include_site_packages, include_site_packages=include_site_packages,
allowed_eggs_from_site_packages=allowed_eggs_from_site_packages, allowed_eggs_from_site_packages=allowed_eggs_from_site_packages,
prefer_final=prefer_final) prefer_final=prefer_final, network_cache=network_cache)
return installer.install(specs, working_set) return installer.install(specs, working_set)
......
##############################################################################
#
# Copyright (c) 2010 ViFiB SARL and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import os
import urllib
import re
import posixpath
import base64
import hashlib
import json
_md5_re = re.compile(r'md5=([a-f0-9]+)')
def _get_md5_from_url(url):
match = _md5_re.search(url)
if match:
return match.group(1)
return None
def _get_hash_from_file(path, hash_object):
"""
Calculate the hash from file.
"""
f = open(path, 'rb')
try:
chunk = f.read(2**16)
while chunk:
hash_object.update(chunk)
chunk = f.read(2**16)
return hash_object.hexdigest()
finally:
f.close()
def _get_sha256_from_file(path):
"""
Return the sha256sum from file.
"""
if not os.path.isfile(path):
raise ValueError, 'Could not extract sha256sum.' \
' The path is not a file: %s ' % path
return _get_hash_from_file(path, hashlib.sha256())
def check_sha256sum(path, sha256sum):
"""Tell whether the SHA256sum checksum of the file at path matches
"""
return sha256sum == _get_hash_from_file(path, hashlib.sha256())
def _update_network_cached_map(network_cache, sha256sum, url, logger):
"""
If the url is not into the map file, it must be updated.
"""
# Keep the log
print os.environ.get('PWD')
map_path = os.path.join(os.environ.get('PWD'), '.networkcached.log')
mode = 'r+'
if not os.path.exists(map_path):
mode = 'w+'
f = open(map_path, mode)
try:
flag = False
for line in f:
if url in line:
flag = True
break
if not flag:
f.seek(0, 2)
f.write('%s %s\n' % (sha256sum, url))
f.truncate(f.tell())
finally:
f.close()
# Update the map at networkcached server.
network_cached_url = os.path.join(network_cache, 'map/add')
try:
result = urllib.urlopen(network_cached_url, \
urllib.urlencode({'sha256sum': sha256sum, 'url': url}))
if int(result.code) != 200:
logger.info('Fail to update the network cache map: %s %s' % \
(sha256sum, url))
except IOError, e:
logger.info('An error occurred to update the map in networkcached.' \
' %s' % str(e))
def get_sha256sum_from_networkcached(network_cache, url, logger):
"""
Return the sha256sum if the url exists in networkcached server.
"""
network_cached_url = os.path.join(network_cache,
'map/get', base64.encodestring(url))
try:
result = urllib.urlopen(network_cached_url)
if int(result.code) == 200:
return json.loads(result.read().strip()).get('sha256sum', None)
except IOError, e:
logger.info('An error occurred to get sha256sum from networkcached:' \
' %s' % str(e))
def download_network_cached(network_cache, path, sha256sum, logger):
"""Download from a network cache provider
If something fail (providor be offline, or hash_string fail), we ignore
network cached files.
return True if download succeeded.
"""
if sha256sum is None:
# Not able to use network cache
return False
url = os.path.join(network_cache, sha256sum)
logger.info('Downloading from network cache %s' % url)
try:
path, headers = urllib.urlretrieve(url, path)
if not check_sha256sum(path, sha256sum):
logger.info('MD5/SHA256 checksum mismatch downloading %r' % url)
return False
except IOError, e:
logger.info('Fail to download from network cache %s' % url)
return False
return True
def upload_network_cached(network_cache, external_url, path, logger):
"""Upload file to a network cache server"""
if network_cache in [None, '']:
logger.debug(
'Upload cache ignored, network-cache was not provided')
return False
sha256sum = _get_sha256_from_file(path)
try:
f = open(path, 'r')
data = f.read()
url = os.path.join(network_cache, sha256sum)
try:
result = urllib.urlopen(url, urllib.urlencode({
"data" : base64.encodestring(data)}))
if result.code == 200 and \
json.loads(result.read()).get('sha') == sha256sum:
_update_network_cached_map(network_cache, sha256sum,
external_url, logger)
except (IOError, EOFError), e:
logger.info('Fail to upload cache on %s. %s' % (external_url, str(e)))
finally:
f.close()
return True
def _get_filename_from_url(url):
"""Inspired how pip get filename from url.
"""
url = url.split('#', 1)[0]
url = url.split('?', 1)[0]
url = url.rstrip('/')
name = posixpath.basename(url)
assert name, (
'URL %r produced no filename' % url)
return name
def fetch_from_network_cache(network_cache, location, tmp, logger=None):
""" Try to download from a network cache and preserve
original filename.
"""
# Make a id from URL
md5sum = _get_md5_from_url(location)
filename = _get_filename_from_url(location)
path = os.path.join(tmp, filename)
sha256sum = get_sha256sum_from_networkcached(network_cache,
location, logger)
if sha256sum is None:
return None
is_downloaded = download_network_cached(network_cache, path, sha256sum, logger)
if is_downloaded:
return path
return None
...@@ -88,6 +88,9 @@ class Eggs(object): ...@@ -88,6 +88,9 @@ class Eggs(object):
kw = {} kw = {}
if 'unzip' in options: if 'unzip' in options:
kw['always_unzip'] = options.query_bool('unzip', None) kw['always_unzip'] = options.query_bool('unzip', None)
if 'network-cache' in b_options:
kw['network_cache'] = b_options.get('network-cache')
ws = zc.buildout.easy_install.install( ws = zc.buildout.easy_install.install(
distributions, options['eggs-directory'], distributions, options['eggs-directory'],
links=self.links, links=self.links,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment