Commit 0690e743 authored by Yingjie Xu's avatar Yingjie Xu

Implement download.

parent f0c93b97
......@@ -82,7 +82,21 @@ class Software(object):
tarname = self.software_url_hash
cache_dir = tempfile.mkdtemp()
tarpath = os.path.join(cache_dir, tarname)
if os.path.exists(self.software_path):
self._install_from_buildout()
else:
if download_network_cached(
self.download_binary_cache_url,
self.download_binary_dir_url,
self.url, self.software_url_hash,
tarpath, self.logger,
self.signature_certificate_list):
tar = tarfile.open(tarpath)
try:
tar.extractall(path=self.software_root)
finally:
tar.close()
else:
tar = tarfile.open(tarpath, "w:gz")
try:
tar.add(self.software_path, arcname=self.software_url_hash)
......@@ -90,8 +104,7 @@ class Software(object):
tar.close()
upload_network_cached(
self.software_root,
self.url,
self.software_url_hash,
self.url, self.software_url_hash,
self.upload_binary_cache_url,
self.upload_binary_dir_url,
tarpath, self.logger,
......@@ -99,8 +112,7 @@ class Software(object):
self.shacache_cert_file,
self.shacache_key_file,
self.shadir_cert_file,
self.shadir_key_file
)
self.shadir_key_file)
def _install_from_buildout(self):
""" Fetches buildout configuration from the server, run buildout with
......
......@@ -21,6 +21,7 @@ import shutil
import urlparse
import traceback
import utils
import json
try:
try:
......@@ -53,29 +54,19 @@ def fallback_call(function):
@fallback_call
def download_network_cached(dir_url, cache_url, path, url, logger,
signature_certificate_list, md5sum=None):
def download_network_cached(cache_url, dir_url, software_url, key, path,
logger, signature_certificate_list):
"""Downloads from a network cache provider
If something fail (providor be offline, or hash_string fail), we ignore
network cached files.
return True if download succeeded.
"""
if not LIBNETWORKCACHE_ENABLED:
return False
if not(dir_url and cache_url):
if not(cache_url and dir_url):
return False
if md5sum is None:
md5sum = _get_md5_from_url(url)
directory_key = get_directory_key(url)
url = os.path.basename(url)
if len(signature_certificate_list) == 0:
# convert [] into None in order to call nc nicely
signature_certificate_list = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
......@@ -84,21 +75,24 @@ def download_network_cached(dir_url, cache_url, path, url, logger,
logger.warning('Incompatible version of networkcache, not using it.')
return False
logger.info('Downloading %s from network cache.' % url)
logger.info('Downloading %s binary from network cache.' % software_url)
try:
file_descriptor = nc.select(directory_key)
json_entry_list = nc.select_generic(key)
for entry in json_entry_list:
json_information, _ = entry
try:
tags = json.loads(json_information)
sha512 = tags.get('sha512')
file_descriptor = nc.download(sha512)
break
except Exception:
continue
f = open(path, 'w+b')
try:
shutil.copyfileobj(file_descriptor, f)
finally:
f.close()
file_descriptor.close()
if not check_md5sum(path, md5sum):
logger.info('MD5 checksum mismatch downloading %s' % url)
return False
except (IOError, DirectoryNotFound), e:
logger.info('Failed to download from network cache %s: %s' % \
(url, str(e)))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment