Commit ad719a4d authored by Thomas Lotze's avatar Thomas Lotze

- changed the download API so that downloading a file returns both the local

  path and a flag indicating whether the downloaded copy is a temporary file
- use this flag to clean up temporary files both when downloading extended
  configuration files and in the tests
parent c1834afe
......@@ -1238,11 +1238,13 @@ def _open(base, filename, seen, dl_options, override):
"""
_update_section(dl_options, override)
_dl_options = _unannotate_section(dl_options.copy())
is_temp = False
download = zc.buildout.download.Download(
_dl_options, cache=_dl_options.get('extends-cache'), fallback=True,
hash_name=True)
if _isurl(filename):
fp = open(download(filename))
path, is_temp = download(filename)
fp = open(path)
base = filename[:filename.rfind('/')]
elif _isurl(base):
if os.path.isabs(filename):
......@@ -1250,7 +1252,8 @@ def _open(base, filename, seen, dl_options, override):
base = os.path.dirname(filename)
else:
filename = base + '/' + filename
fp = open(download(filename))
path, is_temp = download(filename)
fp = open(path)
base = filename[:filename.rfind('/')]
else:
filename = os.path.join(base, filename)
......@@ -1258,6 +1261,8 @@ def _open(base, filename, seen, dl_options, override):
base = os.path.dirname(filename)
if filename in seen:
if is_temp:
os.unlink(path)
raise zc.buildout.UserError("Recursive file include", seen, filename)
root_config_file = not seen
......@@ -1268,6 +1273,9 @@ def _open(base, filename, seen, dl_options, override):
parser = ConfigParser.RawConfigParser()
parser.optionxform = lambda s: s
parser.readfp(fp)
if is_temp:
os.unlink(path)
extends = extended_by = None
for section in parser.sections():
options = dict(parser.items(section))
......
......@@ -84,11 +84,11 @@ class Download(object):
"""
if self.cache:
local_path = self.download_cached(url, md5sum)
local_path, is_temp = self.download_cached(url, md5sum)
else:
local_path = self.download(url, md5sum, path)
local_path, is_temp = self.download(url, md5sum, path)
return locate_at(local_path, path)
return locate_at(local_path, path), is_temp
def download_cached(self, url, md5sum=None):
"""Download a file from a URL using the cache.
......@@ -106,9 +106,10 @@ class Download(object):
self.logger.debug('Searching cache at %s' % cache_dir)
if os.path.isfile(cached_path):
is_temp = False
if self.fallback:
try:
self.download(url, md5sum, cached_path)
_, is_temp = self.download(url, md5sum, cached_path)
except ChecksumError:
raise
except Exception:
......@@ -122,9 +123,9 @@ class Download(object):
else:
self.logger.debug('Cache miss; will cache %s as %s' %
(url, cached_path))
self.download(url, md5sum, cached_path)
_, is_temp = self.download(url, md5sum, cached_path)
return cached_path
return cached_path, is_temp
def download(self, url, md5sum=None, path=None):
"""Download a file from a URL to a given or temporary path.
......@@ -143,7 +144,7 @@ class Download(object):
raise ChecksumError(
'MD5 checksum mismatch for local resource at %r.' %
url_path)
return locate_at(url_path, path)
return locate_at(url_path, path), False
if self.offline:
raise zc.buildout.UserError(
......@@ -152,18 +153,23 @@ class Download(object):
self.logger.info('Downloading %s' % url)
urllib._urlopener = url_opener
handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
try:
try:
tmp_path, headers = urllib.urlretrieve(url, tmp_path)
os.close(handle)
if not check_md5sum(tmp_path, md5sum):
os.remove(tmp_path)
raise ChecksumError(
'MD5 checksum mismatch downloading %r' % url)
except:
os.remove(tmp_path)
raise
finally:
os.close(handle)
if path:
shutil.move(tmp_path, path)
return path
return path, False
else:
return tmp_path
return tmp_path, True
def filename(self, url):
"""Determine a file name from a URL according to the configuration.
......
This diff is collapsed.
......@@ -13,6 +13,13 @@ Also, all of the following will take place inside the sample buildout.
>>> server_url = start_server(server_data)
>>> cd(sample_buildout)
We also use a fresh directory for temporary files in order to make sure that
all temporary files have been cleaned up in the end:
>>> import tempfile
>>> old_tempdir = tempfile.tempdir
>>> tempfile.tempdir = tmpdir('tmp')
Basic use of the extends cache
------------------------------
......@@ -375,3 +382,15 @@ While:
Checking for upgrades.
An internal error occured ...
ValueError: install_from_cache set to true with no download cache
Clean up
--------
We should have cleaned up all temporary files created by downloading things:
>>> ls(tempfile.tempdir)
Reset the global temporary directory:
>>> tempfile.tempdir = old_tempdir
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment