Commit ad719a4d authored by Thomas Lotze's avatar Thomas Lotze

- changed the download API so that downloading a file returns both the local

  path and a flag indicating whether the downloaded copy is a temporary file
- use this flag to clean up temporary files both when downloading extended
  configuration files and in the tests
parent c1834afe
...@@ -1238,11 +1238,13 @@ def _open(base, filename, seen, dl_options, override): ...@@ -1238,11 +1238,13 @@ def _open(base, filename, seen, dl_options, override):
""" """
_update_section(dl_options, override) _update_section(dl_options, override)
_dl_options = _unannotate_section(dl_options.copy()) _dl_options = _unannotate_section(dl_options.copy())
is_temp = False
download = zc.buildout.download.Download( download = zc.buildout.download.Download(
_dl_options, cache=_dl_options.get('extends-cache'), fallback=True, _dl_options, cache=_dl_options.get('extends-cache'), fallback=True,
hash_name=True) hash_name=True)
if _isurl(filename): if _isurl(filename):
fp = open(download(filename)) path, is_temp = download(filename)
fp = open(path)
base = filename[:filename.rfind('/')] base = filename[:filename.rfind('/')]
elif _isurl(base): elif _isurl(base):
if os.path.isabs(filename): if os.path.isabs(filename):
...@@ -1250,7 +1252,8 @@ def _open(base, filename, seen, dl_options, override): ...@@ -1250,7 +1252,8 @@ def _open(base, filename, seen, dl_options, override):
base = os.path.dirname(filename) base = os.path.dirname(filename)
else: else:
filename = base + '/' + filename filename = base + '/' + filename
fp = open(download(filename)) path, is_temp = download(filename)
fp = open(path)
base = filename[:filename.rfind('/')] base = filename[:filename.rfind('/')]
else: else:
filename = os.path.join(base, filename) filename = os.path.join(base, filename)
...@@ -1258,6 +1261,8 @@ def _open(base, filename, seen, dl_options, override): ...@@ -1258,6 +1261,8 @@ def _open(base, filename, seen, dl_options, override):
base = os.path.dirname(filename) base = os.path.dirname(filename)
if filename in seen: if filename in seen:
if is_temp:
os.unlink(path)
raise zc.buildout.UserError("Recursive file include", seen, filename) raise zc.buildout.UserError("Recursive file include", seen, filename)
root_config_file = not seen root_config_file = not seen
...@@ -1268,6 +1273,9 @@ def _open(base, filename, seen, dl_options, override): ...@@ -1268,6 +1273,9 @@ def _open(base, filename, seen, dl_options, override):
parser = ConfigParser.RawConfigParser() parser = ConfigParser.RawConfigParser()
parser.optionxform = lambda s: s parser.optionxform = lambda s: s
parser.readfp(fp) parser.readfp(fp)
if is_temp:
os.unlink(path)
extends = extended_by = None extends = extended_by = None
for section in parser.sections(): for section in parser.sections():
options = dict(parser.items(section)) options = dict(parser.items(section))
......
...@@ -84,11 +84,11 @@ class Download(object): ...@@ -84,11 +84,11 @@ class Download(object):
""" """
if self.cache: if self.cache:
local_path = self.download_cached(url, md5sum) local_path, is_temp = self.download_cached(url, md5sum)
else: else:
local_path = self.download(url, md5sum, path) local_path, is_temp = self.download(url, md5sum, path)
return locate_at(local_path, path) return locate_at(local_path, path), is_temp
def download_cached(self, url, md5sum=None): def download_cached(self, url, md5sum=None):
"""Download a file from a URL using the cache. """Download a file from a URL using the cache.
...@@ -106,9 +106,10 @@ class Download(object): ...@@ -106,9 +106,10 @@ class Download(object):
self.logger.debug('Searching cache at %s' % cache_dir) self.logger.debug('Searching cache at %s' % cache_dir)
if os.path.isfile(cached_path): if os.path.isfile(cached_path):
is_temp = False
if self.fallback: if self.fallback:
try: try:
self.download(url, md5sum, cached_path) _, is_temp = self.download(url, md5sum, cached_path)
except ChecksumError: except ChecksumError:
raise raise
except Exception: except Exception:
...@@ -122,9 +123,9 @@ class Download(object): ...@@ -122,9 +123,9 @@ class Download(object):
else: else:
self.logger.debug('Cache miss; will cache %s as %s' % self.logger.debug('Cache miss; will cache %s as %s' %
(url, cached_path)) (url, cached_path))
self.download(url, md5sum, cached_path) _, is_temp = self.download(url, md5sum, cached_path)
return cached_path return cached_path, is_temp
def download(self, url, md5sum=None, path=None): def download(self, url, md5sum=None, path=None):
"""Download a file from a URL to a given or temporary path. """Download a file from a URL to a given or temporary path.
...@@ -143,7 +144,7 @@ class Download(object): ...@@ -143,7 +144,7 @@ class Download(object):
raise ChecksumError( raise ChecksumError(
'MD5 checksum mismatch for local resource at %r.' % 'MD5 checksum mismatch for local resource at %r.' %
url_path) url_path)
return locate_at(url_path, path) return locate_at(url_path, path), False
if self.offline: if self.offline:
raise zc.buildout.UserError( raise zc.buildout.UserError(
...@@ -152,18 +153,23 @@ class Download(object): ...@@ -152,18 +153,23 @@ class Download(object):
self.logger.info('Downloading %s' % url) self.logger.info('Downloading %s' % url)
urllib._urlopener = url_opener urllib._urlopener = url_opener
handle, tmp_path = tempfile.mkstemp(prefix='buildout-') handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
tmp_path, headers = urllib.urlretrieve(url, tmp_path) try:
os.close(handle) try:
if not check_md5sum(tmp_path, md5sum): tmp_path, headers = urllib.urlretrieve(url, tmp_path)
os.remove(tmp_path) if not check_md5sum(tmp_path, md5sum):
raise ChecksumError( raise ChecksumError(
'MD5 checksum mismatch downloading %r' % url) 'MD5 checksum mismatch downloading %r' % url)
except:
os.remove(tmp_path)
raise
finally:
os.close(handle)
if path: if path:
shutil.move(tmp_path, path) shutil.move(tmp_path, path)
return path return path, False
else: else:
return tmp_path return tmp_path, True
def filename(self, url): def filename(self, url):
"""Determine a file name from a URL according to the configuration. """Determine a file name from a URL according to the configuration.
......
This diff is collapsed.
...@@ -13,6 +13,13 @@ Also, all of the following will take place inside the sample buildout. ...@@ -13,6 +13,13 @@ Also, all of the following will take place inside the sample buildout.
>>> server_url = start_server(server_data) >>> server_url = start_server(server_data)
>>> cd(sample_buildout) >>> cd(sample_buildout)
We also use a fresh directory for temporary files in order to make sure that
all temporary files have been cleaned up in the end:
>>> import tempfile
>>> old_tempdir = tempfile.tempdir
>>> tempfile.tempdir = tmpdir('tmp')
Basic use of the extends cache Basic use of the extends cache
------------------------------ ------------------------------
...@@ -375,3 +382,15 @@ While: ...@@ -375,3 +382,15 @@ While:
Checking for upgrades. Checking for upgrades.
An internal error occured ... An internal error occured ...
ValueError: install_from_cache set to true with no download cache ValueError: install_from_cache set to true with no download cache
Clean up
--------
We should have cleaned up all temporary files created by downloading things:
>>> ls(tempfile.tempdir)
Reset the global temporary directory:
>>> tempfile.tempdir = old_tempdir
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment