- changed the download API so that downloading a file returns both the local

  path and a flag indicating whether the downloaded copy is a temporary file
- use this flag to clean up temporary files both when downloading extended
  configuration files and in the tests


git-svn-id: http://svn.zope.org/repos/main/zc.buildout/trunk@102708 62d5b8a3-27da-0310-9561-8e5933582275

src/zc/buildout/buildout.py

@@ -1238,11 +1238,13 @@ def _open(base, filename, seen, dl_options, override):
     """
     _update_section(dl_options, override)
     _dl_options = _unannotate_section(dl_options.copy())
+    is_temp = False
     download = zc.buildout.download.Download(
         _dl_options, cache=_dl_options.get('extends-cache'), fallback=True,
         hash_name=True)
     if _isurl(filename):
-        fp = open(download(filename))
+        path, is_temp = download(filename)
+        fp = open(path)
         base = filename[:filename.rfind('/')]
     elif _isurl(base):
         if os.path.isabs(filename):
@@ -1250,7 +1252,8 @@ def _open(base, filename, seen, dl_options, override):
             base = os.path.dirname(filename)
         else:
             filename = base + '/' + filename
-            fp = open(download(filename))
+            path, is_temp = download(filename)
+            fp = open(path)
             base = filename[:filename.rfind('/')]
     else:
         filename = os.path.join(base, filename)
@@ -1258,6 +1261,8 @@ def _open(base, filename, seen, dl_options, override):
         base = os.path.dirname(filename)
 
     if filename in seen:
+        if is_temp:
+            os.unlink(path)
         raise zc.buildout.UserError("Recursive file include", seen, filename)
 
     root_config_file = not seen
@@ -1268,6 +1273,9 @@ def _open(base, filename, seen, dl_options, override):
     parser = ConfigParser.RawConfigParser()
     parser.optionxform = lambda s: s
     parser.readfp(fp)
+    if is_temp:
+        os.unlink(path)
+
     extends = extended_by = None
     for section in parser.sections():
         options = dict(parser.items(section))

src/zc/buildout/download.py

@@ -84,11 +84,11 @@ class Download(object):
 
         """
         if self.cache:
-            local_path = self.download_cached(url, md5sum)
+            local_path, is_temp = self.download_cached(url, md5sum)
         else:
-            local_path = self.download(url, md5sum, path)
+            local_path, is_temp = self.download(url, md5sum, path)
 
-        return locate_at(local_path, path)
+        return locate_at(local_path, path), is_temp
 
     def download_cached(self, url, md5sum=None):
         """Download a file from a URL using the cache.
@@ -106,9 +106,10 @@ class Download(object):
 
         self.logger.debug('Searching cache at %s' % cache_dir)
         if os.path.isfile(cached_path):
+            is_temp = False
             if self.fallback:
                 try:
-                    self.download(url, md5sum, cached_path)
+                    _, is_temp = self.download(url, md5sum, cached_path)
                 except ChecksumError:
                     raise
                 except Exception:
@@ -122,9 +123,9 @@ class Download(object):
         else:
             self.logger.debug('Cache miss; will cache %s as %s' %
                               (url, cached_path))
-            self.download(url, md5sum, cached_path)
+            _, is_temp = self.download(url, md5sum, cached_path)
 
-        return cached_path
+        return cached_path, is_temp
 
     def download(self, url, md5sum=None, path=None):
         """Download a file from a URL to a given or temporary path.
@@ -143,7 +144,7 @@ class Download(object):
                 raise ChecksumError(
                     'MD5 checksum mismatch for local resource at %r.' %
                     url_path)
-            return locate_at(url_path, path)
+            return locate_at(url_path, path), False
 
         if self.offline:
             raise zc.buildout.UserError(
@@ -152,18 +153,23 @@ class Download(object):
         self.logger.info('Downloading %s' % url)
         urllib._urlopener = url_opener
         handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
-        tmp_path, headers = urllib.urlretrieve(url, tmp_path)
-        os.close(handle)
-        if not check_md5sum(tmp_path, md5sum):
-            os.remove(tmp_path)
-            raise ChecksumError(
-                'MD5 checksum mismatch downloading %r' % url)
+        try:
+            try:
+                tmp_path, headers = urllib.urlretrieve(url, tmp_path)
+                if not check_md5sum(tmp_path, md5sum):
+                    raise ChecksumError(
+                        'MD5 checksum mismatch downloading %r' % url)
+            except:
+                os.remove(tmp_path)
+                raise
+        finally:
+            os.close(handle)
 
         if path:
             shutil.move(tmp_path, path)
-            return path
+            return path, False
         else:
-            return tmp_path
+            return tmp_path, True
 
     def filename(self, url):
         """Determine a file name from a URL according to the configuration.

src/zc/buildout/extends-cache.txt

@@ -13,6 +13,13 @@ Also, all of the following will take place inside the sample buildout.
 >>> server_url = start_server(server_data)
 >>> cd(sample_buildout)
 
+We also use a fresh directory for temporary files in order to make sure that
+all temporary files have been cleaned up in the end:
+
+>>> import tempfile
+>>> old_tempdir = tempfile.tempdir
+>>> tempfile.tempdir = tmpdir('tmp')
+
 
 Basic use of the extends cache
 ------------------------------
@@ -375,3 +382,15 @@ While:
   Checking for upgrades.
 An internal error occured ...
 ValueError: install_from_cache set to true with no download cache
+
+
+Clean up
+--------
+
+We should have cleaned up all temporary files created by downloading things:
+
+>>> ls(tempfile.tempdir)
+
+Reset the global temporary directory:
+
+>>> tempfile.tempdir = old_tempdir