Fix leaking temporary files with templates from URLs

The expected usage of zc.buildout.download.Download() is that the caller is
supposed to remove the temporary files (cf [1] [2])

[1]: https://github.com/buildout/buildout/blob/3ed7c06bf7be594dcb3c906c2517ee438f059251/src/zc/buildout/download.txt#L55
[2]: https://github.com/buildout/buildout/blob/3da22a7dacb9a4d1f771e7763e014214c31d935b/src/zc/buildout/download.py#L171-L176

slapos/recipe/template/README.jinja2.txt

@@ -264,6 +264,37 @@ And the generated file with have the right permissions::
 Note that Buildout will not allow you to have write permission for others
 and will silently remove it (i.e a 207 mode will become 205).
 
+Fetching resources from URLs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can specify the resource to fetch from an URL::
+
+    >>> server_data = tmpdir('server_data')
+    >>> server_url = start_server(server_data)
+
+    >>> write(server_data, 'foo.in', '{{bar}}')
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = template
+    ...
+    ... [template]
+    ... recipe = slapos.recipe.template:jinja2
+    ... template = %sfoo.in
+    ... rendered = foo
+    ... context = key bar buildout:parts
+    ... md5sum = %s
+    ... ''' % (server_url, md5sum))
+    >>> run_buildout()
+    Uninstalling template.
+    Installing template.
+    Downloading http://localhost/foo.in
+    Cannot download http://localhost/foo.in from network cache.
+
+    >>> cat('foo')
+    template
+
+
 Template imports
 ~~~~~~~~~~~~~~~~
 

slapos/recipe/template/README.txt

@@ -130,6 +130,37 @@ And the generated file with have the right permissions::
     >>> print("0%o" % stat.S_IMODE(os.stat('template.out').st_mode))
     0627
 
+
+Fetching template source from an URL
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can fetch resources from an URL::
+
+    >>> server_data = tmpdir('server_data')
+    >>> server_url = start_server(server_data)
+
+    >>> write(server_data, 'template.in', '${buildout:parts}')
+    >>> write('buildout.cfg', '''
+    ... [buildout]
+    ... parts = template
+    ...
+    ... [template]
+    ... recipe = slapos.recipe.template
+    ... url = %stemplate.in
+    ... md5sum = %s
+    ... output = template.out
+    ...
+    ... ''' % (server_url, md5sum))
+
+    >>> run_buildout()
+    Downloading http://localhost/template.in
+    Cannot download http://localhost/template.in from network cache.
+    Uninstalling template.
+    Installing template.
+
+    >>> cat('template.out')
+    template
+
 Section dependency
 ------------------
 

slapos/recipe/template/__init__.py

@@ -35,18 +35,20 @@ class Recipe(object):
     path, is_temp = download(options.pop('url'),
         md5sum=options.get('md5sum'))
 
-    self.mode = None
-    if 'mode' in options:
-      # Mode is in octal notation
-      self.mode = int(options['mode'], 8)
-
-    hidden_option = '__template_content_%s__' % name
-
-    with open(path) as inputfile:
-      self.output_content = '$'.join(options._sub(s, None)
-        for s in inputfile.read().split('$$'))
-
-    self.output_filename = options['output']
+    try:
+      self.mode = None
+      if 'mode' in options:
+        # Mode is in octal notation
+        self.mode = int(options['mode'], 8)
+
+      with open(path) as inputfile:
+        self.output_content = '$'.join(options._sub(s, None)
+          for s in inputfile.read().split('$$'))
+
+      self.output_filename = options['output']
+    finally:
+      if is_temp:
+        os.remove(path)
 
   def install(self):
     with open(self.output_filename, 'w') as outputfile:

slapos/recipe/template/jinja2_template.py

@@ -219,15 +219,19 @@ class Recipe(object):
             try:
                 compiled_source = compiled_source_cache[template]
             except KeyError:
-                download_path = zc.buildout.download.Download(
+                download_path, is_temp = zc.buildout.download.Download(
                     self.buildout['buildout'],
                     hash_name=True,
                 )(
                     template,
                     md5sum=self.md5sum,
-                )[0]
-                with open(download_path, 'rb') as f:
-                    source = f.read().decode(self.encoding)
+                )
+                try:                
+                    with open(download_path, 'rb') as f:
+                        source = f.read().decode(self.encoding)
+                finally:
+                    if is_temp:
+                        os.remove(download_path)
                 compiled_source_cache[template] = compiled_source = \
                     env.compile(source, filename=download_path)
 

slapos/recipe/template/tests.py

@@ -26,12 +26,19 @@
 ##############################################################################
 from __future__ import print_function
 import doctest
+import os
 import re
 import unittest
+import tempfile
 from zc.buildout import testing
 from zope.testing import renormalizing
 
+
+tempdir = tempfile.mkdtemp()
+
+
 def setUp(test):
+  os.environ['TMPDIR'] = tempdir
   testing.buildoutSetUp(test)
   testing.install_develop('slapos.recipe.template', test)
   (lambda system, buildout, **kw: test.globs.update(
@@ -39,10 +46,24 @@ def setUp(test):
       ))(**test.globs)
 
 
+def tearDown(test):
+  testing.buildoutTearDown(test)
+  leaked_tempfiles = os.listdir(tempdir)
+  assert leaked_tempfiles == [], leaked_tempfiles
+
+
 normalize_setuptools_42 = re.compile(
     'WARNING: The easy_install command is deprecated and will be removed in a future version\\.\n'), ''
 normalize_cryptography_3_on_python2 = re.compile(
     '.*CryptographyDeprecationWarning: Python 2 is no longer supported.*\n.*\n'), ''
+normalize_server = (re.compile('http://localhost:[0-9]{4,5}/'), 'http://localhost/')
+try:
+  import slapos.libnetworkcache
+except ImportError:
+  normalize_networkcache = (
+      re.compile('Cannot download http://localhost/([^\s]+) from network cache.\n'), ''),
+else:
+  normalize_networkcache = ()
 
 
 def test_suite():
@@ -50,13 +71,14 @@ def test_suite():
     doctest.DocFileSuite(
       filename,
       setUp=setUp,
-      tearDown=testing.buildoutTearDown,
+      tearDown=tearDown,
       checker=renormalizing.RENormalizing((
         testing.normalize_path,
         testing.not_found,
         normalize_setuptools_42,
         normalize_cryptography_3_on_python2,
-        )),
+        normalize_server,
+      ) + normalize_networkcache),
     ) for filename in [
       'README.txt',
       'README.jinja2.txt',