Commit f7d36f80 authored by Jérome Perrin's avatar Jérome Perrin

Fix leaking temporary files with templates from URLs

The expected usage of zc.buildout.download.Download() is that the caller is
supposed to remove the temporary files (cf [1] [2])

[1]: https://github.com/buildout/buildout/blob/3ed7c06bf7be594dcb3c906c2517ee438f059251/src/zc/buildout/download.txt#L55
[2]: https://github.com/buildout/buildout/blob/3da22a7dacb9a4d1f771e7763e014214c31d935b/src/zc/buildout/download.py#L171-L176
parent c50ca752
......@@ -264,6 +264,37 @@ And the generated file with have the right permissions::
Note that Buildout will not allow you to have write permission for others
and will silently remove it (i.e a 207 mode will become 205).
Fetching resources from URLs
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
You can specify the resource to fetch from an URL::
>>> server_data = tmpdir('server_data')
>>> server_url = start_server(server_data)
>>> write(server_data, 'foo.in', '{{bar}}')
>>> write('buildout.cfg',
... '''
... [buildout]
... parts = template
...
... [template]
... recipe = slapos.recipe.template:jinja2
... template = %sfoo.in
... rendered = foo
... context = key bar buildout:parts
... md5sum = %s
... ''' % (server_url, md5sum))
>>> run_buildout()
Uninstalling template.
Installing template.
Downloading http://localhost/foo.in
Cannot download http://localhost/foo.in from network cache.
>>> cat('foo')
template
Template imports
~~~~~~~~~~~~~~~~
......
......@@ -130,6 +130,37 @@ And the generated file with have the right permissions::
>>> print("0%o" % stat.S_IMODE(os.stat('template.out').st_mode))
0627
Fetching template source from an URL
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
You can fetch resources from an URL::
>>> server_data = tmpdir('server_data')
>>> server_url = start_server(server_data)
>>> write(server_data, 'template.in', '${buildout:parts}')
>>> write('buildout.cfg', '''
... [buildout]
... parts = template
...
... [template]
... recipe = slapos.recipe.template
... url = %stemplate.in
... md5sum = %s
... output = template.out
...
... ''' % (server_url, md5sum))
>>> run_buildout()
Downloading http://localhost/template.in
Cannot download http://localhost/template.in from network cache.
Uninstalling template.
Installing template.
>>> cat('template.out')
template
Section dependency
------------------
......
......@@ -35,18 +35,20 @@ class Recipe(object):
path, is_temp = download(options.pop('url'),
md5sum=options.get('md5sum'))
self.mode = None
if 'mode' in options:
# Mode is in octal notation
self.mode = int(options['mode'], 8)
hidden_option = '__template_content_%s__' % name
with open(path) as inputfile:
self.output_content = '$'.join(options._sub(s, None)
for s in inputfile.read().split('$$'))
self.output_filename = options['output']
try:
self.mode = None
if 'mode' in options:
# Mode is in octal notation
self.mode = int(options['mode'], 8)
with open(path) as inputfile:
self.output_content = '$'.join(options._sub(s, None)
for s in inputfile.read().split('$$'))
self.output_filename = options['output']
finally:
if is_temp:
os.remove(path)
def install(self):
with open(self.output_filename, 'w') as outputfile:
......
......@@ -219,15 +219,19 @@ class Recipe(object):
try:
compiled_source = compiled_source_cache[template]
except KeyError:
download_path = zc.buildout.download.Download(
download_path, is_temp = zc.buildout.download.Download(
self.buildout['buildout'],
hash_name=True,
)(
template,
md5sum=self.md5sum,
)[0]
with open(download_path, 'rb') as f:
source = f.read().decode(self.encoding)
)
try:
with open(download_path, 'rb') as f:
source = f.read().decode(self.encoding)
finally:
if is_temp:
os.remove(download_path)
compiled_source_cache[template] = compiled_source = \
env.compile(source, filename=download_path)
......
......@@ -26,12 +26,19 @@
##############################################################################
from __future__ import print_function
import doctest
import os
import re
import unittest
import tempfile
from zc.buildout import testing
from zope.testing import renormalizing
tempdir = tempfile.mkdtemp()
def setUp(test):
os.environ['TMPDIR'] = tempdir
testing.buildoutSetUp(test)
testing.install_develop('slapos.recipe.template', test)
(lambda system, buildout, **kw: test.globs.update(
......@@ -39,10 +46,24 @@ def setUp(test):
))(**test.globs)
def tearDown(test):
testing.buildoutTearDown(test)
leaked_tempfiles = os.listdir(tempdir)
assert leaked_tempfiles == [], leaked_tempfiles
normalize_setuptools_42 = re.compile(
'WARNING: The easy_install command is deprecated and will be removed in a future version\\.\n'), ''
normalize_cryptography_3_on_python2 = re.compile(
'.*CryptographyDeprecationWarning: Python 2 is no longer supported.*\n.*\n'), ''
normalize_server = (re.compile('http://localhost:[0-9]{4,5}/'), 'http://localhost/')
try:
import slapos.libnetworkcache
except ImportError:
normalize_networkcache = (
re.compile('Cannot download http://localhost/([^\s]+) from network cache.\n'), ''),
else:
normalize_networkcache = ()
def test_suite():
......@@ -50,13 +71,14 @@ def test_suite():
doctest.DocFileSuite(
filename,
setUp=setUp,
tearDown=testing.buildoutTearDown,
tearDown=tearDown,
checker=renormalizing.RENormalizing((
testing.normalize_path,
testing.not_found,
normalize_setuptools_42,
normalize_cryptography_3_on_python2,
)),
normalize_server,
) + normalize_networkcache),
) for filename in [
'README.txt',
'README.jinja2.txt',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment