Commit e4ceb0ac authored by Jim Fulton's avatar Jim Fulton

Author: Thomas Lotze <tl@gocept.com>

Date:   Wed Feb 16 15:47:25 2011 +0000
made sure to download extended configuration files only once per buildout run (based on patch by Rafael Monnerat)

Cherry-picked from svn trunk
parent 7033957c
...@@ -25,6 +25,9 @@ New features: ...@@ -25,6 +25,9 @@ New features:
- Remove duplicate path from script's sys.path setup. - Remove duplicate path from script's sys.path setup.
- Make sure to download extended configuration files only once per buildout
run even if they are referenced multiple times (patch by Rafael Monnerat).
Bugs fixed: Bugs fixed:
- In the download module, fixed the handling of directories that are pointed - In the download module, fixed the handling of directories that are pointed
......
...@@ -195,12 +195,12 @@ class Buildout(UserDict.DictMixin): ...@@ -195,12 +195,12 @@ class Buildout(UserDict.DictMixin):
'.buildout', 'default.cfg') '.buildout', 'default.cfg')
if os.path.exists(user_config): if os.path.exists(user_config):
_update(data, _open(os.path.dirname(user_config), user_config, _update(data, _open(os.path.dirname(user_config), user_config,
[], data['buildout'].copy(), override)) [], data['buildout'].copy(), override, set()))
# load configuration files # load configuration files
if config_file: if config_file:
_update(data, _open(os.path.dirname(config_file), config_file, [], _update(data, _open(os.path.dirname(config_file), config_file, [],
data['buildout'].copy(), override)) data['buildout'].copy(), override, set()))
# apply command-line options # apply command-line options
_update(data, cloptions) _update(data, cloptions)
...@@ -1286,17 +1286,26 @@ def _save_options(section, options, f): ...@@ -1286,17 +1286,26 @@ def _save_options(section, options, f):
for option, value in items: for option, value in items:
_save_option(option, value, f) _save_option(option, value, f)
def _open(base, filename, seen, dl_options, override): def _convert_bool(name, value):
if value not in ('true', 'false'):
raise zc.buildout.UserError(
'Invalid value for %s option: %s' % (name, value))
else:
return value == 'true'
def _open(base, filename, seen, dl_options, override, downloaded):
"""Open a configuration file and return the result as a dictionary, """Open a configuration file and return the result as a dictionary,
Recursively open other files based on buildout options found. Recursively open other files based on buildout options found.
""" """
_update_section(dl_options, override) _update_section(dl_options, override)
_dl_options = _unannotate_section(dl_options.copy()) _dl_options = _unannotate_section(dl_options.copy())
is_temp = False newest = _convert_bool('newest', _dl_options.get('newest', 'false'))
fallback = newest and not (filename in downloaded)
download = zc.buildout.download.Download( download = zc.buildout.download.Download(
_dl_options, cache=_dl_options.get('extends-cache'), fallback=True, _dl_options, cache=_dl_options.get('extends-cache'),
hash_name=True) fallback=fallback, hash_name=True)
is_temp = False
if _isurl(filename): if _isurl(filename):
path, is_temp = download(filename) path, is_temp = download(filename)
fp = open(path) fp = open(path)
...@@ -1314,6 +1323,7 @@ def _open(base, filename, seen, dl_options, override): ...@@ -1314,6 +1323,7 @@ def _open(base, filename, seen, dl_options, override):
filename = os.path.join(base, filename) filename = os.path.join(base, filename)
fp = open(filename) fp = open(filename)
base = os.path.dirname(filename) base = os.path.dirname(filename)
downloaded.add(filename)
if filename in seen: if filename in seen:
if is_temp: if is_temp:
...@@ -1348,9 +1358,11 @@ def _open(base, filename, seen, dl_options, override): ...@@ -1348,9 +1358,11 @@ def _open(base, filename, seen, dl_options, override):
if extends: if extends:
extends = extends.split() extends = extends.split()
eresult = _open(base, extends.pop(0), seen, dl_options, override) eresult = _open(base, extends.pop(0), seen, dl_options, override,
downloaded)
for fname in extends: for fname in extends:
_update(eresult, _open(base, fname, seen, dl_options, override)) _update(eresult, _open(base, fname, seen, dl_options, override,
downloaded))
result = _update(eresult, result) result = _update(eresult, result)
if extended_by: if extended_by:
...@@ -1358,9 +1370,9 @@ def _open(base, filename, seen, dl_options, override): ...@@ -1358,9 +1370,9 @@ def _open(base, filename, seen, dl_options, override):
"The extendedBy option is deprecated. Stop using it." "The extendedBy option is deprecated. Stop using it."
) )
for fname in extended_by.split(): for fname in extended_by.split():
result = _update(result, result = _update(
_open(base, fname, seen, dl_options, override)) result,
_open(base, fname, seen, dl_options, override, downloaded))
seen.pop() seen.pop()
return result return result
......
...@@ -386,6 +386,105 @@ While: ...@@ -386,6 +386,105 @@ While:
An internal error occured ... An internal error occured ...
ValueError: install_from_cache set to true with no download cache ValueError: install_from_cache set to true with no download cache
>>> rmdir('home', '.buildout')
Newest and non-newest behaviour for extends cache
-------------------------------------------------
While offline mode forbids network access completely, 'newest' mode determines
whether to look for updated versions of a resource even if some version of it
is already present locally. If we run buildout in newest mode
(``newest = true``), the configuration files are updated with each run:
>>> mkdir("cache")
>>> write(server_data, 'base.cfg', """\
... [buildout]
... parts =
... """)
>>> write('buildout.cfg', """\
... [buildout]
... extends-cache = cache
... extends = %sbase.cfg
... """ % server_url)
>>> print system(buildout)
>>> ls('cache')
- 5aedc98d7e769290a29d654a591a3a45
>>> cat('cache', os.listdir(cache)[0])
[buildout]
parts =
A change to ``base.cfg`` is picked up on the next buildout run:
>>> write(server_data, 'base.cfg', """\
... [buildout]
... parts =
... foo = bar
... """)
>>> print system(buildout + " -n")
Unused options for buildout: 'foo'.
>>> cat('cache', os.listdir(cache)[0])
[buildout]
parts =
foo = bar
In contrast, when not using ``newest`` mode (``newest = false``), the files
already present in the extends cache will not be updated:
>>> write(server_data, 'base.cfg', """\
... [buildout]
... parts =
... """)
>>> print system(buildout + " -N")
Unused options for buildout: 'foo'.
>>> cat('cache', os.listdir(cache)[0])
[buildout]
parts =
foo = bar
Even when updating base configuration files with a buildout run, any given
configuration file will be downloaded only once during that particular run. If
some base configuration file is extended more than once, its cached copy is
used:
>>> write(server_data, 'baseA.cfg', """\
... [buildout]
... extends = %sbase.cfg
... foo = bar
... """ % server_url)
>>> write(server_data, 'baseB.cfg', """\
... [buildout]
... extends-cache = cache
... extends = %sbase.cfg
... bar = foo
... """ % server_url)
>>> write('buildout.cfg', """\
... [buildout]
... extends-cache = cache
... newest = true
... extends = %sbaseA.cfg %sbaseB.cfg
... """ % (server_url, server_url))
>>> print system(buildout + " -n")
Unused options for buildout: 'bar' 'foo'.
(XXX We patch download utility's API to produce readable output for the test;
a better solution would utilise the logging already done by the utility.)
>>> import zc.buildout
>>> old_download = zc.buildout.download.Download.download
>>> def wrapper_download(self, url, md5sum=None, path=None):
... print "The URL %s was downloaded." % url
... return old_download(url, md5sum, path)
>>> zc.buildout.download.Download.download = wrapper_download
>>> zc.buildout.buildout.main([])
The URL http://localhost/baseA.cfg was downloaded.
The URL http://localhost/base.cfg was downloaded.
The URL http://localhost/baseB.cfg was downloaded.
Not upgrading because not running a local buildout command.
Unused options for buildout: 'bar' 'foo'.
>>> zc.buildout.download.Download.download = old_download
Clean up Clean up
-------- --------
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment