made sure to download extended configuration files only once per buildout run...

made sure to download extended configuration files only once per buildout run (based on patch by Rafael Monnerat)

git-svn-id: http://svn.zope.org/repos/main/zc.buildout/trunk@120386 62d5b8a3-27da-0310-9561-8e5933582275

CHANGES.txt

@@ -12,6 +12,9 @@ Change History
   due to missing support for the ``-E`` option, which only got added
   afterwards.
 
+- Made sure to download extended configuration files only once per buildout
+  run even if they are referenced multiple times (patch by Rafael Monnerat).
+
 Bugs fixed:
 
 - In the download module, fixed the handling of directories that are pointed

src/zc/buildout/buildout.py

@@ -193,12 +193,12 @@ class Buildout(UserDict.DictMixin):
                                        '.buildout', 'default.cfg')
             if os.path.exists(user_config):
                 _update(data, _open(os.path.dirname(user_config), user_config,
-                                    [], data['buildout'].copy(), override))
+                                    [], data['buildout'].copy(), override, set()))
 
         # load configuration files
         if config_file:
             _update(data, _open(os.path.dirname(config_file), config_file, [],
-                                data['buildout'].copy(), override))
+                                data['buildout'].copy(), override, set()))
 
         # apply command-line options
         _update(data, cloptions)
@@ -1387,17 +1387,19 @@ def _save_options(section, options, f):
     for option, value in items:
         _save_option(option, value, f)
 
-def _open(base, filename, seen, dl_options, override):
+def _open(base, filename, seen, dl_options, override, downloaded):
     """Open a configuration file and return the result as a dictionary,
 
     Recursively open other files based on buildout options found.
     """
     _update_section(dl_options, override)
     _dl_options = _unannotate_section(dl_options.copy())
-    is_temp = False
+    newest = _convert_bool('newest', _dl_options.get('newest', 'false'))
+    fallback = newest and not (filename in downloaded)
     download = zc.buildout.download.Download(
-        _dl_options, cache=_dl_options.get('extends-cache'), fallback=True,
-        hash_name=True)
+        _dl_options, cache=_dl_options.get('extends-cache'),
+        fallback=fallback, hash_name=True)
+    is_temp = False
     if _isurl(filename):
         path, is_temp = download(filename)
         fp = open(path)
@@ -1415,6 +1417,7 @@ def _open(base, filename, seen, dl_options, override):
         filename = os.path.join(base, filename)
         fp = open(filename)
         base = os.path.dirname(filename)
+    downloaded.add(filename)
 
     if filename in seen:
         if is_temp:
@@ -1449,9 +1452,11 @@ def _open(base, filename, seen, dl_options, override):
 
     if extends:
         extends = extends.split()
-        eresult = _open(base, extends.pop(0), seen, dl_options, override)
+        eresult = _open(base, extends.pop(0), seen, dl_options, override,
+                        downloaded)
         for fname in extends:
-            _update(eresult, _open(base, fname, seen, dl_options, override))
+            _update(eresult, _open(base, fname, seen, dl_options, override,
+                    downloaded))
         result = _update(eresult, result)
 
     if extended_by:
@@ -1459,9 +1464,9 @@ def _open(base, filename, seen, dl_options, override):
             "The extendedBy option is deprecated.  Stop using it."
             )
         for fname in extended_by.split():
-            result = _update(result,
-                             _open(base, fname, seen, dl_options, override))
-
+            result = _update(
+                result,
+                _open(base, fname, seen, dl_options, override, downloaded))
     seen.pop()
     return result
 

src/zc/buildout/extends-cache.txt

@@ -386,6 +386,104 @@ While:
 An internal error occurred ...
 ValueError: install_from_cache set to true with no download cache
 
+>>> rmdir('home', '.buildout')
+
+Newest and non-newest behaviour for extends cache
+-------------------------------------------------
+
+While offline mode forbids network access completely, 'newest' mode determines
+whether to look for updated versions of a resource even if some version of it
+is already present locally. If we run buildout in newest mode
+(``newest = true``), the configuration files are updated with each run:
+
+>>> mkdir("cache")
+>>> write(server_data, 'base.cfg', """\
+... [buildout]
+... parts =
+... """)
+>>> write('buildout.cfg', """\
+... [buildout]
+... extends-cache = cache
+... extends = %sbase.cfg
+... """ % server_url)
+>>> print system(buildout)
+>>> ls('cache')
+-  5aedc98d7e769290a29d654a591a3a45
+>>> cat('cache', os.listdir(cache)[0])
+[buildout]
+parts =
+
+A change to ``base.cfg`` is picked up on the next buildout run:
+
+>>> write(server_data, 'base.cfg', """\
+... [buildout]
+... parts =
+... foo = bar
+... """)
+>>> print system(buildout + " -n")
+Unused options for buildout: 'foo'.
+>>> cat('cache', os.listdir(cache)[0])
+[buildout]
+parts =
+foo = bar
+
+In contrast, when not using ``newest`` mode (``newest = false``), the files
+already present in the extends cache will not be updated:
+
+>>> write(server_data, 'base.cfg', """\
+... [buildout]
+... parts =
+... """)
+>>> print system(buildout + " -N")
+Unused options for buildout: 'foo'.
+>>> cat('cache', os.listdir(cache)[0])
+[buildout]
+parts =
+foo = bar
+
+Even when updating base configuration files with a buildout run, any given
+configuration file will be downloaded only once during that particular run. If
+some base configuration file is extended more than once, its cached copy is
+used:
+
+>>> write(server_data, 'baseA.cfg', """\
+... [buildout]
+... extends = %sbase.cfg
+... foo = bar
+... """ % server_url)
+>>> write(server_data, 'baseB.cfg', """\
+... [buildout]
+... extends-cache = cache
+... extends = %sbase.cfg
+... bar = foo
+... """ % server_url)
+>>> write('buildout.cfg', """\
+... [buildout]
+... extends-cache = cache
+... newest = true
+... extends = %sbaseA.cfg %sbaseB.cfg
+... """ % (server_url, server_url))
+>>> print system(buildout + " -n")
+Unused options for buildout: 'bar' 'foo'.
+
+(XXX We patch download utility's API to produce readable output for the test;
+a better solution would utilise the logging already done by the utility.)
+
+>>> import zc.buildout
+>>> old_download = zc.buildout.download.Download.download
+>>> def wrapper_download(self, url, md5sum=None, path=None):
+...   print "The URL %s was downloaded." % url
+...   return old_download(url, md5sum, path)
+>>> zc.buildout.download.Download.download = wrapper_download
+
+>>> zc.buildout.buildout.main([])
+The URL http://localhost/baseA.cfg was downloaded.
+The URL http://localhost/base.cfg was downloaded.
+The URL http://localhost/baseB.cfg was downloaded.
+Unused options for buildout: 'bar' 'foo'.
+
+>>> zc.buildout.download.Download.download = old_download
+
 
 Clean up
 --------