Add referred parts' hash strings in __buildout_signature__, that invokes...

Add referred parts' hash strings in __buildout_signature__, that invokes rebuild of a part when one of its (recursive) dependencies are modified.

Also remove duplicates and sort entries in __buildout_signature__.

src/zc/buildout/buildout.py

@@ -310,6 +310,7 @@ class Buildout(DictMixin):
         self._raw = _unannotate(data)
         self._data = {}
         self._parts = []
+        self._initializing = []
 
         # provide some defaults before options are parsed
         # because while parsing options those attributes might be
@@ -839,14 +840,29 @@ class Buildout(DictMixin):
                     "Unexpected entry, %r, in develop-eggs directory.", f)
 
     def _compute_part_signatures(self, parts):
+        # The same recipe may appear many times.
+        sig_cache= {}
+
         # Compute recipe signature and add to options
         for part in parts:
             options = self.get(part)
             if options is None:
                 options = self[part] = {}
             recipe, entry = _recipe(options)
+
+            try:
+                # Copy cached list, because sig.append is called later
+                sig = sig_cache[recipe][:]
+            except KeyError:
                 req = pkg_resources.Requirement.parse(recipe)
-            sig = _dists_sig(pkg_resources.working_set.resolve([req]))
+                sig_result = sig_cache[recipe] = sorted(set(_dists_sig(pkg_resources.working_set.resolve([req]))))
+                sig = sig_result[:]
+
+            for dependency in sorted(options.depends):
+                m = md5()
+                for item in sorted(self[dependency].items()):
+                    m.update(('%r\0%r\0' % item).encode())
+                sig.append('%s:%s' % (dependency, m.hexdigest()))
             options['__buildout_signature__'] = ' '.join(sig)
 
     def _read_installed_part_options(self):
@@ -1178,13 +1194,19 @@ class Buildout(DictMixin):
                     v = ' '+v
                 print_("%s =%s" % (k, v))
 
+    def initialize(self, options, reqs, entry):
+        recipe_class = _install_and_load(reqs, 'zc.buildout', entry, self)
+        self._initializing.append(options)
+        try:
+            return recipe_class(self, options.name, options)
+        finally:
+            del self._initializing[-1]
+
     def __getitem__(self, section):
         __doing__ = 'Getting section %s.', section
         try:
-            return self._data[section]
+            options = self._data[section]
         except KeyError:
-            pass
-
             try:
                 data = self._raw[section]
             except KeyError:
@@ -1193,6 +1215,11 @@ class Buildout(DictMixin):
             options = self.Options(self, section, data)
             self._data[section] = options
             options._initialize()
+
+        if self._initializing:
+            caller = self._initializing[-1]
+            if 'buildout' != section != caller.name:
+                  caller.depends.add(section)
         return options
 
     def __setitem__(self, name, data):
@@ -1275,6 +1302,7 @@ class Options(DictMixin):
         self._raw = data
         self._cooked = {}
         self._data = {}
+        self.depends = set()
 
     def _initialize(self):
         name = self.name
@@ -1292,17 +1320,9 @@ class Options(DictMixin):
             return # buildout section can never be a part
 
         if self.get('recipe'):
-            self.initialize()
+            self.recipe = self.buildout.initialize(self, *_recipe(self._data))
             self.buildout._parts.append(name)
 
-    def initialize(self):
-        reqs, entry = _recipe(self._data)
-        buildout = self.buildout
-        recipe_class = _install_and_load(reqs, 'zc.buildout', entry, buildout)
-
-        name = self.name
-        self.recipe = recipe_class(buildout, name, self)
-
     def _do_extend_raw(self, name, data, doing):
         if name == 'buildout':
             return data
@@ -1409,7 +1429,14 @@ class Options(DictMixin):
             section, option = s
             if not section:
                 section = self.name
-            v = self.buildout[section].get(option, None, seen, last=last)
+                options = self
+            else:
+                self.buildout._initializing.append(self)
+                try:
+                    options = self.buildout[section]
+                finally:
+                    del self.buildout._initializing[-1]
+            v = options.get(option, None, seen, last=last)
             if v is None:
                 if option == '_buildout_section_name_':
                     v = self.name

src/zc/buildout/buildout.txt

@@ -2544,7 +2544,7 @@ were created.
 The ``.installed.cfg`` is only updated for the recipes that ran::
 
     >>> cat(sample_buildout, '.installed.cfg')
-    ... # doctest: +NORMALIZE_WHITESPACE
+    ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
     [buildout]
     installed_develop_eggs = /sample-buildout/develop-eggs/recipes.egg-link
     parts = debug d1 d2 d3 d4
@@ -2574,7 +2574,7 @@ The ``.installed.cfg`` is only updated for the recipes that ran::
     <BLANKLINE>
     [d4]
     __buildout_installed__ = /sample-buildout/data2-extra
-    __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+    __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg== d2:...
     path = /sample-buildout/data2-extra
     recipe = recipes:mkdir
 
@@ -2587,6 +2587,7 @@ Now, if we run the buildout without the install command::
 
     >>> print_(system(buildout), end='')
     Develop: '/sample-buildout/recipes'
+    Uninstalling d4.
     Uninstalling d2.
     Uninstalling d1.
     Uninstalling debug.
@@ -2596,7 +2597,8 @@ Now, if we run the buildout without the install command::
     Installing d2.
     d2: Creating directory data2
     Updating d3.
-    Updating d4.
+    Installing d4.
+    d4: Creating directory data2-extra
 
 We see the output of the debug recipe, and that ``data2`` was created.  We
 also see that ``d1`` and ``d2`` have gone away::

src/zc/buildout/testing.py

@@ -179,18 +179,14 @@ def wait_until(label, func, *args, **kw):
         time.sleep(0.01)
     raise ValueError('Timed out waiting for: '+label)
 
-class TestOptions(zc.buildout.buildout.Options):
-
-    def initialize(self):
-        pass
-
 class Buildout(zc.buildout.buildout.Buildout):
 
     def __init__(self):
         zc.buildout.buildout.Buildout.__init__(
             self, '', [('buildout', 'directory', os.getcwd())])
 
-    Options = TestOptions
+    def initialize(self, *args):
+        pass
 
 def buildoutSetUp(test):
 

src/zc/buildout/tests.py

@@ -2089,6 +2089,80 @@ def dealing_with_extremely_insane_dependencies():
     Error: Couldn't find a distribution for 'pack5'.
     """
 
+def test_part_pulled_by_recipe():
+    """
+    >>> mkdir(sample_buildout, 'recipes')
+    >>> write(sample_buildout, 'recipes', 'test.py',
+    ... '''
+    ... class Recipe:
+    ...
+    ...     def __init__(self, buildout, name, options):
+    ...         self.x = buildout[options['x']][name]
+    ...
+    ...     def install(self):
+    ...         print self.x
+    ...         return ()
+    ...
+    ...     update = install
+    ... ''')
+
+
+    >>> write(sample_buildout, 'recipes', 'setup.py',
+    ... '''
+    ... from setuptools import setup
+    ... setup(
+    ...     name = "recipes",
+    ...     entry_points = {'zc.buildout': ['test = test:Recipe']},
+    ...     )
+    ... ''')
+
+    >>> write(sample_buildout, 'buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... develop = recipes
+    ... parts = a
+    ... [a]
+    ... recipe = recipes:test
+    ... x = b
+    ... [b]
+    ... <= a
+    ... a = A
+    ... b = B
+    ... c = ${c:x}
+    ... [c]
+    ... x = c
+    ... ''')
+
+    >>> os.chdir(sample_buildout)
+    >>> buildout = os.path.join(sample_buildout, 'bin', 'buildout')
+
+    >>> print_(system(buildout), end='')
+    Develop: '/sample-buildout/recipes'
+    Installing b.
+    B
+    Installing a.
+    A
+
+    >>> print_(system(buildout), end='')
+    Develop: '/sample-buildout/recipes'
+    Updating b.
+    B
+    Updating a.
+    A
+
+    >>> cat('.installed.cfg') # doctest: +ELLIPSIS
+    [buildout]
+    ...
+    [b]
+    __buildout_installed__ = 
+    __buildout_signature__ = recipes-... c:...
+    ...
+    [a]
+    __buildout_installed__ = 
+    __buildout_signature__ = recipes-... b:...
+    ...
+    """
+
 def read_find_links_to_load_extensions():
     r"""
 We'll create a wacky buildout extension that just announces itself when used:

zc.recipe.egg_/src/zc/recipe/egg/api.rst

@@ -97,14 +97,14 @@ of extra requirements to be included in the working set.
 We can see that the options were augmented with additional data
 computed by the egg recipe by looking at .installed.cfg:
 
-    >>> cat(sample_buildout, '.installed.cfg')
+    >>> cat(sample_buildout, '.installed.cfg') # doctest: +ELLIPSIS
     [buildout]
     installed_develop_eggs = /sample-buildout/develop-eggs/sample.egg-link
     parts = sample-part
     <BLANKLINE>
     [sample-part]
     __buildout_installed__ = 
-    __buildout_signature__ = ...
+    __buildout_signature__ = sample-... setuptools-...egg zc.buildout-... zc.recipe.egg-...
     _b = /sample-buildout/bin
     _d = /sample-buildout/develop-eggs
     _e = /sample-buildout/eggs

[Julien Muchembled]

Due to a bug in manuel egg, we had no test results for doctests for a long time and we missed the fact that a more recent commit reverted to the old behaviour. Could it be dffdeffc?

However, the behaviour of buildout at this point is meaningless. Buildout was previously invoked with install d3 d4 command-line arguments and I consider that d2 should have been reinstalled, because a section shall not access the option of another section if the latter is not properly installed.

Does anyone use the install command in a way that fixing it is important?

But we should at least revert this change so that this doctest continues.

/cc @jerome

[Jérome Perrin]

Sorry I did not reply ... I don't have much to say. I don't think we rely on invoking buildout with "install part1 part2" at all, but my opinion is that we should try to keep buildout behavior close to the original.

[Jérome Perrin]

In !21 (merged) there are changes to revert the test to the original behavior

[Jérome Perrin]

I don't see how setuptools signature can be setuptools-...egg here, it's always the egg name and the egg version. Do we have a test result where this test was passing ? In my development branch I changed this to remove the egg prefix here and test pass.

[Jérome Perrin]

this is changed in !21 (merged)