- Now, final distributions are prefered over non-final versions. If

  both final and non-final versions satisfy a requirement, then the
  final version will be used even if it is older.  The normal way to
  override this for specific packages is to specifically require a
  non-final version, either specifically or via a lower bound.

- There is a buildout prefer-final version that can be used with a
  value of "false"::

prefer-final = false

To prefer newer versions, regardless of whether or not they are
  final, buildout-wide.


git-svn-id: http://svn.zope.org/repos/main/zc.buildout/trunk@79039 62d5b8a3-27da-0310-9561-8e5933582275

CHANGES.txt

 Status
 ******
 
-The buildout system is under active development. Some near term
+The buildout system is under active development. Some near-term
 priorities include:
 
 - Fixing `bugs <https://launchpad.net/products/zc.buildout/+bugs>`_
 
 - Adding support for making distributions from buildouts
 
-I recently broke buildout on Windows pretty badly.  My apologies!  I
-plan to get builout working on Windows again in 1.0.0b27. (Actually,
-I'm hoping to end this prolonged "beta" cycle and produce a 1.0.0
-final soon.) 
-
 Change History
 **************
 
 1.0.0b29 (unreleased)
 =====================
 
+Feature Changes
+---------------
+
+- Now, final distributions are prefered over non-final versions.  If
+  both final and non-final versions satisfy a requirement, then the
+  final version will be used even if it is older.  The normal way to
+  override this for specific packages is to specifically require a
+  non-final version, either specifically or via a lower bound.
+
+- There is a buildout prefer-final version that can be used with a
+  value of "false"::
+
+    prefer-final = false
+
+  To prefer newer versions, regardless of whether or not they are
+  final, buildout-wide.
+
 Bugs Fixed
 ----------
 

src/zc/buildout/buildout.py

@@ -152,6 +152,13 @@ class Buildout(UserDict.DictMixin):
         if versions:
             zc.buildout.easy_install.default_versions(dict(self[versions]))
 
+        prefer_final = options.get('prefer-final', 'true')
+        if prefer_final not in ('true', 'false'):
+            self._error('Invalid value for prefer-final option: %s',
+                        prefer_final)
+        zc.buildout.easy_install.prefer_final(prefer_final=='true')
+        
+
         download_cache = options.get('download-cache')
         if download_cache:
             download_cache = os.path.join(options['directory'], download_cache)

src/zc/buildout/buildout.txt

@@ -2060,6 +2060,17 @@ running the buildout when not connected to the internet.  It also
 makes buildouts run much faster. This option is typically set using
 the buildout -o option.
 
+Final releases prefered
+-----------------------
+
+By default, when searching for new releases, final releases are
+prefered.  If there are final releases that satisfy distribution
+requirements, then those releases are used even if newer non-final
+releases are available.  The buildout prefer-final option can be used
+to override this behavior.  If a buildout prefer-final option is given
+with the value "false", then, when looking for distributions, the
+newest will be used regardless of whether or not they are final. 
+
 Controlling the installation database
 -------------------------------------
 

src/zc/buildout/downloadcache.txt

@@ -36,8 +36,10 @@ download:
     <a href="demo-0.1-py2.4.egg">demo-0.1-py2.4.egg</a><br>
     <a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br>
     <a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br>
+    <a href="demo-0.4c1-py2.4.egg">demo-0.4c1-py2.4.egg</a><br>
     <a href="demoneeded-1.0.zip">demoneeded-1.0.zip</a><br>
     <a href="demoneeded-1.1.zip">demoneeded-1.1.zip</a><br>
+    <a href="demoneeded-1.2c1.zip">demoneeded-1.2c1.zip</a><br>
     <a href="extdemo-1.4.zip">extdemo-1.4.zip</a><br>
     <a href="index/">index/</a><br>
     <a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>

src/zc/buildout/easy_install.py

@@ -116,6 +116,7 @@ class Installer:
     _versions = {}
     _download_cache = None
     _install_from_cache = False
+    _prefer_final = True
 
     def __init__(self,
                  dest=None,
@@ -166,6 +167,7 @@ class Installer:
                          req.project_name, str(req))
             return None, self._obtain(req, source)
 
+
         # Note that dists are sorted from best to worst, as promised by
         # env.__getitem__
 
@@ -174,12 +176,6 @@ class Installer:
                 logger.debug('We have a develop egg: %s', dist)
                 return dist, None
 
-        if not self._newest:
-            # We don't need the newest, so we'll use the newest one we
-            # find, which is the first returned by
-            # Environment.__getitem__.
-            return dists[0], None
-
         # Special common case, we have a specification for a single version:
         specs = req.specs
         if len(specs) == 1 and specs[0][0] == '==':
@@ -187,6 +183,20 @@ class Installer:
                          str(req))
             return dists[0], None
 
+        if self._prefer_final:
+            fdists = [dist for dist in dists
+                      if _final_version(dist.parsed_version)
+                      ]
+            if fdists:
+                # There are final dists, so only use those
+                dists = fdists
+
+        if not self._newest:
+            # We don't need the newest, so we'll use the newest one we
+            # find, which is the first returned by
+            # Environment.__getitem__.
+            return dists[0], None
+
         best_we_have = dists[0] # Because dists are sorted from best to worst
 
         # We have some installed distros.  There might, theoretically, be
@@ -207,16 +217,37 @@ class Installer:
                 'Using our best, %s.',
                 str(req), best_available)
             return best_we_have, None
-        else:
-            # Let's find out if we already have the best available:
-            if best_we_have.parsed_version >= best_available.parsed_version:
-                # Yup. Use it.
-                logger.debug(
-                    'We have the best distribution that satisfies %r.',
-                    str(req))
-                return best_we_have, None
 
-        return None, best_available
+        if self._prefer_final:
+            if _final_version(best_available.parsed_version):
+                if _final_version(best_we_have.parsed_version):
+                    if (best_we_have.parsed_version
+                        <
+                        best_available.parsed_version
+                        ):
+                        return None, best_available
+                else:
+                    return None, best_available
+            else:
+                if (not _final_version(best_we_have.parsed_version)
+                    and
+                    (best_we_have.parsed_version
+                     <
+                     best_available.parsed_version
+                     )
+                    ):
+                    return None, best_available
+        else:
+            if (best_we_have.parsed_version
+                <
+                best_available.parsed_version
+                ):
+                return None, best_available
+            
+        logger.debug(
+            'We have the best distribution that satisfies %r.',
+            str(req))
+        return best_we_have, None
 
     def _load_dist(self, dist):
         dists = pkg_resources.Environment(
@@ -316,17 +347,37 @@ class Installer:
             shutil.rmtree(tmp)
             
     def _obtain(self, requirement, source=None):
+
+        # initialize out index for this project:
         index = self._index
         if index.obtain(requirement) is None:
+            # Nothing is available.
             return None
-        
+
+        # Filter the available dists for the requirement and source flag
+        dists = [dist for dist in index[requirement.project_name]
+                 if ((dist in requirement)
+                     and
+                     ((not source) or
+                      (dist.precedence == pkg_resources.SOURCE_DIST)
+                      )
+                     )
+                 ]
+
+        # If we prefer final dists, filter for final and use the
+        # result if it is non empty.
+        if self._prefer_final:
+            fdists = [dist for dist in dists
+                      if _final_version(dist.parsed_version)
+                      ]
+            if fdists:
+                # There are final dists, so only use those
+                dists = fdists
+
+        # Now find the best one:
         best = []
         bestv = ()
-        for dist in index[requirement.project_name]:
-            if dist not in requirement:
-                continue
-            if source and dist.precedence != pkg_resources.SOURCE_DIST:
-                continue
+        for dist in dists:
             distv = dist.parsed_version
             if distv > bestv:
                 best = [dist]
@@ -647,6 +698,12 @@ def install_from_cache(setting=None):
         Installer._install_from_cache = bool(setting)
     return old
 
+def prefer_final(setting=None):
+    old = Installer._prefer_final
+    if setting is not None:
+        Installer._prefer_final = bool(setting)
+    return old
+
 def install(specs, dest,
             links=(), index=None,
             executable=sys.executable, always_unzip=False,
@@ -976,5 +1033,9 @@ def _fix_file_links(links):
                 link += '/'
         yield link
 
-
-
+_final_parts = '*final-', '*final'
+def _final_version(parsed_version):
+    for part in parsed_version:
+        if (part[:1] == '*') and (part not in _final_parts):
+            return False
+    return True

src/zc/buildout/easy_install.txt

@@ -93,8 +93,10 @@ We have a link server that has a number of eggs:
     <a href="demo-0.1-py2.4.egg">demo-0.1-py2.4.egg</a><br>
     <a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br>
     <a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br>
+    <a href="demo-0.4c1-py2.4.egg">demo-0.4c1-py2.4.egg</a><br>
     <a href="demoneeded-1.0.zip">demoneeded-1.0.zip</a><br>
     <a href="demoneeded-1.1.zip">demoneeded-1.1.zip</a><br>
+    <a href="demoneeded-1.2c1.zip">demoneeded-1.2c1.zip</a><br>
     <a href="extdemo-1.4.zip">extdemo-1.4.zip</a><br>
     <a href="index/">index/</a><br>
     <a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
@@ -120,6 +122,8 @@ The working set contains the distributions we retrieved.
     demo 0.2
     demoneeded 1.1
 
+We got demoneeded because it was a dependency of demo.
+
 And the actual eggs were added to the eggs directory.
 
     >>> ls(dest)
@@ -127,7 +131,7 @@ And the actual eggs were added to the eggs directory.
     -  demoneeded-1.1-py2.4.egg
 
 If we remove the version restriction on demo, but specify a false
-value for newest, no new didstributions will be installed:
+value for newest, no new distributions will be installed:
 
     >>> ws = zc.buildout.easy_install.install(
     ...     ['demo'], dest, links=[link_server], index=link_server+'index/',
@@ -145,6 +149,36 @@ If we leave off the newst option, we'll get an update for demo:
     -  demo-0.3-py2.4.egg
     -  demoneeded-1.1-py2.4.egg
 
+Note that we didn't get the newest versions available.  There were
+release candidated for newer versions of both packages. By default,
+final releases are prefered.  We can change this behavior using the
+prefer_final function:
+
+    >>> zc.buildout.easy_install.prefer_final(False)
+    True
+
+The old setting is returned.
+
+    >>> ws = zc.buildout.easy_install.install(
+    ...     ['demo'], dest, links=[link_server], index=link_server+'index/')
+    >>> for dist in ws:
+    ...     print dist
+    demo 0.4c1
+    demoneeded 1.2c1
+
+    >>> ls(dest)
+    -  demo-0.2-py2.4.egg
+    -  demo-0.3-py2.4.egg
+    -  demo-0.4c1-py2.4.egg
+    -  demoneeded-1.1-py2.4.egg
+    -  demoneeded-1.2c1-py2.4.egg
+
+Let's put the setting back to the default.
+
+    >>> zc.buildout.easy_install.prefer_final(True)
+    False
+
+
 We can supply additional distributions.  We can also supply
 specifications for distributions that would normally be found via
 dependencies.  We might do this to specify a sprcific version.
@@ -162,8 +196,10 @@ dependencies.  We might do this to specify a sprcific version.
     >>> ls(dest)
     -  demo-0.2-py2.4.egg
     -  demo-0.3-py2.4.egg
+    -  demo-0.4c1-py2.4.egg
     -  demoneeded-1.0-py2.4.egg
     -  demoneeded-1.1-py2.4.egg
+    -  demoneeded-1.2c1-py2.4.egg
     d  other-1.0-py2.4.egg
 
 We can request that eggs be unzipped even if they are zip safe.  This
@@ -685,8 +721,10 @@ Let's update our link server with a new version of extdemo:
     <a href="demo-0.1-py2.4.egg">demo-0.1-py2.4.egg</a><br>
     <a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br>
     <a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br>
+    <a href="demo-0.4c1-py2.4.egg">demo-0.4c1-py2.4.egg</a><br>
     <a href="demoneeded-1.0.zip">demoneeded-1.0.zip</a><br>
     <a href="demoneeded-1.1.zip">demoneeded-1.1.zip</a><br>
+    <a href="demoneeded-1.2c1.zip">demoneeded-1.2c1.zip</a><br>
     <a href="extdemo-1.4.zip">extdemo-1.4.zip</a><br>
     <a href="extdemo-1.5.zip">extdemo-1.5.zip</a><br>
     <a href="index/">index/</a><br>

src/zc/buildout/tests.py

@@ -558,23 +558,6 @@ def create_sections_on_command_line():
     
     """
 
-# Why?
-## def error_for_undefined_install_parts():
-##     """
-## Any parts we pass to install on the command line must be
-## listed in the configuration.
-
-##     >>> print system(join('bin', 'buildout') + ' install foo'),
-##     Invalid install parts: foo.
-##     Install parts must be listed in the configuration.
-
-##     >>> print system(join('bin', 'buildout') + ' install foo bar'),
-##     Invalid install parts: foo bar.
-##     Install parts must be listed in the configuration.
-    
-##     """
-
-
 bootstrap_py = os.path.join(
        os.path.dirname(
           os.path.dirname(
@@ -2073,6 +2056,241 @@ directory and then use the wacky extension to load the demo package
     
     """
 
+
+def create_egg(name, version, dest):
+    d = tempfile.mkdtemp()
+    if dest=='available':
+        extras = dict(x=['x'])
+    else:
+        extras = {}
+        
+    try:
+        open(os.path.join(d, 'setup.py'), 'w').write(
+            'from setuptools import setup\n'
+            'setup(name=%r, version=%r, extras_require=%r, zip_safe=True,\n'
+            '      py_modules=["setup"]\n)'
+            % (name, str(version), extras)
+            )
+        zc.buildout.testing.bdist_egg(d, sys.executable, os.path.abspath(dest))
+    finally:
+        shutil.rmtree(d)
+
+def prefer_final_permutation(existing, available):
+    for d in ('existing', 'available'):
+        if os.path.exists(d):
+            shutil.rmtree(d)
+        os.mkdir(d)
+    for version in existing:
+        create_egg('spam', version, 'existing')
+    for version in available:
+        create_egg('spam', version, 'available')
+
+    zc.buildout.easy_install.clear_index_cache()
+    [dist] = list(
+        zc.buildout.easy_install.install(['spam'], 'existing', ['available'],
+                                         always_unzip=True)
+        )
+
+    if dist.extras:
+        print 'downloaded', dist.version
+    else:
+        print 'had', dist.version
+    sys.path_importer_cache.clear()
+
+def prefer_final():
+    """
+This test tests several permutations:
+
+Using different version numbers to work around zip impporter cache problems. :(
+
+- With prefer final:
+
+    - no existing and newer dev available
+    >>> prefer_final_permutation((), [1, '2a1'])
+    downloaded 1
+
+    - no existing and only dev available
+    >>> prefer_final_permutation((), ['3a1'])
+    downloaded 3a1
+
+    - final existing and only dev acailable
+    >>> prefer_final_permutation([4], ['5a1'])
+    had 4
+
+    - final existing and newer final available
+    >>> prefer_final_permutation([6], [7])
+    downloaded 7
+
+    - final existing and same final available
+    >>> prefer_final_permutation([8], [8])
+    had 8
+
+    - final existing and older final available
+    >>> prefer_final_permutation([10], [9])
+    had 10
+
+    - only dev existing and final available
+    >>> prefer_final_permutation(['12a1'], [11])
+    downloaded 11
+
+    - only dev existing and no final available newer dev available
+    >>> prefer_final_permutation(['13a1'], ['13a2'])
+    downloaded 13a2
+
+    - only dev existing and no final available older dev available
+    >>> prefer_final_permutation(['15a1'], ['14a1'])
+    had 15a1
+
+    - only dev existing and no final available same dev available
+    >>> prefer_final_permutation(['16a1'], ['16a1'])
+    had 16a1
+
+- Without prefer final:
+
+    >>> _ = zc.buildout.easy_install.prefer_final(False)
+
+    - no existing and newer dev available
+    >>> prefer_final_permutation((), [18, '19a1'])
+    downloaded 19a1
+
+    - no existing and only dev available
+    >>> prefer_final_permutation((), ['20a1'])
+    downloaded 20a1
+
+    - final existing and only dev acailable
+    >>> prefer_final_permutation([21], ['22a1'])
+    downloaded 22a1
+
+    - final existing and newer final available
+    >>> prefer_final_permutation([23], [24])
+    downloaded 24
+
+    - final existing and same final available
+    >>> prefer_final_permutation([25], [25])
+    had 25
+
+    - final existing and older final available
+    >>> prefer_final_permutation([27], [26])
+    had 27
+
+    - only dev existing and final available
+    >>> prefer_final_permutation(['29a1'], [28])
+    had 29a1
+
+    - only dev existing and no final available newer dev available
+    >>> prefer_final_permutation(['30a1'], ['30a2'])
+    downloaded 30a2
+
+    - only dev existing and no final available older dev available
+    >>> prefer_final_permutation(['32a1'], ['31a1'])
+    had 32a1
+
+    - only dev existing and no final available same dev available
+    >>> prefer_final_permutation(['33a1'], ['33a1'])
+    had 33a1
+
+    >>> _ = zc.buildout.easy_install.prefer_final(True)
+    
+    """
+
+def buildout_prefer_final_option():
+    """
+The prefer-final buildout option can be used for override the default
+preference for newer distributions. 
+
+The default is prefer-final = true:
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = eggs
+    ... find-links = %(link_server)s
+    ...
+    ... [eggs]
+    ... recipe = zc.recipe.egg:eggs
+    ... eggs = demo
+    ... ''' % globals())
+
+    >>> print system(buildout+' -v'), # doctest: +ELLIPSIS
+    Installing 'zc.buildout', 'setuptools'.
+    ...
+    Picked: demo = 0.3
+    ...
+    Picked: demoneeded = 1.1
+
+
+Here we see that the final versions of demo and demoneeded are used.
+We get the same behavior if we add prefer-final = true
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = eggs
+    ... find-links = %(link_server)s
+    ... prefer-final = true
+    ...
+    ... [eggs]
+    ... recipe = zc.recipe.egg:eggs
+    ... eggs = demo
+    ... ''' % globals())
+
+    >>> print system(buildout+' -v'), # doctest: +ELLIPSIS
+    Installing 'zc.buildout', 'setuptools'.
+    ...
+    Picked: demo = 0.3
+    ...
+    Picked: demoneeded = 1.1
+
+If we specify prefer-final = false, we'll get the newest
+distributions:
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = eggs
+    ... find-links = %(link_server)s
+    ... prefer-final = false
+    ...
+    ... [eggs]
+    ... recipe = zc.recipe.egg:eggs
+    ... eggs = demo
+    ... ''' % globals())
+
+    >>> print system(buildout+' -v'), # doctest: +ELLIPSIS
+    Installing 'zc.buildout', 'setuptools'.
+    ...
+    Picked: demo = 0.4c1
+    ...
+    Picked: demoneeded = 1.2c1
+
+We get an error if we specify anything but true or false:
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = eggs
+    ... find-links = %(link_server)s
+    ... prefer-final = no
+    ...
+    ... [eggs]
+    ... recipe = zc.recipe.egg:eggs
+    ... eggs = demo
+    ... ''' % globals())
+
+    >>> print system(buildout+' -v'), # doctest: +ELLIPSIS
+    While:
+      Initializing.
+    Error: Invalid value for prefer-final option: no
+
+    """
+
+
+# XXX Tests needed:
+
+# Link added from package meta data
+
+
+
 ######################################################################
     
 def create_sample_eggs(test, executable=sys.executable):
@@ -2082,15 +2300,16 @@ def create_sample_eggs(test, executable=sys.executable):
     try:
         write(tmp, 'README.txt', '')
 
-        for i in (0, 1):
+        for i in (0, 1, 2):
             write(tmp, 'eggrecipedemobeeded.py', 'y=%s\n' % i)
+            c1 = i==2 and 'c1' or ''
             write(
                 tmp, 'setup.py',
                 "from setuptools import setup\n"
                 "setup(name='demoneeded', py_modules=['eggrecipedemobeeded'],"
-                " zip_safe=True, version='1.%s', author='bob', url='bob', "
+                " zip_safe=True, version='1.%s%s', author='bob', url='bob', "
                 "author_email='bob')\n"
-                % i
+                % (i, c1)
                 )
             zc.buildout.testing.sdist(tmp, dest)
 
@@ -2104,13 +2323,14 @@ def create_sample_eggs(test, executable=sys.executable):
 
         os.remove(os.path.join(tmp, 'eggrecipedemobeeded.py'))
 
-        for i in (1, 2, 3):
+        for i in (1, 2, 3, 4):
             write(
                 tmp, 'eggrecipedemo.py',
                 'import eggrecipedemobeeded\n'
                 'x=%s\n'
                 'def main(): print x, eggrecipedemobeeded.y\n'
                 % i)
+            c1 = i==4 and 'c1' or ''
             write(
                 tmp, 'setup.py',
                 "from setuptools import setup\n"
@@ -2118,7 +2338,7 @@ def create_sample_eggs(test, executable=sys.executable):
                 " install_requires = 'demoneeded',"
                 " entry_points={'console_scripts': "
                      "['demo = eggrecipedemo:main']},"
-                " zip_safe=True, version='0.%s')\n" % i
+                " zip_safe=True, version='0.%s%s')\n" % (i, c1)
                 )
             zc.buildout.testing.bdist_egg(tmp, executable, dest)
     finally:

zc.recipe.egg_/src/zc/recipe/egg/README.txt

@@ -36,8 +36,10 @@ We have a link server that has a number of distributions:
     <a href="demo-0.1-py2.3.egg">demo-0.1-py2.3.egg</a><br>
     <a href="demo-0.2-py2.3.egg">demo-0.2-py2.3.egg</a><br>
     <a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br>
+    <a href="demo-0.4c1-py2.3.egg">demo-0.4c1-py2.3.egg</a><br>
     <a href="demoneeded-1.0.zip">demoneeded-1.0.zip</a><br>
     <a href="demoneeded-1.1.zip">demoneeded-1.1.zip</a><br>
+    <a href="demoneeded-1.2c1.zip">demoneeded-1.2c1.zip</a><br>
     <a href="extdemo-1.4.zip">extdemo-1.4.zip</a><br>
     <a href="index/">index/</a><br>
     <a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br>

zc.recipe.egg_/src/zc/recipe/egg/selecting-python.txt

@@ -14,8 +14,10 @@ We have a link server:
     <a href="demo-0.1-py2.3.egg">demo-0.1-py2.3.egg</a><br>
     <a href="demo-0.2-py2.3.egg">demo-0.2-py2.3.egg</a><br>
     <a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br>
+    <a href="demo-0.4c1-py2.3.egg">demo-0.4c1-py2.3.egg</a><br>
     <a href="demoneeded-1.0.zip">demoneeded-1.0.zip</a><br>
     <a href="demoneeded-1.1.zip">demoneeded-1.1.zip</a><br>
+    <a href="demoneeded-1.2c1.zip">demoneeded-1.2c1.zip</a><br>
     <a href="extdemo-1.4.zip">extdemo-1.4.zip</a><br>
     <a href="index/">index/</a><br>
     <a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br>