Commit 33dcf8ca authored by Jim Fulton's avatar Jim Fulton

- Now, final distributions are prefered over non-final versions. If

  both final and non-final versions satisfy a requirement, then the
  final version will be used even if it is older.  The normal way to
  override this for specific packages is to specifically require a
  non-final version, either specifically or via a lower bound.

- There is a buildout prefer-final version that can be used with a
  value of "false"::

prefer-final = false

To prefer newer versions, regardless of whether or not they are
  final, buildout-wide.
parent 2e3031ce
Status
******
The buildout system is under active development. Some near term
The buildout system is under active development. Some near-term
priorities include:
- Fixing `bugs <https://launchpad.net/products/zc.buildout/+bugs>`_
- Adding support for making distributions from buildouts
I recently broke buildout on Windows pretty badly. My apologies! I
plan to get builout working on Windows again in 1.0.0b27. (Actually,
I'm hoping to end this prolonged "beta" cycle and produce a 1.0.0
final soon.)
Change History
**************
1.0.0b29 (unreleased)
=====================
Feature Changes
---------------
- Now, final distributions are prefered over non-final versions. If
both final and non-final versions satisfy a requirement, then the
final version will be used even if it is older. The normal way to
override this for specific packages is to specifically require a
non-final version, either specifically or via a lower bound.
- There is a buildout prefer-final version that can be used with a
value of "false"::
prefer-final = false
To prefer newer versions, regardless of whether or not they are
final, buildout-wide.
Bugs Fixed
----------
......
......@@ -152,6 +152,13 @@ class Buildout(UserDict.DictMixin):
if versions:
zc.buildout.easy_install.default_versions(dict(self[versions]))
prefer_final = options.get('prefer-final', 'true')
if prefer_final not in ('true', 'false'):
self._error('Invalid value for prefer-final option: %s',
prefer_final)
zc.buildout.easy_install.prefer_final(prefer_final=='true')
download_cache = options.get('download-cache')
if download_cache:
download_cache = os.path.join(options['directory'], download_cache)
......
......@@ -2060,6 +2060,17 @@ running the buildout when not connected to the internet. It also
makes buildouts run much faster. This option is typically set using
the buildout -o option.
Final releases prefered
-----------------------
By default, when searching for new releases, final releases are
prefered. If there are final releases that satisfy distribution
requirements, then those releases are used even if newer non-final
releases are available. The buildout prefer-final option can be used
to override this behavior. If a buildout prefer-final option is given
with the value "false", then, when looking for distributions, the
newest will be used regardless of whether or not they are final.
Controlling the installation database
-------------------------------------
......
......@@ -36,8 +36,10 @@ download:
<a href="demo-0.1-py2.4.egg">demo-0.1-py2.4.egg</a><br>
<a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br>
<a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br>
<a href="demo-0.4c1-py2.4.egg">demo-0.4c1-py2.4.egg</a><br>
<a href="demoneeded-1.0.zip">demoneeded-1.0.zip</a><br>
<a href="demoneeded-1.1.zip">demoneeded-1.1.zip</a><br>
<a href="demoneeded-1.2c1.zip">demoneeded-1.2c1.zip</a><br>
<a href="extdemo-1.4.zip">extdemo-1.4.zip</a><br>
<a href="index/">index/</a><br>
<a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
......
......@@ -116,6 +116,7 @@ class Installer:
_versions = {}
_download_cache = None
_install_from_cache = False
_prefer_final = True
def __init__(self,
dest=None,
......@@ -166,6 +167,7 @@ class Installer:
req.project_name, str(req))
return None, self._obtain(req, source)
# Note that dists are sorted from best to worst, as promised by
# env.__getitem__
......@@ -174,12 +176,6 @@ class Installer:
logger.debug('We have a develop egg: %s', dist)
return dist, None
if not self._newest:
# We don't need the newest, so we'll use the newest one we
# find, which is the first returned by
# Environment.__getitem__.
return dists[0], None
# Special common case, we have a specification for a single version:
specs = req.specs
if len(specs) == 1 and specs[0][0] == '==':
......@@ -187,6 +183,20 @@ class Installer:
str(req))
return dists[0], None
if self._prefer_final:
fdists = [dist for dist in dists
if _final_version(dist.parsed_version)
]
if fdists:
# There are final dists, so only use those
dists = fdists
if not self._newest:
# We don't need the newest, so we'll use the newest one we
# find, which is the first returned by
# Environment.__getitem__.
return dists[0], None
best_we_have = dists[0] # Because dists are sorted from best to worst
# We have some installed distros. There might, theoretically, be
......@@ -207,17 +217,38 @@ class Installer:
'Using our best, %s.',
str(req), best_available)
return best_we_have, None
if self._prefer_final:
if _final_version(best_available.parsed_version):
if _final_version(best_we_have.parsed_version):
if (best_we_have.parsed_version
<
best_available.parsed_version
):
return None, best_available
else:
return None, best_available
else:
if (not _final_version(best_we_have.parsed_version)
and
(best_we_have.parsed_version
<
best_available.parsed_version
)
):
return None, best_available
else:
# Let's find out if we already have the best available:
if best_we_have.parsed_version >= best_available.parsed_version:
# Yup. Use it.
if (best_we_have.parsed_version
<
best_available.parsed_version
):
return None, best_available
logger.debug(
'We have the best distribution that satisfies %r.',
str(req))
return best_we_have, None
return None, best_available
def _load_dist(self, dist):
dists = pkg_resources.Environment(
dist.location,
......@@ -316,17 +347,37 @@ class Installer:
shutil.rmtree(tmp)
def _obtain(self, requirement, source=None):
# initialize out index for this project:
index = self._index
if index.obtain(requirement) is None:
# Nothing is available.
return None
# Filter the available dists for the requirement and source flag
dists = [dist for dist in index[requirement.project_name]
if ((dist in requirement)
and
((not source) or
(dist.precedence == pkg_resources.SOURCE_DIST)
)
)
]
# If we prefer final dists, filter for final and use the
# result if it is non empty.
if self._prefer_final:
fdists = [dist for dist in dists
if _final_version(dist.parsed_version)
]
if fdists:
# There are final dists, so only use those
dists = fdists
# Now find the best one:
best = []
bestv = ()
for dist in index[requirement.project_name]:
if dist not in requirement:
continue
if source and dist.precedence != pkg_resources.SOURCE_DIST:
continue
for dist in dists:
distv = dist.parsed_version
if distv > bestv:
best = [dist]
......@@ -647,6 +698,12 @@ def install_from_cache(setting=None):
Installer._install_from_cache = bool(setting)
return old
def prefer_final(setting=None):
old = Installer._prefer_final
if setting is not None:
Installer._prefer_final = bool(setting)
return old
def install(specs, dest,
links=(), index=None,
executable=sys.executable, always_unzip=False,
......@@ -976,5 +1033,9 @@ def _fix_file_links(links):
link += '/'
yield link
_final_parts = '*final-', '*final'
def _final_version(parsed_version):
for part in parsed_version:
if (part[:1] == '*') and (part not in _final_parts):
return False
return True
......@@ -93,8 +93,10 @@ We have a link server that has a number of eggs:
<a href="demo-0.1-py2.4.egg">demo-0.1-py2.4.egg</a><br>
<a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br>
<a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br>
<a href="demo-0.4c1-py2.4.egg">demo-0.4c1-py2.4.egg</a><br>
<a href="demoneeded-1.0.zip">demoneeded-1.0.zip</a><br>
<a href="demoneeded-1.1.zip">demoneeded-1.1.zip</a><br>
<a href="demoneeded-1.2c1.zip">demoneeded-1.2c1.zip</a><br>
<a href="extdemo-1.4.zip">extdemo-1.4.zip</a><br>
<a href="index/">index/</a><br>
<a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
......@@ -120,6 +122,8 @@ The working set contains the distributions we retrieved.
demo 0.2
demoneeded 1.1
We got demoneeded because it was a dependency of demo.
And the actual eggs were added to the eggs directory.
>>> ls(dest)
......@@ -127,7 +131,7 @@ And the actual eggs were added to the eggs directory.
- demoneeded-1.1-py2.4.egg
If we remove the version restriction on demo, but specify a false
value for newest, no new didstributions will be installed:
value for newest, no new distributions will be installed:
>>> ws = zc.buildout.easy_install.install(
... ['demo'], dest, links=[link_server], index=link_server+'index/',
......@@ -145,6 +149,36 @@ If we leave off the newst option, we'll get an update for demo:
- demo-0.3-py2.4.egg
- demoneeded-1.1-py2.4.egg
Note that we didn't get the newest versions available. There were
release candidated for newer versions of both packages. By default,
final releases are prefered. We can change this behavior using the
prefer_final function:
>>> zc.buildout.easy_install.prefer_final(False)
True
The old setting is returned.
>>> ws = zc.buildout.easy_install.install(
... ['demo'], dest, links=[link_server], index=link_server+'index/')
>>> for dist in ws:
... print dist
demo 0.4c1
demoneeded 1.2c1
>>> ls(dest)
- demo-0.2-py2.4.egg
- demo-0.3-py2.4.egg
- demo-0.4c1-py2.4.egg
- demoneeded-1.1-py2.4.egg
- demoneeded-1.2c1-py2.4.egg
Let's put the setting back to the default.
>>> zc.buildout.easy_install.prefer_final(True)
False
We can supply additional distributions. We can also supply
specifications for distributions that would normally be found via
dependencies. We might do this to specify a sprcific version.
......@@ -162,8 +196,10 @@ dependencies. We might do this to specify a sprcific version.
>>> ls(dest)
- demo-0.2-py2.4.egg
- demo-0.3-py2.4.egg
- demo-0.4c1-py2.4.egg
- demoneeded-1.0-py2.4.egg
- demoneeded-1.1-py2.4.egg
- demoneeded-1.2c1-py2.4.egg
d other-1.0-py2.4.egg
We can request that eggs be unzipped even if they are zip safe. This
......@@ -685,8 +721,10 @@ Let's update our link server with a new version of extdemo:
<a href="demo-0.1-py2.4.egg">demo-0.1-py2.4.egg</a><br>
<a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br>
<a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br>
<a href="demo-0.4c1-py2.4.egg">demo-0.4c1-py2.4.egg</a><br>
<a href="demoneeded-1.0.zip">demoneeded-1.0.zip</a><br>
<a href="demoneeded-1.1.zip">demoneeded-1.1.zip</a><br>
<a href="demoneeded-1.2c1.zip">demoneeded-1.2c1.zip</a><br>
<a href="extdemo-1.4.zip">extdemo-1.4.zip</a><br>
<a href="extdemo-1.5.zip">extdemo-1.5.zip</a><br>
<a href="index/">index/</a><br>
......
......@@ -558,23 +558,6 @@ def create_sections_on_command_line():
"""
# Why?
## def error_for_undefined_install_parts():
## """
## Any parts we pass to install on the command line must be
## listed in the configuration.
## >>> print system(join('bin', 'buildout') + ' install foo'),
## Invalid install parts: foo.
## Install parts must be listed in the configuration.
## >>> print system(join('bin', 'buildout') + ' install foo bar'),
## Invalid install parts: foo bar.
## Install parts must be listed in the configuration.
## """
bootstrap_py = os.path.join(
os.path.dirname(
os.path.dirname(
......@@ -2073,6 +2056,241 @@ directory and then use the wacky extension to load the demo package
"""
def create_egg(name, version, dest):
d = tempfile.mkdtemp()
if dest=='available':
extras = dict(x=['x'])
else:
extras = {}
try:
open(os.path.join(d, 'setup.py'), 'w').write(
'from setuptools import setup\n'
'setup(name=%r, version=%r, extras_require=%r, zip_safe=True,\n'
' py_modules=["setup"]\n)'
% (name, str(version), extras)
)
zc.buildout.testing.bdist_egg(d, sys.executable, os.path.abspath(dest))
finally:
shutil.rmtree(d)
def prefer_final_permutation(existing, available):
for d in ('existing', 'available'):
if os.path.exists(d):
shutil.rmtree(d)
os.mkdir(d)
for version in existing:
create_egg('spam', version, 'existing')
for version in available:
create_egg('spam', version, 'available')
zc.buildout.easy_install.clear_index_cache()
[dist] = list(
zc.buildout.easy_install.install(['spam'], 'existing', ['available'],
always_unzip=True)
)
if dist.extras:
print 'downloaded', dist.version
else:
print 'had', dist.version
sys.path_importer_cache.clear()
def prefer_final():
"""
This test tests several permutations:
Using different version numbers to work around zip impporter cache problems. :(
- With prefer final:
- no existing and newer dev available
>>> prefer_final_permutation((), [1, '2a1'])
downloaded 1
- no existing and only dev available
>>> prefer_final_permutation((), ['3a1'])
downloaded 3a1
- final existing and only dev acailable
>>> prefer_final_permutation([4], ['5a1'])
had 4
- final existing and newer final available
>>> prefer_final_permutation([6], [7])
downloaded 7
- final existing and same final available
>>> prefer_final_permutation([8], [8])
had 8
- final existing and older final available
>>> prefer_final_permutation([10], [9])
had 10
- only dev existing and final available
>>> prefer_final_permutation(['12a1'], [11])
downloaded 11
- only dev existing and no final available newer dev available
>>> prefer_final_permutation(['13a1'], ['13a2'])
downloaded 13a2
- only dev existing and no final available older dev available
>>> prefer_final_permutation(['15a1'], ['14a1'])
had 15a1
- only dev existing and no final available same dev available
>>> prefer_final_permutation(['16a1'], ['16a1'])
had 16a1
- Without prefer final:
>>> _ = zc.buildout.easy_install.prefer_final(False)
- no existing and newer dev available
>>> prefer_final_permutation((), [18, '19a1'])
downloaded 19a1
- no existing and only dev available
>>> prefer_final_permutation((), ['20a1'])
downloaded 20a1
- final existing and only dev acailable
>>> prefer_final_permutation([21], ['22a1'])
downloaded 22a1
- final existing and newer final available
>>> prefer_final_permutation([23], [24])
downloaded 24
- final existing and same final available
>>> prefer_final_permutation([25], [25])
had 25
- final existing and older final available
>>> prefer_final_permutation([27], [26])
had 27
- only dev existing and final available
>>> prefer_final_permutation(['29a1'], [28])
had 29a1
- only dev existing and no final available newer dev available
>>> prefer_final_permutation(['30a1'], ['30a2'])
downloaded 30a2
- only dev existing and no final available older dev available
>>> prefer_final_permutation(['32a1'], ['31a1'])
had 32a1
- only dev existing and no final available same dev available
>>> prefer_final_permutation(['33a1'], ['33a1'])
had 33a1
>>> _ = zc.buildout.easy_install.prefer_final(True)
"""
def buildout_prefer_final_option():
"""
The prefer-final buildout option can be used for override the default
preference for newer distributions.
The default is prefer-final = true:
>>> write('buildout.cfg',
... '''
... [buildout]
... parts = eggs
... find-links = %(link_server)s
...
... [eggs]
... recipe = zc.recipe.egg:eggs
... eggs = demo
... ''' % globals())
>>> print system(buildout+' -v'), # doctest: +ELLIPSIS
Installing 'zc.buildout', 'setuptools'.
...
Picked: demo = 0.3
...
Picked: demoneeded = 1.1
Here we see that the final versions of demo and demoneeded are used.
We get the same behavior if we add prefer-final = true
>>> write('buildout.cfg',
... '''
... [buildout]
... parts = eggs
... find-links = %(link_server)s
... prefer-final = true
...
... [eggs]
... recipe = zc.recipe.egg:eggs
... eggs = demo
... ''' % globals())
>>> print system(buildout+' -v'), # doctest: +ELLIPSIS
Installing 'zc.buildout', 'setuptools'.
...
Picked: demo = 0.3
...
Picked: demoneeded = 1.1
If we specify prefer-final = false, we'll get the newest
distributions:
>>> write('buildout.cfg',
... '''
... [buildout]
... parts = eggs
... find-links = %(link_server)s
... prefer-final = false
...
... [eggs]
... recipe = zc.recipe.egg:eggs
... eggs = demo
... ''' % globals())
>>> print system(buildout+' -v'), # doctest: +ELLIPSIS
Installing 'zc.buildout', 'setuptools'.
...
Picked: demo = 0.4c1
...
Picked: demoneeded = 1.2c1
We get an error if we specify anything but true or false:
>>> write('buildout.cfg',
... '''
... [buildout]
... parts = eggs
... find-links = %(link_server)s
... prefer-final = no
...
... [eggs]
... recipe = zc.recipe.egg:eggs
... eggs = demo
... ''' % globals())
>>> print system(buildout+' -v'), # doctest: +ELLIPSIS
While:
Initializing.
Error: Invalid value for prefer-final option: no
"""
# XXX Tests needed:
# Link added from package meta data
######################################################################
def create_sample_eggs(test, executable=sys.executable):
......@@ -2082,15 +2300,16 @@ def create_sample_eggs(test, executable=sys.executable):
try:
write(tmp, 'README.txt', '')
for i in (0, 1):
for i in (0, 1, 2):
write(tmp, 'eggrecipedemobeeded.py', 'y=%s\n' % i)
c1 = i==2 and 'c1' or ''
write(
tmp, 'setup.py',
"from setuptools import setup\n"
"setup(name='demoneeded', py_modules=['eggrecipedemobeeded'],"
" zip_safe=True, version='1.%s', author='bob', url='bob', "
" zip_safe=True, version='1.%s%s', author='bob', url='bob', "
"author_email='bob')\n"
% i
% (i, c1)
)
zc.buildout.testing.sdist(tmp, dest)
......@@ -2104,13 +2323,14 @@ def create_sample_eggs(test, executable=sys.executable):
os.remove(os.path.join(tmp, 'eggrecipedemobeeded.py'))
for i in (1, 2, 3):
for i in (1, 2, 3, 4):
write(
tmp, 'eggrecipedemo.py',
'import eggrecipedemobeeded\n'
'x=%s\n'
'def main(): print x, eggrecipedemobeeded.y\n'
% i)
c1 = i==4 and 'c1' or ''
write(
tmp, 'setup.py',
"from setuptools import setup\n"
......@@ -2118,7 +2338,7 @@ def create_sample_eggs(test, executable=sys.executable):
" install_requires = 'demoneeded',"
" entry_points={'console_scripts': "
"['demo = eggrecipedemo:main']},"
" zip_safe=True, version='0.%s')\n" % i
" zip_safe=True, version='0.%s%s')\n" % (i, c1)
)
zc.buildout.testing.bdist_egg(tmp, executable, dest)
finally:
......
......@@ -36,8 +36,10 @@ We have a link server that has a number of distributions:
<a href="demo-0.1-py2.3.egg">demo-0.1-py2.3.egg</a><br>
<a href="demo-0.2-py2.3.egg">demo-0.2-py2.3.egg</a><br>
<a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br>
<a href="demo-0.4c1-py2.3.egg">demo-0.4c1-py2.3.egg</a><br>
<a href="demoneeded-1.0.zip">demoneeded-1.0.zip</a><br>
<a href="demoneeded-1.1.zip">demoneeded-1.1.zip</a><br>
<a href="demoneeded-1.2c1.zip">demoneeded-1.2c1.zip</a><br>
<a href="extdemo-1.4.zip">extdemo-1.4.zip</a><br>
<a href="index/">index/</a><br>
<a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br>
......
......@@ -14,8 +14,10 @@ We have a link server:
<a href="demo-0.1-py2.3.egg">demo-0.1-py2.3.egg</a><br>
<a href="demo-0.2-py2.3.egg">demo-0.2-py2.3.egg</a><br>
<a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br>
<a href="demo-0.4c1-py2.3.egg">demo-0.4c1-py2.3.egg</a><br>
<a href="demoneeded-1.0.zip">demoneeded-1.0.zip</a><br>
<a href="demoneeded-1.1.zip">demoneeded-1.1.zip</a><br>
<a href="demoneeded-1.2c1.zip">demoneeded-1.2c1.zip</a><br>
<a href="extdemo-1.4.zip">extdemo-1.4.zip</a><br>
<a href="index/">index/</a><br>
<a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br>
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment