Skip to content

S
slapos.buildout
Commit 8a087b51 authored by Jim Fulton's avatar Jim Fulton
Browse files
Options

normalize the way default options are handled and documented 

Cherry-picked from trunk. Thanks Gary.
parent 8a3f1e9f
Hide whitespace changes
Inline Side-by-side
Showing with 259 additions and 161 deletions
src/zc/buildout/buildout.py
View file @ 8a087b51
......@@ -108,13 +108,24 @@ def _unannotate(data):
return data
_buildout_default_options = _annotate_section({
'eggs-directory': 'eggs',
'develop-eggs-directory': 'develop-eggs',
'allow-hosts': '*',
'allow-picked-versions': 'true',
'bin-directory': 'bin',
'parts-directory': 'parts',
'develop-eggs-directory': 'develop-eggs',
'eggs-directory': 'eggs',
'executable': sys.executable,
'find-links': '',
'install-from-cache': 'false',
'installed': '.installed.cfg',
'log-level': 'INFO',
'log-format': '',
'log-level': 'INFO',
'newest': 'true',
'offline': 'false',
'parts-directory': 'parts',
'prefer-final': 'false',
'python': 'buildout',
'unzip': 'false',
'use-dependency-links': 'true',
}, 'DEFAULT_VALUE')
# _buildout_version and _buildout_1_4_default_versions are part of a
......@@ -201,7 +212,7 @@ class Buildout(UserDict.DictMixin):
# provide some defaults before options are parsed
# because while parsing options those attributes might be
# used already (Gottfried Ganssauge)
buildout_section = data.get('buildout')
buildout_section = data['buildout']
# Try to make sure we have absolute paths for standard
# directories. We do this before doing substitutions, in case
......@@ -214,22 +225,28 @@ class Buildout(UserDict.DictMixin):
d = self._buildout_path(buildout_section[name+'-directory'])
buildout_section[name+'-directory'] = d
links = buildout_section and buildout_section.get('find-links', '')
# Attributes on this buildout object shouldn't be used by
# recipes in their __init__. It can cause bugs, because the
# recipes will be instantiated below (``options = self['buildout']``)
# before this has completed initializing. These attributes are
# left behind for legacy support but recipe authors should
# beware of using them. A better practice is for a recipe to
# use the buildout['buildout'] options.
links = buildout_section['find-links']
self._links = links and links.split() or ()
allow_hosts = buildout_section and buildout_section.get(
'allow-hosts', '*').split('\n')
allow_hosts = buildout_section['allow-hosts'].split('\n')
self._allow_hosts = tuple([host.strip() for host in allow_hosts
if host.strip() != ''])
self._logger = logging.getLogger('zc.buildout')
self.offline = False
self.newest = True
self.offline = (buildout_section['offline'] == 'true')
self.newest = (buildout_section['newest'] == 'true')
##################################################################
## WARNING!!!
## ALL ATTRIBUTES MUST HAVE REASONABLE DEFAULTS AT THIS POINT
## OTHERWISE ATTRIBUTEERRORS MIGHT HAPPEN ANY TIME
## OTHERWISE ATTRIBUTEERRORS MIGHT HAPPEN ANY TIME FROM RECIPES.
## RECIPES SHOULD GENERALLY USE buildout['buildout'] OPTIONS, NOT
## BUILDOUT ATTRIBUTES.
##################################################################
# initialize some attrs and buildout directories.
options = self['buildout']
......@@ -238,7 +255,7 @@ class Buildout(UserDict.DictMixin):
links = options.get('find-links', '')
self._links = links and links.split() or ()
allow_hosts = options.get('allow-hosts', '*').split('\n')
allow_hosts = options['allow-hosts'].split('\n')
self._allow_hosts = tuple([host.strip() for host in allow_hosts
if host.strip() != ''])
......@@ -256,20 +273,18 @@ class Buildout(UserDict.DictMixin):
self._setup_logging()
offline = options.get('offline', 'false')
offline = options['offline']
if offline not in ('true', 'false'):
self._error('Invalid value for offline option: %s', offline)
options['offline'] = offline
self.offline = offline == 'true'
self.offline = (offline == 'true')
if self.offline:
newest = options['newest'] = 'false'
else:
newest = options.get('newest', 'true')
newest = options['newest']
if newest not in ('true', 'false'):
self._error('Invalid value for newest option: %s', newest)
options['newest'] = newest
self.newest = newest == 'true'
self.newest = (newest == 'true')
# This is a hacked version of zc.buildout for 1.4.4.
# This means that buildout uses the defaults set up above. The point
......@@ -282,25 +297,25 @@ class Buildout(UserDict.DictMixin):
versions.update(dict(self[versions_section]))
zc.buildout.easy_install.default_versions(versions)
prefer_final = options.get('prefer-final', 'false')
prefer_final = options['prefer-final']
if prefer_final not in ('true', 'false'):
self._error('Invalid value for prefer-final option: %s',
prefer_final)
zc.buildout.easy_install.prefer_final(prefer_final=='true')
use_dependency_links = options.get('use-dependency-links', 'true')
use_dependency_links = options['use-dependency-links']
if use_dependency_links not in ('true', 'false'):
self._error('Invalid value for use-dependency-links option: %s',
use_dependency_links)
zc.buildout.easy_install.use_dependency_links(
use_dependency_links == 'true')
allow_picked_versions = options.get('allow-picked-versions', 'true')
allow_picked_versions = options['allow-picked-versions']
if allow_picked_versions not in ('true', 'false'):
self._error('Invalid value for allow-picked-versions option: %s',
allow_picked_versions)
zc.buildout.easy_install.allow_picked_versions(
allow_picked_versions=='true')
allow_picked_versions == 'true')
download_cache = options.get('download-cache')
if download_cache:
......@@ -317,13 +332,12 @@ class Buildout(UserDict.DictMixin):
zc.buildout.easy_install.download_cache(download_cache)
install_from_cache = options.get('install-from-cache')
if install_from_cache:
if install_from_cache not in ('true', 'false'):
self._error('Invalid value for install-from-cache option: %s',
install_from_cache)
if install_from_cache == 'true':
zc.buildout.easy_install.install_from_cache(True)
install_from_cache = options['install-from-cache']
if install_from_cache not in ('true', 'false'):
self._error('Invalid value for install-from-cache option: %s',
install_from_cache)
zc.buildout.easy_install.install_from_cache(
install_from_cache=='true')
# "Use" each of the defaults so they aren't reported as unused options.
for name in _buildout_default_options:
......
src/zc/buildout/buildout.txt
View file @ 8a087b51
......@@ -504,7 +504,7 @@ Let's fix the recipe:
>>> write(sample_buildout, 'recipes', 'mkdir.py',
... """
... import logging, os, zc.buildout
... import logging, os, zc.buildout, sys
...
... class Mkdir:
...
......@@ -533,13 +533,15 @@ Let's fix the recipe:
... 'Creating directory %s', os.path.basename(path))
... os.mkdir(path)
... created.append(path)
... except:
... except Exception:
... for d in created:
... os.rmdir(d)
... assert not os.path.exists(d)
... logging.getLogger(self.name).info(
... 'Removed %s due to error',
... os.path.basename(d))
... sys.stderr.flush()
... sys.stdout.flush()
... raise
...
... return paths
......@@ -581,7 +583,7 @@ When we rerun the buildout:
.. Wait for the file to really disappear. My linux is weird.
>>> wait_until("foo goes away", lambda : not os.path.exists('foo'),
... timeout=100)
... timeout=200)
we get the same error, but we don't get the directory left behind:
......@@ -729,6 +731,10 @@ COMMAND_LINE_VALUE).
==================
<BLANKLINE>
[buildout]
allow-hosts= *
DEFAULT_VALUE
allow-picked-versions= true
DEFAULT_VALUE
bin-directory= bin
DEFAULT_VALUE
develop= recipes
......@@ -739,16 +745,34 @@ COMMAND_LINE_VALUE).
COMPUTED_VALUE
eggs-directory= eggs
DEFAULT_VALUE
executable= ...
DEFAULT_VALUE
find-links=
DEFAULT_VALUE
install-from-cache= false
DEFAULT_VALUE
installed= .installed.cfg
DEFAULT_VALUE
log-format=
DEFAULT_VALUE
log-level= INFO
DEFAULT_VALUE
newest= true
DEFAULT_VALUE
offline= false
DEFAULT_VALUE
parts= data-dir
/sample-buildout/buildout.cfg
parts-directory= parts
DEFAULT_VALUE
prefer-final= false
DEFAULT_VALUE
python= buildout
DEFAULT_VALUE
unzip= false
DEFAULT_VALUE
use-dependency-links= true
DEFAULT_VALUE
<BLANKLINE>
[data-dir]
path= foo bins
......@@ -2201,10 +2225,15 @@ database is shown.
<BLANKLINE>
Configuration data:
[buildout]
allow-hosts = *
allow-picked-versions = true
bin-directory = /sample-buildout/bin
develop-eggs-directory = /sample-buildout/develop-eggs
directory = /sample-buildout
eggs-directory = /sample-buildout/eggs
executable = python
find-links =
install-from-cache = false
installed = /sample-buildout/.installed.cfg
log-format =
log-level = INFO
......@@ -2212,6 +2241,10 @@ database is shown.
offline = false
parts =
parts-directory = /sample-buildout/parts
prefer-final = false
python = buildout
unzip = false
use-dependency-links = true
verbosity = 20
<BLANKLINE>
......@@ -2219,6 +2252,37 @@ All of these options can be overridden by configuration files or by
command-line assignments. We've discussed most of these options
already, but let's review them and touch on some we haven't discussed:
allow-hosts
On some environments the links visited by `zc.buildout` can be forbidden by
paranoid firewalls. These URLs might be in the chain of links visited by
`zc.buildout` as defined by buildout's `find-links` option, or as defined
by various eggs in their `url`, `download_url`, `dependency_links` metadata.
The fact that package_index works like a spider and might visit links and
go to other locations makes this even harder.
The `allow-hosts` option provides a way to prevent this, and
works exactly like the one provided in `easy_install`.
You can provide a list of allowed host, together with wildcards::
[buildout]
...
allow-hosts =
*.python.org
example.com
All URLs that does not match these hosts will not be visited.
allow-picked-versions
By default, the buildout will choose the best match for a given requirement
if the requirement is not specified precisely (for instance, using the
"versions" option. This behavior corresponds to the
"allow-picked-versions" being set to its default value, "true". If
"allow-picked-versions" is "false," instead of picking the best match,
buildout will raise an error. This helps enforce repeatability.
bin-directory
The directory path where scripts are written. This can be a
relative path, which is interpreted relative to the directory
......@@ -2239,6 +2303,51 @@ eggs-directory
*never* be modified. This can be a relative path, which is
interpreted relative to the directory option.
executable
The Python executable used to run the buildout. See the python
option below.
find-links
You can specify more locations to search for distributions using the
`find-links` option. All locations specified will be searched for
distributions along with the package index as described before.
Locations can be urls::
[buildout]
...
find-links = http://download.zope.org/distribution/
They can also be directories on disk::
[buildout]
...
find-links = /some/path
Finally, they can also be direct paths to distributions::
[buildout]
...
find-links = /some/path/someegg-1.0.0-py2.3.egg
Any number of locations can be specified in the `find-links` option::
[buildout]
...
find-links =
http://download.zope.org/distribution/
/some/otherpath
/some/path/someegg-1.0.0-py2.3.egg
install-from-cache
A download cache can be used as the basis of application source releases.
In an application source release, we want to distribute an application that
can be built without making any network accesses. In this case, we
distribute a buildout with download cache and tell the buildout to install
from the download cache only, without making network accesses. The
buildout install-from-cache option can be used to signal that packages
should be installed only from the download cache.
installed
The file path where information about the results of the previous
buildout run is written. This can be a relative path, which is
......@@ -2252,12 +2361,101 @@ log-format
log-level
The log level before verbosity adjustment
newest
By default buildout and recipes will try to find the newest versions of
distributions needed to satisfy requirements. This can be very time
consuming, especially when incrementally working on setting up a buildout
or working on a recipe. The buildout "newest" option can be used to to
suppress this. If the "newest" option is set to false, then new
distributions won't be sought if an installed distribution meets
requirements. The "newest" option can also be set to false using the -N
command-line option. See also the "offline" option.
offline
The "offline" option goes a bit further than the "newest" option. If the
buildout "offline" option is given a value of "true", the buildout and
recipes that are aware of the option will avoid doing network access. This
is handy when running the buildout when not connected to the internet. It
also makes buildouts run much faster. This option is typically set using
the buildout -o option.
parts
A white space separated list of parts to be installed.
parts-directory
A working directory that parts can used to store data.
prefer-final
Currently, when searching for new releases, the newest available
release is used. This isn't usually ideal, as you may get a
development release or alpha releases not ready to be widely used.
You can request that final releases be preferred using the prefer
final option in the buildout section::
[buildout]
...
prefer-final = true
When the prefer-final option is set to true, then when searching for
new releases, final releases are preferred. If there are final
releases that satisfy distribution requirements, then those releases
are used even if newer non-final releases are available. The buildout
prefer-final option can be used to override this behavior.
In buildout version 2, final releases will be preferred by default.
You will then need to use a false value for prefer-final to get the
newest releases.
python
The name of a section containing information about the default
Python interpreter. Recipes that need a installation
typically have options to tell them which Python installation to
use. By convention, if a section-specific option isn't used, the
option is looked for in the buildout section. The option must
point to a section with an executable option giving the path to a
Python executable. By default, the buildout section defines the
default Python as the Python used to run the buildout.
unzip
By default, zc.buildout doesn't unzip zip-safe eggs ("unzip = false").
This follows the policy followed by setuptools itself. Experience shows
this policy to to be inconvenient. Zipped eggs make debugging more
difficult and often import more slowly. You can include an unzip option in
the buildout section to change the default unzipping policy ("unzip =
true").
use-dependency-links
By default buildout will obey the setuptools dependency_links metadata
when it looks for dependencies. This behavior can be controlled with
the use-dependency-links buildout option::
[buildout]
...
use-dependency-links = false
The option defaults to true. If you set it to false, then dependency
links are only looked for in the locations specified by find-links.
unzip
By default, zc.buildout doesn't unzip zip-safe eggs ("unzip = false").
This follows the policy followed by setuptools itself. Experience shows
this policy to to be inconvenient. Zipped eggs make debugging more
difficult and often import more slowly. You can include an unzip option in
the buildout section to change the default unzipping policy ("unzip =
true").
use-dependency-links
By default buildout will obey the setuptools dependency_links metadata
when it looks for dependencies. This behavior can be controlled with
the use-dependency-links buildout option::
[buildout]
...
use-dependency-links = false
The option defaults to true. If you set it to false, then dependency
links are only looked for in the locations specified by find-links.
verbosity
A log-level adjustment. Typically, this is set via the -q and -v
command-line options.
......@@ -2336,48 +2534,6 @@ if there isn't a configuration file:
Generated script '/sample-bootstrapped2/bin/buildout'.
Newest and Offline Modes
------------------------
By default buildout and recipes will try to find the newest versions
of distributions needed to satisfy requirements. This can be very
time consuming, especially when incrementally working on setting up a
buildout or working on a recipe. The buildout newest option can be
used to to suppress this. If the newest option is set to false, then
new distributions won't be sought if an installed distribution meets
requirements. The newest option can be set to false using the -N
command-line option.
The offline option goes a bit further. If the buildout offline option
is given a value of "true", the buildout and recipes that are aware of
the option will avoid doing network access. This is handy when
running the buildout when not connected to the internet. It also
makes buildouts run much faster. This option is typically set using
the buildout -o option.
Preferring Final Releases
-------------------------
Currently, when searching for new releases, the newest available
release is used. This isn't usually ideal, as you may get a
development release or alpha releases not ready to be widely used.
You can request that final releases be preferred using the prefer
final option in the buildout section::
[buildout]
...
prefer-final = true
When the prefer-final option is set to true, then when searching for
new releases, final releases are preferred. If there are final
releases that satisfy distribution requirements, then those releases
are used even if newer non-final releases are available. The buildout
prefer-final option can be used to override this behavior.
In buildout version 2, final releases will be preferred by default.
You will then need to use a false value for prefer-final to get the
newest releases.
Finding distributions
---------------------
......@@ -2396,49 +2552,7 @@ distributions. The latest version of the distribution that meets the
requirements of the buildout will always be used.
You can also specify more locations to search for distributions using
the `find-links` option. All locations specified will be searched for
distributions along with the package index as described before.
Locations can be urls::
[buildout]
...
find-links = http://download.zope.org/distribution/
They can also be directories on disk::
[buildout]
...
find-links = /some/path
Finally, they can also be direct paths to distributions::
[buildout]
...
find-links = /some/path/someegg-1.0.0-py2.3.egg
Any number of locations can be specified in the `find-links` option::
[buildout]
...
find-links =
http://download.zope.org/distribution/
/some/otherpath
/some/path/someegg-1.0.0-py2.3.egg
Dependency links
----------------
By default buildout will obey the setuptools dependency_links metadata
when it looks for dependencies. This behavior can be controlled with
the use-dependency-links buildout option::
[buildout]
...
use-dependency-links = false
The option defaults to true. If you set it to false, then dependency
links are only looked for in the locations specified by find-links.
the `find-links` option. See its description above.
Controlling the installation database
-------------------------------------
......@@ -2599,38 +2713,3 @@ We see that our extension is loaded and executed:
ext ['buildout']
Develop: '/sample-bootstrapped/demo'
unload ['buildout']
Allow hosts
-----------
On some environments the links visited by `zc.buildout` can be forbidden
by paranoiac firewalls. These URL might be on the chain of links
visited by `zc.buildout` wheter they are defined in the `find-links` option,
wheter they are defined by various eggs in their `url`, `download_url`,
`dependency_links` metadata.
It is even harder to track that package_index works like a spider and
might visit links and go to other location.
The `allow-hosts` option provides a way to prevent this, and
works exactly like the one provided in `easy_install`.
You can provide a list of allowed host, together with wildcards::
[buildout]
...
allow-hosts =
*.python.org
example.com
All urls that does not match these hosts will not be visited.
.. [#future_recipe_methods] In the future, additional methods may be
added. Older recipes with fewer methods will still be
supported.
.. [#packaging_info] If we wanted to create a distribution from this
package, we would need specify much more information. See the
`setuptools documentation
<http://peak.telecommunity.com/DevCenter/setuptools>`_.
src/zc/buildout/tests.py
View file @ 8a087b51
......@@ -2851,6 +2851,8 @@ def test_suite():
r'when that file already exists: '),
'[Errno 17] File exists: '
),
(re.compile('executable = %s' % re.escape(sys.executable)),
'executable = python'),
])
),
doctest.DocFileSuite(
......@@ -2947,6 +2949,9 @@ def test_suite():
'-q develop -mxN -d /sample-buildout/develop-eggs'
),
(re.compile(r'^[*]...'), '...'),
# for bug_92891_bootstrap_crashes_with_egg_recipe_in_buildout_section
(re.compile(r"Unused options for buildout: 'eggs' 'scripts'\."),
"Unused options for buildout: 'scripts' 'eggs'."),
]),
),
zc.buildout.rmtree.test_suite(),
......
zc.recipe.egg_/src/zc/recipe/egg/egg.py
View file @ 8a087b51
......@@ -27,8 +27,8 @@ class Eggs(object):
self.buildout = buildout
self.name = name
self.options = options
links = options.get('find-links',
buildout['buildout'].get('find-links'))
b_options = buildout['buildout']
links = options.get('find-links', b_options['find-links'])
if links:
links = links.split()
options['find-links'] = '\n'.join(links)
......@@ -36,20 +36,19 @@ class Eggs(object):
links = ()
self.links = links
index = options.get('index', buildout['buildout'].get('index'))
index = options.get('index', b_options.get('index'))
if index is not None:
options['index'] = index
self.index = index
allow_hosts = buildout['buildout'].get('allow-hosts', '*')
allow_hosts = b_options['allow-hosts']
allow_hosts = tuple([host.strip() for host in allow_hosts.split('\n')
if host.strip()!=''])
self.allow_hosts = allow_hosts
options['eggs-directory'] = buildout['buildout']['eggs-directory']
options['eggs-directory'] = b_options['eggs-directory']
options['_e'] = options['eggs-directory'] # backward compat.
options['develop-eggs-directory'
] = buildout['buildout']['develop-eggs-directory']
options['develop-eggs-directory'] = b_options['develop-eggs-directory']
options['_d'] = options['develop-eggs-directory'] # backward compat.
def working_set(self, extra=()):
......@@ -58,6 +57,7 @@ class Eggs(object):
This is intended for reuse by similar recipes.
"""
options = self.options
b_options = self.buildout['buildout']
# Backward compat. :(
options['executable'] = sys.executable
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7