Skip to content

S
slapos.buildout
Commit 98b7c55c authored by Gary Poster's avatar Gary Poster
Browse files
Options

support limiting packages from site-packages

parent e73c70bb
Hide whitespace changes
Inline Side-by-side
Showing with 628 additions and 130 deletions
.bzrignore 0 → 100644
View file @ 98b7c55c
.installed.cfg
bin
build
develop-eggs
eggs
parts
src/zc.buildout.egg-info
z3c.recipe.scripts_/src/z3c.recipe.scripts.egg-info
zc.recipe.egg_/src/zc.recipe.egg.egg-info
src/zc/buildout/easy_install.py
View file @ 98b7c55c
......@@ -19,6 +19,7 @@ installed.
"""
import distutils.errors
import fnmatch
import glob
import logging
import os
......@@ -67,6 +68,64 @@ buildout_and_setuptools_path = [
pkg_resources.Requirement.parse('zc.buildout')).location,
]
def _get_system_paths(executable):
"""return lists of standard lib and site paths for executable.
"""
# We want to get a list of the site packages, which is not easy.
# The canonical way to do this is to use
# distutils.sysconfig.get_python_lib(), but that only returns a
# single path, which does not reflect reality for many system
# Pythons, which have multiple additions. Instead, we start Python
# with -S, which does not import site.py and set up the extra paths
# like site-packages or (Ubuntu/Debian) dist-packages and
# python-support. We then compare that sys.path with the normal one
# (minus user packages if this is Python 2.6, because we don't
# support those (yet?). The set of the normal one minus the set of
# the ones in ``python -S`` is the set of packages that are
# effectively site-packages.
#
# The given executable might not be the current executable, so it is
# appropriate to do another subprocess to figure out what the
# additional site-package paths are. Moreover, even if this
# executable *is* the current executable, this code might be run in
# the context of code that has manipulated the sys.path--for
# instance, to add local zc.buildout or setuptools eggs.
def get_sys_path(*args, **kwargs):
cmd = [executable]
cmd.extend(args)
cmd.extend([
"-c", "import sys, os;"
"print repr([os.path.normpath(p) for p in sys.path if p])"])
# Windows needs some (as yet to be determined) part of the real env.
env = os.environ.copy()
env.update(kwargs)
_proc = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
stdout, stderr = _proc.communicate();
if _proc.returncode:
raise RuntimeError(
'error trying to get system packages:\n%s' % (stderr,))
res = eval(stdout.strip())
try:
res.remove('.')
except ValueError:
pass
return res
stdlib = get_sys_path('-S') # stdlib only
no_user_paths = get_sys_path(PYTHONNOUSERSITE='x')
site_paths = [p for p in no_user_paths if p not in stdlib]
return (stdlib, site_paths)
def _get_version_info(executable):
cmd = [executable, '-Sc', 'import sys; print repr(sys.version_info)']
_proc = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = _proc.communicate();
if _proc.returncode:
raise RuntimeError(
'error trying to get system packages:\n%s' % (stderr,))
return eval(stdout.strip())
class IncompatibleVersionError(zc.buildout.UserError):
"""A specified version is incompatible with a given requirement.
......@@ -109,7 +168,12 @@ class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
_indexes = {}
def _get_index(executable, index_url, find_links, allow_hosts=('*',)):
def _get_index(executable, index_url, find_links, allow_hosts=('*',),
path=None):
# If path is None, the index will use sys.path. If you provide an empty
# path ([]), it will complain uselessly about missing index pages for
# packages found in the paths that you expect to use. Therefore, this path
# is always the same as the _env path in the Installer.
key = executable, index_url, tuple(find_links)
index = _indexes.get(key)
if index is not None:
......@@ -118,7 +182,8 @@ def _get_index(executable, index_url, find_links, allow_hosts=('*',)):
if index_url is None:
index_url = default_index_url
index = AllowHostsPackageIndex(
index_url, hosts=allow_hosts, python=_get_version(executable)
index_url, hosts=allow_hosts, search_path=path,
python=_get_version(executable)
)
if find_links:
......@@ -192,6 +257,8 @@ class Installer:
_use_dependency_links = True
_allow_picked_versions = True
_always_unzip = False
_include_site_packages = True
_allowed_eggs_from_site_packages = ('*',)
def __init__(self,
dest=None,
......@@ -203,6 +270,8 @@ class Installer:
newest=True,
versions=None,
use_dependency_links=None,
include_site_packages=None,
allowed_eggs_from_site_packages=None,
allow_hosts=('*',)
):
self._dest = dest
......@@ -225,7 +294,28 @@ class Installer:
self._executable = executable
if always_unzip is not None:
self._always_unzip = always_unzip
path = (path and path[:] or []) + buildout_and_setuptools_path
path = (path and path[:] or [])
if include_site_packages is not None:
self._include_site_packages = include_site_packages
if allowed_eggs_from_site_packages is not None:
self._allowed_eggs_from_site_packages = tuple(
allowed_eggs_from_site_packages)
stdlib, self._site_packages = _get_system_paths(executable)
version_info = _get_version_info(executable)
if version_info == sys.version_info:
# Maybe we can add the buildout and setuptools path. If we
# are including site_packages, we only have to include the extra
# bits here, so we don't duplicate. On the other hand, if we
# are not including site_packages, we only want to include the
# parts that are not in site_packages, so the code is the same.
path.extend(
set(buildout_and_setuptools_path).difference(
self._site_packages))
if self._include_site_packages:
path.extend(self._site_packages)
# else we could try to still include the buildout_and_setuptools_path
# if the elements are not in site_packages, but we're not bothering
# with this optimization for now, in the name of code simplicity.
if dest is not None and dest not in path:
path.insert(0, dest)
self._path = path
......@@ -234,13 +324,42 @@ class Installer:
self._newest = newest
self._env = pkg_resources.Environment(path,
python=_get_version(executable))
self._index = _get_index(executable, index, links, self._allow_hosts)
self._index = _get_index(executable, index, links, self._allow_hosts,
self._path)
if versions is not None:
self._versions = versions
_allowed_eggs_from_site_packages_regex = None
def allow_site_package_egg(self, name):
if (not self._include_site_packages or
not self._allowed_eggs_from_site_packages):
# If the answer is a blanket "no," perform a shortcut.
return False
if self._allowed_eggs_from_site_packages_regex is None:
pattern = '(%s)' % (
'|'.join(
fnmatch.translate(name)
for name in self._allowed_eggs_from_site_packages),
)
self._allowed_eggs_from_site_packages_regex = re.compile(pattern)
return bool(self._allowed_eggs_from_site_packages_regex.match(name))
def _satisfied(self, req, source=None):
dists = [dist for dist in self._env[req.project_name] if dist in req]
# We get all distributions that match the given requirement. If we are
# not supposed to include site-packages for the given egg, we also
# filter those out. Even if include_site_packages is False and so we
# have excluded site packages from the _env's paths (see
# Installer.__init__), we need to do the filtering here because an
# .egg-link, such as one for setuptools or zc.buildout installed by
# zc.buildout.buildout.Buildout.bootstrap, can indirectly include a
# path in our _site_packages.
dists = [dist for dist in self._env[req.project_name] if (
dist in req and (
dist.location not in self._site_packages or
self.allow_site_package_egg(dist.project_name))
)
]
if not dists:
logger.debug('We have no distributions for %s that satisfies %r.',
req.project_name, str(req))
......@@ -441,14 +560,22 @@ class Installer:
# Nothing is available.
return None
# Filter the available dists for the requirement and source flag
dists = [dist for dist in index[requirement.project_name]
if ((dist in requirement)
and
((not source) or
(dist.precedence == pkg_resources.SOURCE_DIST)
)
)
# Filter the available dists for the requirement and source flag. If
# we are not supposed to include site-packages for the given egg, we
# also filter those out. Even if include_site_packages is False and so
# we have excluded site packages from the _env's paths (see
# Installer.__init__), we need to do the filtering here because an
# .egg-link, such as one for setuptools or zc.buildout installed by
# zc.buildout.buildout.Buildout.bootstrap, can indirectly include a
# path in our _site_packages.
dists = [dist for dist in index[requirement.project_name] if (
dist in requirement and (
dist.location not in self._site_packages or
self.allow_site_package_egg(dist.project_name))
and (
(not source) or
(dist.precedence == pkg_resources.SOURCE_DIST))
)
]
# If we prefer final dists, filter for final and use the
......@@ -608,7 +735,7 @@ class Installer:
self._links.append(link)
self._index = _get_index(self._executable,
self._index_url, self._links,
self._allow_hosts)
self._allow_hosts, self._path)
for dist in dists:
# Check whether we picked a version and, if we did, report it:
......@@ -689,35 +816,52 @@ class Installer:
self._maybe_add_setuptools(ws, dist)
# OK, we have the requested distributions and they're in the working
# set, but they may have unmet requirements. We'll simply keep
# trying to resolve requirements, adding missing requirements as they
# are reported.
#
# Note that we don't pass in the environment, because we want
# set, but they may have unmet requirements. We'll resolve these
# requirements. This is code modified from
# pkg_resources.WorkingSet.resolve. We can't reuse that code directly
# because we have to constrain our requirements (see
# versions_section_ignored_for_dependency_in_favor_of_site_packages in
# zc.buildout.tests).
requirements.reverse() # Set up the stack.
processed = {} # This is a set of processed requirements.
best = {} # This is a mapping of key -> dist.
# Note that we don't use the existing environment, because we want
# to look for new eggs unless what we have is the best that
# matches the requirement.
while 1:
try:
ws.resolve(requirements)
except pkg_resources.DistributionNotFound, err:
[requirement] = err
requirement = self._constrain(requirement)
if destination:
logger.debug('Getting required %r', str(requirement))
else:
logger.debug('Adding required %r', str(requirement))
_log_requirement(ws, requirement)
for dist in self._get_dist(requirement, ws, self._always_unzip
):
ws.add(dist)
self._maybe_add_setuptools(ws, dist)
except pkg_resources.VersionConflict, err:
raise VersionConflict(err, ws)
else:
break
env = pkg_resources.Environment(ws.entries)
while requirements:
# Process dependencies breadth-first.
req = self._constrain(requirements.pop(0))
if req in processed:
# Ignore cyclic or redundant dependencies.
continue
dist = best.get(req.key)
if dist is None:
# Find the best distribution and add it to the map.
dist = ws.by_key.get(req.key)
if dist is None:
try:
dist = best[req.key] = env.best_match(req, ws)
except pkg_resources.VersionConflict, err:
raise VersionConflict(err, ws)
if dist is None:
if destination:
logger.debug('Getting required %r', str(req))
else:
logger.debug('Adding required %r', str(req))
_log_requirement(ws, req)
for dist in self._get_dist(req,
ws, self._always_unzip):
ws.add(dist)
self._maybe_add_setuptools(ws, dist)
if dist not in req:
# Oops, the "best" so far conflicts with a dependency.
raise VersionConflict(
pkg_resources.VersionConflict(dist, req), ws)
requirements.extend(dist.requires(req.extras)[::-1])
processed[req] = True
if dist.location in self._site_packages:
logger.debug('Egg from site-packages: %s', dist)
return ws
def build(self, spec, build_ext):
......@@ -812,6 +956,18 @@ def prefer_final(setting=None):
Installer._prefer_final = bool(setting)
return old
def include_site_packages(setting=None):
old = Installer._include_site_packages
if setting is not None:
Installer._include_site_packages = bool(setting)
return old
def allowed_eggs_from_site_packages(setting=None):
old = Installer._allowed_eggs_from_site_packages
if setting is not None:
Installer._allowed_eggs_from_site_packages = tuple(setting)
return old
def use_dependency_links(setting=None):
old = Installer._use_dependency_links
if setting is not None:
......@@ -834,9 +990,13 @@ def install(specs, dest,
links=(), index=None,
executable=sys.executable, always_unzip=None,
path=None, working_set=None, newest=True, versions=None,
use_dependency_links=None, allow_hosts=('*',)):
use_dependency_links=None, include_site_packages=None,
allowed_eggs_from_site_packages=None, allow_hosts=('*',)):
installer = Installer(dest, links, index, executable, always_unzip, path,
newest, versions, use_dependency_links,
include_site_packages=include_site_packages,
allowed_eggs_from_site_packages=
allowed_eggs_from_site_packages,
allow_hosts=allow_hosts)
return installer.install(specs, working_set)
......@@ -844,9 +1004,14 @@ def install(specs, dest,
def build(spec, dest, build_ext,
links=(), index=None,
executable=sys.executable,
path=None, newest=True, versions=None, allow_hosts=('*',)):
path=None, newest=True, versions=None, include_site_packages=None,
allowed_eggs_from_site_packages=None, allow_hosts=('*',)):
installer = Installer(dest, links, index, executable, True, path, newest,
versions, allow_hosts=allow_hosts)
versions,
include_site_packages=include_site_packages,
allowed_eggs_from_site_packages=
allowed_eggs_from_site_packages,
allow_hosts=allow_hosts)
return installer.build(spec, build_ext)
......@@ -941,9 +1106,12 @@ def develop(setup, dest,
undo.reverse()
[f() for f in undo]
def working_set(specs, executable, path):
return install(specs, None, executable=executable, path=path)
def working_set(specs, executable, path, include_site_packages=None,
allowed_eggs_from_site_packages=None):
return install(
specs, None, executable=executable, path=path,
include_site_packages=include_site_packages,
allowed_eggs_from_site_packages=allowed_eggs_from_site_packages)
############################################################################
# Script generation functions
......@@ -1276,54 +1444,6 @@ if _interactive:
# These are used only by the newer ``generate_scripts`` function.
def _get_system_paths(executable):
"""return lists of standard lib and site paths for executable.
"""
# We want to get a list of the site packages, which is not easy.
# The canonical way to do this is to use
# distutils.sysconfig.get_python_lib(), but that only returns a
# single path, which does not reflect reality for many system
# Pythons, which have multiple additions. Instead, we start Python
# with -S, which does not import site.py and set up the extra paths
# like site-packages or (Ubuntu/Debian) dist-packages and
# python-support. We then compare that sys.path with the normal one
# (minus user packages if this is Python 2.6, because we don't
# support those (yet?). The set of the normal one minus the set of
# the ones in ``python -S`` is the set of packages that are
# effectively site-packages.
#
# The given executable might not be the current executable, so it is
# appropriate to do another subprocess to figure out what the
# additional site-package paths are. Moreover, even if this
# executable *is* the current executable, this code might be run in
# the context of code that has manipulated the sys.path--for
# instance, to add local zc.buildout or setuptools eggs.
def get_sys_path(*args, **kwargs):
cmd = [executable]
cmd.extend(args)
cmd.extend([
"-c", "import sys, os;"
"print repr([os.path.normpath(p) for p in sys.path if p])"])
# Windows needs some (as yet to be determined) part of the real env.
env = os.environ.copy()
env.update(kwargs)
_proc = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
stdout, stderr = _proc.communicate();
if _proc.returncode:
raise RuntimeError(
'error trying to get system packages:\n%s' % (stderr,))
res = eval(stdout.strip())
try:
res.remove('.')
except ValueError:
pass
return res
stdlib = get_sys_path('-S') # stdlib only
no_user_paths = get_sys_path(PYTHONNOUSERSITE='x')
site_paths = [p for p in no_user_paths if p not in stdlib]
return (stdlib, site_paths)
def _get_module_file(executable, name):
"""Return a module's file path.
......
src/zc/buildout/easy_install.txt
View file @ 98b7c55c
......@@ -89,6 +89,14 @@ use_dependency_links
for using dependency_links in preference to other
locations. Defaults to true.
include_site_packages
A flag indicating whether Python's non-standard-library packages should
be available for finding dependencies. Defaults to true.
Paths outside of Python's standard library--or more precisely, those that
are not included when Python is started with the -S argument--are loosely
referred to as "site-packages" here.
relative_paths
Adjust egg paths so they are relative to the script path. This
allows scripts to work when scripts and eggs are moved, as long as
......@@ -399,6 +407,68 @@ dictionary:
>>> [d.version for d in ws]
['0.3', '1.1']
Dependencies in Site Packages
-----------------------------
Paths outside of Python's standard library--or more precisely, those that are
not included when Python is started with the -S argument--are loosely referred
to as "site-packages" here. These site-packages are searched by default for
distributions. This can be disabled, so that, for instance, a system Python
can be used with buildout, cleaned of any packages installed by a user or
system package manager.
The default behavior can be controlled and introspected using
zc.buildout.easy_install.include_site_packages.
>>> zc.buildout.easy_install.include_site_packages()
True
Here's an example of using a Python executable that includes our dependencies.
Our "py_path" will have the "demoneeded," and "demo" packages available.
We'll simply be asking for "demoneeded" here, but without any external
index or links.
>>> from zc.buildout.tests import create_sample_sys_install
>>> py_path, site_packages_path = make_py()
>>> create_sample_sys_install(site_packages_path)
>>> example_dest = tmpdir('site-packages-example-install')
>>> workingset = zc.buildout.easy_install.install(
... ['demoneeded'], example_dest, links=[], executable=py_path,
... index=None)
>>> [dist.project_name for dist in workingset]
['demoneeded']
That worked fine. Let's try again with site packages not allowed. We'll
change the policy by changing the default. Notice that the function for
changing the default value returns the previous value.
>>> zc.buildout.easy_install.include_site_packages(False)
True
>>> zc.buildout.easy_install.include_site_packages()
False
>>> zc.buildout.easy_install.clear_index_cache()
>>> rmdir(example_dest)
>>> example_dest = tmpdir('site-packages-example-install')
>>> workingset = zc.buildout.easy_install.install(
... ['demoneeded'], example_dest, links=[], executable=py_path,
... index=None)
Traceback (most recent call last):
...
MissingDistribution: Couldn't find a distribution for 'demoneeded'.
>>> zc.buildout.easy_install.clear_index_cache()
Now we'll reset the default.
>>> zc.buildout.easy_install.include_site_packages(True)
False
>>> zc.buildout.easy_install.include_site_packages()
True
Dependency links
----------------
......@@ -1259,6 +1329,7 @@ call to another text fixture to create.
>>> namespace_eggs = tmpdir('namespace_eggs')
>>> create_sample_namespace_eggs(namespace_eggs)
>>> reset_interpreter()
>>> ws = zc.buildout.easy_install.install(
... ['demo', 'tellmy.fortune'], join(interpreter_dir, 'eggs'),
... links=[link_server, namespace_eggs], index=link_server+'index/')
......@@ -1319,6 +1390,7 @@ The most complex that this function gets is if you use namespace packages,
include site-packages, and use relative paths. For completeness, we'll look
at that result.
>>> reset_interpreter()
>>> generated = zc.buildout.easy_install.generate_scripts(
... interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
... interpreter='py', add_site_packages=True,
......
src/zc/buildout/testing.py
View file @ 98b7c55c
......@@ -222,11 +222,37 @@ def wait_until(label, func, *args, **kw):
time.sleep(0.01)
raise ValueError('Timed out waiting for: '+label)
def get_installer_values():
"""Get the current values for the easy_install module.
This is necessary because instantiating a Buildout will force the
Buildout's values on the installer.
Returns a dict of names-values suitable for set_installer_values."""
names = ('default_versions', 'download_cache', 'install_from_cache',
'prefer_final', 'include_site_packages',
'allowed_eggs_from_site_packages', 'use_dependency_links',
'allow_picked_versions', 'always_unzip'
)
values = {}
for name in names:
values[name] = getattr(zc.buildout.easy_install, name)()
return values
def set_installer_values(values):
"""Set the given values on the installer."""
for name, value in values.items():
getattr(zc.buildout.easy_install, name)(value)
def make_buildout():
# Create a basic buildout.cfg to avoid a warning from buildout:
"""Make a buildout that uses this version of zc.buildout."""
# Create a basic buildout.cfg to avoid a warning from buildout.
open('buildout.cfg', 'w').write(
"[buildout]\nparts =\n"
)
# Get state of installer defaults so we can reinstate them (instantiating
# a Buildout will force the Buildout's defaults on the installer).
installer_values = get_installer_values()
# Use the buildout bootstrap command to create a buildout
zc.buildout.buildout.Buildout(
'buildout.cfg',
......@@ -234,20 +260,23 @@ def make_buildout():
# trick bootstrap into putting the buildout develop egg
# in the eggs dir.
('buildout', 'develop-eggs-directory', 'eggs'),
]
],
user_defaults=False,
).bootstrap([])
# Create the develop-eggs dir, which didn't get created the usual
# way due to the trick above:
os.mkdir('develop-eggs')
# Reinstate the default values of the installer.
set_installer_values(installer_values)
def buildoutSetUp(test):
test.globs['__tear_downs'] = __tear_downs = []
test.globs['register_teardown'] = register_teardown = __tear_downs.append
prefer_final = zc.buildout.easy_install.prefer_final()
installer_values = get_installer_values()
register_teardown(
lambda: zc.buildout.easy_install.prefer_final(prefer_final)
lambda: set_installer_values(installer_values)
)
here = os.getcwd()
......@@ -367,8 +396,6 @@ def buildoutSetUp(test):
make_py = make_py
))
zc.buildout.easy_install.prefer_final(prefer_final)
def buildoutTearDown(test):
for f in test.globs['__tear_downs']:
f()
......
src/zc/buildout/tests.py
View file @ 98b7c55c
......@@ -385,6 +385,64 @@ buildout will tell us who's asking for something that we can't find.
Error: Couldn't find a distribution for 'demoneeded'.
"""
def show_eggs_from_site_packages():
"""
Sometimes you want to know what eggs are coming from site-packages. This
might be for a diagnostic, or so that you can get a starting value for the
allowed-eggs-from-site-packages option. The -v flag will also include this
information.
Our "py_path" has the "demoneeded," "demo"
packages available. We'll ask for "bigdemo," which will get both of them.
Here's our set up.
>>> py_path, site_packages_path = make_py()
>>> create_sample_sys_install(site_packages_path)
>>> write('buildout.cfg',
... '''
... [buildout]
... parts = eggs
... prefer-final = true
... find-links = %(link_server)s
...
... [primed_python]
... executable = %(py_path)s
...
... [eggs]
... recipe = zc.recipe.egg:eggs
... python = primed_python
... eggs = bigdemo
... ''' % globals())
Now here is the output. The lines that begin with "Egg from site-packages:"
indicate the eggs from site-packages that have been selected. You'll see
we have two: demo 0.3 and demoneeded 1.1.
>>> print system(py_path+" "+buildout+" -v")
Installing 'zc.buildout', 'setuptools'.
We have a develop egg: zc.buildout V
We have the best distribution that satisfies 'setuptools'.
Picked: setuptools = V
Installing 'zc.recipe.egg'.
We have a develop egg: zc.recipe.egg V
Installing eggs.
Installing 'bigdemo'.
We have no distributions for bigdemo that satisfies 'bigdemo'.
Getting distribution for 'bigdemo'.
Got bigdemo 0.1.
Picked: bigdemo = 0.1
Getting required 'demo'
required by bigdemo 0.1.
We have a develop egg: demo V
Egg from site-packages: demo 0.3
Getting required 'demoneeded'
required by demo 0.3.
We have a develop egg: demoneeded V
Egg from site-packages: demoneeded 1.1
<BLANKLINE>
"""
def test_comparing_saved_options_with_funny_characters():
"""
......@@ -2003,6 +2061,197 @@ Now the install works correctly, as seen here.
"""
def isolated_include_site_packages():
"""
This is an isolated test of the include_site_packages functionality, passing
the argument directly to install, overriding a default.
Our "py_path" has the "demoneeded" and "demo" packages available. We'll
simply be asking for "demoneeded" here.
>>> py_path, site_packages_path = make_py()
>>> create_sample_sys_install(site_packages_path)
>>> zc.buildout.easy_install.include_site_packages(False)
True
>>> example_dest = tmpdir('site-packages-example-install')
>>> workingset = zc.buildout.easy_install.install(
... ['demoneeded'], example_dest, links=[], executable=py_path,
... index=None, include_site_packages=True)
>>> [dist.project_name for dist in workingset]
['demoneeded']
That worked fine. Let's try again with site packages not allowed (and
reversing the default).
>>> zc.buildout.easy_install.include_site_packages(True)
False
>>> zc.buildout.easy_install.clear_index_cache()
>>> rmdir(example_dest)
>>> example_dest = tmpdir('site-packages-example-install')
>>> workingset = zc.buildout.easy_install.install(
... ['demoneeded'], example_dest, links=[], executable=py_path,
... index=None, include_site_packages=False)
Traceback (most recent call last):
...
MissingDistribution: Couldn't find a distribution for 'demoneeded'.
That's a failure, as expected.
Now we explore an important edge case.
Some system Pythons include setuptools (and other Python packages) in their
site-packages (or equivalent) using a .egg-info directory. The pkg_resources
module (from setuptools) considers a package installed using .egg-info to be a
develop egg.
zc.buildout.buildout.Buildout.bootstrap will make setuptools and zc.buildout
available to the buildout via the eggs directory, for normal eggs; or the
develop-eggs directory, for develop-eggs.
If setuptools or zc.buildout is found in site-packages and considered by
pkg_resources to be a develop egg, then the bootstrap code will use a .egg-link
in the local develop-eggs, pointing to site-packages, in its entirety. Because
develop-eggs must always be available for searching for distributions, this
indirectly brings site-packages back into the search path for distributions.
Because of this, we have to take special care that we still exclude
site-packages even in this case. See the comments about site packages in the
Installer._satisfied and Installer._obtain methods for the implementation
(as of this writing).
In this demonstration, we insert a link to the "demoneeded" distribution
in our develop-eggs, which would bring the package back in, except for
the special care we have taken to exclude it.
>>> zc.buildout.easy_install.clear_index_cache()
>>> rmdir(example_dest)
>>> example_dest = tmpdir('site-packages-example-install')
>>> mkdir(example_dest, 'develop-eggs')
>>> write(example_dest, 'develop-eggs', 'demoneeded.egg-link',
... site_packages_path)
>>> workingset = zc.buildout.easy_install.install(
... ['demoneeded'], example_dest, links=[],
... path=[join(example_dest, 'develop-eggs')],
... executable=py_path,
... index=None, include_site_packages=False)
Traceback (most recent call last):
...
MissingDistribution: Couldn't find a distribution for 'demoneeded'.
The MissingDistribution error shows that buildout correctly excluded the
"site-packages" source even though it was indirectly included in the path
via a .egg-link file.
"""
def allowed_eggs_from_site_packages():
"""
Sometimes you need or want to control what eggs from site-packages are used.
The allowed-eggs-from-site-packages option allows you to specify a whitelist of
project names that may be included from site-packages. You can use globs to
specify the value. It defaults to a single value of '*', indicating that any
package may come from site-packages.
This option interacts with include-site-packages in the following ways.
If include-site-packages is true, then allowed-eggs-from-site-packages filters
what eggs from site-packages may be chosen. If allowed-eggs-from-site-packages
is an empty list, then no eggs from site-packages are chosen, but site-packages
will still be included at the end of path lists.
If include-site-packages is false, allowed-eggs-from-site-packages is
irrelevant.
This test shows the interaction with the zc.buildout.easy_install API. Another
test below (allow_site_package_eggs_option) shows using it with a buildout.cfg.
Our "py_path" has the "demoneeded" and "demo" packages available. We'll
simply be asking for "demoneeded" here.
>>> py_path, site_packages_path = make_py()
>>> create_sample_sys_install(site_packages_path)
>>> example_dest = tmpdir('site-packages-example-install')
>>> workingset = zc.buildout.easy_install.install(
... ['demoneeded'], example_dest, links=[], executable=py_path,
... index=None,
... allowed_eggs_from_site_packages=['demoneeded', 'other'])
>>> [dist.project_name for dist in workingset]
['demoneeded']
That worked fine. It would work fine for a glob too.
>>> zc.buildout.easy_install.clear_index_cache()
>>> rmdir(example_dest)
>>> example_dest = tmpdir('site-packages-example-install')
>>> workingset = zc.buildout.easy_install.install(
... ['demoneeded'], example_dest, links=[], executable=py_path,
... index=None,
... allowed_eggs_from_site_packages=['?emon*', 'other'])
>>> [dist.project_name for dist in workingset]
['demoneeded']
But now let's try again with 'demoneeded' not allowed.
>>> zc.buildout.easy_install.clear_index_cache()
>>> rmdir(example_dest)
>>> example_dest = tmpdir('site-packages-example-install')
>>> workingset = zc.buildout.easy_install.install(
... ['demoneeded'], example_dest, links=[], executable=py_path,
... index=None,
... allowed_eggs_from_site_packages=['demo'])
Traceback (most recent call last):
...
MissingDistribution: Couldn't find a distribution for 'demoneeded'.
Here's the same, but with an empty list.
>>> zc.buildout.easy_install.clear_index_cache()
>>> rmdir(example_dest)
>>> example_dest = tmpdir('site-packages-example-install')
>>> workingset = zc.buildout.easy_install.install(
... ['demoneeded'], example_dest, links=[], executable=py_path,
... index=None,
... allowed_eggs_from_site_packages=[])
Traceback (most recent call last):
...
MissingDistribution: Couldn't find a distribution for 'demoneeded'.
Of course, this doesn't stop us from getting a package from elsewhere. Here,
we add a link server.
>>> zc.buildout.easy_install.clear_index_cache()
>>> rmdir(example_dest)
>>> example_dest = tmpdir('site-packages-example-install')
>>> workingset = zc.buildout.easy_install.install(
... ['demoneeded'], example_dest, executable=py_path,
... links=[link_server], index=link_server+'index/',
... allowed_eggs_from_site_packages=['other'])
>>> [dist.project_name for dist in workingset]
['demoneeded']
>>> [dist.location for dist in workingset]
['/site-packages-example-install/demoneeded-1.1-py2.6.egg']
Finally, here's an example of an interaction: we say that it is OK to
allow the "demoneeded" egg to come from site-packages, but we don't
include-site-packages.
>>> zc.buildout.easy_install.clear_index_cache()
>>> rmdir(example_dest)
>>> example_dest = tmpdir('site-packages-example-install')
>>> workingset = zc.buildout.easy_install.install(
... ['demoneeded'], example_dest, links=[], executable=py_path,
... index=None, include_site_packages=False,
... allowed_eggs_from_site_packages=['demoneeded'])
Traceback (most recent call last):
...
MissingDistribution: Couldn't find a distribution for 'demoneeded'.
"""
if sys.version_info > (2, 4):
def test_exit_codes():
"""
......@@ -2930,24 +3179,59 @@ def create_sample_namespace_eggs(dest, site_packages_path=None):
finally:
shutil.rmtree(tmp)
def _write_eggrecipedemoneeded(tmp, minor_version, suffix=''):
from zc.buildout.testing import write
write(tmp, 'README.txt', '')
write(tmp, 'eggrecipedemoneeded.py',
'y=%s\ndef f():\n pass' % minor_version)
write(
tmp, 'setup.py',
"from setuptools import setup\n"
"setup(name='demoneeded', py_modules=['eggrecipedemoneeded'],"
" zip_safe=True, version='1.%s%s', author='bob', url='bob', "
"author_email='bob')\n"
% (minor_version, suffix)
)
def _write_eggrecipedemo(tmp, minor_version, suffix=''):
from zc.buildout.testing import write
write(tmp, 'README.txt', '')
write(
tmp, 'eggrecipedemo.py',
'import eggrecipedemoneeded\n'
'x=%s\n'
'def main(): print x, eggrecipedemoneeded.y\n'
% minor_version)
write(
tmp, 'setup.py',
"from setuptools import setup\n"
"setup(name='demo', py_modules=['eggrecipedemo'],"
" install_requires = 'demoneeded',"
" entry_points={'console_scripts': "
"['demo = eggrecipedemo:main']},"
" zip_safe=True, version='0.%s%s')\n" % (minor_version, suffix)
)
def create_sample_sys_install(site_packages_path):
for creator, minor_version in (
(_write_eggrecipedemoneeded, 1),
(_write_eggrecipedemo, 3)):
# Write the files and install in site_packages_path.
tmp = tempfile.mkdtemp()
try:
creator(tmp, minor_version)
zc.buildout.testing.sys_install(tmp, site_packages_path)
finally:
shutil.rmtree(tmp)
def create_sample_eggs(test, executable=sys.executable):
write = test.globs['write']
from zc.buildout.testing import write
dest = test.globs['sample_eggs']
tmp = tempfile.mkdtemp()
try:
write(tmp, 'README.txt', '')
for i in (0, 1, 2):
write(tmp, 'eggrecipedemoneeded.py', 'y=%s\ndef f():\n pass' % i)
c1 = i==2 and 'c1' or ''
write(
tmp, 'setup.py',
"from setuptools import setup\n"
"setup(name='demoneeded', py_modules=['eggrecipedemoneeded'],"
" zip_safe=True, version='1.%s%s', author='bob', url='bob', "
"author_email='bob')\n"
% (i, c1)
)
suffix = i==2 and 'c1' or ''
_write_eggrecipedemoneeded(tmp, i, suffix)
zc.buildout.testing.sdist(tmp, dest)
write(
......@@ -2961,22 +3245,8 @@ def create_sample_eggs(test, executable=sys.executable):
os.remove(os.path.join(tmp, 'eggrecipedemoneeded.py'))
for i in (1, 2, 3, 4):
write(
tmp, 'eggrecipedemo.py',
'import eggrecipedemoneeded\n'
'x=%s\n'
'def main(): print x, eggrecipedemoneeded.y\n'
% i)
c1 = i==4 and 'c1' or ''
write(
tmp, 'setup.py',
"from setuptools import setup\n"
"setup(name='demo', py_modules=['eggrecipedemo'],"
" install_requires = 'demoneeded',"
" entry_points={'console_scripts': "
"['demo = eggrecipedemo:main']},"
" zip_safe=True, version='0.%s%s')\n" % (i, c1)
)
suffix = i==4 and 'c1' or ''
_write_eggrecipedemo(tmp, i, suffix)
zc.buildout.testing.bdist_egg(tmp, executable, dest)
write(tmp, 'eggrecipebigdemo.py', 'import eggrecipedemo')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7