[feat] Add buildout:extra-paths option

This option determines what paths zc.buildout will scan for already
installed distributions. It can be set to an empty value to enable
isolation. The special values 'sys.path', 'legacy' and 'zc.buildout'
expand to particular paths. The default is 'zc.buildout'.

The 'sys.path' mode simply expands to the currrent sys.path.

The 'legacy' mode reproduces the previous behavior of specifically
scanning the paths of the current zc.buildout distribution and its
dependencies using a weird ordering logic that bears no relation to the
order they appear in sys.path - which can produce unexpected results as
other distributions for zc.buildout, pip, wheel and setuptools may take
precedence over the ones currently running.

The 'zc.buildout' mode is similar to 'legacy' as it uses only the paths
of the currently running zc.buildout and dependencies, but unlike
'legacy' mode it respects the order in which these appear in sys.path,
avoiding unexpected results.

This mode is set to the default because it is closest to 'legacy' mode
and because it has a nice property: running in succession

  `buildout buildout:extra-paths= bootstrap` (1)
  `bin/buildout` (2)

will result in (1) installing zc.buildout and its dependencies in
./eggs in isolation from the environment, and (2) using only the
paths of these in ./eggs, i.e. continuing to operate in isolation,
even without setting extra-paths explictly.

doc/reference.rst

@@ -358,6 +358,19 @@ extends-cache
   substitutions, and the result is a relative path, then it will be
   interpreted relative to the buildout directory.)
 
+.. _extra-paths-buildout-option
+
+extra-paths, default: 'zc.buildout'
+  Extra paths to scan for already installed distributions.
+
+  Setting this to an empty value enables isolation of buildout.
+  Setting this to 'legacy' enables the legacy behavior of
+  scanning the paths of the distributions of zc.buildout itself
+  and its dependencies, which may contain sites-packages or not.
+  Setting this to 'zc.buildout' also scans the paths of the
+  current zc.buildout and dependencies, but respects the order
+  they appear in sys.path, avoiding unexpected results.
+
 .. _find-links-option:
 
 find-links, default: ''

src/zc/buildout/buildout.py

@@ -292,6 +292,7 @@ _buildout_default_options = _annotate_section({
     'develop-eggs-directory': 'develop-eggs',
     'eggs-directory': 'eggs',
     'executable': sys.executable,
+    'extra-paths': 'zc.buildout',
     'find-links': '',
     'install-from-cache': 'false',
     'installed': '.installed.cfg',
@@ -533,6 +534,42 @@ class Buildout(DictMixin):
             options['installed'] = os.path.join(options['directory'],
                                                 options['installed'])
 
+        # Extra paths to scan for already installed distributions.
+        extra_paths = options['extra-paths']
+        if extra_paths == 'sys.path':
+            # special case: sys.path
+            extra_paths = sys.path
+            options['extra-paths'] = ' '.join(extra_paths)
+        elif extra_paths == 'legacy':
+            # special case: legacy behavior
+            # this case is why this is done before setting easy_install
+            # versions and other options, to get the legacy behavior.
+            # XXX: These 'sorted' calls correspond to the original behavior,
+            # but they are quite problematic, as other distributions for
+            # zc.buildout, pip, wheel and setuptools may take precedence
+            # over the ones currently running.
+            old_extra_paths = zc.buildout.easy_install.extra_paths(
+              sorted({d.location for d in pkg_resources.working_set}))
+            try:
+                buildout_and_setuptools_dists = list(
+                    zc.buildout.easy_install.install(['zc.buildout'], None,
+                        check_picked=False))
+            finally:
+                zc.buildout.easy_install.extra_paths(old_extra_paths)
+            extra_paths = sorted(
+                {d.location for d in buildout_and_setuptools_dists})
+            options['extra-paths'] = ' '.join(extra_paths)
+        elif extra_paths == 'zc.buildout':
+            # special case: only zc.buildout and its dependencies
+            # but in the order they appear in sys.path, unlike legacy
+            buildout_dists = pkg_resources.require('zc.buildout')
+            buildout_paths = {d.location for d in buildout_dists}
+            extra_paths = [p for p in sys.path if p in buildout_paths]
+            options['extra-paths'] = ' '.join(extra_paths)
+        else:
+            extra_paths = extra_paths.split()
+        zc.buildout.easy_install.extra_paths(extra_paths)
+
         self._setup_logging()
         self._setup_socket_timeout()
 

src/zc/buildout/easy_install.py

@@ -91,13 +91,15 @@ if has_distribute and not has_setuptools:
     sys.exit("zc.buildout 3 needs setuptools, not distribute."
              "Did you properly install with pip in a virtualenv ?")
 
-# Include buildout and setuptools eggs in paths.  We get this
-# initially from the entire working set.  Later, we'll use the install
-# function to narrow to just the buildout and setuptools paths.
-buildout_and_setuptools_path = sorted({d.location for d in pkg_resources.working_set})
-setuptools_path = buildout_and_setuptools_path
-pip_path = buildout_and_setuptools_path
-logger.debug('before restricting versions: pip_path %r', pip_path)
+# XXX Take care to respect the sys.path order, as otherwise other
+# distributions for pip, wheel and setuptools may take precedence
+# over the ones currently running.
+pip_path = setuptools_path = [
+  dist.location
+  for dist in pkg_resources.working_set
+  if dist.project_name in ('pip', 'wheel', 'setuptools')
+  ]
+pip_pythonpath = setuptools_pythonpath = os.pathsep.join(pip_path)
 
 FILE_SCHEME = re.compile('file://', re.I).match
 DUNDER_FILE_PATTERN = re.compile(r"__file__ = '(?P<filename>.+)'$")
@@ -241,6 +243,7 @@ class Installer(object):
     _allow_picked_versions = True
     _store_required_by = False
     _allow_unknown_extras = False
+    _extra_paths = []
 
     def __init__(self,
                  dest=None,
@@ -275,7 +278,7 @@ class Installer(object):
             links.insert(0, self._download_cache)
 
         self._index_url = index
-        path = (path and path[:] or []) + buildout_and_setuptools_path
+        path = (path and path[:] or []) + self._extra_paths
         self._path = path
         if self._dest is None:
             newest = False
@@ -950,6 +953,12 @@ def get_picked_versions():
     required_by = Installer._required_by
     return (picked_versions, required_by)
 
+def extra_paths(setting=None):
+    old = Installer._extra_paths
+    if setting is not None:
+        Installer._extra_paths = setting
+    return old
+
 
 def install(specs, dest,
             links=(), index=None,
@@ -974,19 +983,6 @@ def install(specs, dest,
                           allow_unknown_extras=allow_unknown_extras)
     return installer.install(specs, working_set)
 
-buildout_and_setuptools_dists = list(install(['zc.buildout'], None,
-                                             check_picked=False))
-buildout_and_setuptools_path = sorted({d.location
-                                for d in buildout_and_setuptools_dists})
-
-pip_dists = [d for d in buildout_and_setuptools_dists if d.project_name != 'zc.buildout']
-pip_path = sorted({d.location for d in pip_dists})
-logger.debug('after restricting versions: pip_path %r', pip_path)
-pip_pythonpath = os.pathsep.join(pip_path)
-
-setuptools_path = pip_path
-setuptools_pythonpath = pip_pythonpath
-
 
 def build(spec, dest, build_ext,
           links=(), index=None,

src/zc/buildout/tests/buildout.txt

@@ -2917,6 +2917,7 @@ database is shown::
     directory = /sample-buildout
     eggs-directory = /sample-buildout/eggs
     executable = python
+    extra-paths = ...
     find-links =
     install-from-cache = false
     installed = /sample-buildout/.installed.cfg

src/zc/buildout/tests/test_all.py

@@ -2677,7 +2677,8 @@ honoring our version specification.
     ... eggs = foo
     ... ''' % ('\n'.join(
     ...     '%s = %s' % (d.key, d.version)
-    ...     for d in zc.buildout.easy_install.buildout_and_setuptools_dists)))
+    ...     for d in pkg_resources.working_set.resolve(
+    ...         pkg_resources.parse_requirements('zc.buildout')))))
 
     >>> print_(system(buildout), end='')
     Installing foo.