[feat] Enable isolation from preexisting packages

Add option `isolate-from-buildout-and-setuptools-path` to prevent
buildout from scanning the directories where zc.buildout's or its
dependencies's currently running packages are located.

In apparently intended design, `buildout bootstrap` should already
provide isolation by either installing pinned versions of zc.buildout
and dependencies in local ./eggs directory, or copying them from the
currently running egg into ./eggs if the running versions match.

However, when the currently running versions are not eggs - which is
the case when they are installed by pip in site-packages, e.g. as is
the case in a virtual environment - then buildout merely creates
`.egg-link` references to the locations of the running packages, which
is often site-packages.

The presence of .egg-link referencing site-packages can cause packages
available in site-packages to interfere when considering which eggs to
install and what path items to insert in sys.path when creating
scripts.

This is in some way a workaround for the fact that buildout only knows
how to handle eggs and not .dist-info packages such as produced by pip.

With `isolate-from-buildout-and-setuptools-path`, buildout bootstrap
will allways install a fresh version of zc.buildout & co in ./eggs

src/zc/buildout/buildout.py

@@ -377,6 +377,7 @@ _buildout_default_options = _annotate_section({
     'find-links': '',
     'install-from-cache': 'false',
     'installed': '.installed.cfg',
+    'isolate-from-buildout-and-setuptools-path': 'false',
     'log-format': '',
     'log-level': 'INFO',
     'newest': 'true',
@@ -620,6 +621,9 @@ class Buildout(DictMixin):
             bool_option(options, 'use-dependency-links'))
         zc.buildout.easy_install.allow_picked_versions(
                 bool_option(options, 'allow-picked-versions'))
+        zc.buildout.easy_install.isolate_from_buildout_and_setuptools_path(
+                bool_option(options,
+                            'isolate-from-buildout-and-setuptools-path'))
         self.show_picked_versions = bool_option(options,
                                                 'show-picked-versions')
         self.update_versions_file = options['update-versions-file']

src/zc/buildout/easy_install.py

@@ -154,8 +154,8 @@ class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
 
 
 _indexes = {}
-def _get_index(index_url, find_links, allow_hosts=('*',)):
-    key = index_url, tuple(find_links)
+def _get_index(index_url, find_links, allow_hosts=('*',), isolate=False):
+    key = index_url, tuple(find_links), tuple(allow_hosts), isolate
     index = _indexes.get(key)
     if index is not None:
         return index
@@ -164,6 +164,13 @@ def _get_index(index_url, find_links, allow_hosts=('*',)):
         index_url = default_index_url
     if index_url.startswith('file://'):
         index_url = index_url[7:]
+    if isolate:
+        index = AllowHostsPackageIndex(
+          index_url,
+          hosts=allow_hosts,
+          search_path=[], # do not look at sys.path
+        )
+    else:
         index = AllowHostsPackageIndex(index_url, hosts=allow_hosts)
 
     if find_links:
@@ -265,6 +272,7 @@ class Installer(object):
     _allow_picked_versions = True
     _store_required_by = False
     _allow_unknown_extras = False
+    _isolate_from_buildout_and_setuptools_path = False
 
     def __init__(self,
                  dest=None,
@@ -301,13 +309,16 @@ class Installer(object):
             links.insert(0, self._download_cache)
 
         self._index_url = index
-        path = (path and path[:] or []) + buildout_and_setuptools_path
+        isolate = self._isolate_from_buildout_and_setuptools_path
+        path = path and path[:] or []
+        if not isolate:
+            path += buildout_and_setuptools_path
         self._path = path
         if self._dest is None:
             newest = False
         self._newest = newest
         self._env = self._make_env()
-        self._index = _get_index(index, links, self._allow_hosts)
+        self._index = _get_index(index, links, self._allow_hosts, isolate)
         self._requirements_and_constraints = []
         self._check_picked = check_picked
 
@@ -694,7 +705,9 @@ class Installer(object):
                         links.append(link)
                         reindex = True
         if reindex:
-            self._index = _get_index(self._index_url, links, self._allow_hosts)
+            isolate = self._isolate_from_buildout_and_setuptools_path
+            self._index = _get_index(
+                self._index_url, links, self._allow_hosts, isolate)
 
     def _check_picked_requirement_versions(self, requirement, dists):
         """ Check whether we picked a version and, if we did, report it """
@@ -1045,6 +1058,12 @@ def allow_picked_versions(setting=None):
         Installer._allow_picked_versions = bool(setting)
     return old
 
+def isolate_from_buildout_and_setuptools_path(setting=None):
+    old = Installer._isolate_from_buildout_and_setuptools_path
+    if setting is not None:
+        Installer._isolate_from_buildout_and_setuptools_path = bool(setting)
+    return old
+
 def store_required_by(setting=None):
     old = Installer._store_required_by
     if setting is not None:

src/zc/buildout/tests/__init__.py

@@ -14,6 +14,7 @@
 ##############################################################################
 import re
 import os
+import pkg_resources
 import sys
 import shutil
 import tempfile
@@ -161,6 +162,33 @@ def create_sample_eggs(test, executable=sys.executable):
             "import builddep"
             )
         zc.buildout.testing.sdist(tmp, dest)
+        shutil.os.unlink(os.path.join(tmp, 'pyproject.toml'))
+
+        # Create distributions for zc.buildout and fake dependencies
+        # for bootstrapping in isolation tests
+        dist = pkg_resources.working_set.find(
+            pkg_resources.Requirement.parse('zc.buildout'))
+        test.globs['mkdir'](dest, 'zc.buildout')
+        zc.buildout.testing.sdist(
+            os.path.dirname(dist.location),
+            os.path.join(dest, 'zc.buildout'),
+            )
+        fakes = {
+          'setuptools': '38.2.3',
+          'pip': '24.0',
+          'wheel': '0.43.0'
+        }
+        for name, version in fakes.items():
+          module = 'fake_%s' % name.replace('.', '_')
+          write(tmp, module + '.py', '')
+          write(
+              tmp, 'setup.py',
+              "from setuptools import setup\n"
+              "setup(name='%s', "
+              " py_modules=['%s'], "
+              " zip_safe=True, version='%s')\n" % (name, module, version)
+              )
+          zc.buildout.testing.sdist(tmp, dest)
 
     finally:
         shutil.rmtree(tmp)

src/zc/buildout/tests/buildout.txt

@@ -836,6 +836,8 @@ the origin of the value (file name or ``COMPUTED_VALUE``, ``DEFAULT_VALUE``,
         DEFAULT_VALUE
     installed= .installed.cfg
         DEFAULT_VALUE
+    isolate-from-buildout-and-setuptools-path= false
+        DEFAULT_VALUE
     log-format=
         DEFAULT_VALUE
     log-level= INFO
@@ -953,6 +955,11 @@ You get more information about the way values are computed::
        AS DEFAULT_VALUE
        SET VALUE = .installed.cfg
     <BLANKLINE>
+    isolate-from-buildout-and-setuptools-path= false
+    <BLANKLINE>
+       AS DEFAULT_VALUE
+       SET VALUE = false
+    <BLANKLINE>
     log-format=
     <BLANKLINE>
        AS DEFAULT_VALUE
@@ -3032,6 +3039,7 @@ database is shown::
     find-links =
     install-from-cache = false
     installed = /sample-buildout/.installed.cfg
+    isolate-from-buildout-and-setuptools-path = false
     log-format =
     log-level = INFO
     newest = true
@@ -3336,6 +3344,55 @@ exists::
       Initializing.
     Error: '/sample-bootstrapped/setup.cfg' already exists.
 
+
+Bootstrapping and running in isolation
+--------------------------------------
+
+We can set the ``isolate-from-buildout-and-setuptools-path`` option to make
+buildout install its own local copies of eggs even if they are available at
+the locations of the ``zc.buildout`` and ``setuptools`` eggs.
+
+Note that bootstrapping like this is a workaround for the fact that when the
+currently running ``zc.buildout`` and ``setuptools`` are not eggs, e.g. when
+they are installed by pip, bootstrap without isolation creates ``.egg-link``
+files in ``develop-eggs`` instead of local copies in ``eggs``.
+
+    >>> sample_bootstrapped3 = tmpdir('sample-bootstrapped3')
+
+    >>> write(sample_bootstrapped3, 'buildout.cfg',
+    ... """
+    ... [buildout]
+    ... isolate-from-buildout-and-setuptools-path = true
+    ... parts =
+    ... """)
+
+    >>> print_(system(buildout
+    ...              +' -c'+os.path.join(sample_bootstrapped3, 'buildout.cfg')
+    ...              +' bootstrap'), end='')
+    ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    Creating directory '/sample-bootstrapped3/eggs'.
+    Creating directory '/sample-bootstrapped3/bin'.
+    Creating directory '/sample-bootstrapped3/parts'.
+    Creating directory '/sample-bootstrapped3/develop-eggs'.
+    Getting distribution for 'zc.buildout...'.
+    Got zc.buildout ...
+    Getting distribution for 'wheel'.
+    Got wheel ...
+    Getting distribution for 'pip'.
+    Got pip ...
+    Getting distribution for 'setuptools...'.
+    Got setuptools ...
+    Generated script '/sample-bootstrapped3/bin/buildout'.
+
+    >>> _ = ls(sample_bootstrapped3, 'eggs')
+    d pip.egg
+    setuptools.egg
+    d wheel.egg
+    d zc.buildout.egg
+
+    >>> _ = ls(sample_bootstrapped3, 'develop-eggs')
+
+
 Initial eggs
 ------------
 

src/zc/buildout/tests/downloadcache.txt

@@ -47,8 +47,12 @@ download:
     <a href="index/">index/</a><br>
     <a href="mixedcase-0.5.zip">mixedcase-0.5.zip</a><br>
     <a href="other-1.0-pyN.N.egg">other-1.0-pyN.N.egg</a><br>
+    <a href="pip-24.0.zip">pip-24.0.zip</a><br>
+    <a href="setuptools-38.2.3.zip">setuptools-38.2.3.zip</a><br>
+    <a href="wheel-0.43.0.zip">wheel-0.43.0.zip</a><br>
     <a href="withbuildsystemrequires-0.1.zip">withbuildsystemrequires-0.1.zip</a><br>
     <a href="withsetuprequires-0.1.zip">withsetuprequires-0.1.zip</a><br>
+    <a href="zc.buildout/">zc.buildout/</a><br>
     </body></html>
 
 

src/zc/buildout/tests/easy_install.txt

@@ -111,8 +111,12 @@ We have a link server that has a number of eggs:
     <a href="index/">index/</a><br>
     <a href="mixedcase-0.5.zip">mixedcase-0.5.zip</a><br>
     <a href="other-1.0-pyN.N.egg">other-1.0-pyN.N.egg</a><br>
+    <a href="pip-24.0.zip">pip-24.0.zip</a><br>
+    <a href="setuptools-38.2.3.zip">setuptools-38.2.3.zip</a><br>
+    <a href="wheel-0.43.0.zip">wheel-0.43.0.zip</a><br>
     <a href="withbuildsystemrequires-0.1.zip">withbuildsystemrequires-0.1.zip</a><br>
     <a href="withsetuprequires-0.1.zip">withsetuprequires-0.1.zip</a><br>
+    <a href="zc.buildout/">zc.buildout/</a><br>
     </body></html>
 
 Let's make a directory and install the demo egg to it, using the demo:
@@ -1251,8 +1255,12 @@ Let's update our link server with a new version of extdemo:
     <a href="index/">index/</a><br>
     <a href="mixedcase-0.5.zip">mixedcase-0.5.zip</a><br>
     <a href="other-1.0-pyN.N.egg">other-1.0-pyN.N.egg</a><br>
+    <a href="pip-24.0.zip">pip-24.0.zip</a><br>
+    <a href="setuptools-38.2.3.zip">setuptools-38.2.3.zip</a><br>
+    <a href="wheel-0.43.0.zip">wheel-0.43.0.zip</a><br>
     <a href="withbuildsystemrequires-0.1.zip">withbuildsystemrequires-0.1.zip</a><br>
     <a href="withsetuprequires-0.1.zip">withsetuprequires-0.1.zip</a><br>
+    <a href="zc.buildout/">zc.buildout/</a><br>
     </body></html>
 
 The easy_install caches information about servers to reduce network