dist: re-implement `fetch_build_egg` to use pip

docs/setuptools.txt

@@ -282,10 +282,11 @@ unless you need the associated ``setuptools`` feature.
 ``setup_requires``
     A string or list of strings specifying what other distributions need to
     be present in order for the *setup script* to run.  ``setuptools`` will
-    attempt to obtain these before processing the rest of the setup script or
-    commands.  This argument is needed if you are using distutils extensions as
-    part of your build process; for example, extensions that process setup()
-    arguments and turn them into EGG-INFO metadata files.
+    attempt to obtain these (using pip if available) before processing the
+    rest of the setup script or commands.  This argument is needed if you
+    are using distutils extensions as part of your build process; for
+    example, extensions that process setup() arguments and turn them into
+    EGG-INFO metadata files.
 
     (Note: projects listed in ``setup_requires`` will NOT be automatically
     installed on the system where the setup script is being run.  They are
@@ -332,10 +333,10 @@ unless you need the associated ``setuptools`` feature.
     needed to install it, you can use this option to specify them.  It should
     be a string or list of strings specifying what other distributions need to
     be present for the package's tests to run.  When you run the ``test``
-    command, ``setuptools`` will  attempt to obtain these.  Note that these
-    required projects will *not* be installed on the system where the tests
-    are run, but only downloaded to the project's setup directory if they're
-    not already installed locally.
+    command, ``setuptools`` will  attempt to obtain these (using pip if
+    available).  Note that these required projects will *not* be installed on
+    the system where the tests are run, but only downloaded to the project's setup
+    directory if they're not already installed locally.
 
     New in 41.5.0: Deprecated the test command.
 

setuptools/dist.py

@@ -759,32 +759,8 @@ class Distribution(_Distribution):
 
     def fetch_build_egg(self, req):
         """Fetch an egg needed for building"""
-        from setuptools.command.easy_install import easy_install
-        dist = self.__class__({'script_args': ['easy_install']})
-        opts = dist.get_option_dict('easy_install')
-        opts.clear()
-        opts.update(
-            (k, v)
-            for k, v in self.get_option_dict('easy_install').items()
-            if k in (
-                # don't use any other settings
-                'find_links', 'site_dirs', 'index_url',
-                'optimize', 'site_dirs', 'allow_hosts',
-            ))
-        if self.dependency_links:
-            links = self.dependency_links[:]
-            if 'find_links' in opts:
-                links = opts['find_links'][1] + links
-            opts['find_links'] = ('setup', links)
-        install_dir = self.get_egg_cache_dir()
-        cmd = easy_install(
-            dist, args=["x"], install_dir=install_dir,
-            exclude_scripts=True,
-            always_copy=False, build_directory=None, editable=False,
-            upgrade=False, multi_version=True, no_report=True, user=False
-        )
-        cmd.ensure_finalized()
-        return cmd.easy_install(req)
+        from setuptools.installer import fetch_build_egg
+        return fetch_build_egg(self, req)
 
     def _set_global_opts_from_features(self):
         """Add --with-X/--without-X options based on optional features"""

setuptools/installer.py

+import glob
+import os
+import subprocess
+import sys
+from distutils import log
+from distutils.errors import DistutilsError
+
+import pkg_resources
+from setuptools.command.easy_install import easy_install
+from setuptools.wheel import Wheel
+
+from .py31compat import TemporaryDirectory
+
+
+def _legacy_fetch_build_egg(dist, req):
+    """Fetch an egg needed for building.
+
+    Legacy path using EasyInstall.
+    """
+    tmp_dist = dist.__class__({'script_args': ['easy_install']})
+    opts = tmp_dist.get_option_dict('easy_install')
+    opts.clear()
+    opts.update(
+        (k, v)
+        for k, v in dist.get_option_dict('easy_install').items()
+        if k in (
+            # don't use any other settings
+            'find_links', 'site_dirs', 'index_url',
+            'optimize', 'site_dirs', 'allow_hosts',
+        ))
+    if dist.dependency_links:
+        links = dist.dependency_links[:]
+        if 'find_links' in opts:
+            links = opts['find_links'][1] + links
+        opts['find_links'] = ('setup', links)
+    install_dir = dist.get_egg_cache_dir()
+    cmd = easy_install(
+        tmp_dist, args=["x"], install_dir=install_dir,
+        exclude_scripts=True,
+        always_copy=False, build_directory=None, editable=False,
+        upgrade=False, multi_version=True, no_report=True, user=False
+    )
+    cmd.ensure_finalized()
+    return cmd.easy_install(req)
+
+
+def fetch_build_egg(dist, req):
+    """Fetch an egg needed for building.
+
+    Use pip/wheel to fetch/build a wheel."""
+    # Check pip is available.
+    try:
+        pkg_resources.get_distribution('pip')
+    except pkg_resources.DistributionNotFound:
+        dist.announce(
+            'WARNING: The pip package is not available, falling back '
+            'to EasyInstall for handling setup_requires/test_requires; '
+            'this is deprecated and will be removed in a future version.'
+            , log.WARN
+        )
+        return _legacy_fetch_build_egg(dist, req)
+    # Warn if wheel is not.
+    try:
+        pkg_resources.get_distribution('wheel')
+    except pkg_resources.DistributionNotFound:
+        dist.announce('WARNING: The wheel package is not available.', log.WARN)
+    if not isinstance(req, pkg_resources.Requirement):
+        req = pkg_resources.Requirement.parse(req)
+    # Take easy_install options into account, but do not override relevant
+    # pip environment variables (like PIP_INDEX_URL or PIP_QUIET); they'll
+    # take precedence.
+    opts = dist.get_option_dict('easy_install')
+    if 'allow_hosts' in opts:
+        raise DistutilsError('the `allow-hosts` option is not supported '
+                             'when using pip to install requirements.')
+    if 'PIP_QUIET' in os.environ or 'PIP_VERBOSE' in os.environ:
+        quiet = False
+    else:
+        quiet = True
+    if 'PIP_INDEX_URL' in os.environ:
+        index_url = None
+    elif 'index_url' in opts:
+        index_url = opts['index_url'][1]
+    else:
+        index_url = None
+    if 'find_links' in opts:
+        find_links = opts['find_links'][1][:]
+    else:
+        find_links = []
+    if dist.dependency_links:
+        find_links.extend(dist.dependency_links)
+    eggs_dir = os.path.realpath(dist.get_egg_cache_dir())
+    environment = pkg_resources.Environment()
+    for egg_dist in pkg_resources.find_distributions(eggs_dir):
+        if egg_dist in req and environment.can_add(egg_dist):
+            return egg_dist
+    with TemporaryDirectory() as tmpdir:
+        cmd = [
+            sys.executable, '-m', 'pip',
+            '--disable-pip-version-check',
+            'wheel', '--no-deps',
+            '-w', tmpdir,
+        ]
+        if quiet:
+            cmd.append('--quiet')
+        if index_url is not None:
+            cmd.extend(('--index-url', index_url))
+        if find_links is not None:
+            for link in find_links:
+                cmd.extend(('--find-links', link))
+        # If requirement is a PEP 508 direct URL, directly pass
+        # the URL to pip, as `req @ url` does not work on the
+        # command line.
+        if req.url:
+            cmd.append(req.url)
+        else:
+            cmd.append(str(req))
+        try:
+            subprocess.check_call(cmd)
+        except subprocess.CalledProcessError as e:
+            raise DistutilsError(str(e))
+        wheel = Wheel(glob.glob(os.path.join(tmpdir, '*.whl'))[0])
+        dist_location = os.path.join(eggs_dir, wheel.egg_name())
+        wheel.install_as_egg(dist_location)
+        dist_metadata = pkg_resources.PathMetadata(
+            dist_location, os.path.join(dist_location, 'EGG-INFO'))
+        dist = pkg_resources.Distribution.from_filename(
+            dist_location, metadata=dist_metadata)
+        return dist

setuptools/tests/server.py

 """Basic http server for tests to simulate PyPI or custom indexes
 """
 
+import os
 import time
 import threading
 
 from setuptools.extern.six.moves import BaseHTTPServer, SimpleHTTPServer
+from setuptools.extern.six.moves.urllib_parse import urljoin
+from setuptools.extern.six.moves.urllib.request import pathname2url
 
 
 class IndexServer(BaseHTTPServer.HTTPServer):
@@ -69,6 +72,20 @@ class MockServer(BaseHTTPServer.HTTPServer, threading.Thread):
     def run(self):
         self.serve_forever()
 
+    @property
+    def netloc(self):
+        return 'localhost:%s' % self.server_port
+
     @property
     def url(self):
-        return 'http://localhost:%(server_port)s/' % vars(self)
+        return 'http://%s/' % self.netloc
+
+
+def path_to_url(path, authority=None):
+    """ Convert a path to a file: URL. """
+    path = os.path.normpath(os.path.abspath(path))
+    base = 'file:'
+    if authority is not None:
+        base += '//' + authority
+    url = urljoin(base, pathname2url(path))
+    return url

setuptools/tests/test_virtualenv.py

@@ -121,14 +121,12 @@ def test_pip_upgrade_from_source(pip_version, virtualenv):
     virtualenv.run('pip install --no-cache-dir --upgrade ' + sdist)
 
 
-def test_test_command_install_requirements(bare_virtualenv, tmpdir):
+def _check_test_command_install_requirements(virtualenv, tmpdir):
     """
     Check the test command will install all required dependencies.
     """
-    bare_virtualenv.run(' && '.join((
-        'cd {source}',
-        'python setup.py develop',
-    )).format(source=SOURCE_DIR))
+    # Install setuptools.
+    virtualenv.run('python setup.py develop', cd=SOURCE_DIR)
 
     def sdist(distname, version):
         dist_path = tmpdir.join('%s-%s.tar.gz' % (distname, version))
@@ -179,12 +177,20 @@ def test_test_command_install_requirements(bare_virtualenv, tmpdir):
             open('success', 'w').close()
             '''))
     # Run test command for test package.
-    bare_virtualenv.run(' && '.join((
+    virtualenv.run(' && '.join((
         'cd {tmpdir}',
         'python setup.py test -s test',
     )).format(tmpdir=tmpdir))
     assert tmpdir.join('success').check()
 
+def test_test_command_install_requirements(virtualenv, tmpdir):
+    # Ensure pip/wheel packages are installed.
+    virtualenv.run("python -c \"__import__('pkg_resources').require(['pip', 'wheel'])\"")
+    _check_test_command_install_requirements(virtualenv, tmpdir)
+
+def test_test_command_install_requirements_when_using_easy_install(bare_virtualenv, tmpdir):
+    _check_test_command_install_requirements(bare_virtualenv, tmpdir)
+
 
 def test_no_missing_dependencies(bare_virtualenv):
     """

tests/requirements.txt

@@ -9,3 +9,4 @@ coverage>=4.5.1
 pytest-cov>=2.5.1
 paver; python_version>="3.6"
 futures; python_version=="2.7"
+pip>=19.1 # For proper file:// URLs support.