dist: re-implement `fetch_build_egg` to use pip

docs/setuptools.txt

@@ -282,10 +282,11 @@ unless you need the associated ``setuptools`` feature.
 ``setup_requires``
     A string or list of strings specifying what other distributions need to
     be present in order for the *setup script* to run.  ``setuptools`` will
-    attempt to obtain these before processing the rest of the setup script or
-    commands.  This argument is needed if you are using distutils extensions as
-    part of your build process; for example, extensions that process setup()
-    arguments and turn them into EGG-INFO metadata files.
+    attempt to obtain these (using pip if available) before processing the
+    rest of the setup script or commands.  This argument is needed if you
+    are using distutils extensions as part of your build process; for
+    example, extensions that process setup() arguments and turn them into
+    EGG-INFO metadata files.
 
     (Note: projects listed in ``setup_requires`` will NOT be automatically
     installed on the system where the setup script is being run.  They are
@@ -332,10 +333,10 @@ unless you need the associated ``setuptools`` feature.
     needed to install it, you can use this option to specify them.  It should
     be a string or list of strings specifying what other distributions need to
     be present for the package's tests to run.  When you run the ``test``
-    command, ``setuptools`` will  attempt to obtain these.  Note that these
-    required projects will *not* be installed on the system where the tests
-    are run, but only downloaded to the project's setup directory if they're
-    not already installed locally.
+    command, ``setuptools`` will  attempt to obtain these (using pip if
+    available).  Note that these required projects will *not* be installed on
+    the system where the tests are run, but only downloaded to the project's setup
+    directory if they're not already installed locally.
 
     New in 41.5.0: Deprecated the test command.
 

setuptools/dist.py

@@ -759,32 +759,8 @@ class Distribution(_Distribution):
 
     def fetch_build_egg(self, req):
         """Fetch an egg needed for building"""
-        from setuptools.command.easy_install import easy_install
-        dist = self.__class__({'script_args': ['easy_install']})
-        opts = dist.get_option_dict('easy_install')
-        opts.clear()
-        opts.update(
-            (k, v)
-            for k, v in self.get_option_dict('easy_install').items()
-            if k in (
-                # don't use any other settings
-                'find_links', 'site_dirs', 'index_url',
-                'optimize', 'site_dirs', 'allow_hosts',
-            ))
-        if self.dependency_links:
-            links = self.dependency_links[:]
-            if 'find_links' in opts:
-                links = opts['find_links'][1] + links
-            opts['find_links'] = ('setup', links)
-        install_dir = self.get_egg_cache_dir()
-        cmd = easy_install(
-            dist, args=["x"], install_dir=install_dir,
-            exclude_scripts=True,
-            always_copy=False, build_directory=None, editable=False,
-            upgrade=False, multi_version=True, no_report=True, user=False
-        )
-        cmd.ensure_finalized()
-        return cmd.easy_install(req)
+        from setuptools.installer import fetch_build_egg
+        return fetch_build_egg(self, req)
 
     def _set_global_opts_from_features(self):
         """Add --with-X/--without-X options based on optional features"""

setuptools/installer.py

+import glob
+import os
+import subprocess
+import sys
+from distutils import log
+from distutils.errors import DistutilsError
+
+import pkg_resources
+from setuptools.command.easy_install import easy_install
+from setuptools.wheel import Wheel
+
+from .py31compat import TemporaryDirectory
+
+
+def _legacy_fetch_build_egg(dist, req):
+    """Fetch an egg needed for building.
+
+    Legacy path using EasyInstall.
+    """
+    tmp_dist = dist.__class__({'script_args': ['easy_install']})
+    opts = tmp_dist.get_option_dict('easy_install')
+    opts.clear()
+    opts.update(
+        (k, v)
+        for k, v in dist.get_option_dict('easy_install').items()
+        if k in (
+            # don't use any other settings
+            'find_links', 'site_dirs', 'index_url',
+            'optimize', 'site_dirs', 'allow_hosts',
+        ))
+    if dist.dependency_links:
+        links = dist.dependency_links[:]
+        if 'find_links' in opts:
+            links = opts['find_links'][1] + links
+        opts['find_links'] = ('setup', links)
+    install_dir = dist.get_egg_cache_dir()
+    cmd = easy_install(
+        tmp_dist, args=["x"], install_dir=install_dir,
+        exclude_scripts=True,
+        always_copy=False, build_directory=None, editable=False,
+        upgrade=False, multi_version=True, no_report=True, user=False
+    )
+    cmd.ensure_finalized()
+    return cmd.easy_install(req)
+
+
+def fetch_build_egg(dist, req):
+    """Fetch an egg needed for building.
+
+    Use pip/wheel to fetch/build a wheel."""
+    # Check pip is available.
+    try:
+        pkg_resources.get_distribution('pip')
+    except pkg_resources.DistributionNotFound:
+        dist.announce(
+            'WARNING: The pip package is not available, falling back '
+            'to EasyInstall for handling setup_requires/test_requires; '
+            'this is deprecated and will be removed in a future version.'
+            , log.WARN
+        )
+        return _legacy_fetch_build_egg(dist, req)
+    # Warn if wheel is not.
+    try:
+        pkg_resources.get_distribution('wheel')
+    except pkg_resources.DistributionNotFound:
+        dist.announce('WARNING: The wheel package is not available.', log.WARN)
+    if not isinstance(req, pkg_resources.Requirement):
+        req = pkg_resources.Requirement.parse(req)
+    # Take easy_install options into account, but do not override relevant
+    # pip environment variables (like PIP_INDEX_URL or PIP_QUIET); they'll
+    # take precedence.
+    opts = dist.get_option_dict('easy_install')
+    if 'allow_hosts' in opts:
+        raise DistutilsError('the `allow-hosts` option is not supported '
+                             'when using pip to install requirements.')
+    if 'PIP_QUIET' in os.environ or 'PIP_VERBOSE' in os.environ:
+        quiet = False
+    else:
+        quiet = True
+    if 'PIP_INDEX_URL' in os.environ:
+        index_url = None
+    elif 'index_url' in opts:
+        index_url = opts['index_url'][1]
+    else:
+        index_url = None
+    if 'find_links' in opts:
+        find_links = opts['find_links'][1][:]
+    else:
+        find_links = []
+    if dist.dependency_links:
+        find_links.extend(dist.dependency_links)
+    eggs_dir = os.path.realpath(dist.get_egg_cache_dir())
+    environment = pkg_resources.Environment()
+    for egg_dist in pkg_resources.find_distributions(eggs_dir):
+        if egg_dist in req and environment.can_add(egg_dist):
+            return egg_dist
+    with TemporaryDirectory() as tmpdir:
+        cmd = [
+            sys.executable, '-m', 'pip',
+            '--disable-pip-version-check',
+            'wheel', '--no-deps',
+            '-w', tmpdir,
+        ]
+        if quiet:
+            cmd.append('--quiet')
+        if index_url is not None:
+            cmd.extend(('--index-url', index_url))
+        if find_links is not None:
+            for link in find_links:
+                cmd.extend(('--find-links', link))
+        # If requirement is a PEP 508 direct URL, directly pass
+        # the URL to pip, as `req @ url` does not work on the
+        # command line.
+        if req.url:
+            cmd.append(req.url)
+        else:
+            cmd.append(str(req))
+        try:
+            subprocess.check_call(cmd)
+        except subprocess.CalledProcessError as e:
+            raise DistutilsError(str(e))
+        wheel = Wheel(glob.glob(os.path.join(tmpdir, '*.whl'))[0])
+        dist_location = os.path.join(eggs_dir, wheel.egg_name())
+        wheel.install_as_egg(dist_location)
+        dist_metadata = pkg_resources.PathMetadata(
+            dist_location, os.path.join(dist_location, 'EGG-INFO'))
+        dist = pkg_resources.Distribution.from_filename(
+            dist_location, metadata=dist_metadata)
+        return dist

setuptools/tests/server.py

 """Basic http server for tests to simulate PyPI or custom indexes
 """
 
+import os
 import time
 import threading
 
 from setuptools.extern.six.moves import BaseHTTPServer, SimpleHTTPServer
+from setuptools.extern.six.moves.urllib_parse import urljoin
+from setuptools.extern.six.moves.urllib.request import pathname2url
 
 
 class IndexServer(BaseHTTPServer.HTTPServer):
@@ -69,6 +72,20 @@ class MockServer(BaseHTTPServer.HTTPServer, threading.Thread):
     def run(self):
         self.serve_forever()
 
+    @property
+    def netloc(self):
+        return 'localhost:%s' % self.server_port
+
     @property
     def url(self):
-        return 'http://localhost:%(server_port)s/' % vars(self)
+        return 'http://%s/' % self.netloc
+
+
+def path_to_url(path, authority=None):
+    """ Convert a path to a file: URL. """
+    path = os.path.normpath(os.path.abspath(path))
+    base = 'file:'
+    if authority is not None:
+        base += '//' + authority
+    url = urljoin(base, pathname2url(path))
+    return url

setuptools/tests/test_easy_install.py

@@ -15,24 +15,24 @@ import distutils.errors
 import io
 import zipfile
 import mock
-from setuptools.command.easy_install import (
-    EasyInstallDeprecationWarning, ScriptWriter, WindowsScriptWriter,
-)
 import time
+
 from setuptools.extern import six
-from setuptools.extern.six.moves import urllib
 
 import pytest
 
 from setuptools import sandbox
 from setuptools.sandbox import run_setup
 import setuptools.command.easy_install as ei
-from setuptools.command.easy_install import PthDistributions
+from setuptools.command.easy_install import (
+    EasyInstallDeprecationWarning, ScriptWriter, PthDistributions,
+    WindowsScriptWriter,
+)
 from setuptools.command import easy_install as easy_install_pkg
 from setuptools.dist import Distribution
 from pkg_resources import normalize_path, working_set
 from pkg_resources import Distribution as PRDistribution
-import setuptools.tests.server
+from setuptools.tests.server import MockServer, path_to_url
 from setuptools.tests import fail_on_ascii
 import pkg_resources
 
@@ -440,35 +440,40 @@ def distutils_package():
         yield
 
 
+@pytest.fixture
+def mock_index():
+    # set up a server which will simulate an alternate package index.
+    p_index = MockServer()
+    if p_index.server_port == 0:
+        # Some platforms (Jython) don't find a port to which to bind,
+        # so skip test for them.
+        pytest.skip("could not find a valid port")
+    p_index.start()
+    return p_index
+
+
 class TestDistutilsPackage:
     def test_bdist_egg_available_on_distutils_pkg(self, distutils_package):
         run_setup('setup.py', ['bdist_egg'])
 
 
 class TestSetupRequires:
-    def test_setup_requires_honors_fetch_params(self):
+
+    def test_setup_requires_honors_fetch_params(self, mock_index, monkeypatch):
         """
         When easy_install installs a source distribution which specifies
         setup_requires, it should honor the fetch parameters (such as
-        allow-hosts, index-url, and find-links).
+        index-url, and find-links).
         """
-        # set up a server which will simulate an alternate package index.
-        p_index = setuptools.tests.server.MockServer()
-        p_index.start()
-        netloc = 1
-        p_index_loc = urllib.parse.urlparse(p_index.url)[netloc]
-        if p_index_loc.endswith(':0'):
-            # Some platforms (Jython) don't find a port to which to bind,
-            #  so skip this test for them.
-            return
+        monkeypatch.setenv(str('PIP_RETRIES'), str('0'))
+        monkeypatch.setenv(str('PIP_TIMEOUT'), str('0'))
         with contexts.quiet():
             # create an sdist that has a build-time dependency.
             with TestSetupRequires.create_sdist() as dist_file:
                 with contexts.tempdir() as temp_install_dir:
                     with contexts.environment(PYTHONPATH=temp_install_dir):
                         ei_params = [
-                            '--index-url', p_index.url,
-                            '--allow-hosts', p_index_loc,
+                            '--index-url', mock_index.url,
                             '--exclude-scripts',
                             '--install-dir', temp_install_dir,
                             dist_file,
@@ -478,10 +483,8 @@ class TestSetupRequires:
                             # fail because it doesn't exist.
                             with pytest.raises(SystemExit):
                                 easy_install_pkg.main(ei_params)
-        # there should have been two or three requests to the server
-        #  (three happens on Python 3.3a)
-        assert 2 <= len(p_index.requests) <= 3
-        assert p_index.requests[0].path == '/does-not-exist/'
+        # there should have been one requests to the server
+        assert [r.path for r in mock_index.requests] == ['/does-not-exist/']
 
     @staticmethod
     @contextlib.contextmanager
@@ -500,7 +503,9 @@ class TestSetupRequires:
                         version="1.0",
                         setup_requires = ['does-not-exist'],
                     )
-                """))])
+                """)),
+                ('setup.cfg', ''),
+            ])
             yield dist_path
 
     use_setup_cfg = (
@@ -632,6 +637,113 @@ class TestSetupRequires:
                 assert len(lines) > 0
                 assert lines[-1].strip() == '42'
 
+    def test_setup_requires_honors_pip_env(self, mock_index, monkeypatch):
+        monkeypatch.setenv(str('PIP_RETRIES'), str('0'))
+        monkeypatch.setenv(str('PIP_TIMEOUT'), str('0'))
+        monkeypatch.setenv(str('PIP_INDEX_URL'), mock_index.url)
+        with contexts.save_pkg_resources_state():
+            with contexts.tempdir() as temp_dir:
+                test_pkg = create_setup_requires_package(
+                    temp_dir, 'python-xlib', '0.19',
+                    setup_attrs=dict(dependency_links=[]))
+                test_setup_cfg = os.path.join(test_pkg, 'setup.cfg')
+                with open(test_setup_cfg, 'w') as fp:
+                    fp.write(DALS(
+                        '''
+                        [easy_install]
+                        index_url = https://pypi.org/legacy/
+                        '''))
+                test_setup_py = os.path.join(test_pkg, 'setup.py')
+                with pytest.raises(distutils.errors.DistutilsError):
+                    run_setup(test_setup_py, [str('--version')])
+        assert len(mock_index.requests) == 1
+        assert mock_index.requests[0].path == '/python-xlib/'
+
+    def test_setup_requires_with_pep508_url(self, mock_index, monkeypatch):
+        monkeypatch.setenv(str('PIP_RETRIES'), str('0'))
+        monkeypatch.setenv(str('PIP_TIMEOUT'), str('0'))
+        monkeypatch.setenv(str('PIP_INDEX_URL'), mock_index.url)
+        with contexts.save_pkg_resources_state():
+            with contexts.tempdir() as temp_dir:
+                dep_sdist = os.path.join(temp_dir, 'dep.tar.gz')
+                make_trivial_sdist(dep_sdist, 'dependency', '42')
+                dep_url = path_to_url(dep_sdist, authority='localhost')
+                test_pkg = create_setup_requires_package(
+                    temp_dir,
+                    'python-xlib', '0.19', # Ignored (overriden by setup_attrs).
+                    setup_attrs=dict(setup_requires='dependency @ %s' % dep_url))
+                test_setup_py = os.path.join(test_pkg, 'setup.py')
+                run_setup(test_setup_py, [str('--version')])
+        assert len(mock_index.requests) == 0
+
+    def test_setup_requires_with_allow_hosts(self, mock_index):
+        ''' The `allow-hosts` option in not supported anymore. '''
+        with contexts.save_pkg_resources_state():
+            with contexts.tempdir() as temp_dir:
+                test_pkg = os.path.join(temp_dir, 'test_pkg')
+                test_setup_py = os.path.join(test_pkg, 'setup.py')
+                test_setup_cfg = os.path.join(test_pkg, 'setup.cfg')
+                os.mkdir(test_pkg)
+                with open(test_setup_py, 'w') as fp:
+                    fp.write(DALS(
+                        '''
+                        from setuptools import setup
+                        setup(setup_requires='python-xlib')
+                        '''))
+                with open(test_setup_cfg, 'w') as fp:
+                    fp.write(DALS(
+                        '''
+                        [easy_install]
+                        allow_hosts = *
+                        '''))
+                with pytest.raises(distutils.errors.DistutilsError):
+                    run_setup(test_setup_py, [str('--version')])
+        assert len(mock_index.requests) == 0
+
+    def test_setup_requires_with_python_requires(self, monkeypatch, tmpdir):
+        ''' Check `python_requires` is honored. '''
+        monkeypatch.setenv(str('PIP_RETRIES'), str('0'))
+        monkeypatch.setenv(str('PIP_TIMEOUT'), str('0'))
+        monkeypatch.setenv(str('PIP_NO_INDEX'), str('1'))
+        monkeypatch.setenv(str('PIP_VERBOSE'), str('1'))
+        dep_1_0_sdist = 'dep-1.0.tar.gz'
+        dep_1_0_url = path_to_url(str(tmpdir / dep_1_0_sdist))
+        dep_1_0_python_requires = '>=2.7'
+        make_python_requires_sdist(str(tmpdir / dep_1_0_sdist), 'dep', '1.0', dep_1_0_python_requires)
+        dep_2_0_sdist = 'dep-2.0.tar.gz'
+        dep_2_0_url = path_to_url(str(tmpdir / dep_2_0_sdist))
+        dep_2_0_python_requires = '!=' + '.'.join(map(str, sys.version_info[:2])) + '.*'
+        make_python_requires_sdist(str(tmpdir / dep_2_0_sdist), 'dep', '2.0', dep_2_0_python_requires)
+        index = tmpdir / 'index.html'
+        index.write_text(DALS(
+            '''
+            <!DOCTYPE html>
+            <html><head><title>Links for dep</title></head>
+            <body>
+                <h1>Links for dep</h1>
+                <a href="{dep_1_0_url}" data-requires-python="{dep_1_0_python_requires}">{dep_1_0_sdist}</a><br/>
+                <a href="{dep_2_0_url}" data-requires-python="{dep_2_0_python_requires}">{dep_2_0_sdist}</a><br/>
+            </body>
+            </html>
+            ''').format(
+                dep_1_0_url=dep_1_0_url,
+                dep_1_0_sdist=dep_1_0_sdist,
+                dep_1_0_python_requires=dep_1_0_python_requires,
+                dep_2_0_url=dep_2_0_url,
+                dep_2_0_sdist=dep_2_0_sdist,
+                dep_2_0_python_requires=dep_2_0_python_requires,
+            ), 'utf-8')
+        index_url = path_to_url(str(index))
+        with contexts.save_pkg_resources_state():
+            test_pkg = create_setup_requires_package(
+                str(tmpdir),
+                'python-xlib', '0.19', # Ignored (overriden by setup_attrs).
+                setup_attrs=dict(setup_requires='dep', dependency_links=[index_url]))
+            test_setup_py = os.path.join(test_pkg, 'setup.py')
+            run_setup(test_setup_py, [str('--version')])
+        eggs = list(map(str, pkg_resources.find_distributions(os.path.join(test_pkg, '.eggs'))))
+        assert eggs == ['dep 1.0']
+
 
 def make_trivial_sdist(dist_path, distname, version):
     """
@@ -647,7 +759,9 @@ def make_trivial_sdist(dist_path, distname, version):
                  name=%r,
                  version=%r
              )
-         """ % (distname, version)))])
+         """ % (distname, version))),
+        ('setup.cfg', ''),
+    ])
 
 
 def make_nspkg_sdist(dist_path, distname, version):
@@ -683,12 +797,29 @@ def make_nspkg_sdist(dist_path, distname, version):
     make_sdist(dist_path, files)
 
 
+def make_python_requires_sdist(dist_path, distname, version, python_requires):
+    make_sdist(dist_path, [
+        ('setup.py', DALS("""\
+                          import setuptools
+                          setuptools.setup(
+                              name={name!r},
+                              version={version!r},
+                              python_requires={python_requires!r},
+                          )
+                          """).format(name=distname, version=version,
+                                      python_requires=python_requires)),
+         ('setup.cfg', ''),
+    ])
+
+
 def make_sdist(dist_path, files):
     """
     Create a simple sdist tarball at dist_path, containing the files
     listed in ``files`` as ``(filename, content)`` tuples.
     """
 
+    # Distributions with only one file don't play well with pip.
+    assert len(files) > 1
     with tarfile.open(dist_path, 'w:gz') as dist:
         for filename, content in files:
             file_bytes = io.BytesIO(content.encode('utf-8'))
@@ -721,8 +852,8 @@ def create_setup_requires_package(path, distname='foobar', version='0.1',
     test_pkg = os.path.join(path, 'test_pkg')
     os.mkdir(test_pkg)
 
+    # setup.cfg
     if use_setup_cfg:
-        test_setup_cfg = os.path.join(test_pkg, 'setup.cfg')
         options = []
         metadata = []
         for name in use_setup_cfg:
@@ -734,8 +865,7 @@ def create_setup_requires_package(path, distname='foobar', version='0.1',
             if isinstance(value, (tuple, list)):
                 value = ';'.join(value)
             section.append('%s: %s' % (name, value))
-        with open(test_setup_cfg, 'w') as f:
-            f.write(DALS(
+        test_setup_cfg_contents = DALS(
                 """
                 [metadata]
                 {metadata}
@@ -745,16 +875,19 @@ def create_setup_requires_package(path, distname='foobar', version='0.1',
             ).format(
                 options='\n'.join(options),
                 metadata='\n'.join(metadata),
-            ))
-
-    test_setup_py = os.path.join(test_pkg, 'setup.py')
+            )
+    else:
+        test_setup_cfg_contents = ''
+    with open(os.path.join(test_pkg, 'setup.cfg'), 'w') as f:
+        f.write(test_setup_cfg_contents)
 
+    # setup.py
     if setup_py_template is None:
         setup_py_template = DALS("""\
             import setuptools
             setuptools.setup(**%r)
         """)
-    with open(test_setup_py, 'w') as f:
+    with open(os.path.join(test_pkg, 'setup.py'), 'w') as f:
         f.write(setup_py_template % test_setup_attrs)
 
     foobar_path = os.path.join(path, '%s-%s.tar.gz' % (distname, version))

setuptools/tests/test_virtualenv.py

@@ -121,14 +121,12 @@ def test_pip_upgrade_from_source(pip_version, virtualenv):
     virtualenv.run('pip install --no-cache-dir --upgrade ' + sdist)
 
 
-def test_test_command_install_requirements(bare_virtualenv, tmpdir):
+def _check_test_command_install_requirements(virtualenv, tmpdir):
     """
     Check the test command will install all required dependencies.
     """
-    bare_virtualenv.run(' && '.join((
-        'cd {source}',
-        'python setup.py develop',
-    )).format(source=SOURCE_DIR))
+    # Install setuptools.
+    virtualenv.run('python setup.py develop', cd=SOURCE_DIR)
 
     def sdist(distname, version):
         dist_path = tmpdir.join('%s-%s.tar.gz' % (distname, version))
@@ -179,12 +177,20 @@ def test_test_command_install_requirements(bare_virtualenv, tmpdir):
             open('success', 'w').close()
             '''))
     # Run test command for test package.
-    bare_virtualenv.run(' && '.join((
+    virtualenv.run(' && '.join((
         'cd {tmpdir}',
         'python setup.py test -s test',
     )).format(tmpdir=tmpdir))
     assert tmpdir.join('success').check()
 
+def test_test_command_install_requirements(virtualenv, tmpdir):
+    # Ensure pip/wheel packages are installed.
+    virtualenv.run("python -c \"__import__('pkg_resources').require(['pip', 'wheel'])\"")
+    _check_test_command_install_requirements(virtualenv, tmpdir)
+
+def test_test_command_install_requirements_when_using_easy_install(bare_virtualenv, tmpdir):
+    _check_test_command_install_requirements(bare_virtualenv, tmpdir)
+
 
 def test_no_missing_dependencies(bare_virtualenv):
     """

tests/requirements.txt

@@ -9,3 +9,4 @@ coverage>=4.5.1
 pytest-cov>=2.5.1
 paver; python_version>="3.6"
 futures; python_version=="2.7"
+pip>=19.1 # For proper file:// URLs support.