Merge branch 'master' into feature/deterministic-provides-extras

.bumpversion.cfg

+[bumpversion]
+current_version = 41.2.0
+commit = True
+tag = True
+
+[bumpversion:file:setup.cfg]
+

.travis.yml

-dist: trusty
+dist: xenial
 language: python
 
 jobs:
@@ -18,12 +18,11 @@ jobs:
     python: 3.6
   - &latest_py3
     python: 3.7
-    dist: xenial
   - <<: *latest_py3
     env: LANG=C
   - python: 3.8-dev
-    dist: xenial
-    env: DISABLE_COVERAGE=1  # Ignore invalid coverage data.
+  - <<: *latest_py3
+    env: TOXENV=docs DISABLE_COVERAGE=1
   - <<: *default_py
     stage: deploy (to PyPI for tagged commits)
     if: tag IS present
@@ -38,9 +37,9 @@ jobs:
       on:
         tags: true
         all_branches: true
-      user: jaraco
+      user: __token__
       password:
-        secure: tfWrsQMH2bHrWjqnP+08IX1WlkbW94Q30f4d7lCyhWS1FIf/jBDx4jrEILNfMxQ1NCwuBRje5sihj1Ow0BFf0vVrkaeff2IdvnNDEGFduMejaEQJL3s3QrLfpiAvUbtqwyWaHfAdGfk48PovDKTx0ZTvXZKYGXZhxGCYSlG2CE6Y6RDvnEl6Tk8e+LqUohkcSOwxrRwUoyxSnUaavdGohXxDT8MJlfWOXgr2u+KsRrriZqp3l6Fdsnk4IGvy6pXpy42L1HYQyyVu9XyJilR2JTbC6eCp5f8p26093m1Qas49+t6vYb0VLqQe12dO+Jm3v4uztSS5pPQzS7PFyjEYd2Rdb6ijsdbsy1074S4q7G9Sz+T3RsPUwYEJ07lzez8cxP64dtj5j94RL8m35A1Fb1OE8hHN+4c1yLG1gudfXbem+fUhi2eqhJrzQo5vsvDv1xS5x5GIS5ZHgKHCsWcW1Tv+dsFkrhaup3uU6VkOuc9UN+7VPsGEY7NvquGpTm8O1CnGJRzuJg6nbYRGj8ORwDpI0KmrExx6akV92P72fMC/I5TCgbSQSZn370H3Jj40gz1SM30WAli9M+wFHFd4ddMVY65yxj0NLmrP+m1tvnWdKtNh/RHuoW92d9/UFtiA5IhMf1/3djfsjBq6S9NT1uaLkVkTttqrPYJ7hOql8+g=
+        secure: FSp9KU+pdvWPxBOaxe6BNmcJ9y8259G3/NdTJ00r0qx/xMLpSneGjpuLqoD6BL2JoM6gRwurwakWoH/9Ah+Di7afETjMnL6WJKtDZ+Uu3YLx3ss7/FlhVz6zmVTaDJUzuo9dGr//qLBQTIxVjGYfQelRJyfMAXtrYWdeT/4489E45lMw+86Z/vnSBOxs4lWekeQW5Gem0cDViWu67RRiGkAEvrYVwuImMr2Dyhpv+l/mQGQIS/ezXuAEFToE6+q8VUVe/aK498Qovdc+O4M7OYk1JouFpffZ3tVZ6iWHQFcR11480UdI6VCIcFpPvGC/J8MWUWLjq7YOm0X9jPXgdYMUQLAP4clFgUr2qNoRSKWfuQlNdVVuS2htYcjJ3eEl90FhcIZKp+WVMrypRPOQJ8CBielZEs0dhytRrZSaJC1BNq25O/BPzws8dL8hYtoXsM6I3Zv5cZgdyqyq/eOEMCX7Cetv6do0U41VGEV5UohvyyuwH5l9GCuPREpY3sXayPg8fw7XcPjvvzSVyjcUT/ePW8sfnAyWZnngjweAn6dK8IFGPuSPQdlos78uxeUOvCVUW0xv/0m4lX73yoHdVVdLbu1MJTyibFGec86Bew9JqIcDlhHaIJ9ihZ9Z9tOtvp1cuNyKYE4kvmOtumDDicEw4DseYn2z5sZDTYTBsKY=
       distributions: release
       skip_cleanup: true
       skip_upload_docs: true
@@ -63,6 +62,7 @@ install:
 - "! grep pyc setuptools.egg-info/SOURCES.txt"
 
 script:
+  - export NETWORK_REQUIRED=1
   - |
     ( # Run testsuite.
       if [ -z "$DISABLE_COVERAGE" ]

CHANGES.rst

+v41.2.0
+-------
+
+* #479: Remove some usage of the deprecated ``imp`` module.
+* #1565: Changed html_sidebars from string to list of string as per
+  https://www.sphinx-doc.org/en/master/changes.html#id58
+
+
+v41.1.0
+-------
+
+* #1697: Moved most of the constants from setup.py to setup.cfg
+* #1749: Fixed issue with the PEP 517 backend where building a source distribution would fail if any tarball existed in the destination directory.
+* #1750: Fixed an issue with PEP 517 backend where wheel builds would fail if the destination directory did not already exist.
+* #1756: Force metadata-version >= 1.2. when project urls are present.
+* #1769: Improve ``package_data`` check: ensure the dictionary values are lists/tuples of strings.
+* #1788: Changed compatibility fallback logic for ``html.unescape`` to avoid accessing ``HTMLParser.unescape`` when not necessary. ``HTMLParser.unescape`` is deprecated and will be removed in Python 3.9.
+* #1790: Added the file path to the error message when a ``UnicodeDecodeError`` occurs while reading a metadata file.
+* #1776: Use license classifiers rather than the license field.
+
+
+v41.0.1
+-------
+
+* #1671: Fixed issue with the PEP 517 backend that prevented building a wheel when the ``dist/`` directory contained existing ``.whl`` files.
+* #1709: In test.paths_on_python_path, avoid adding unnecessary duplicates to the PYTHONPATH.
+* #1741: In package_index, now honor "current directory" during a checkout of git and hg repositories under Windows
+
+
+v41.0.0
+-------
+
+* #1735: When parsing setup.cfg files, setuptools now requires the files to be encoded as UTF-8. Any other encoding will lead to a UnicodeDecodeError. This change removes support for specifying an encoding using a 'coding: ' directive in the header of the file, a feature that was introduces in 40.7. Given the recent release of the aforementioned feature, it is assumed that few if any projects are utilizing the feature to specify an encoding other than UTF-8.
+
+
+v40.9.0
+-------
+
+* #1675: Added support for ``setup.cfg``-only projects when using the ``setuptools.build_meta`` backend. Projects that have enabled PEP 517 no longer need to have a ``setup.py`` and can use the purely declarative ``setup.cfg`` configuration file instead.
+* #1720: Added support for ``pkg_resources.parse_requirements``-style requirements in ``setup_requires`` when ``setup.py`` is invoked from the ``setuptools.build_meta`` build backend.
+* #1664: Added the path to the ``PKG-INFO`` or ``METADATA`` file in the exception
+  text when the ``Version:`` header can't be found.
+* #1705: Removed some placeholder documentation sections referring to deprecated features.
+
+
 v40.8.0
 -------
 
@@ -1480,8 +1525,8 @@ v20.6.0
 19.7
 ----
 
-* `Off-project PR <https://github.com/jaraco/setuptools/pull/32>`_:
-  For FreeBSD, also honor root certificates from ca_root_nss.
+* Off-project PR: `0dcee79 <https://github.com/pypa/setuptools/commit/0dcee791dfdcfacddaaec79b29f30a347a147413>`_ and `f9bd9b9 <https://github.com/pypa/setuptools/commit/f9bd9b9f5df54ef5a0bf8d16c3a889ab8c640580>`_
+  For FreeBSD, also `honor root certificates from ca_root_nss <https://github.com/pypa/setuptools/commit/3ae46c30225eb46e1f5aada1a19e88b79f04dc72>`_.
 
 19.6.2
 ------
@@ -1645,9 +1690,9 @@ v20.6.0
   now logged when pkg_resources is imported on Python 3.2 or earlier
   Python 3 versions.
 * `Add support for python_platform_implementation environment marker
-  <https://github.com/jaraco/setuptools/pull/28>`_.
+  <https://github.com/pypa/setuptools/commit/94416707fd59a65f4a8f7f70541d6b3fc018b626>`_.
 * `Fix dictionary mutation during iteration
-  <https://github.com/jaraco/setuptools/pull/29>`_.
+  <https://github.com/pypa/setuptools/commit/57ebfa41e0f96b97e599ecd931b7ae8a143e096e>`_.
 
 18.4
 ----
@@ -2007,7 +2052,7 @@ process to fail and PyPI uploads no longer accept files for 13.0.
 ---
 
 * Prefer vendored packaging library `as recommended
-  <https://github.com/jaraco/setuptools/commit/170657b68f4b92e7e1bf82f5e19a831f5744af67#commitcomment-9109448>`_.
+  <https://github.com/pypa/setuptools/commit/170657b68f4b92e7e1bf82f5e19a831f5744af67>`_.
 
 9.0.1
 -----

README.rst

@@ -29,6 +29,10 @@ Bug reports and especially tested patches may be
 submitted directly to the `bug tracker
 <https://github.com/pypa/setuptools/issues>`_.
 
+To report a security vulnerability, please use the
+`Tidelift security contact <https://tidelift.com/security>`_.
+Tidelift will coordinate the fix and disclosure.
+
 
 Code of Conduct
 ---------------

appveyor.yml

@@ -3,6 +3,7 @@ clone_depth: 50
 environment:
 
   APPVEYOR: True
+  NETWORK_REQUIRED: True
   CODECOV_ENV: APPVEYOR_JOB_NAME
 
   matrix:
@@ -25,7 +26,7 @@ cache:
 test_script:
   - python --version
   - python -m pip install --disable-pip-version-check --upgrade pip setuptools wheel
-  - pip install --upgrade tox tox-venv
+  - pip install --upgrade tox tox-venv virtualenv
   - pip freeze --all
   - python bootstrap.py
   - tox -- --cov

changelog.d/1675.change.rst

-Added support for ``setup.cfg``-only projects when using the ``setuptools.build_meta`` backend. Projects that have enabled PEP 517 no longer need to have a ``setup.py`` and can use the purely declarative ``setup.cfg`` configuration file instead.

docs/conf.py

@@ -69,7 +69,7 @@ html_theme_path = ['_theme']
 html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-html_sidebars = {'index': 'indexsidebar.html'}
+html_sidebars = {'index': ['relations.html', 'sourcelink.html', 'indexsidebar.html', 'searchbox.html']}
 
 # If false, no module index is generated.
 html_use_modindex = False

docs/developer-guide.txt

@@ -137,3 +137,17 @@ To build the docs locally, use tox::
 
 .. _Sphinx: http://www.sphinx-doc.org/en/master/
 .. _published documentation: https://setuptools.readthedocs.io/en/latest/
+
+---------------------
+Vendored Dependencies
+---------------------
+
+Setuptools has some dependencies, but due to `bootstrapping issues
+<https://github.com/pypa/setuptools/issues/980>`, those dependencies
+cannot be declared as they won't be resolved soon enough to build
+setuptools from source. Eventually, this limitation may be lifted as
+PEP 517/518 reach ubiquitous adoption, but for now, Setuptools
+cannot declare dependencies other than through
+``setuptools/_vendor/vendored.txt`` and
+``pkg_reosurces/_vendor/vendored.txt`` and refreshed by way of
+``paver update_vendored`` (pavement.py).

docs/setuptools.txt

@@ -136,16 +136,18 @@ dependencies, and perhaps some data files and scripts::
         author="Me",
         author_email="me@example.com",
         description="This is an Example Package",
-        license="PSF",
         keywords="hello world example examples",
         url="http://example.com/HelloWorld/",   # project home page, if any
         project_urls={
             "Bug Tracker": "https://bugs.example.com/HelloWorld/",
             "Documentation": "https://docs.example.com/HelloWorld/",
             "Source Code": "https://code.example.com/HelloWorld/",
-        }
+        },
+        classifiers=[
+            'License :: OSI Approved :: Python Software Foundation License'
+        ]
 
-        # could also include long_description, download_url, classifiers, etc.
+        # could also include long_description, download_url, etc.
     )
 
 In the sections that follow, we'll explain what most of these ``setup()``
@@ -2234,6 +2236,7 @@ boilerplate code in some cases.
     license = BSD 3-Clause License
     classifiers =
         Framework :: Django
+        License :: OSI Approved :: BSD License
         Programming Language :: Python :: 3
         Programming Language :: Python :: 3.5
 
@@ -2679,42 +2682,6 @@ A few important points for writing revision control file finders:
   inform the user of the missing program(s).
 
 
-Subclassing ``Command``
------------------------
-
-Sorry, this section isn't written yet, and neither is a lot of what's below
-this point.
-
-XXX
-
-
-Reusing ``setuptools`` Code
-===========================
-
-``ez_setup``
-------------
-
-XXX
-
-
-``setuptools.archive_util``
----------------------------
-
-XXX
-
-
-``setuptools.sandbox``
-----------------------
-
-XXX
-
-
-``setuptools.package_index``
-----------------------------
-
-XXX
-
-
 Mailing List and Bug Tracker
 ============================
 

launcher.c

@@ -37,6 +37,7 @@
 #include <windows.h>
 #include <tchar.h>
 #include <fcntl.h>
+#include <process.h>
 
 int child_pid=0;
 

pkg_resources/__init__.py

@@ -1403,14 +1403,30 @@ class NullProvider:
     def has_resource(self, resource_name):
         return self._has(self._fn(self.module_path, resource_name))
 
+    def _get_metadata_path(self, name):
+        return self._fn(self.egg_info, name)
+
     def has_metadata(self, name):
-        return self.egg_info and self._has(self._fn(self.egg_info, name))
+        if not self.egg_info:
+            return self.egg_info
+
+        path = self._get_metadata_path(name)
+        return self._has(path)
 
     def get_metadata(self, name):
         if not self.egg_info:
             return ""
-        value = self._get(self._fn(self.egg_info, name))
-        return value.decode('utf-8') if six.PY3 else value
+        path = self._get_metadata_path(name)
+        value = self._get(path)
+        if six.PY2:
+            return value
+        try:
+            return value.decode('utf-8')
+        except UnicodeDecodeError as exc:
+            # Include the path in the error message to simplify
+            # troubleshooting, and without changing the exception type.
+            exc.reason += ' in {} file at path: {}'.format(name, path)
+            raise
 
     def get_metadata_lines(self, name):
         return yield_lines(self.get_metadata(name))
@@ -1868,6 +1884,9 @@ class FileMetadata(EmptyProvider):
     def __init__(self, path):
         self.path = path
 
+    def _get_metadata_path(self, name):
+        return self.path
+
     def has_metadata(self, name):
         return name == 'PKG-INFO' and os.path.isfile(self.path)
 
@@ -2661,10 +2680,14 @@ class Distribution:
         try:
             return self._version
         except AttributeError:
-            version = _version_from_file(self._get_metadata(self.PKG_INFO))
+            version = self._get_version()
             if version is None:
-                tmpl = "Missing 'Version:' header and/or %s file"
-                raise ValueError(tmpl % self.PKG_INFO, self)
+                path = self._get_metadata_path_for_display(self.PKG_INFO)
+                msg = (
+                    "Missing 'Version:' header and/or {} file at path: {}"
+                ).format(self.PKG_INFO, path)
+                raise ValueError(msg, self)
+
             return version
 
     @property
@@ -2722,11 +2745,34 @@ class Distribution:
                 )
         return deps
 
+    def _get_metadata_path_for_display(self, name):
+        """
+        Return the path to the given metadata file, if available.
+        """
+        try:
+            # We need to access _get_metadata_path() on the provider object
+            # directly rather than through this class's __getattr__()
+            # since _get_metadata_path() is marked private.
+            path = self._provider._get_metadata_path(name)
+
+        # Handle exceptions e.g. in case the distribution's metadata
+        # provider doesn't support _get_metadata_path().
+        except Exception:
+            return '[could not detect]'
+
+        return path
+
     def _get_metadata(self, name):
         if self.has_metadata(name):
             for line in self.get_metadata_lines(name):
                 yield line
 
+    def _get_version(self):
+        lines = self._get_metadata(self.PKG_INFO)
+        version = _version_from_file(lines)
+
+        return version
+
     def activate(self, path=None, replace=False):
         """Ensure distribution is importable on `path` (default=sys.path)"""
         if path is None:
@@ -2945,7 +2991,7 @@ class EggInfoDistribution(Distribution):
         take an extra step and try to get the version number from
         the metadata file itself instead of the filename.
         """
-        md_version = _version_from_file(self._get_metadata(self.PKG_INFO))
+        md_version = self._get_version()
         if md_version:
             self._version = md_version
         return self

pkg_resources/tests/test_pkg_resources.py

@@ -17,6 +17,8 @@ try:
 except ImportError:
     import mock
 
+from pkg_resources import DistInfoDistribution, Distribution, EggInfoDistribution
+from setuptools.extern import six
 from pkg_resources.extern.six.moves import map
 from pkg_resources.extern.six import text_type, string_types
 
@@ -190,6 +192,142 @@ class TestResourceManager:
         subprocess.check_call(cmd)
 
 
+def make_test_distribution(metadata_path, metadata):
+    """
+    Make a test Distribution object, and return it.
+
+    :param metadata_path: the path to the metadata file that should be
+        created. This should be inside a distribution directory that should
+        also be created. For example, an argument value might end with
+        "<project>.dist-info/METADATA".
+    :param metadata: the desired contents of the metadata file, as bytes.
+    """
+    dist_dir = os.path.dirname(metadata_path)
+    os.mkdir(dist_dir)
+    with open(metadata_path, 'wb') as f:
+        f.write(metadata)
+    dists = list(pkg_resources.distributions_from_metadata(dist_dir))
+    dist, = dists
+
+    return dist
+
+
+def test_get_metadata__bad_utf8(tmpdir):
+    """
+    Test a metadata file with bytes that can't be decoded as utf-8.
+    """
+    filename = 'METADATA'
+    # Convert the tmpdir LocalPath object to a string before joining.
+    metadata_path = os.path.join(str(tmpdir), 'foo.dist-info', filename)
+    # Encode a non-ascii string with the wrong encoding (not utf-8).
+    metadata = 'née'.encode('iso-8859-1')
+    dist = make_test_distribution(metadata_path, metadata=metadata)
+
+    if six.PY2:
+        # In Python 2, get_metadata() doesn't do any decoding.
+        actual = dist.get_metadata(filename)
+        assert actual == metadata
+        return
+
+    # Otherwise, we are in the Python 3 case.
+    with pytest.raises(UnicodeDecodeError) as excinfo:
+        dist.get_metadata(filename)
+
+    exc = excinfo.value
+    actual = str(exc)
+    expected = (
+        # The error message starts with "'utf-8' codec ..." However, the
+        # spelling of "utf-8" can vary (e.g. "utf8") so we don't include it
+        "codec can't decode byte 0xe9 in position 1: "
+        'invalid continuation byte in METADATA file at path: '
+    )
+    assert expected in actual, 'actual: {}'.format(actual)
+    assert actual.endswith(metadata_path), 'actual: {}'.format(actual)
+
+
+# TODO: remove this in favor of Path.touch() when Python 2 is dropped.
+def touch_file(path):
+    """
+    Create an empty file.
+    """
+    with open(path, 'w'):
+        pass
+
+
+def make_distribution_no_version(tmpdir, basename):
+    """
+    Create a distribution directory with no file containing the version.
+    """
+    # Convert the LocalPath object to a string before joining.
+    dist_dir = os.path.join(str(tmpdir), basename)
+    os.mkdir(dist_dir)
+    # Make the directory non-empty so distributions_from_metadata()
+    # will detect it and yield it.
+    touch_file(os.path.join(dist_dir, 'temp.txt'))
+
+    dists = list(pkg_resources.distributions_from_metadata(dist_dir))
+    assert len(dists) == 1
+    dist, = dists
+
+    return dist, dist_dir
+
+
+@pytest.mark.parametrize(
+    'suffix, expected_filename, expected_dist_type',
+    [
+        ('egg-info', 'PKG-INFO', EggInfoDistribution),
+        ('dist-info', 'METADATA', DistInfoDistribution),
+    ],
+)
+def test_distribution_version_missing(tmpdir, suffix, expected_filename,
+    expected_dist_type):
+    """
+    Test Distribution.version when the "Version" header is missing.
+    """
+    basename = 'foo.{}'.format(suffix)
+    dist, dist_dir = make_distribution_no_version(tmpdir, basename)
+
+    expected_text = (
+        "Missing 'Version:' header and/or {} file at path: "
+    ).format(expected_filename)
+    metadata_path = os.path.join(dist_dir, expected_filename)
+
+    # Now check the exception raised when the "version" attribute is accessed.
+    with pytest.raises(ValueError) as excinfo:
+        dist.version
+
+    err = str(excinfo.value)
+    # Include a string expression after the assert so the full strings
+    # will be visible for inspection on failure.
+    assert expected_text in err, str((expected_text, err))
+
+    # Also check the args passed to the ValueError.
+    msg, dist = excinfo.value.args
+    assert expected_text in msg
+    # Check that the message portion contains the path.
+    assert metadata_path in msg, str((metadata_path, msg))
+    assert type(dist) == expected_dist_type
+
+
+def test_distribution_version_missing_undetected_path():
+    """
+    Test Distribution.version when the "Version" header is missing and
+    the path can't be detected.
+    """
+    # Create a Distribution object with no metadata argument, which results
+    # in an empty metadata provider.
+    dist = Distribution('/foo')
+    with pytest.raises(ValueError) as excinfo:
+        dist.version
+
+    msg, dist = excinfo.value.args
+    expected = (
+        "Missing 'Version:' header and/or PKG-INFO file at path: "
+        '[could not detect]'
+    )
+    assert msg == expected
+
+
 class TestDeepVersionLookupDistutils:
     @pytest.fixture
     def env(self, tmpdir):

pytest.ini

 [pytest]
-addopts=--doctest-modules --doctest-glob=pkg_resources/api_tests.txt -rsxX
+addopts=--doctest-modules --doctest-glob=pkg_resources/api_tests.txt -r sxX
 norecursedirs=dist build *.egg setuptools/extern pkg_resources/extern .*
 flake8-ignore =
     setuptools/site-patch.py F821

setup.cfg

-[bumpversion]
-current_version = 40.8.0
-commit = True
-tag = True
-
 [egg_info]
 tag_build = .post
 tag_date = 1
@@ -23,7 +18,46 @@ formats = zip
 universal = 1
 
 [metadata]
+name = setuptools
+version = 41.2.0
+description = Easily download, build, install, upgrade, and uninstall Python packages
+author = Python Packaging Authority
+author_email = distutils-sig@python.org
+long_description = file: README.rst
+long_description_content_type = text/x-rst; charset=UTF-8
 license_file = LICENSE
-
-[bumpversion:file:setup.py]
-
+keywords = CPAN PyPI distutils eggs package management
+url = https://github.com/pypa/setuptools
+project_urls =
+    Documentation = https://setuptools.readthedocs.io/
+classifiers =
+    Development Status :: 5 - Production/Stable
+    Intended Audience :: Developers
+    License :: OSI Approved :: MIT License
+    Operating System :: OS Independent
+    Programming Language :: Python :: 2
+    Programming Language :: Python :: 2.7
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.4
+    Programming Language :: Python :: 3.5
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Topic :: Software Development :: Libraries :: Python Modules
+    Topic :: System :: Archiving :: Packaging
+    Topic :: System :: Systems Administration
+    Topic :: Utilities
+
+[options]
+zip_safe = True
+python_requires = >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*
+py_modules = easy_install
+packages = find:
+
+[options.packages.find]
+exclude = *.tests
+
+[options.extras_require]
+ssl =
+    wincertstore==0.2; sys_platform=='win32'
+certs =
+    certifi==2016.9.26

setup.py

@@ -3,10 +3,8 @@
 Distutils setup file, used to install or test 'setuptools'
 """
 
-import io
 import os
 import sys
-import textwrap
 
 import setuptools
 
@@ -49,10 +47,6 @@ def _gen_console_scripts():
     yield tmpl.format(shortver=sys.version[:3])
 
 
-readme_path = os.path.join(here, 'README.rst')
-with io.open(readme_path, encoding='utf-8') as readme_file:
-    long_description = readme_file.read()
-
 package_data = dict(
     setuptools=['script (dev).tmpl', 'script.tmpl', 'site-patch.py'],
 )
@@ -88,26 +82,8 @@ def pypi_link(pkg_filename):
 
 
 setup_params = dict(
-    name="setuptools",
-    version="40.8.0",
-    description=(
-        "Easily download, build, install, upgrade, and uninstall "
-        "Python packages"
-    ),
-    author="Python Packaging Authority",
-    author_email="distutils-sig@python.org",
-    long_description=long_description,
-    long_description_content_type='text/x-rst; charset=UTF-8',
-    keywords="CPAN PyPI distutils eggs package management",
-    url="https://github.com/pypa/setuptools",
-    project_urls={
-        "Documentation": "https://setuptools.readthedocs.io/",
-    },
     src_root=None,
-    packages=setuptools.find_packages(exclude=['*.tests']),
     package_data=package_data,
-    py_modules=['easy_install'],
-    zip_safe=True,
     entry_points={
         "distutils.commands": [
             "%(cmd)s = setuptools.command.%(cmd)s:%(cmd)s" % locals()
@@ -153,28 +129,6 @@ setup_params = dict(
         "setuptools.installation":
             ['eggsecutable = setuptools.command.easy_install:bootstrap'],
     },
-    classifiers=textwrap.dedent("""
-        Development Status :: 5 - Production/Stable
-        Intended Audience :: Developers
-        License :: OSI Approved :: MIT License
-        Operating System :: OS Independent
-        Programming Language :: Python :: 2
-        Programming Language :: Python :: 2.7
-        Programming Language :: Python :: 3
-        Programming Language :: Python :: 3.4
-        Programming Language :: Python :: 3.5
-        Programming Language :: Python :: 3.6
-        Programming Language :: Python :: 3.7
-        Topic :: Software Development :: Libraries :: Python Modules
-        Topic :: System :: Archiving :: Packaging
-        Topic :: System :: Systems Administration
-        Topic :: Utilities
-        """).strip().splitlines(),
-    python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*',
-    extras_require={
-        "ssl:sys_platform=='win32'": "wincertstore==0.2",
-        "certs": "certifi==2016.9.26",
-    },
     dependency_links=[
         pypi_link(
             'certifi-2016.9.26.tar.gz#md5=baa81e951a29958563689d868ef1064d',
@@ -183,7 +137,6 @@ setup_params = dict(
             'wincertstore-0.2.zip#md5=ae728f2f007185648d0c7a8679b361e2',
         ),
     ],
-    scripts=[],
     setup_requires=[
     ] + wheel,
 )

setuptools/build_meta.py

@@ -35,6 +35,10 @@ import contextlib
 
 import setuptools
 import distutils
+from setuptools.py31compat import TemporaryDirectory
+
+from pkg_resources import parse_requirements
+from pkg_resources.py31compat import makedirs
 
 __all__ = ['get_requires_for_build_sdist',
            'get_requires_for_build_wheel',
@@ -51,7 +55,9 @@ class SetupRequirementsError(BaseException):
 
 class Distribution(setuptools.dist.Distribution):
     def fetch_build_eggs(self, specifiers):
-        raise SetupRequirementsError(specifiers)
+        specifier_list = list(map(str, parse_requirements(specifiers)))
+
+        raise SetupRequirementsError(specifier_list)
 
     @classmethod
     @contextlib.contextmanager
@@ -174,28 +180,38 @@ class _BuildMetaBackend(object):
 
         return dist_infos[0]
 
-    def build_wheel(self, wheel_directory, config_settings=None,
-                    metadata_directory=None):
+    def _build_with_temp_dir(self, setup_command, result_extension,
+                             result_directory, config_settings):
         config_settings = self._fix_config(config_settings)
-        wheel_directory = os.path.abspath(wheel_directory)
-        sys.argv = sys.argv[:1] + ['bdist_wheel'] + \
-            config_settings["--global-option"]
-        self.run_setup()
-        if wheel_directory != 'dist':
-            shutil.rmtree(wheel_directory)
-            shutil.copytree('dist', wheel_directory)
+        result_directory = os.path.abspath(result_directory)
 
-        return _file_with_extension(wheel_directory, '.whl')
+        # Build in a temporary directory, then copy to the target.
+        makedirs(result_directory, exist_ok=True)
+        with TemporaryDirectory(dir=result_directory) as tmp_dist_dir:
+            sys.argv = (sys.argv[:1] + setup_command +
+                        ['--dist-dir', tmp_dist_dir] +
+                        config_settings["--global-option"])
+            self.run_setup()
 
-    def build_sdist(self, sdist_directory, config_settings=None):
-        config_settings = self._fix_config(config_settings)
-        sdist_directory = os.path.abspath(sdist_directory)
-        sys.argv = sys.argv[:1] + ['sdist', '--formats', 'gztar'] + \
-            config_settings["--global-option"] + \
-            ["--dist-dir", sdist_directory]
-        self.run_setup()
+            result_basename = _file_with_extension(tmp_dist_dir, result_extension)
+            result_path = os.path.join(result_directory, result_basename)
+            if os.path.exists(result_path):
+                # os.rename will fail overwriting on non-Unix.
+                os.remove(result_path)
+            os.rename(os.path.join(tmp_dist_dir, result_basename), result_path)
+
+        return result_basename
 
-        return _file_with_extension(sdist_directory, '.tar.gz')
+
+    def build_wheel(self, wheel_directory, config_settings=None,
+                    metadata_directory=None):
+        return self._build_with_temp_dir(['bdist_wheel'], '.whl',
+                                         wheel_directory, config_settings)
+
+    def build_sdist(self, sdist_directory, config_settings=None):
+        return self._build_with_temp_dir(['sdist', '--formats', 'gztar'],
+                                         '.tar.gz', sdist_directory,
+                                         config_settings)
 
 
 class _BuildMetaLegacyBackend(_BuildMetaBackend):

setuptools/command/build_ext.py

 import os
 import sys
 import itertools
-import imp
 from distutils.command.build_ext import build_ext as _du_build_ext
 from distutils.file_util import copy_file
 from distutils.ccompiler import new_compiler
@@ -12,6 +11,13 @@ from distutils import log
 from setuptools.extension import Library
 from setuptools.extern import six
 
+if six.PY2:
+    import imp
+
+    EXTENSION_SUFFIXES = [s for s, _, tp in imp.get_suffixes() if tp == imp.C_EXTENSION]
+else:
+    from importlib.machinery import EXTENSION_SUFFIXES
+
 try:
     # Attempt to use Cython for building extensions, if available
     from Cython.Distutils.build_ext import build_ext as _build_ext
@@ -64,7 +70,7 @@ if_dl = lambda s: s if have_rtld else ''
 
 def get_abi3_suffix():
     """Return the file extension for an abi3-compliant Extension()"""
-    for suffix, _, _ in (s for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION):
+    for suffix in EXTENSION_SUFFIXES:
         if '.abi3' in suffix:  # Unix
             return suffix
         elif suffix == '.pyd':  # Windows

setuptools/command/install_lib.py

 import os
-import imp
+import sys
 from itertools import product, starmap
 import distutils.command.install_lib as orig
 
@@ -74,10 +74,10 @@ class install_lib(orig.install_lib):
         yield '__init__.pyc'
         yield '__init__.pyo'
 
-        if not hasattr(imp, 'get_tag'):
+        if not hasattr(sys, 'implementation'):
             return
 
-        base = os.path.join('__pycache__', '__init__.' + imp.get_tag())
+        base = os.path.join('__pycache__', '__init__.' + sys.implementation.cache_tag)
         yield base + '.pyc'
         yield base + '.pyo'
         yield base + '.opt-1.pyc'

setuptools/command/test.py

@@ -15,6 +15,7 @@ from pkg_resources import (resource_listdir, resource_exists, normalize_path,
                            working_set, _namespace_packages, evaluate_marker,
                            add_activation_listener, require, EntryPoint)
 from setuptools import Command
+from .build_py import _unique_everseen
 
 __metaclass__ = type
 
@@ -186,7 +187,7 @@ class test(Command):
         orig_pythonpath = os.environ.get('PYTHONPATH', nothing)
         current_pythonpath = os.environ.get('PYTHONPATH', '')
         try:
-            prefix = os.pathsep.join(paths)
+            prefix = os.pathsep.join(_unique_everseen(paths))
             to_join = filter(None, [prefix, current_pythonpath])
             new_path = os.pathsep.join(to_join)
             if new_path:

setuptools/dist.py

@@ -11,7 +11,6 @@ import distutils.log
 import distutils.core
 import distutils.cmd
 import distutils.dist
-from distutils.errors import DistutilsOptionError
 from distutils.util import strtobool
 from distutils.debug import DEBUG
 from distutils.fancy_getopt import translate_longopt
@@ -37,7 +36,6 @@ from setuptools.depends import Require
 from setuptools import windows_support
 from setuptools.monkey import get_unpatched
 from setuptools.config import parse_configuration
-from .unicode_utils import detect_encoding
 import pkg_resources
 
 __import__('setuptools.extern.packaging.specifiers')
@@ -57,7 +55,8 @@ def get_metadata_version(self):
             mv = StrictVersion('2.1')
         elif (self.maintainer is not None or
               self.maintainer_email is not None or
-              getattr(self, 'python_requires', None) is not None):
+              getattr(self, 'python_requires', None) is not None or
+              self.project_urls):
             mv = StrictVersion('1.2')
         elif (self.provides or self.requires or self.obsoletes or
                 self.classifiers or self.download_url):
@@ -136,7 +135,6 @@ def write_pkg_file(self, file):
         def write_field(key, value):
             file.write("%s: %s\n" % (key, value))
 
-
     write_field('Metadata-Version', str(version))
     write_field('Name', self.get_name())
     write_field('Version', self.get_version())
@@ -216,8 +214,12 @@ def check_importable(dist, attr, value):
 
 
 def assert_string_list(dist, attr, value):
-    """Verify that value is a string list or None"""
+    """Verify that value is a string list"""
     try:
+        # verify that value is a list or tuple to exclude unordered
+        # or single-use iterables
+        assert isinstance(value, (list, tuple))
+        # verify that elements of value are strings
         assert ''.join(value) != value
     except (TypeError, ValueError, AttributeError, AssertionError):
         raise DistutilsSetupError(
@@ -310,20 +312,17 @@ def check_test_suite(dist, attr, value):
 
 def check_package_data(dist, attr, value):
     """Verify that value is a dictionary of package names to glob lists"""
-    if isinstance(value, dict):
-        for k, v in value.items():
-            if not isinstance(k, str):
-                break
-            try:
-                iter(v)
-            except TypeError:
-                break
-        else:
-            return
-    raise DistutilsSetupError(
-        attr + " must be a dictionary mapping package names to lists of "
-        "wildcard patterns"
-    )
+    if not isinstance(value, dict):
+        raise DistutilsSetupError(
+            "{!r} must be a dictionary mapping package names to lists of "
+            "string wildcard patterns".format(attr))
+    for k, v in value.items():
+        if not isinstance(k, six.string_types):
+            raise DistutilsSetupError(
+                "keys of {!r} dict must be strings (got {!r})"
+                .format(attr, k)
+            )
+        assert_string_list(dist, 'values of {!r} dict'.format(attr), v)
 
 
 def check_packages(dist, attr, value):
@@ -590,13 +589,9 @@ class Distribution(_Distribution):
 
         parser = ConfigParser()
         for filename in filenames:
-            with io.open(filename, 'rb') as fp:
-                encoding = detect_encoding(fp)
+            with io.open(filename, encoding='utf-8') as reader:
                 if DEBUG:
-                    self.announce("  reading %s [%s]" % (
-                        filename, encoding or 'locale')
-                    )
-                reader = io.TextIOWrapper(fp, encoding=encoding)
+                    self.announce("  reading {filename}".format(**locals()))
                 (parser.read_file if six.PY3 else parser.readfp)(reader)
             for section in parser.sections():
                 options = parser.options(section)
@@ -886,7 +881,7 @@ class Distribution(_Distribution):
     def include(self, **attrs):
         """Add items to distribution that are named in keyword arguments
 
-        For example, 'dist.exclude(py_modules=["x"])' would add 'x' to
+        For example, 'dist.include(py_modules=["x"])' would add 'x' to
         the distribution's 'py_modules' attribute, if it was not already
         there.
 
@@ -1104,7 +1099,6 @@ class Distribution(_Distribution):
             return _Distribution.handle_display_options(self, option_order)
 
         # Stdout may be StringIO (e.g. in tests)
-        import io
         if not isinstance(sys.stdout, io.TextIOWrapper):
             return _Distribution.handle_display_options(self, option_order)
 
@@ -1283,4 +1277,5 @@ class Feature:
 
 
 class DistDeprecationWarning(SetuptoolsDeprecationWarning):
-    """Class for warning about deprecations in dist in setuptools. Not ignored by default, unlike DeprecationWarning."""
+    """Class for warning about deprecations in dist in
+    setuptools. Not ignored by default, unlike DeprecationWarning."""

setuptools/package_index.py

@@ -897,7 +897,7 @@ class PackageIndex(Environment):
 
         if rev is not None:
             self.info("Checking out %s", rev)
-            os.system("(cd %s && git checkout --quiet %s)" % (
+            os.system("git -C %s checkout --quiet %s" % (
                 filename,
                 rev,
             ))
@@ -913,7 +913,7 @@ class PackageIndex(Environment):
 
         if rev is not None:
             self.info("Updating to %s", rev)
-            os.system("(cd %s && hg up -C -r %s -q)" % (
+            os.system("hg --cwd %s up -C -r %s -q" % (
                 filename,
                 rev,
             ))

setuptools/py31compat.py

@@ -17,9 +17,9 @@ except ImportError:
         errors on deletion.
         """
 
-        def __init__(self):
+        def __init__(self, **kwargs):
             self.name = None  # Handle mkdtemp raising an exception
-            self.name = tempfile.mkdtemp()
+            self.name = tempfile.mkdtemp(**kwargs)
 
         def __enter__(self):
             return self.name

setuptools/py33compat.py

@@ -52,4 +52,8 @@ class Bytecode_compat:
 Bytecode = getattr(dis, 'Bytecode', Bytecode_compat)
 
 
-unescape = getattr(html, 'unescape', html_parser.HTMLParser().unescape)
+unescape = getattr(html, 'unescape', None)
+if unescape is None:
+    # HTMLParser.unescape is deprecated since Python 3.4, and will be removed
+    # from 3.9.
+    unescape = html_parser.HTMLParser().unescape

setuptools/tests/test_build_meta.py

@@ -28,7 +28,7 @@ class BuildBackend(BuildBackendBase):
 
     def __init__(self, *args, **kwargs):
         super(BuildBackend, self).__init__(*args, **kwargs)
-        self.pool = futures.ProcessPoolExecutor()
+        self.pool = futures.ProcessPoolExecutor(max_workers=1)
 
     def __getattr__(self, name):
         """Handles aribrary function invocations on the build backend."""
@@ -157,6 +157,53 @@ class TestBuildMetaBackend:
 
         assert os.path.isfile(os.path.join(dist_dir, wheel_name))
 
+    @pytest.mark.parametrize('build_type', ('wheel', 'sdist'))
+    def test_build_with_existing_file_present(self, build_type, tmpdir_cwd):
+        # Building a sdist/wheel should still succeed if there's
+        # already a sdist/wheel in the destination directory.
+        files = {
+            'setup.py': "from setuptools import setup\nsetup()",
+            'VERSION': "0.0.1",
+            'setup.cfg': DALS("""
+                [metadata]
+                name = foo
+                version = file: VERSION
+            """),
+            'pyproject.toml': DALS("""
+                [build-system]
+                requires = ["setuptools", "wheel"]
+                build-backend = "setuptools.build_meta
+            """),
+        }
+
+        build_files(files)
+
+        dist_dir = os.path.abspath('preexisting-' + build_type)
+
+        build_backend = self.get_build_backend()
+        build_method = getattr(build_backend, 'build_' + build_type)
+
+        # Build a first sdist/wheel.
+        # Note: this also check the destination directory is
+        # successfully created if it does not exist already.
+        first_result = build_method(dist_dir)
+
+        # Change version.
+        with open("VERSION", "wt") as version_file:
+            version_file.write("0.0.2")
+
+        # Build a *second* sdist/wheel.
+        second_result = build_method(dist_dir)
+
+        assert os.path.isfile(os.path.join(dist_dir, first_result))
+        assert first_result != second_result
+
+        # And if rebuilding the exact same sdist/wheel?
+        open(os.path.join(dist_dir, second_result), 'w').close()
+        third_result = build_method(dist_dir)
+        assert third_result == second_result
+        assert os.path.getsize(os.path.join(dist_dir, third_result)) > 0
+
     def test_build_sdist(self, build_backend):
         dist_dir = os.path.abspath('pip-sdist')
         os.makedirs(dist_dir)
@@ -287,6 +334,57 @@ class TestBuildMetaBackend:
         with pytest.raises(ImportError):
             build_backend.build_sdist("temp")
 
+    @pytest.mark.parametrize('setup_literal, requirements', [
+        ("'foo'", ['foo']),
+        ("['foo']", ['foo']),
+        (r"'foo\n'", ['foo']),
+        (r"'foo\n\n'", ['foo']),
+        ("['foo', 'bar']", ['foo', 'bar']),
+        (r"'# Has a comment line\nfoo'", ['foo']),
+        (r"'foo # Has an inline comment'", ['foo']),
+        (r"'foo \\\n >=3.0'", ['foo>=3.0']),
+        (r"'foo\nbar'", ['foo', 'bar']),
+        (r"'foo\nbar\n'", ['foo', 'bar']),
+        (r"['foo\n', 'bar\n']", ['foo', 'bar']),
+    ])
+    @pytest.mark.parametrize('use_wheel', [True, False])
+    def test_setup_requires(self, setup_literal, requirements, use_wheel,
+                            tmpdir_cwd):
+
+        files = {
+            'setup.py': DALS("""
+                from setuptools import setup
+
+                setup(
+                    name="qux",
+                    version="0.0.0",
+                    py_modules=["hello.py"],
+                    setup_requires={setup_literal},
+                )
+            """).format(setup_literal=setup_literal),
+            'hello.py': DALS("""
+            def run():
+                print('hello')
+            """),
+        }
+
+        build_files(files)
+
+        build_backend = self.get_build_backend()
+
+        if use_wheel:
+            base_requirements = ['wheel']
+            get_requires = build_backend.get_requires_for_build_wheel
+        else:
+            base_requirements = []
+            get_requires = build_backend.get_requires_for_build_sdist
+
+        # Ensure that the build requirements are properly parsed
+        expected = sorted(base_requirements + requirements)
+        actual = get_requires()
+
+        assert expected == sorted(actual)
+
 
 class TestBuildMetaLegacyBackend(TestBuildMetaBackend):
     backend_name = 'setuptools.build_meta:__legacy__'

setuptools/tests/test_config.py

@@ -8,8 +8,7 @@ from distutils.errors import DistutilsOptionError, DistutilsFileError
 from mock import patch
 from setuptools.dist import Distribution, _Distribution
 from setuptools.config import ConfigHandler, read_configuration
-from setuptools.extern.six.moves.configparser import InterpolationMissingOptionError
-from setuptools.tests import is_ascii
+from setuptools.extern.six.moves import configparser
 from . import py2_only, py3_only
 from .textwrap import DALS
 
@@ -29,7 +28,9 @@ def make_package_dir(name, base_dir, ns=False):
     return dir_package, init_file
 
 
-def fake_env(tmpdir, setup_cfg, setup_py=None, encoding='ascii', package_path='fake_package'):
+def fake_env(
+        tmpdir, setup_cfg, setup_py=None,
+        encoding='ascii', package_path='fake_package'):
 
     if setup_py is None:
         setup_py = (
@@ -440,13 +441,10 @@ class TestMetadata:
             '[metadata]\n'
             'description = %(message)s\n'
         )
-        with pytest.raises(InterpolationMissingOptionError):
+        with pytest.raises(configparser.InterpolationMissingOptionError):
             with get_dist(tmpdir):
                 pass
 
-    skip_if_not_ascii = pytest.mark.skipif(not is_ascii, reason='Test not supported with this locale')
-
-    @skip_if_not_ascii
     def test_non_ascii_1(self, tmpdir):
         fake_env(
             tmpdir,
@@ -454,18 +452,8 @@ class TestMetadata:
             'description = éàïôñ\n',
             encoding='utf-8'
         )
-        with pytest.raises(UnicodeDecodeError):
-            with get_dist(tmpdir):
-                pass
-
-    def test_non_ascii_2(self, tmpdir):
-        fake_env(
-            tmpdir,
-            '# -*- coding: invalid\n'
-        )
-        with pytest.raises(LookupError):
-            with get_dist(tmpdir):
-                pass
+        with get_dist(tmpdir):
+            pass
 
     def test_non_ascii_3(self, tmpdir):
         fake_env(
@@ -476,7 +464,6 @@ class TestMetadata:
         with get_dist(tmpdir):
             pass
 
-    @skip_if_not_ascii
     def test_non_ascii_4(self, tmpdir):
         fake_env(
             tmpdir,
@@ -488,8 +475,10 @@ class TestMetadata:
         with get_dist(tmpdir) as dist:
             assert dist.metadata.description == 'éàïôñ'
 
-    @skip_if_not_ascii
-    def test_non_ascii_5(self, tmpdir):
+    def test_not_utf8(self, tmpdir):
+        """
+        Config files encoded not in UTF-8 will fail
+        """
         fake_env(
             tmpdir,
             '# vim: set fileencoding=iso-8859-15 :\n'
@@ -497,8 +486,9 @@ class TestMetadata:
             'description = éàïôñ\n',
             encoding='iso-8859-15'
         )
-        with get_dist(tmpdir) as dist:
-            assert dist.metadata.description == 'éàïôñ'
+        with pytest.raises(UnicodeDecodeError):
+            with get_dist(tmpdir):
+                pass
 
 
 class TestOptions:

setuptools/tests/test_dist.py

@@ -4,8 +4,13 @@ from __future__ import unicode_literals
 
 import io
 import collections
-
-from setuptools.dist import DistDeprecationWarning, _get_unpatched
+import re
+from distutils.errors import DistutilsSetupError
+from setuptools.dist import (
+    _get_unpatched,
+    check_package_data,
+    DistDeprecationWarning,
+)
 from setuptools import Distribution
 from setuptools.extern.six.moves.urllib.request import pathname2url
 from setuptools.extern.six.moves.urllib_parse import urljoin
@@ -278,3 +283,48 @@ def test_provides_extras_deterministic_order():
         reversed(list(attrs['extras_require'].items())))
     dist = Distribution(attrs)
     assert dist.metadata.provides_extras == ['b', 'a']
+
+   
+CHECK_PACKAGE_DATA_TESTS = (
+    # Valid.
+    ({
+        '': ['*.txt', '*.rst'],
+        'hello': ['*.msg'],
+    }, None),
+    # Not a dictionary.
+    ((
+        ('', ['*.txt', '*.rst']),
+        ('hello', ['*.msg']),
+    ), (
+        "'package_data' must be a dictionary mapping package"
+        " names to lists of string wildcard patterns"
+    )),
+    # Invalid key type.
+    ({
+        400: ['*.txt', '*.rst'],
+    }, (
+        "keys of 'package_data' dict must be strings (got 400)"
+    )),
+    # Invalid value type.
+    ({
+        'hello': str('*.msg'),
+    }, (
+        "\"values of 'package_data' dict\" must be a list of strings (got '*.msg')"
+    )),
+    # Invalid value type (generators are single use)
+    ({
+        'hello': (x for x in "generator"),
+    }, (
+        "\"values of 'package_data' dict\" must be a list of strings "
+        "(got <generator object"
+    )),
+)
+
+
+@pytest.mark.parametrize('package_data, expected_message', CHECK_PACKAGE_DATA_TESTS)
+def test_check_package_data(package_data, expected_message):
+    if expected_message is None:
+        assert check_package_data(None, 'package_data', package_data) is None
+    else:
+        with pytest.raises(DistutilsSetupError, match=re.escape(expected_message)):
+            check_package_data(None, str('package_data'), package_data)

setuptools/tests/test_egg_info.py

@@ -622,6 +622,7 @@ class TestEggInfo:
         assert expected_line in pkg_info_lines
         expected_line = 'Project-URL: Link Two, https://example.com/two/'
         assert expected_line in pkg_info_lines
+        assert 'Metadata-Version: 1.2' in pkg_info_lines
 
     def test_python_requires_egg_info(self, tmpdir_cwd, env):
         self._setup_script_with_requires(

setuptools/tests/test_integration.py

@@ -141,6 +141,7 @@ def test_build_deps_on_distutils(request, tmpdir_factory, build_dep):
         allowed_unknowns = [
             'test_suite',
             'tests_require',
+            'python_requires',
             'install_requires',
         ]
         assert not match or match.group(1).strip('"\'') in allowed_unknowns
@@ -149,8 +150,8 @@ def test_build_deps_on_distutils(request, tmpdir_factory, build_dep):
 def install(pkg_dir, install_dir):
     with open(os.path.join(pkg_dir, 'setuptools.py'), 'w') as breaker:
         breaker.write('raise ImportError()')
-    cmd = [sys.executable, 'setup.py', 'install', '--prefix', install_dir]
-    env = dict(os.environ, PYTHONPATH=pkg_dir)
+    cmd = [sys.executable, 'setup.py', 'install', '--prefix', str(install_dir)]
+    env = dict(os.environ, PYTHONPATH=str(pkg_dir))
     output = subprocess.check_output(
         cmd, cwd=pkg_dir, env=env, stderr=subprocess.STDOUT)
     return output.decode('utf-8')

setuptools/tests/test_packageindex.py

@@ -249,7 +249,7 @@ class TestPackageIndex:
         first_call_args = os_system_mock.call_args_list[0][0]
         assert first_call_args == (expected,)
 
-        tmpl = '(cd {expected_dir} && git checkout --quiet master)'
+        tmpl = 'git -C {expected_dir} checkout --quiet master'
         expected = tmpl.format(**locals())
         assert os_system_mock.call_args_list[1][0] == (expected,)
         assert result == expected_dir

setuptools/tests/test_setopt.py

+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import io
+
+import six
+
+from setuptools.command import setopt
+from setuptools.extern.six.moves import configparser
+
+
+class TestEdit:
+    @staticmethod
+    def parse_config(filename):
+        parser = configparser.ConfigParser()
+        with io.open(filename, encoding='utf-8') as reader:
+            (parser.read_file if six.PY3 else parser.readfp)(reader)
+        return parser
+
+    @staticmethod
+    def write_text(file, content):
+        with io.open(file, 'wb') as strm:
+            strm.write(content.encode('utf-8'))
+
+    def test_utf8_encoding_retained(self, tmpdir):
+        """
+        When editing a file, non-ASCII characters encoded in
+        UTF-8 should be retained.
+        """
+        config = tmpdir.join('setup.cfg')
+        self.write_text(str(config), '[names]\njaraco=джарако')
+        setopt.edit_config(str(config), dict(names=dict(other='yes')))
+        parser = self.parse_config(str(config))
+        assert parser.get('names', 'jaraco') == 'джарако'
+        assert parser.get('names', 'other') == 'yes'

setuptools/tests/test_virtualenv.py

@@ -8,6 +8,8 @@ from pytest_fixture_config import yield_requires_config
 
 import pytest_virtualenv
 
+from setuptools.extern import six
+
 from .textwrap import DALS
 from .test_easy_install import make_nspkg_sdist
 
@@ -52,10 +54,58 @@ def test_clean_env_install(bare_virtualenv):
     )).format(source=SOURCE_DIR))
 
 
-def test_pip_upgrade_from_source(virtualenv):
+def _get_pip_versions():
+    # This fixture will attempt to detect if tests are being run without
+    # network connectivity and if so skip some tests
+
+    network = True
+    if not os.environ.get('NETWORK_REQUIRED', False):  # pragma: nocover
+        try:
+            from urllib.request import urlopen
+            from urllib.error import URLError
+        except ImportError:
+            from urllib2 import urlopen, URLError # Python 2.7 compat
+
+        try:
+            urlopen('https://pypi.org', timeout=1)
+        except URLError:
+            # No network, disable most of these tests
+            network = False
+
+    network_versions = [
+        'pip==9.0.3',
+        'pip==10.0.1',
+        'pip==18.1',
+        'pip==19.0.1',
+    ]
+
+    # Pip's master dropped support for 3.4.
+    if not six.PY34:
+        network_versions.append('https://github.com/pypa/pip/archive/master.zip')
+
+    versions = [None] + [
+        pytest.param(v, **({} if network else {'marks': pytest.mark.skip}))
+        for v in network_versions
+    ]
+
+    return versions
+
+
+@pytest.mark.parametrize('pip_version', _get_pip_versions())
+def test_pip_upgrade_from_source(pip_version, virtualenv):
     """
     Check pip can upgrade setuptools from source.
     """
+    # Install pip/wheel, and remove setuptools (as it
+    # should not be needed for bootstraping from source)
+    if pip_version is None:
+        upgrade_pip = ()
+    else:
+        upgrade_pip = ('python -m pip install -U {pip_version} --retries=1',)
+    virtualenv.run(' && '.join((
+        'pip uninstall -y setuptools',
+        'pip install -U wheel',
+    ) + upgrade_pip).format(pip_version=pip_version))
     dist_dir = virtualenv.workspace
     # Generate source distribution / wheel.
     virtualenv.run(' && '.join((

setuptools/unicode_utils.py

 import unicodedata
 import sys
-import re
 
 from setuptools.extern import six
 
@@ -43,15 +42,3 @@ def try_encode(string, enc):
         return string.encode(enc)
     except UnicodeEncodeError:
         return None
-
-
-CODING_RE = re.compile(br'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)')
-
-
-def detect_encoding(fp):
-    first_line = fp.readline()
-    fp.seek(0)
-    m = CODING_RE.match(first_line)
-    if m is None:
-        return None
-    return m.group(1).decode('ascii')

tests/requirements.txt

@@ -3,8 +3,7 @@ pytest-flake8; python_version!="3.4"
 pytest-flake8<=1.0.0; python_version=="3.4"
 virtualenv>=13.0.0
 pytest-virtualenv>=1.2.7
-# pytest pinned to <4 due to #1638
-pytest>=3.7,<4
+pytest>=3.7
 wheel
 coverage>=4.5.1
 pytest-cov>=2.5.1

tox.ini

@@ -14,11 +14,11 @@ deps=-rtests/requirements.txt
 # from being added to `sys.path`.
 install_command=python -c 'import sys; sys.path.remove(""); from pkg_resources import load_entry_point; load_entry_point("pip", "console_scripts", "pip")()' install {opts} {packages}
 # Same as above.
-list_dependencies_command={envbindir}/pip freeze
+list_dependencies_command={envbindir}/pip freeze --all
 setenv=COVERAGE_FILE={toxworkdir}/.coverage.{envname}
 # TODO: The passed environment variables came from copying other tox.ini files
 # These should probably be individually annotated to explain what needs them.
-passenv=APPDATA HOMEDRIVE HOMEPATH windir APPVEYOR APPVEYOR_* CI CODECOV_* TRAVIS TRAVIS_*
+passenv=APPDATA HOMEDRIVE HOMEPATH windir APPVEYOR APPVEYOR_* CI CODECOV_* TRAVIS TRAVIS_* NETWORK_REQUIRED
 commands=pytest --cov-config={toxinidir}/tox.ini --cov-report= {posargs}
 usedevelop=True