Merge branch 'master' into add_markdown_to_possible_readmes

CHANGES.rst

-v36.3.1
+v36.4.0
+-------
+
+* #1075: Add new ``Description-Content-Type`` metadata field. `See here for
+  documentation on how to use this field.
+  <https://packaging.python.org/specifications/#description-content-type>`_
+
+* #1068: Sort files and directories when building eggs for
+  deterministic order.
+
+* #196: Remove caching of easy_install command in fetch_build_egg.
+  Fixes issue where ``pytest-runner-N.N`` would satisfy the installation
+  of ``pytest``.
+
+* #1129: Fix working set dependencies handling when replacing conflicting
+  distributions (e.g. when using ``setup_requires`` with a conflicting
+  transitive dependency, fix #1124).
 
-* Improved handling of README files extensions and added 
+* #1133: Improved handling of README files extensions and added 
   Markdown to the list of searched READMES. 
 
 v36.3.0

docs/setuptools.txt

@@ -2394,27 +2394,28 @@ Metadata
     Aliases given below are supported for compatibility reasons,
     but not advised.
 
-=================  =================  =====
-Key                Aliases            Accepted value type
-=================  =================  =====
-name                                  str
-version                               attr:, str
-url                home-page          str
-download_url       download-url       str
-author                                str
-author_email       author-email       str
-maintainer                            str
-maintainer_email   maintainer-email   str
-classifiers        classifier         file:, list-comma
-license                               file:, str
-description        summary            file:, str
-long_description   long-description   file:, str
-keywords                              list-comma
-platforms          platform           list-comma
-provides                              list-comma
-requires                              list-comma
-obsoletes                             list-comma
-=================  =================  =====
+==============================  =================  =====
+Key                             Aliases            Accepted value type
+==============================  =================  =====
+name                                               str
+version                                            attr:, str
+url                             home-page          str
+download_url                    download-url       str
+author                                             str
+author_email                    author-email       str
+maintainer                                         str
+maintainer_email                maintainer-email   str
+classifiers                     classifier         file:, list-comma
+license                                            file:, str
+description                     summary            file:, str
+long_description                long-description   file:, str
+long_description_content_type                      str
+keywords                                           list-comma
+platforms                       platform           list-comma
+provides                                           list-comma
+requires                                           list-comma
+obsoletes                                          list-comma
+==============================  =================  =====
 
 .. note::
 

pkg_resources/__init__.py

@@ -852,7 +852,10 @@ class WorkingSet(object):
                             # distribution
                             env = Environment([])
                             ws = WorkingSet([])
-                    dist = best[req.key] = env.best_match(req, ws, installer)
+                    dist = best[req.key] = env.best_match(
+                        req, ws, installer,
+                        replace_conflicting=replace_conflicting
+                    )
                     if dist is None:
                         requirers = required_by.get(req, None)
                         raise DistributionNotFound(req, requirers)
@@ -1104,7 +1107,7 @@ class Environment(object):
                 dists.append(dist)
                 dists.sort(key=operator.attrgetter('hashcmp'), reverse=True)
 
-    def best_match(self, req, working_set, installer=None):
+    def best_match(self, req, working_set, installer=None, replace_conflicting=False):
         """Find distribution best matching `req` and usable on `working_set`
 
         This calls the ``find(req)`` method of the `working_set` to see if a
@@ -1117,7 +1120,12 @@ class Environment(object):
         calling the environment's ``obtain(req, installer)`` method will be
         returned.
         """
-        dist = working_set.find(req)
+        try:
+            dist = working_set.find(req)
+        except VersionConflict:
+            if not replace_conflicting:
+                raise
+            dist = None
         if dist is not None:
             return dist
         for dist in self[req.key]:

pkg_resources/tests/test_working_set.py

+import inspect
+import re
+import textwrap
+
+import pytest
+
+import pkg_resources
+
+from .test_resources import Metadata
+
+
+def strip_comments(s):
+    return '\n'.join(
+        l for l in s.split('\n')
+        if l.strip() and not l.strip().startswith('#')
+    )
+
+def parse_distributions(s):
+    '''
+    Parse a series of distribution specs of the form:
+    {project_name}-{version}
+       [optional, indented requirements specification]
+
+    Example:
+
+        foo-0.2
+        bar-1.0
+          foo>=3.0
+          [feature]
+          baz
+
+    yield 2 distributions:
+        - project_name=foo, version=0.2
+        - project_name=bar, version=1.0, requires=['foo>=3.0', 'baz; extra=="feature"']
+    '''
+    s = s.strip()
+    for spec in re.split('\n(?=[^\s])', s):
+        if not spec:
+            continue
+        fields = spec.split('\n', 1)
+        assert 1 <= len(fields) <= 2
+        name, version = fields.pop(0).split('-')
+        if fields:
+            requires = textwrap.dedent(fields.pop(0))
+            metadata=Metadata(('requires.txt', requires))
+        else:
+            metadata = None
+        dist = pkg_resources.Distribution(project_name=name,
+                                          version=version,
+                                          metadata=metadata)
+        yield dist
+
+
+class FakeInstaller(object):
+
+    def __init__(self, installable_dists):
+        self._installable_dists = installable_dists
+
+    def __call__(self, req):
+        return next(iter(filter(lambda dist: dist in req,
+                                self._installable_dists)), None)
+
+
+def parametrize_test_working_set_resolve(*test_list):
+    idlist = []
+    argvalues = []
+    for test in test_list:
+        (
+            name,
+            installed_dists,
+            installable_dists,
+            requirements,
+            expected1, expected2
+        ) = [
+            strip_comments(s.lstrip()) for s in
+            textwrap.dedent(test).lstrip().split('\n\n', 5)
+        ]
+        installed_dists = list(parse_distributions(installed_dists))
+        installable_dists = list(parse_distributions(installable_dists))
+        requirements = list(pkg_resources.parse_requirements(requirements))
+        for id_, replace_conflicting, expected in (
+            (name, False, expected1),
+            (name + '_replace_conflicting', True, expected2),
+        ):
+            idlist.append(id_)
+            expected = strip_comments(expected.strip())
+            if re.match('\w+$', expected):
+                expected = getattr(pkg_resources, expected)
+                assert issubclass(expected, Exception)
+            else:
+                expected = list(parse_distributions(expected))
+            argvalues.append(pytest.param(installed_dists, installable_dists,
+                                          requirements, replace_conflicting,
+                                          expected))
+    return pytest.mark.parametrize('installed_dists,installable_dists,'
+                                   'requirements,replace_conflicting,'
+                                   'resolved_dists_or_exception',
+                                   argvalues, ids=idlist)
+
+
+@parametrize_test_working_set_resolve(
+    '''
+    # id
+    noop
+
+    # installed
+
+    # installable
+
+    # wanted
+
+    # resolved
+
+    # resolved [replace conflicting]
+    ''',
+
+    '''
+    # id
+    already_installed
+
+    # installed
+    foo-3.0
+
+    # installable
+
+    # wanted
+    foo>=2.1,!=3.1,<4
+
+    # resolved
+    foo-3.0
+
+    # resolved [replace conflicting]
+    foo-3.0
+    ''',
+
+    '''
+    # id
+    installable_not_installed
+
+    # installed
+
+    # installable
+    foo-3.0
+    foo-4.0
+
+    # wanted
+    foo>=2.1,!=3.1,<4
+
+    # resolved
+    foo-3.0
+
+    # resolved [replace conflicting]
+    foo-3.0
+    ''',
+
+    '''
+    # id
+    not_installable
+
+    # installed
+
+    # installable
+
+    # wanted
+    foo>=2.1,!=3.1,<4
+
+    # resolved
+    DistributionNotFound
+
+    # resolved [replace conflicting]
+    DistributionNotFound
+    ''',
+
+    '''
+    # id
+    no_matching_version
+
+    # installed
+
+    # installable
+    foo-3.1
+
+    # wanted
+    foo>=2.1,!=3.1,<4
+
+    # resolved
+    DistributionNotFound
+
+    # resolved [replace conflicting]
+    DistributionNotFound
+    ''',
+
+    '''
+    # id
+    installable_with_installed_conflict
+
+    # installed
+    foo-3.1
+
+    # installable
+    foo-3.5
+
+    # wanted
+    foo>=2.1,!=3.1,<4
+
+    # resolved
+    VersionConflict
+
+    # resolved [replace conflicting]
+    foo-3.5
+    ''',
+
+    '''
+    # id
+    not_installable_with_installed_conflict
+
+    # installed
+    foo-3.1
+
+    # installable
+
+    # wanted
+    foo>=2.1,!=3.1,<4
+
+    # resolved
+    VersionConflict
+
+    # resolved [replace conflicting]
+    DistributionNotFound
+    ''',
+
+    '''
+    # id
+    installed_with_installed_require
+
+    # installed
+    foo-3.9
+    baz-0.1
+        foo>=2.1,!=3.1,<4
+
+    # installable
+
+    # wanted
+    baz
+
+    # resolved
+    foo-3.9
+    baz-0.1
+
+    # resolved [replace conflicting]
+    foo-3.9
+    baz-0.1
+    ''',
+
+    '''
+    # id
+    installed_with_conflicting_installed_require
+
+    # installed
+    foo-5
+    baz-0.1
+        foo>=2.1,!=3.1,<4
+
+    # installable
+
+    # wanted
+    baz
+
+    # resolved
+    VersionConflict
+
+    # resolved [replace conflicting]
+    DistributionNotFound
+    ''',
+
+    '''
+    # id
+    installed_with_installable_conflicting_require
+
+    # installed
+    foo-5
+    baz-0.1
+        foo>=2.1,!=3.1,<4
+
+    # installable
+    foo-2.9
+
+    # wanted
+    baz
+
+    # resolved
+    VersionConflict
+
+    # resolved [replace conflicting]
+    baz-0.1
+    foo-2.9
+    ''',
+
+    '''
+    # id
+    installed_with_installable_require
+
+    # installed
+    baz-0.1
+        foo>=2.1,!=3.1,<4
+
+    # installable
+    foo-3.9
+
+    # wanted
+    baz
+
+    # resolved
+    foo-3.9
+    baz-0.1
+
+    # resolved [replace conflicting]
+    foo-3.9
+    baz-0.1
+    ''',
+
+    '''
+    # id
+    installable_with_installed_require
+
+    # installed
+    foo-3.9
+
+    # installable
+    baz-0.1
+        foo>=2.1,!=3.1,<4
+
+    # wanted
+    baz
+
+    # resolved
+    foo-3.9
+    baz-0.1
+
+    # resolved [replace conflicting]
+    foo-3.9
+    baz-0.1
+    ''',
+
+    '''
+    # id
+    installable_with_installable_require
+
+    # installed
+
+    # installable
+    foo-3.9
+    baz-0.1
+        foo>=2.1,!=3.1,<4
+
+    # wanted
+    baz
+
+    # resolved
+    foo-3.9
+    baz-0.1
+
+    # resolved [replace conflicting]
+    foo-3.9
+    baz-0.1
+    ''',
+
+    '''
+    # id
+    installable_with_conflicting_installable_require
+
+    # installed
+    foo-5
+
+    # installable
+    foo-2.9
+    baz-0.1
+        foo>=2.1,!=3.1,<4
+
+    # wanted
+    baz
+
+    # resolved
+    VersionConflict
+
+    # resolved [replace conflicting]
+    baz-0.1
+    foo-2.9
+    ''',
+
+    '''
+    # id
+    conflicting_installables
+
+    # installed
+
+    # installable
+    foo-2.9
+    foo-5.0
+
+    # wanted
+    foo>=2.1,!=3.1,<4
+    foo>=4
+
+    # resolved
+    VersionConflict
+
+    # resolved [replace conflicting]
+    VersionConflict
+    ''',
+
+    '''
+    # id
+    installables_with_conflicting_requires
+
+    # installed
+
+    # installable
+    foo-2.9
+        dep==1.0
+    baz-5.0
+        dep==2.0
+    dep-1.0
+    dep-2.0
+
+    # wanted
+    foo
+    baz
+
+    # resolved
+    VersionConflict
+
+    # resolved [replace conflicting]
+    VersionConflict
+    ''',
+
+    '''
+    # id
+    installables_with_conflicting_nested_requires
+
+    # installed
+
+    # installable
+    foo-2.9
+        dep1
+    dep1-1.0
+        subdep<1.0
+    baz-5.0
+        dep2
+    dep2-1.0
+        subdep>1.0
+    subdep-0.9
+    subdep-1.1
+
+    # wanted
+    foo
+    baz
+
+    # resolved
+    VersionConflict
+
+    # resolved [replace conflicting]
+    VersionConflict
+    ''',
+)
+def test_working_set_resolve(installed_dists, installable_dists, requirements,
+                             replace_conflicting, resolved_dists_or_exception):
+    ws = pkg_resources.WorkingSet([])
+    list(map(ws.add, installed_dists))
+    resolve_call = lambda: ws.resolve(
+        requirements, installer=FakeInstaller(installable_dists),
+        replace_conflicting=replace_conflicting,
+    )
+    if inspect.isclass(resolved_dists_or_exception):
+        with pytest.raises(resolved_dists_or_exception):
+            resolve_call()
+    else:
+        assert sorted(resolve_call()) == sorted(resolved_dists_or_exception)

setup.py

@@ -95,6 +95,7 @@ setup_params = dict(
     author="Python Packaging Authority",
     author_email="distutils-sig@python.org",
     long_description=long_description,
+    long_description_content_type='text/x-rst; charset=UTF-8',
     keywords="CPAN PyPI distutils eggs package management",
     url="https://github.com/pypa/setuptools",
     src_root=None,

setuptools/command/bdist_egg.py

@@ -38,6 +38,14 @@ def strip_module(filename):
         filename = filename[:-6]
     return filename
 
+def sorted_walk(dir):
+    """Do os.walk in a reproducible way,
+    independent of indeterministic filesystem readdir order
+    """
+    for base, dirs, files in os.walk(dir):
+        dirs.sort()
+        files.sort()
+        yield base, dirs, files
 
 def write_stub(resource, pyfile):
     _stub_template = textwrap.dedent("""
@@ -302,7 +310,7 @@ class bdist_egg(Command):
         ext_outputs = []
 
         paths = {self.bdist_dir: ''}
-        for base, dirs, files in os.walk(self.bdist_dir):
+        for base, dirs, files in sorted_walk(self.bdist_dir):
             for filename in files:
                 if os.path.splitext(filename)[1].lower() in NATIVE_EXTENSIONS:
                     all_outputs.append(paths[base] + filename)
@@ -329,7 +337,7 @@ NATIVE_EXTENSIONS = dict.fromkeys('.dll .so .dylib .pyd'.split())
 
 def walk_egg(egg_dir):
     """Walk an unpacked egg's contents, skipping the metadata directory"""
-    walker = os.walk(egg_dir)
+    walker = sorted_walk(egg_dir)
     base, dirs, files = next(walker)
     if 'EGG-INFO' in dirs:
         dirs.remove('EGG-INFO')
@@ -463,10 +471,10 @@ def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=True,
     compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED
     if not dry_run:
         z = zipfile.ZipFile(zip_filename, mode, compression=compression)
-        for dirname, dirs, files in os.walk(base_dir):
+        for dirname, dirs, files in sorted_walk(base_dir):
             visit(z, dirname, files)
         z.close()
     else:
-        for dirname, dirs, files in os.walk(base_dir):
+        for dirname, dirs, files in sorted_walk(base_dir):
             visit(None, dirname, files)
     return zip_filename

setuptools/command/egg_info.py

@@ -599,6 +599,10 @@ def write_pkg_info(cmd, basename, filename):
         metadata = cmd.distribution.metadata
         metadata.version, oldver = cmd.egg_version, metadata.version
         metadata.name, oldname = cmd.egg_name, metadata.name
+        metadata.long_description_content_type = getattr(
+            cmd.distribution,
+            'long_description_content_type'
+        )
         try:
             # write unescaped data to PKG-INFO, so older pkg_resources
             # can still parse it

setuptools/dist.py

@@ -58,6 +58,13 @@ def write_pkg_file(self, file):
     if self.download_url:
         file.write('Download-URL: %s\n' % self.download_url)
 
+    long_desc_content_type = getattr(
+        self,
+        'long_description_content_type',
+        None
+    ) or 'UNKNOWN'
+    file.write('Description-Content-Type: %s\n' % long_desc_content_type)
+
     long_desc = rfc822_escape(self.get_long_description())
     file.write('Description: %s\n' % long_desc)
 
@@ -317,6 +324,9 @@ class Distribution(Distribution_parse_config_files, _Distribution):
         self.dist_files = []
         self.src_root = attrs and attrs.pop("src_root", None)
         self.patch_missing_pkg_info(attrs)
+        self.long_description_content_type = _attrs_dict.get(
+            'long_description_content_type'
+        )
         # Make sure we have any eggs needed to interpret 'attrs'
         if attrs is not None:
             self.dependency_links = attrs.pop('dependency_links', [])
@@ -485,36 +495,30 @@ class Distribution(Distribution_parse_config_files, _Distribution):
 
     def fetch_build_egg(self, req):
         """Fetch an egg needed for building"""
-
-        try:
-            cmd = self._egg_fetcher
-            cmd.package_index.to_scan = []
-        except AttributeError:
-            from setuptools.command.easy_install import easy_install
-            dist = self.__class__({'script_args': ['easy_install']})
-            dist.parse_config_files()
-            opts = dist.get_option_dict('easy_install')
-            keep = (
-                'find_links', 'site_dirs', 'index_url', 'optimize',
-                'site_dirs', 'allow_hosts'
-            )
-            for key in list(opts):
-                if key not in keep:
-                    del opts[key]  # don't use any other settings
-            if self.dependency_links:
-                links = self.dependency_links[:]
-                if 'find_links' in opts:
-                    links = opts['find_links'][1].split() + links
-                opts['find_links'] = ('setup', links)
-            install_dir = self.get_egg_cache_dir()
-            cmd = easy_install(
-                dist, args=["x"], install_dir=install_dir,
-                exclude_scripts=True,
-                always_copy=False, build_directory=None, editable=False,
-                upgrade=False, multi_version=True, no_report=True, user=False
-            )
-            cmd.ensure_finalized()
-            self._egg_fetcher = cmd
+        from setuptools.command.easy_install import easy_install
+        dist = self.__class__({'script_args': ['easy_install']})
+        dist.parse_config_files()
+        opts = dist.get_option_dict('easy_install')
+        keep = (
+            'find_links', 'site_dirs', 'index_url', 'optimize',
+            'site_dirs', 'allow_hosts'
+        )
+        for key in list(opts):
+            if key not in keep:
+                del opts[key]  # don't use any other settings
+        if self.dependency_links:
+            links = self.dependency_links[:]
+            if 'find_links' in opts:
+                links = opts['find_links'][1].split() + links
+            opts['find_links'] = ('setup', links)
+        install_dir = self.get_egg_cache_dir()
+        cmd = easy_install(
+            dist, args=["x"], install_dir=install_dir,
+            exclude_scripts=True,
+            always_copy=False, build_directory=None, editable=False,
+            upgrade=False, multi_version=True, no_report=True, user=False
+        )
+        cmd.ensure_finalized()
         return cmd.easy_install(req)
 
     def _set_global_opts_from_features(self):

setuptools/tests/test_dist.py

+from setuptools import Distribution
+from setuptools.extern.six.moves.urllib.request import pathname2url
+from setuptools.extern.six.moves.urllib_parse import urljoin
+
+from .textwrap import DALS
+from .test_easy_install import make_nspkg_sdist
+
+
+def test_dist_fetch_build_egg(tmpdir):
+    """
+    Check multiple calls to `Distribution.fetch_build_egg` work as expected.
+    """
+    index = tmpdir.mkdir('index')
+    index_url = urljoin('file://', pathname2url(str(index)))
+    def sdist_with_index(distname, version):
+        dist_dir = index.mkdir(distname)
+        dist_sdist = '%s-%s.tar.gz' % (distname, version)
+        make_nspkg_sdist(str(dist_dir.join(dist_sdist)), distname, version)
+        with dist_dir.join('index.html').open('w') as fp:
+            fp.write(DALS(
+                '''
+                <!DOCTYPE html><html><body>
+                <a href="{dist_sdist}" rel="internal">{dist_sdist}</a><br/>
+                </body></html>
+                '''
+            ).format(dist_sdist=dist_sdist))
+    sdist_with_index('barbazquux', '3.2.0')
+    sdist_with_index('barbazquux-runner', '2.11.1')
+    with tmpdir.join('setup.cfg').open('w') as fp:
+        fp.write(DALS(
+            '''
+            [easy_install]
+            index_url = {index_url}
+            '''
+        ).format(index_url=index_url))
+    reqs = '''
+    barbazquux-runner
+    barbazquux
+    '''.split()
+    with tmpdir.as_cwd():
+        dist = Distribution()
+        resolved_dists = [
+            dist.fetch_build_egg(r)
+            for r in reqs
+        ]
+    assert [dist.key for dist in resolved_dists if dist] == reqs

setuptools/tests/test_egg_info.py

@@ -398,6 +398,31 @@ class TestEggInfo(object):
             self._run_install_command(tmpdir_cwd, env)
         assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == []
 
+    def test_long_description_content_type(self, tmpdir_cwd, env):
+        # Test that specifying a `long_description_content_type` keyword arg to
+        # the `setup` function results in writing a `Description-Content-Type`
+        # line to the `PKG-INFO` file in the `<distribution>.egg-info`
+        # directory.
+        # `Description-Content-Type` is described at
+        # https://github.com/pypa/python-packaging-user-guide/pull/258
+
+        self._setup_script_with_requires(
+            """long_description_content_type='text/markdown',""")
+        environ = os.environ.copy().update(
+            HOME=env.paths['home'],
+        )
+        code, data = environment.run_setup_py(
+            cmd=['egg_info'],
+            pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]),
+            data_stream=1,
+            env=environ,
+        )
+        egg_info_dir = os.path.join('.', 'foo.egg-info')
+        with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file:
+            pkg_info_lines = pkginfo_file.read().split('\n')
+        expected_line = 'Description-Content-Type: text/markdown'
+        assert expected_line in pkg_info_lines
+
     def test_python_requires_egg_info(self, tmpdir_cwd, env):
         self._setup_script_with_requires(
             """python_requires='>=2.7.12',""")