bpo-34632: Add importlib.metadata (GH-12547)

Add importlib.metadata module as forward port of the standalone importlib_metadata.

Doc/library/modules.rst

@@ -17,3 +17,4 @@ The full list of modules described in this chapter is:
    modulefinder.rst
    runpy.rst
    importlib.rst
+   importlib.metadata.rst

Doc/tools/susp-ignored.csv

@@ -350,3 +350,6 @@ whatsnew/3.7,,::,error::BytesWarning
 whatsnew/changelog,,::,error::BytesWarning
 whatsnew/changelog,,::,default::BytesWarning
 whatsnew/changelog,,::,default::DeprecationWarning
+library/importlib.metadata,,.. highlight:,.. highlight:: none
+library/importlib.metadata,,:main,"EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts')"
+library/importlib.metadata,,`,of directories ``path`` (defaults to sys.path).

Lib/importlib/_bootstrap_external.py

@@ -1363,6 +1363,58 @@ class PathFinder:
             return None
         return spec.loader
 
+    search_template = r'(?:{pattern}(-.*)?\.(dist|egg)-info|EGG-INFO)'
+
+    @classmethod
+    def find_distributions(cls, name=None, path=None):
+        """
+        Find distributions.
+
+        Return an iterable of all Distribution instances capable of
+        loading the metadata for packages matching the ``name``
+        (or all names if not supplied) along the paths in the list
+        of directories ``path`` (defaults to sys.path).
+        """
+        import re
+        from importlib.metadata import PathDistribution
+        if path is None:
+            path = sys.path
+        pattern = '.*' if name is None else re.escape(name)
+        found = cls._search_paths(pattern, path)
+        return map(PathDistribution, found)
+
+    @classmethod
+    def _search_paths(cls, pattern, paths):
+        """Find metadata directories in paths heuristically."""
+        import itertools
+        return itertools.chain.from_iterable(
+            cls._search_path(path, pattern)
+            for path in map(cls._switch_path, paths)
+            )
+
+    @staticmethod
+    def _switch_path(path):
+        from contextlib import suppress
+        import zipfile
+        from pathlib import Path
+        with suppress(Exception):
+            return zipfile.Path(path)
+        return Path(path)
+
+    @classmethod
+    def _predicate(cls, pattern, root, item):
+        import re
+        return re.match(pattern, str(item.name), flags=re.IGNORECASE)
+
+    @classmethod
+    def _search_path(cls, root, pattern):
+        if not root.is_dir():
+            return ()
+        normalized = pattern.replace('-', '_')
+        matcher = cls.search_template.format(pattern=normalized)
+        return (item for item in root.iterdir()
+                if cls._predicate(matcher, root, item))
+
 
 class FileFinder:
 

Lib/test/test_importlib/fixtures.py

+from __future__ import unicode_literals
+
+import os
+import sys
+import shutil
+import tempfile
+import textwrap
+import contextlib
+
+try:
+    from contextlib import ExitStack
+except ImportError:
+    from contextlib2 import ExitStack
+
+try:
+    import pathlib
+except ImportError:
+    import pathlib2 as pathlib
+
+
+__metaclass__ = type
+
+
+@contextlib.contextmanager
+def tempdir():
+    tmpdir = tempfile.mkdtemp()
+    try:
+        yield pathlib.Path(tmpdir)
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+@contextlib.contextmanager
+def save_cwd():
+    orig = os.getcwd()
+    try:
+        yield
+    finally:
+        os.chdir(orig)
+
+
+@contextlib.contextmanager
+def tempdir_as_cwd():
+    with tempdir() as tmp:
+        with save_cwd():
+            os.chdir(str(tmp))
+            yield tmp
+
+
+class SiteDir:
+    def setUp(self):
+        self.fixtures = ExitStack()
+        self.addCleanup(self.fixtures.close)
+        self.site_dir = self.fixtures.enter_context(tempdir())
+
+
+class OnSysPath:
+    @staticmethod
+    @contextlib.contextmanager
+    def add_sys_path(dir):
+        sys.path[:0] = [str(dir)]
+        try:
+            yield
+        finally:
+            sys.path.remove(str(dir))
+
+    def setUp(self):
+        super(OnSysPath, self).setUp()
+        self.fixtures.enter_context(self.add_sys_path(self.site_dir))
+
+
+class DistInfoPkg(OnSysPath, SiteDir):
+    files = {
+        "distinfo_pkg-1.0.0.dist-info": {
+            "METADATA": """
+                Name: distinfo-pkg
+                Author: Steven Ma
+                Version: 1.0.0
+                Requires-Dist: wheel >= 1.0
+                Requires-Dist: pytest; extra == 'test'
+                """,
+            "RECORD": "mod.py,sha256=abc,20\n",
+            "entry_points.txt": """
+                [entries]
+                main = mod:main
+            """
+            },
+        "mod.py": """
+            def main():
+                print("hello world")
+            """,
+        }
+
+    def setUp(self):
+        super(DistInfoPkg, self).setUp()
+        build_files(DistInfoPkg.files, self.site_dir)
+
+
+class DistInfoPkgOffPath(SiteDir):
+    def setUp(self):
+        super(DistInfoPkgOffPath, self).setUp()
+        build_files(DistInfoPkg.files, self.site_dir)
+
+
+class EggInfoPkg(OnSysPath, SiteDir):
+    files = {
+        "egginfo_pkg.egg-info": {
+            "PKG-INFO": """
+                Name: egginfo-pkg
+                Author: Steven Ma
+                License: Unknown
+                Version: 1.0.0
+                Classifier: Intended Audience :: Developers
+                Classifier: Topic :: Software Development :: Libraries
+                """,
+            "SOURCES.txt": """
+                mod.py
+                egginfo_pkg.egg-info/top_level.txt
+            """,
+            "entry_points.txt": """
+                [entries]
+                main = mod:main
+            """,
+            "requires.txt": """
+                wheel >= 1.0; python_version >= "2.7"
+                [test]
+                pytest
+            """,
+            "top_level.txt": "mod\n"
+            },
+        "mod.py": """
+            def main():
+                print("hello world")
+            """,
+        }
+
+    def setUp(self):
+        super(EggInfoPkg, self).setUp()
+        build_files(EggInfoPkg.files, prefix=self.site_dir)
+
+
+class EggInfoFile(OnSysPath, SiteDir):
+    files = {
+        "egginfo_file.egg-info": """
+            Metadata-Version: 1.0
+            Name: egginfo_file
+            Version: 0.1
+            Summary: An example package
+            Home-page: www.example.com
+            Author: Eric Haffa-Vee
+            Author-email: eric@example.coms
+            License: UNKNOWN
+            Description: UNKNOWN
+            Platform: UNKNOWN
+            """,
+        }
+
+    def setUp(self):
+        super(EggInfoFile, self).setUp()
+        build_files(EggInfoFile.files, prefix=self.site_dir)
+
+
+def build_files(file_defs, prefix=pathlib.Path()):
+    """Build a set of files/directories, as described by the
+
+    file_defs dictionary.  Each key/value pair in the dictionary is
+    interpreted as a filename/contents pair.  If the contents value is a
+    dictionary, a directory is created, and the dictionary interpreted
+    as the files within it, recursively.
+
+    For example:
+
+    {"README.txt": "A README file",
+     "foo": {
+        "__init__.py": "",
+        "bar": {
+            "__init__.py": "",
+        },
+        "baz.py": "# Some code",
+     }
+    }
+    """
+    for name, contents in file_defs.items():
+        full_name = prefix / name
+        if isinstance(contents, dict):
+            full_name.mkdir()
+            build_files(contents, prefix=full_name)
+        else:
+            if isinstance(contents, bytes):
+                with full_name.open('wb') as f:
+                    f.write(contents)
+            else:
+                with full_name.open('w') as f:
+                    f.write(DALS(contents))
+
+
+def DALS(str):
+    "Dedent and left-strip"
+    return textwrap.dedent(str).lstrip()

Lib/test/test_importlib/test_main.py

+# coding: utf-8
+
+import re
+import textwrap
+import unittest
+import importlib.metadata
+
+from . import fixtures
+from importlib.metadata import (
+    Distribution, EntryPoint,
+    PackageNotFoundError, distributions,
+    entry_points, metadata, version,
+    )
+
+
+class BasicTests(fixtures.DistInfoPkg, unittest.TestCase):
+    version_pattern = r'\d+\.\d+(\.\d)?'
+
+    def test_retrieves_version_of_self(self):
+        dist = Distribution.from_name('distinfo-pkg')
+        assert isinstance(dist.version, str)
+        assert re.match(self.version_pattern, dist.version)
+
+    def test_for_name_does_not_exist(self):
+        with self.assertRaises(PackageNotFoundError):
+            Distribution.from_name('does-not-exist')
+
+    def test_new_style_classes(self):
+        self.assertIsInstance(Distribution, type)
+
+
+class ImportTests(fixtures.DistInfoPkg, unittest.TestCase):
+    def test_import_nonexistent_module(self):
+        # Ensure that the MetadataPathFinder does not crash an import of a
+        # non-existant module.
+        with self.assertRaises(ImportError):
+            importlib.import_module('does_not_exist')
+
+    def test_resolve(self):
+        entries = dict(entry_points()['entries'])
+        ep = entries['main']
+        self.assertEqual(ep.load().__name__, "main")
+
+    def test_resolve_without_attr(self):
+        ep = EntryPoint(
+            name='ep',
+            value='importlib.metadata',
+            group='grp',
+            )
+        assert ep.load() is importlib.metadata
+
+
+class NameNormalizationTests(
+        fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
+    @staticmethod
+    def pkg_with_dashes(site_dir):
+        """
+        Create minimal metadata for a package with dashes
+        in the name (and thus underscores in the filename).
+        """
+        metadata_dir = site_dir / 'my_pkg.dist-info'
+        metadata_dir.mkdir()
+        metadata = metadata_dir / 'METADATA'
+        with metadata.open('w') as strm:
+            strm.write('Version: 1.0\n')
+        return 'my-pkg'
+
+    def test_dashes_in_dist_name_found_as_underscores(self):
+        """
+        For a package with a dash in the name, the dist-info metadata
+        uses underscores in the name. Ensure the metadata loads.
+        """
+        pkg_name = self.pkg_with_dashes(self.site_dir)
+        assert version(pkg_name) == '1.0'
+
+    @staticmethod
+    def pkg_with_mixed_case(site_dir):
+        """
+        Create minimal metadata for a package with mixed case
+        in the name.
+        """
+        metadata_dir = site_dir / 'CherryPy.dist-info'
+        metadata_dir.mkdir()
+        metadata = metadata_dir / 'METADATA'
+        with metadata.open('w') as strm:
+            strm.write('Version: 1.0\n')
+        return 'CherryPy'
+
+    def test_dist_name_found_as_any_case(self):
+        """
+        Ensure the metadata loads when queried with any case.
+        """
+        pkg_name = self.pkg_with_mixed_case(self.site_dir)
+        assert version(pkg_name) == '1.0'
+        assert version(pkg_name.lower()) == '1.0'
+        assert version(pkg_name.upper()) == '1.0'
+
+
+class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
+    @staticmethod
+    def pkg_with_non_ascii_description(site_dir):
+        """
+        Create minimal metadata for a package with non-ASCII in
+        the description.
+        """
+        metadata_dir = site_dir / 'portend.dist-info'
+        metadata_dir.mkdir()
+        metadata = metadata_dir / 'METADATA'
+        with metadata.open('w', encoding='utf-8') as fp:
+            fp.write('Description: pôrˈtend\n')
+        return 'portend'
+
+    @staticmethod
+    def pkg_with_non_ascii_description_egg_info(site_dir):
+        """
+        Create minimal metadata for an egg-info package with
+        non-ASCII in the description.
+        """
+        metadata_dir = site_dir / 'portend.dist-info'
+        metadata_dir.mkdir()
+        metadata = metadata_dir / 'METADATA'
+        with metadata.open('w', encoding='utf-8') as fp:
+            fp.write(textwrap.dedent("""
+                Name: portend
+
+                pôrˈtend
+                """).lstrip())
+        return 'portend'
+
+    def test_metadata_loads(self):
+        pkg_name = self.pkg_with_non_ascii_description(self.site_dir)
+        meta = metadata(pkg_name)
+        assert meta['Description'] == 'pôrˈtend'
+
+    def test_metadata_loads_egg_info(self):
+        pkg_name = self.pkg_with_non_ascii_description_egg_info(self.site_dir)
+        meta = metadata(pkg_name)
+        assert meta.get_payload() == 'pôrˈtend\n'
+
+
+class DiscoveryTests(fixtures.EggInfoPkg,
+                     fixtures.DistInfoPkg,
+                     unittest.TestCase):
+
+    def test_package_discovery(self):
+        dists = list(distributions())
+        assert all(
+            isinstance(dist, Distribution)
+            for dist in dists
+            )
+        assert any(
+            dist.metadata['Name'] == 'egginfo-pkg'
+            for dist in dists
+            )
+        assert any(
+            dist.metadata['Name'] == 'distinfo-pkg'
+            for dist in dists
+            )

Lib/test/test_importlib/test_metadata_api.py

+import re
+import textwrap
+import unittest
+import itertools
+
+from collections.abc import Iterator
+
+from . import fixtures
+from importlib.metadata import (
+    Distribution, PackageNotFoundError, distribution,
+    entry_points, files, metadata, requires, version,
+    )
+
+
+class APITests(
+        fixtures.EggInfoPkg,
+        fixtures.DistInfoPkg,
+        fixtures.EggInfoFile,
+        unittest.TestCase):
+
+    version_pattern = r'\d+\.\d+(\.\d)?'
+
+    def test_retrieves_version_of_self(self):
+        pkg_version = version('egginfo-pkg')
+        assert isinstance(pkg_version, str)
+        assert re.match(self.version_pattern, pkg_version)
+
+    def test_retrieves_version_of_distinfo_pkg(self):
+        pkg_version = version('distinfo-pkg')
+        assert isinstance(pkg_version, str)
+        assert re.match(self.version_pattern, pkg_version)
+
+    def test_for_name_does_not_exist(self):
+        with self.assertRaises(PackageNotFoundError):
+            distribution('does-not-exist')
+
+    def test_for_top_level(self):
+        self.assertEqual(
+            distribution('egginfo-pkg').read_text('top_level.txt').strip(),
+            'mod')
+
+    def test_read_text(self):
+        top_level = [
+            path for path in files('egginfo-pkg')
+            if path.name == 'top_level.txt'
+            ][0]
+        self.assertEqual(top_level.read_text(), 'mod\n')
+
+    def test_entry_points(self):
+        entries = dict(entry_points()['entries'])
+        ep = entries['main']
+        self.assertEqual(ep.value, 'mod:main')
+        self.assertEqual(ep.extras, [])
+
+    def test_metadata_for_this_package(self):
+        md = metadata('egginfo-pkg')
+        assert md['author'] == 'Steven Ma'
+        assert md['LICENSE'] == 'Unknown'
+        assert md['Name'] == 'egginfo-pkg'
+        classifiers = md.get_all('Classifier')
+        assert 'Topic :: Software Development :: Libraries' in classifiers
+
+    @staticmethod
+    def _test_files(files_iter):
+        assert isinstance(files_iter, Iterator), files_iter
+        files = list(files_iter)
+        root = files[0].root
+        for file in files:
+            assert file.root == root
+            assert not file.hash or file.hash.value
+            assert not file.hash or file.hash.mode == 'sha256'
+            assert not file.size or file.size >= 0
+            assert file.locate().exists()
+            assert isinstance(file.read_binary(), bytes)
+            if file.name.endswith('.py'):
+                file.read_text()
+
+    def test_file_hash_repr(self):
+        assertRegex = self.assertRegex
+
+        util = [
+            p for p in files('distinfo-pkg')
+            if p.name == 'mod.py'
+            ][0]
+        assertRegex(
+            repr(util.hash),
+            '<FileHash mode: sha256 value: .*>')
+
+    def test_files_dist_info(self):
+        self._test_files(files('distinfo-pkg'))
+
+    def test_files_egg_info(self):
+        self._test_files(files('egginfo-pkg'))
+
+    def test_version_egg_info_file(self):
+        self.assertEqual(version('egginfo-file'), '0.1')
+
+    def test_requires_egg_info_file(self):
+        requirements = requires('egginfo-file')
+        self.assertIsNone(requirements)
+
+    def test_requires(self):
+        deps = requires('egginfo-pkg')
+        assert any(
+            dep == 'wheel >= 1.0; python_version >= "2.7"'
+            for dep in deps
+            )
+
+    def test_requires_dist_info(self):
+        deps = list(requires('distinfo-pkg'))
+        assert deps and all(deps)
+
+    def test_more_complex_deps_requires_text(self):
+        requires = textwrap.dedent("""
+            dep1
+            dep2
+
+            [:python_version < "3"]
+            dep3
+
+            [extra1]
+            dep4
+
+            [extra2:python_version < "3"]
+            dep5
+            """)
+        deps = sorted(Distribution._deps_from_requires_text(requires))
+        expected = [
+            'dep1',
+            'dep2',
+            'dep3; python_version < "3"',
+            'dep4; extra == "extra1"',
+            'dep5; (python_version < "3") and extra == "extra2"',
+            ]
+        # It's important that the environment marker expression be
+        # wrapped in parentheses to avoid the following 'and' binding more
+        # tightly than some other part of the environment expression.
+
+        assert deps == expected
+
+
+class OffSysPathTests(fixtures.DistInfoPkgOffPath, unittest.TestCase):
+    def test_find_distributions_specified_path(self):
+        dists = itertools.chain.from_iterable(
+            resolver(path=[str(self.site_dir)])
+            for resolver in Distribution._discover_resolvers()
+            )
+        assert any(
+            dist.metadata['Name'] == 'distinfo-pkg'
+            for dist in dists
+            )

Lib/test/test_importlib/test_zip.py

+import sys
+import unittest
+
+from contextlib import ExitStack
+from importlib.metadata import distribution, entry_points, files, version
+from importlib.resources import path
+
+
+class TestZip(unittest.TestCase):
+    root = 'test.test_importlib.data'
+
+    def setUp(self):
+        # Find the path to the example-*.whl so we can add it to the front of
+        # sys.path, where we'll then try to find the metadata thereof.
+        self.resources = ExitStack()
+        self.addCleanup(self.resources.close)
+        wheel = self.resources.enter_context(
+            path(self.root, 'example-21.12-py3-none-any.whl'))
+        sys.path.insert(0, str(wheel))
+        self.resources.callback(sys.path.pop, 0)
+
+    def test_zip_version(self):
+        self.assertEqual(version('example'), '21.12')
+
+    def test_zip_entry_points(self):
+        scripts = dict(entry_points()['console_scripts'])
+        entry_point = scripts['example']
+        self.assertEqual(entry_point.value, 'example:main')
+
+    def test_missing_metadata(self):
+        self.assertIsNone(distribution('example').read_text('does not exist'))
+
+    def test_case_insensitive(self):
+        self.assertEqual(version('Example'), '21.12')
+
+    def test_files(self):
+        for file in files('example'):
+            path = str(file.dist.locate_file(file))
+            assert '.whl/' in path, path
+
+
+class TestEgg(TestZip):
+    def setUp(self):
+        # Find the path to the example-*.egg so we can add it to the front of
+        # sys.path, where we'll then try to find the metadata thereof.
+        self.resources = ExitStack()
+        self.addCleanup(self.resources.close)
+        egg = self.resources.enter_context(
+            path(self.root, 'example-21.12-py3.6.egg'))
+        sys.path.insert(0, str(egg))
+        self.resources.callback(sys.path.pop, 0)
+
+    def test_files(self):
+        for file in files('example'):
+            path = str(file.dist.locate_file(file))
+            assert '.egg/' in path, path

Misc/NEWS.d/next/Library/2019-05-11-02-30-45.bpo-34632.8MXa7T.rst

+Introduce the ``importlib.metadata`` module with (provisional) support for reading metadata from third-party packages.
\ No newline at end of file

Python.framework/Resources

+Versions/Current/Resources
\ No newline at end of file