Merge pull request #1200 from benoit-pierre/wheels_support

easy_install: add support for installing from wheels

setuptools/command/easy_install.py

@@ -53,6 +53,7 @@ from setuptools.package_index import (
     PackageIndex, parse_requirement_arg, URL_SCHEME,
 )
 from setuptools.command import bdist_egg, egg_info
+from setuptools.wheel import Wheel
 from pkg_resources import (
     yield_lines, normalize_path, resource_string, ensure_directory,
     get_distribution, find_distributions, Environment, Requirement,
@@ -842,6 +843,8 @@ class easy_install(Command):
             return [self.install_egg(dist_filename, tmpdir)]
         elif dist_filename.lower().endswith('.exe'):
             return [self.install_exe(dist_filename, tmpdir)]
+        elif dist_filename.lower().endswith('.whl'):
+            return [self.install_wheel(dist_filename, tmpdir)]
 
         # Anything else, try to extract and build
         setup_base = tmpdir
@@ -1038,6 +1041,35 @@ class easy_install(Command):
                     f.write('\n'.join(locals()[name]) + '\n')
                     f.close()
 
+    def install_wheel(self, wheel_path, tmpdir):
+        wheel = Wheel(wheel_path)
+        assert wheel.is_compatible()
+        destination = os.path.join(self.install_dir, wheel.egg_name())
+        destination = os.path.abspath(destination)
+        if not self.dry_run:
+            ensure_directory(destination)
+        if os.path.isdir(destination) and not os.path.islink(destination):
+            dir_util.remove_tree(destination, dry_run=self.dry_run)
+        elif os.path.exists(destination):
+            self.execute(
+                os.unlink,
+                (destination,),
+                "Removing " + destination,
+            )
+        try:
+            self.execute(
+                wheel.install_as_egg,
+                (destination,),
+                ("Installing %s to %s") % (
+                    os.path.basename(wheel_path),
+                    os.path.dirname(destination)
+                ),
+            )
+        finally:
+            update_dist_caches(destination, fix_zipimporter_caches=False)
+        self.add_output(destination)
+        return self.egg_distribution(destination)
+
     __mv_warning = textwrap.dedent("""
         Because this distribution was installed --multi-version, before you can
         import modules from this package in an application, you will need to

setuptools/glibc.py

+# This file originally from pip:
+# https://github.com/pypa/pip/blob/8f4f15a5a95d7d5b511ceaee9ed261176c181970/src/pip/_internal/utils/glibc.py
+from __future__ import absolute_import
+
+import ctypes
+import re
+import warnings
+
+
+def glibc_version_string():
+    "Returns glibc version string, or None if not using glibc."
+
+    # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen
+    # manpage says, "If filename is NULL, then the returned handle is for the
+    # main program". This way we can let the linker do the work to figure out
+    # which libc our process is actually using.
+    process_namespace = ctypes.CDLL(None)
+    try:
+        gnu_get_libc_version = process_namespace.gnu_get_libc_version
+    except AttributeError:
+        # Symbol doesn't exist -> therefore, we are not linked to
+        # glibc.
+        return None
+
+    # Call gnu_get_libc_version, which returns a string like "2.5"
+    gnu_get_libc_version.restype = ctypes.c_char_p
+    version_str = gnu_get_libc_version()
+    # py2 / py3 compatibility:
+    if not isinstance(version_str, str):
+        version_str = version_str.decode("ascii")
+
+    return version_str
+
+
+# Separated out from have_compatible_glibc for easier unit testing
+def check_glibc_version(version_str, required_major, minimum_minor):
+    # Parse string and check against requested version.
+    #
+    # We use a regexp instead of str.split because we want to discard any
+    # random junk that might come after the minor version -- this might happen
+    # in patched/forked versions of glibc (e.g. Linaro's version of glibc
+    # uses version strings like "2.20-2014.11"). See gh-3588.
+    m = re.match(r"(?P<major>[0-9]+)\.(?P<minor>[0-9]+)", version_str)
+    if not m:
+        warnings.warn("Expected glibc version with 2 components major.minor,"
+                      " got: %s" % version_str, RuntimeWarning)
+        return False
+    return (int(m.group("major")) == required_major and
+            int(m.group("minor")) >= minimum_minor)
+
+
+def have_compatible_glibc(required_major, minimum_minor):
+    version_str = glibc_version_string()
+    if version_str is None:
+        return False
+    return check_glibc_version(version_str, required_major, minimum_minor)
+
+
+# platform.libc_ver regularly returns completely nonsensical glibc
+# versions. E.g. on my computer, platform says:
+#
+#   ~$ python2.7 -c 'import platform; print(platform.libc_ver())'
+#   ('glibc', '2.7')
+#   ~$ python3.5 -c 'import platform; print(platform.libc_ver())'
+#   ('glibc', '2.9')
+#
+# But the truth is:
+#
+#   ~$ ldd --version
+#   ldd (Debian GLIBC 2.22-11) 2.22
+#
+# This is unfortunate, because it means that the linehaul data on libc
+# versions that was generated by pip 8.1.2 and earlier is useless and
+# misleading. Solution: instead of using platform, use our code that actually
+# works.
+def libc_ver():
+    """Try to determine the glibc version
+
+    Returns a tuple of strings (lib, version) which default to empty strings
+    in case the lookup fails.
+    """
+    glibc_version = glibc_version_string()
+    if glibc_version is None:
+        return ("", "")
+    else:
+        return ("glibc", glibc_version)

setuptools/package_index.py

@@ -21,13 +21,14 @@ import setuptools
 from pkg_resources import (
     CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST,
     Environment, find_distributions, safe_name, safe_version,
-    to_filename, Requirement, DEVELOP_DIST,
+    to_filename, Requirement, DEVELOP_DIST, EGG_DIST,
 )
 from setuptools import ssl_support
 from distutils import log
 from distutils.errors import DistutilsError
 from fnmatch import translate
 from setuptools.py27compat import get_all_headers
+from setuptools.wheel import Wheel
 
 EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$')
 HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I)
@@ -115,6 +116,17 @@ def distros_for_location(location, basename, metadata=None):
     if basename.endswith('.egg') and '-' in basename:
         # only one, unambiguous interpretation
         return [Distribution.from_location(location, basename, metadata)]
+    if basename.endswith('.whl') and '-' in basename:
+        wheel = Wheel(basename)
+        if not wheel.is_compatible():
+            return []
+        return [Distribution(
+            location=location,
+            project_name=wheel.project_name,
+            version=wheel.version,
+            # Increase priority over eggs.
+            precedence=EGG_DIST + 1,
+        )]
     if basename.endswith('.exe'):
         win_base, py_ver, platform = parse_bdist_wininst(basename)
         if win_base is not None:

setuptools/wheel.py

+'''Wheels support.'''
+
+from distutils.util import get_platform
+import email
+import itertools
+import os
+import re
+import zipfile
+
+from pkg_resources import Distribution, PathMetadata, parse_version
+from setuptools import Distribution as SetuptoolsDistribution
+from setuptools import pep425tags
+from setuptools.command.egg_info import write_requirements
+
+
+WHEEL_NAME = re.compile(
+    r"""^(?P<project_name>.+?)-(?P<version>\d.*?)
+    ((-(?P<build>\d.*?))?-(?P<py_version>.+?)-(?P<abi>.+?)-(?P<platform>.+?)
+    )\.whl$""",
+re.VERBOSE).match
+
+
+class Wheel(object):
+
+    def __init__(self, filename):
+        match = WHEEL_NAME(os.path.basename(filename))
+        if match is None:
+            raise ValueError('invalid wheel name: %r' % filename)
+        self.filename = filename
+        for k, v in match.groupdict().items():
+            setattr(self, k, v)
+
+    def tags(self):
+        '''List tags (py_version, abi, platform) supported by this wheel.'''
+        return itertools.product(self.py_version.split('.'),
+                                 self.abi.split('.'),
+                                 self.platform.split('.'))
+
+    def is_compatible(self):
+        '''Is the wheel is compatible with the current platform?'''
+        supported_tags = pep425tags.get_supported()
+        return next((True for t in self.tags() if t in supported_tags), False)
+
+    def egg_name(self):
+        return Distribution(
+            project_name=self.project_name, version=self.version,
+            platform=(None if self.platform == 'any' else get_platform()),
+        ).egg_name() + '.egg'
+
+    def install_as_egg(self, destination_eggdir):
+        '''Install wheel as an egg directory.'''
+        with zipfile.ZipFile(self.filename) as zf:
+            dist_basename = '%s-%s' % (self.project_name, self.version)
+            dist_info = '%s.dist-info' % dist_basename
+            dist_data = '%s.data' % dist_basename
+            def get_metadata(name):
+                with zf.open('%s/%s' % (dist_info, name)) as fp:
+                    value = fp.read().decode('utf-8')
+                    return email.parser.Parser().parsestr(value)
+            wheel_metadata = get_metadata('WHEEL')
+            dist_metadata = get_metadata('METADATA')
+            # Check wheel format version is supported.
+            wheel_version = parse_version(wheel_metadata.get('Wheel-Version'))
+            if not parse_version('1.0') <= wheel_version < parse_version('2.0dev0'):
+                raise ValueError('unsupported wheel format version: %s' % wheel_version)
+            # Extract to target directory.
+            os.mkdir(destination_eggdir)
+            zf.extractall(destination_eggdir)
+            # Convert metadata.
+            dist_info = os.path.join(destination_eggdir, dist_info)
+            dist = Distribution.from_location(
+                destination_eggdir, dist_info,
+                metadata=PathMetadata(destination_eggdir, dist_info)
+            )
+            # Note: we need to evaluate and strip markers now,
+            # as we can't easily convert back from the syntax:
+            # foobar; "linux" in sys_platform and extra == 'test'
+            def raw_req(req):
+                req.marker = None
+                return str(req)
+            install_requires = list(sorted(map(raw_req, dist.requires())))
+            extras_require = {
+                extra: list(sorted(
+                    req
+                    for req in map(raw_req, dist.requires((extra,)))
+                    if req not in install_requires
+                ))
+                for extra in dist.extras
+            }
+            egg_info = os.path.join(destination_eggdir, 'EGG-INFO')
+            os.rename(dist_info, egg_info)
+            os.rename(os.path.join(egg_info, 'METADATA'),
+                      os.path.join(egg_info, 'PKG-INFO'))
+            setup_dist = SetuptoolsDistribution(attrs=dict(
+                install_requires=install_requires,
+                extras_require=extras_require,
+            ))
+            write_requirements(setup_dist.get_command_obj('egg_info'),
+                               None, os.path.join(egg_info, 'requires.txt'))
+            # Move data entries to their correct location.
+            dist_data = os.path.join(destination_eggdir, dist_data)
+            dist_data_scripts = os.path.join(dist_data, 'scripts')
+            if os.path.exists(dist_data_scripts):
+                egg_info_scripts = os.path.join(destination_eggdir,
+                                                'EGG-INFO', 'scripts')
+                os.mkdir(egg_info_scripts)
+                for entry in os.listdir(dist_data_scripts):
+                    # Remove bytecode, as it's not properly handled
+                    # during easy_install scripts install phase.
+                    if entry.endswith('.pyc'):
+                        os.unlink(os.path.join(dist_data_scripts, entry))
+                    else:
+                        os.rename(os.path.join(dist_data_scripts, entry),
+                                  os.path.join(egg_info_scripts, entry))
+                os.rmdir(dist_data_scripts)
+            for subdir in filter(os.path.exists, (
+                os.path.join(dist_data, d)
+                for d in ('data', 'headers', 'purelib', 'platlib')
+            )):
+                for entry in os.listdir(subdir):
+                    os.rename(os.path.join(subdir, entry),
+                              os.path.join(destination_eggdir, entry))
+                os.rmdir(subdir)
+            if os.path.exists(dist_data):
+                os.rmdir(dist_data)

tests/requirements.txt

@@ -4,3 +4,4 @@ pytest-flake8; python_version>="2.7"
 virtualenv>=13.0.0
 pytest-virtualenv>=1.2.7
 pytest>=3.0.2
+wheel