__init__.py 103 KB
Newer Older
1
# coding: utf-8
2 3
"""
Package resource API
4 5 6 7 8 9 10 11 12 13 14 15 16 17
--------------------

A resource is a logical file contained within a package, or a logical
subdirectory thereof.  The package resource API expects resource names
to have their path parts separated with ``/``, *not* whatever the local
path separator is.  Do not use os.path operations to manipulate resource
names being passed into the API.

The package resource API is designed to work with normal filesystem packages,
.egg files, and unpacked .egg files.  It can also work in a limited way with
.zip files and with custom PEP 302 loaders that support the ``get_data()``
method.
"""

18 19
from __future__ import absolute_import

Jason R. Coombs's avatar
Jason R. Coombs committed
20 21
import sys
import os
22
import io
Jason R. Coombs's avatar
Jason R. Coombs committed
23 24
import time
import re
25
import types
Jason R. Coombs's avatar
Jason R. Coombs committed
26 27
import zipfile
import zipimport
28 29
import warnings
import stat
30
import functools
31
import pkgutil
32
import operator
33
import platform
34
import collections
Jason R. Coombs's avatar
Jason R. Coombs committed
35 36
import plistlib
import email.parser
37
import errno
Jason R. Coombs's avatar
Jason R. Coombs committed
38
import tempfile
39
import textwrap
40
import itertools
41
import inspect
42
from pkgutil import get_importer
43

44 45 46 47 48 49
try:
    import _imp
except ImportError:
    # Python 3.2 compatibility
    import imp as _imp

50 51
from pkg_resources.extern import six
from pkg_resources.extern.six.moves import urllib, map, filter
52

53
# capture these to bypass sandboxing
54 55 56 57 58 59 60 61
from os import utime
try:
    from os import mkdir, rename, unlink
    WRITE_SUPPORT = True
except ImportError:
    # no write support, probably under GAE
    WRITE_SUPPORT = False

62
from os import open as os_open
63
from os.path import isdir, split
64

65
try:
66
    import importlib.machinery as importlib_machinery
67 68 69
    # access attribute to force import under delayed import mechanisms.
    importlib_machinery.__name__
except ImportError:
70
    importlib_machinery = None
71

72
from . import py31compat
73 74 75 76 77 78
from pkg_resources.extern import appdirs
from pkg_resources.extern import packaging
__import__('pkg_resources.extern.packaging.version')
__import__('pkg_resources.extern.packaging.specifiers')
__import__('pkg_resources.extern.packaging.requirements')
__import__('pkg_resources.extern.packaging.markers')
79

80

81
if (3, 0) < sys.version_info < (3, 3):
82
    raise RuntimeError("Python 3.3 or later is required")
83

84 85 86 87 88
if six.PY2:
    # Those builtin exceptions are only defined in Python 3
    PermissionError = None
    NotADirectoryError = None

89 90 91 92 93 94
# declare some globals that will be defined later to
# satisfy the linters.
require = None
working_set = None


95 96 97 98 99 100 101
class PEP440Warning(RuntimeWarning):
    """
    Used when there is an issue with a version or specifier not complying with
    PEP 440.
    """


102
class _SetuptoolsVersionMixin(object):
103 104 105
    def __hash__(self):
        return super(_SetuptoolsVersionMixin, self).__hash__()

106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
    def __lt__(self, other):
        if isinstance(other, tuple):
            return tuple(self) < other
        else:
            return super(_SetuptoolsVersionMixin, self).__lt__(other)

    def __le__(self, other):
        if isinstance(other, tuple):
            return tuple(self) <= other
        else:
            return super(_SetuptoolsVersionMixin, self).__le__(other)

    def __eq__(self, other):
        if isinstance(other, tuple):
            return tuple(self) == other
        else:
            return super(_SetuptoolsVersionMixin, self).__eq__(other)

    def __ge__(self, other):
        if isinstance(other, tuple):
            return tuple(self) >= other
        else:
            return super(_SetuptoolsVersionMixin, self).__ge__(other)

    def __gt__(self, other):
        if isinstance(other, tuple):
            return tuple(self) > other
        else:
            return super(_SetuptoolsVersionMixin, self).__gt__(other)

    def __ne__(self, other):
        if isinstance(other, tuple):
            return tuple(self) != other
        else:
            return super(_SetuptoolsVersionMixin, self).__ne__(other)

    def __getitem__(self, key):
        return tuple(self)[key]

145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163
    def __iter__(self):
        component_re = re.compile(r'(\d+ | [a-z]+ | \.| -)', re.VERBOSE)
        replace = {
            'pre': 'c',
            'preview': 'c',
            '-': 'final-',
            'rc': 'c',
            'dev': '@',
        }.get

        def _parse_version_parts(s):
            for part in component_re.split(s):
                part = replace(part, part)
                if not part or part == '.':
                    continue
                if part[:1] in '0123456789':
                    # pad for numeric comparison
                    yield part.zfill(8)
                else:
164
                    yield '*' + part
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187

            # ensure that alpha/beta/candidate are before final
            yield '*final'

        def old_parse_version(s):
            parts = []
            for part in _parse_version_parts(s.lower()):
                if part.startswith('*'):
                    # remove '-' before a prerelease tag
                    if part < '*final':
                        while parts and parts[-1] == '*final-':
                            parts.pop()
                    # remove trailing zeros from each series of numeric parts
                    while parts and parts[-1] == '00000000':
                        parts.pop()
                parts.append(part)
            return tuple(parts)

        # Warn for use of this function
        warnings.warn(
            "You have iterated over the result of "
            "pkg_resources.parse_version. This is a legacy behavior which is "
            "inconsistent with the new version class introduced in setuptools "
188 189 190 191
            "8.0. In most cases, conversion to a tuple is unnecessary. For "
            "comparison of versions, sort the Version instances directly. If "
            "you have another use case requiring the tuple, please file a "
            "bug with the setuptools project describing that need.",
192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213
            RuntimeWarning,
            stacklevel=1,
        )

        for part in old_parse_version(str(self)):
            yield part


class SetuptoolsVersion(_SetuptoolsVersionMixin, packaging.version.Version):
    pass


class SetuptoolsLegacyVersion(_SetuptoolsVersionMixin,
                              packaging.version.LegacyVersion):
    pass


def parse_version(v):
    try:
        return SetuptoolsVersion(v)
    except packaging.version.InvalidVersion:
        return SetuptoolsLegacyVersion(v)
214

215

Tarek Ziade's avatar
Tarek Ziade committed
216
_state_vars = {}
217

218

Tarek Ziade's avatar
Tarek Ziade committed
219
def _declare_state(vartype, **kw):
220
    globals().update(kw)
221
    _state_vars.update(dict.fromkeys(kw, vartype))
Tarek Ziade's avatar
Tarek Ziade committed
222

223

Tarek Ziade's avatar
Tarek Ziade committed
224 225 226
def __getstate__():
    state = {}
    g = globals()
227
    for k, v in _state_vars.items():
228
        state[k] = g['_sget_' + v](g[k])
Tarek Ziade's avatar
Tarek Ziade committed
229 230
    return state

231

Tarek Ziade's avatar
Tarek Ziade committed
232 233
def __setstate__(state):
    g = globals()
234
    for k, v in state.items():
235
        g['_sset_' + _state_vars[k]](k, g[k], v)
Tarek Ziade's avatar
Tarek Ziade committed
236 237
    return state

238

Tarek Ziade's avatar
Tarek Ziade committed
239 240 241
def _sget_dict(val):
    return val.copy()

242

Tarek Ziade's avatar
Tarek Ziade committed
243 244 245 246
def _sset_dict(key, ob, state):
    ob.clear()
    ob.update(state)

247

Tarek Ziade's avatar
Tarek Ziade committed
248 249
def _sget_object(val):
    return val.__getstate__()
250

251

Tarek Ziade's avatar
Tarek Ziade committed
252 253
def _sset_object(key, ob, state):
    ob.__setstate__(state)
254

255

Tarek Ziade's avatar
Tarek Ziade committed
256
_sget_none = _sset_none = lambda *args: None
257 258


259
def get_supported_platform():
260 261 262 263 264 265 266 267 268 269 270 271
    """Return this platform's maximum compatible version.

    distutils.util.get_platform() normally reports the minimum version
    of Mac OS X that would be required to *use* extensions produced by
    distutils.  But what we want when checking compatibility is to know the
    version of Mac OS X that we are *running*.  To allow usage of packages that
    explicitly require a newer version of Mac OS X, we must also know the
    current version of the OS.

    If this condition occurs for any other platform with a version in its
    platform strings, this function should be extended accordingly.
    """
Jason R. Coombs's avatar
Jason R. Coombs committed
272 273
    plat = get_build_platform()
    m = macosVersionString.match(plat)
274 275 276 277
    if m is not None and sys.platform == "darwin":
        try:
            plat = 'macosx-%s-%s' % ('.'.join(_macosx_vers()[:2]), m.group(3))
        except ValueError:
Jason R. Coombs's avatar
Jason R. Coombs committed
278 279
            # not Mac OS X
            pass
280
    return plat
281

282

283 284
__all__ = [
    # Basic resource access and distribution/entry point discovery
285
    'require', 'run_script', 'get_provider', 'get_distribution',
Jason R. Coombs's avatar
Jason R. Coombs committed
286 287
    'load_entry_point', 'get_entry_map', 'get_entry_info',
    'iter_entry_points',
288 289 290 291 292 293 294 295 296
    'resource_string', 'resource_stream', 'resource_filename',
    'resource_listdir', 'resource_exists', 'resource_isdir',

    # Environmental control
    'declare_namespace', 'working_set', 'add_activation_listener',
    'find_distributions', 'set_extraction_path', 'cleanup_resources',
    'get_default_cache',

    # Primary implementation classes
297
    'Environment', 'WorkingSet', 'ResourceManager',
298 299 300
    'Distribution', 'Requirement', 'EntryPoint',

    # Exceptions
Jason R. Coombs's avatar
Jason R. Coombs committed
301 302
    'ResolutionError', 'VersionConflict', 'DistributionNotFound',
    'UnknownExtra', 'ExtractionError',
303

304 305 306
    # Warnings
    'PEP440Warning',

307 308 309
    # Parsing functions and string utilities
    'parse_requirements', 'parse_version', 'safe_name', 'safe_version',
    'get_platform', 'compatible_platforms', 'yield_lines', 'split_sections',
310
    'safe_extra', 'to_filename', 'invalid_marker', 'evaluate_marker',
311 312 313 314 315

    # filesystem utilities
    'ensure_directory', 'normalize_path',

    # Distribution "precedence" constants
316
    'EGG_DIST', 'BINARY_DIST', 'SOURCE_DIST', 'CHECKOUT_DIST', 'DEVELOP_DIST',
317 318

    # "Provider" interfaces, implementations, and registration/lookup APIs
319
    'IMetadataProvider', 'IResourceProvider', 'FileMetadata',
320 321 322 323 324 325
    'PathMetadata', 'EggMetadata', 'EmptyProvider', 'empty_provider',
    'NullProvider', 'EggProvider', 'DefaultProvider', 'ZipProvider',
    'register_finder', 'register_namespace_handler', 'register_loader_type',
    'fixup_namespace_packages', 'get_importer',

    # Deprecated/backward compatibility only
326
    'run_main', 'AvailableDistributions',
327
]
Jason R. Coombs's avatar
Jason R. Coombs committed
328

329

330
class ResolutionError(Exception):
331
    """Abstract base for dependency resolution errors"""
332

333
    def __repr__(self):
334
        return self.__class__.__name__ + repr(self.args)
335

Jason R. Coombs's avatar
Jason R. Coombs committed
336

337
class VersionConflict(ResolutionError):
Jason R. Coombs's avatar
Jason R. Coombs committed
338 339 340
    """
    An already-installed version conflicts with the requested version.

341 342
    Should be initialized with the installed Distribution and the requested
    Requirement.
Jason R. Coombs's avatar
Jason R. Coombs committed
343 344
    """

345 346
    _template = "{self.dist} is installed but {self.req} is required"

Jason R. Coombs's avatar
Jason R. Coombs committed
347 348 349 350 351 352 353 354
    @property
    def dist(self):
        return self.args[0]

    @property
    def req(self):
        return self.args[1]

355 356 357
    def report(self):
        return self._template.format(**locals())

358 359 360 361 362 363 364 365 366 367
    def with_context(self, required_by):
        """
        If required_by is non-empty, return a version of self that is a
        ContextualVersionConflict.
        """
        if not required_by:
            return self
        args = self.args + (required_by,)
        return ContextualVersionConflict(*args)

368 369 370

class ContextualVersionConflict(VersionConflict):
    """
371
    A VersionConflict that accepts a third parameter, the set of the
372 373 374 375 376
    requirements that required the installed Distribution.
    """

    _template = VersionConflict._template + ' by {self.required_by}'

Jason R. Coombs's avatar
Jason R. Coombs committed
377 378
    @property
    def required_by(self):
379
        return self.args[2]
Jason R. Coombs's avatar
Jason R. Coombs committed
380

381

382 383
class DistributionNotFound(ResolutionError):
    """A requested distribution was not found"""
384

385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
    _template = ("The '{self.req}' distribution was not found "
                 "and is required by {self.requirers_str}")

    @property
    def req(self):
        return self.args[0]

    @property
    def requirers(self):
        return self.args[1]

    @property
    def requirers_str(self):
        if not self.requirers:
            return 'the application'
        return ', '.join(self.requirers)

    def report(self):
        return self._template.format(**locals())

    def __str__(self):
        return self.report()


409 410
class UnknownExtra(ResolutionError):
    """Distribution doesn't have an "extra feature" of the given name"""
411 412


413
_provider_factories = {}
414

415
PY_MAJOR = sys.version[:3]
Jason R. Coombs's avatar
Jason R. Coombs committed
416
EGG_DIST = 3
417
BINARY_DIST = 2
418
SOURCE_DIST = 1
419
CHECKOUT_DIST = 0
420
DEVELOP_DIST = -1
421

422

423 424 425 426 427 428 429 430 431
def register_loader_type(loader_type, provider_factory):
    """Register `provider_factory` to make providers for `loader_type`

    `loader_type` is the type or class of a PEP 302 ``module.__loader__``,
    and `provider_factory` is a function that, passed a *module* object,
    returns an ``IResourceProvider`` for that module.
    """
    _provider_factories[loader_type] = provider_factory

432

433 434
def get_provider(moduleOrReq):
    """Return an IResourceProvider for the named module or requirement"""
435
    if isinstance(moduleOrReq, Requirement):
436
        return working_set.find(moduleOrReq) or require(str(moduleOrReq))[0]
437
    try:
438
        module = sys.modules[moduleOrReq]
439
    except KeyError:
440 441
        __import__(moduleOrReq)
        module = sys.modules[moduleOrReq]
442 443
    loader = getattr(module, '__loader__', None)
    return _find_adapter(_provider_factories, loader)(module)
444

445

446 447
def _macosx_vers(_cache=[]):
    if not _cache:
448
        version = platform.mac_ver()[0]
449 450
        # fallback for MacPorts
        if version == '':
451 452
            plist = '/System/Library/CoreServices/SystemVersion.plist'
            if os.path.exists(plist):
453 454 455 456
                if hasattr(plistlib, 'readPlist'):
                    plist_content = plistlib.readPlist(plist)
                    if 'ProductVersion' in plist_content:
                        version = plist_content['ProductVersion']
457

458
        _cache.append(version.split('.'))
459 460
    return _cache[0]

461

462
def _macosx_arch(machine):
Jason R. Coombs's avatar
Jason R. Coombs committed
463
    return {'PowerPC': 'ppc', 'Power_Macintosh': 'ppc'}.get(machine, machine)
464

465

466
def get_build_platform():
467 468 469 470 471
    """Return this platform's string for platform-specific distributions

    XXX Currently this is the same as ``distutils.util.get_platform()``, but it
    needs some hacks for Linux and Mac OS X.
    """
tarek's avatar
tarek committed
472
    try:
473
        # Python 2.7 or >=3.2
tarek's avatar
tarek committed
474
        from sysconfig import get_platform
475 476
    except ImportError:
        from distutils.util import get_platform
tarek's avatar
tarek committed
477

478 479
    plat = get_platform()
    if sys.platform == "darwin" and not plat.startswith('macosx-'):
480 481 482
        try:
            version = _macosx_vers()
            machine = os.uname()[4].replace(" ", "_")
483
            return "macosx-%d.%d-%s" % (int(version[0]), int(version[1]),
484
                _macosx_arch(machine))
485 486 487 488
        except ValueError:
            # if someone is running a non-Mac darwin system, this will fall
            # through to the default implementation
            pass
489
    return plat
490

491

492
macosVersionString = re.compile(r"macosx-(\d+)\.(\d+)-(.*)")
493
darwinVersionString = re.compile(r"darwin-(\d+)\.(\d+)\.(\d+)-(.*)")
Jason R. Coombs's avatar
Jason R. Coombs committed
494 495
# XXX backward compat
get_platform = get_build_platform
496 497


498
def compatible_platforms(provided, required):
499 500 501 502
    """Can code for the `provided` platform run on the `required` platform?

    Returns true if either platform is ``None``, or the platforms are equal.

503
    XXX Needs compatibility checks for Linux and other unixy OSes.
504
    """
505
    if provided is None or required is None or provided == required:
Jason R. Coombs's avatar
Jason R. Coombs committed
506 507
        # easy case
        return True
508

509 510 511 512
    # Mac OS X special cases
    reqMac = macosVersionString.match(required)
    if reqMac:
        provMac = macosVersionString.match(provided)
513

514 515
        # is this a Mac package?
        if not provMac:
516 517 518 519 520 521 522 523
            # this is backwards compatibility for packages built before
            # setuptools 0.6. All packages built after this point will
            # use the new macosx designation.
            provDarwin = darwinVersionString.match(provided)
            if provDarwin:
                dversion = int(provDarwin.group(1))
                macosversion = "%s.%s" % (reqMac.group(1), reqMac.group(2))
                if dversion == 7 and macosversion >= "10.3" or \
Jason R. Coombs's avatar
Jason R. Coombs committed
524
                        dversion == 8 and macosversion >= "10.4":
525
                    return True
526 527
            # egg isn't macosx or legacy darwin
            return False
528

529 530
        # are they the same major version and machine type?
        if provMac.group(1) != reqMac.group(1) or \
Jason R. Coombs's avatar
Jason R. Coombs committed
531
                provMac.group(3) != reqMac.group(3):
532
            return False
533

534 535 536
        # is the required OS major update >= the provided one?
        if int(provMac.group(2)) > int(reqMac.group(2)):
            return False
537

538 539
        return True

540
    # XXX Linux and other platforms' special cases should go here
541 542
    return False

543

544
def run_script(dist_spec, script_name):
545
    """Locate distribution `dist_spec` and run its `script_name` script"""
546 547 548 549
    ns = sys._getframe(1).f_globals
    name = ns['__name__']
    ns.clear()
    ns['__name__'] = name
550
    require(dist_spec)[0].run_script(script_name, ns)
551

552

Jason R. Coombs's avatar
Jason R. Coombs committed
553 554
# backward compatibility
run_main = run_script
555

556

557 558
def get_distribution(dist):
    """Return a current distribution object for a Requirement or string"""
559
    if isinstance(dist, six.string_types):
560 561 562
        dist = Requirement.parse(dist)
    if isinstance(dist, Requirement):
        dist = get_provider(dist)
563
    if not isinstance(dist, Distribution):
564 565
        raise TypeError("Expected string, Requirement, or Distribution", dist)
    return dist
566

567

568 569 570
def load_entry_point(dist, group, name):
    """Return `name` entry point of `group` for `dist` or raise ImportError"""
    return get_distribution(dist).load_entry_point(group, name)
571

572

573 574 575
def get_entry_map(dist, group=None):
    """Return the entry point map for `group`, or the full entry map"""
    return get_distribution(dist).get_entry_map(group)
576

577

578 579 580
def get_entry_info(dist, group, name):
    """Return the EntryPoint object for `group`+`name`, or ``None``"""
    return get_distribution(dist).get_entry_info(group, name)
581

582

583 584 585 586 587 588 589 590 591 592 593
class IMetadataProvider:
    def has_metadata(name):
        """Does the package's distribution contain the named metadata?"""

    def get_metadata(name):
        """The named metadata resource as a string"""

    def get_metadata_lines(name):
        """Yield named metadata resource as list of non-blank non-comment lines

       Leading and trailing whitespace is stripped from each line, and lines
594
       with ``#`` as the first non-blank character are omitted."""
595

596 597 598 599 600 601 602 603 604 605
    def metadata_isdir(name):
        """Is the named metadata a directory?  (like ``os.path.isdir()``)"""

    def metadata_listdir(name):
        """List of metadata names in the directory (like ``os.listdir()``)"""

    def run_script(script_name, namespace):
        """Execute the named script in the supplied namespace dictionary"""


606
class IResourceProvider(IMetadataProvider):
607 608
    """An object that provides access to package resources"""

609
    def get_resource_filename(manager, resource_name):
610 611 612 613
        """Return a true filesystem path for `resource_name`

        `manager` must be an ``IResourceManager``"""

614
    def get_resource_stream(manager, resource_name):
615 616 617 618
        """Return a readable file-like object for `resource_name`

        `manager` must be an ``IResourceManager``"""

619
    def get_resource_string(manager, resource_name):
620 621 622 623 624 625 626
        """Return a string containing the contents of `resource_name`

        `manager` must be an ``IResourceManager``"""

    def has_resource(resource_name):
        """Does the package contain the named resource?"""

627 628
    def resource_isdir(resource_name):
        """Is the named resource a directory?  (like ``os.path.isdir()``)"""
629

630 631
    def resource_listdir(resource_name):
        """List of resource names in the directory (like ``os.listdir()``)"""
632

633

634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649
class WorkingSet(object):
    """A collection of active distributions on sys.path (or a similar list)"""

    def __init__(self, entries=None):
        """Create working set from list of path entries (default=sys.path)"""
        self.entries = []
        self.entry_keys = {}
        self.by_key = {}
        self.callbacks = []

        if entries is None:
            entries = sys.path

        for entry in entries:
            self.add_entry(entry)

650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678
    @classmethod
    def _build_master(cls):
        """
        Prepare the master working set.
        """
        ws = cls()
        try:
            from __main__ import __requires__
        except ImportError:
            # The main program does not list any requirements
            return ws

        # ensure the requirements are met
        try:
            ws.require(__requires__)
        except VersionConflict:
            return cls._build_from_requirements(__requires__)

        return ws

    @classmethod
    def _build_from_requirements(cls, req_spec):
        """
        Build a working set from a requirement spec. Rewrites sys.path.
        """
        # try it without defaults already on sys.path
        # by starting with an empty path
        ws = cls([])
        reqs = parse_requirements(req_spec)
679
        dists = ws.resolve(reqs, Environment())
680 681 682 683 684 685 686 687 688 689 690 691
        for dist in dists:
            ws.add(dist)

        # add any missing entries from sys.path
        for entry in sys.path:
            if entry not in ws.entries:
                ws.add_entry(entry)

        # then copy back to sys.path
        sys.path[:] = ws.entries
        return ws

692 693 694
    def add_entry(self, entry):
        """Add a path item to ``.entries``, finding any distributions on it

695
        ``find_distributions(entry, True)`` is used to find distributions
696 697 698 699 700 701 702 703
        corresponding to the path entry, and they are added.  `entry` is
        always appended to ``.entries``, even if it is already present.
        (This is because ``sys.path`` can contain the same value more than
        once, and the ``.entries`` of the ``sys.path`` WorkingSet should always
        equal ``sys.path``.)
        """
        self.entry_keys.setdefault(entry, [])
        self.entries.append(entry)
704
        for dist in find_distributions(entry, True):
705
            self.add(dist, entry, False)
706

707
    def __contains__(self, dist):
708 709 710 711 712 713 714 715 716 717 718 719 720 721 722
        """True if `dist` is the active distribution for its project"""
        return self.by_key.get(dist.key) == dist

    def find(self, req):
        """Find a distribution matching requirement `req`

        If there is an active distribution for the requested project, this
        returns it as long as it meets the version requirement specified by
        `req`.  But, if there is an active distribution for the project and it
        does *not* meet the `req` requirement, ``VersionConflict`` is raised.
        If there is no active distribution for the requested project, ``None``
        is returned.
        """
        dist = self.by_key.get(req.key)
        if dist is not None and dist not in req:
Jason R. Coombs's avatar
Jason R. Coombs committed
723 724
            # XXX add more info
            raise VersionConflict(dist, req)
Jason R. Coombs's avatar
Jason R. Coombs committed
725
        return dist
726

727 728
    def iter_entry_points(self, group, name=None):
        """Yield entry point objects from `group` matching `name`
729

730 731 732 733 734 735 736 737 738 739 740
        If `name` is None, yields all entry points in `group` from all
        distributions in the working set, otherwise only ones matching
        both `group` and `name` are yielded (in distribution order).
        """
        for dist in self:
            entries = dist.get_entry_map(group)
            if name is None:
                for ep in entries.values():
                    yield ep
            elif name in entries:
                yield entries[name]
741

742 743 744 745 746 747 748
    def run_script(self, requires, script_name):
        """Locate distribution for `requires` and run `script_name` script"""
        ns = sys._getframe(1).f_globals
        name = ns['__name__']
        ns.clear()
        ns['__name__'] = name
        self.require(requires)[0].run_script(script_name, ns)
749 750 751 752 753 754 755 756 757

    def __iter__(self):
        """Yield distributions for non-duplicate projects in the working set

        The yield order is the order in which the items' path entries were
        added to the working set.
        """
        seen = {}
        for item in self.entries:
758 759 760 761
            if item not in self.entry_keys:
                # workaround a cache issue
                continue

762 763
            for key in self.entry_keys[item]:
                if key not in seen:
764
                    seen[key] = 1
765 766
                    yield self.by_key[key]

767
    def add(self, dist, entry=None, insert=True, replace=False):
768 769 770 771 772 773 774
        """Add `dist` to working set, associated with `entry`

        If `entry` is unspecified, it defaults to the ``.location`` of `dist`.
        On exit from this routine, `entry` is added to the end of the working
        set's ``.entries`` (if it wasn't already present).

        `dist` is only added to the working set if it's for a project that
775 776 777
        doesn't already have a distribution in the set, unless `replace=True`.
        If it's added, any callbacks registered with the ``subscribe()`` method
        will be called.
778
        """
779
        if insert:
780
            dist.insert_on(self.entries, entry, replace=replace)
781

782 783
        if entry is None:
            entry = dist.location
784 785
        keys = self.entry_keys.setdefault(entry, [])
        keys2 = self.entry_keys.setdefault(dist.location, [])
786
        if not replace and dist.key in self.by_key:
Jason R. Coombs's avatar
Jason R. Coombs committed
787 788
            # ignore hidden distros
            return
789 790 791 792

        self.by_key[dist.key] = dist
        if dist.key not in keys:
            keys.append(dist.key)
793 794
        if dist.key not in keys2:
            keys2.append(dist.key)
795 796
        self._added_new(dist)

797
    def resolve(self, requirements, env=None, installer=None,
798
                replace_conflicting=False, extras=None):
799 800 801
        """List all distributions needed to (recursively) meet `requirements`

        `requirements` must be a sequence of ``Requirement`` objects.  `env`,
802
        if supplied, should be an ``Environment`` instance.  If
803 804 805 806 807
        not supplied, it defaults to all distributions available within any
        entry or distribution in the working set.  `installer`, if supplied,
        will be invoked with each requirement that cannot be met by an
        already-installed distribution; it should return a ``Distribution`` or
        ``None``.
808 809 810 811 812 813

        Unless `replace_conflicting=True`, raises a VersionConflict exception if
        any requirements are found on the path that have the correct name but
        the wrong version.  Otherwise, if an `installer` is supplied it will be
        invoked to obtain the correct version of the requirement and activate
        it.
814 815 816 817 818 819

        `extras` is a list of the extras to be used with these requirements.
        This is important because extra requirements may look like `my_req;
        extra = "my_extra"`, which would otherwise be interpreted as a purely
        optional requirement.  Instead, we want to be able to assert that these
        requirements are truly required.
820 821
        """

Jason R. Coombs's avatar
Jason R. Coombs committed
822 823 824 825 826 827
        # set up the stack
        requirements = list(requirements)[::-1]
        # set of processed requirements
        processed = {}
        # key -> dist
        best = {}
828
        to_activate = []
829 830

        req_extras = _ReqExtras()
831

Jason R. Coombs's avatar
Jason R. Coombs committed
832 833
        # Mapping of requirement to set of distributions that required it;
        # useful for reporting info about conflicts.
834
        required_by = collections.defaultdict(set)
835 836

        while requirements:
Jason R. Coombs's avatar
Jason R. Coombs committed
837 838
            # process dependencies breadth-first
            req = requirements.pop(0)
839 840 841
            if req in processed:
                # Ignore cyclic or redundant dependencies
                continue
842

843
            if not req_extras.markers_pass(req, extras):
844 845
                continue

846 847 848
            dist = best.get(req.key)
            if dist is None:
                # Find the best distribution and add it to the map
849
                dist = self.by_key.get(req.key)
850 851
                if dist is None or (dist not in req and replace_conflicting):
                    ws = self
852
                    if env is None:
853 854 855 856 857 858 859 860
                        if dist is None:
                            env = Environment(self.entries)
                        else:
                            # Use an empty environment and workingset to avoid
                            # any further conflicts with the conflicting
                            # distribution
                            env = Environment([])
                            ws = WorkingSet([])
861 862 863 864
                    dist = best[req.key] = env.best_match(
                        req, ws, installer,
                        replace_conflicting=replace_conflicting
                    )
865
                    if dist is None:
866
                        requirers = required_by.get(req, None)
867
                        raise DistributionNotFound(req, requirers)
868
                to_activate.append(dist)
869
            if dist not in req:
870
                # Oops, the "best" so far conflicts with a dependency
871
                dependent_req = required_by[req]
872
                raise VersionConflict(dist, req).with_context(dependent_req)
873 874

            # push the new requirements onto the stack
875 876
            new_requirements = dist.requires(req.extras)[::-1]
            requirements.extend(new_requirements)
877 878

            # Register the new requirements needed by req
879 880
            for new_requirement in new_requirements:
                required_by[new_requirement].add(req.project_name)
881
                req_extras[new_requirement] = req.extras
882

883 884
            processed[req] = True

Jason R. Coombs's avatar
Jason R. Coombs committed
885 886
        # return list of distros to activate
        return to_activate
887

Jason R. Coombs's avatar
Jason R. Coombs committed
888 889
    def find_plugins(self, plugin_env, full_env=None, installer=None,
            fallback=True):
890 891 892 893 894 895 896
        """Find all activatable distributions in `plugin_env`

        Example usage::

            distributions, errors = working_set.find_plugins(
                Environment(plugin_dirlist)
            )
897 898 899 900
            # add plugins+libs to sys.path
            map(working_set.add, distributions)
            # display errors
            print('Could not load', errors)
901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924

        The `plugin_env` should be an ``Environment`` instance that contains
        only distributions that are in the project's "plugin directory" or
        directories. The `full_env`, if supplied, should be an ``Environment``
        contains all currently-available distributions.  If `full_env` is not
        supplied, one is created automatically from the ``WorkingSet`` this
        method is called on, which will typically mean that every directory on
        ``sys.path`` will be scanned for distributions.

        `installer` is a standard installer callback as used by the
        ``resolve()`` method. The `fallback` flag indicates whether we should
        attempt to resolve older versions of a plugin if the newest version
        cannot be resolved.

        This method returns a 2-tuple: (`distributions`, `error_info`), where
        `distributions` is a list of the distributions found in `plugin_env`
        that were loadable, along with any other distributions that are needed
        to resolve their dependencies.  `error_info` is a dictionary mapping
        unloadable plugin distributions to an exception instance describing the
        error that occurred. Usually this will be a ``DistributionNotFound`` or
        ``VersionConflict`` instance.
        """

        plugin_projects = list(plugin_env)
Jason R. Coombs's avatar
Jason R. Coombs committed
925 926
        # scan project names in alphabetic order
        plugin_projects.sort()
927 928 929 930 931 932 933 934 935 936 937

        error_info = {}
        distributions = {}

        if full_env is None:
            env = Environment(self.entries)
            env += plugin_env
        else:
            env = full_env + plugin_env

        shadow_set = self.__class__([])
Jason R. Coombs's avatar
Jason R. Coombs committed
938 939
        # put all our entries in shadow_set
        list(map(shadow_set.add, self))
940 941 942 943 944 945 946 947 948 949

        for project_name in plugin_projects:

            for dist in plugin_env[project_name]:

                req = [dist.as_requirement()]

                try:
                    resolvees = shadow_set.resolve(req, env, installer)

950
                except ResolutionError as v:
Jason R. Coombs's avatar
Jason R. Coombs committed
951 952
                    # save error info
                    error_info[dist] = v
953
                    if fallback:
Jason R. Coombs's avatar
Jason R. Coombs committed
954 955
                        # try the next older version of project
                        continue
956
                    else:
Jason R. Coombs's avatar
Jason R. Coombs committed
957 958
                        # give up on this project, keep going
                        break
959 960

                else:
961
                    list(map(shadow_set.add, resolvees))
962 963 964 965 966 967 968 969 970 971
                    distributions.update(dict.fromkeys(resolvees))

                    # success, no need to try any more versions of this project
                    break

        distributions = list(distributions)
        distributions.sort()

        return distributions, error_info

972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987
    def require(self, *requirements):
        """Ensure that distributions matching `requirements` are activated

        `requirements` must be a string or a (possibly-nested) sequence
        thereof, specifying the distributions and versions required.  The
        return value is a sequence of the distributions that needed to be
        activated to fulfill the requirements; all relevant distributions are
        included, even if they were already activated in this working set.
        """
        needed = self.resolve(parse_requirements(requirements))

        for dist in needed:
            self.add(dist)

        return needed

988 989 990 991 992 993
    def subscribe(self, callback, existing=True):
        """Invoke `callback` for all distributions

        If `existing=True` (default),
        call on all existing ones, as well.
        """
994 995 996
        if callback in self.callbacks:
            return
        self.callbacks.append(callback)
997 998
        if not existing:
            return
999 1000 1001 1002 1003 1004 1005
        for dist in self:
            callback(dist)

    def _added_new(self, dist):
        for callback in self.callbacks:
            callback(dist)

Tarek Ziade's avatar
Tarek Ziade committed
1006
    def __getstate__(self):
1007 1008 1009 1010
        return (
            self.entries[:], self.entry_keys.copy(), self.by_key.copy(),
            self.callbacks[:]
        )
1011

1012 1013
    def __setstate__(self, e_k_b_c):
        entries, keys, by_key, callbacks = e_k_b_c
Tarek Ziade's avatar
Tarek Ziade committed
1014
        self.entries = entries[:]
1015
        self.entry_keys = keys.copy()
Tarek Ziade's avatar
Tarek Ziade committed
1016 1017
        self.by_key = by_key.copy()
        self.callbacks = callbacks[:]
1018 1019


1020 1021 1022 1023 1024
class _ReqExtras(dict):
    """
    Map each requirement to the extras that demanded it.
    """

1025
    def markers_pass(self, req, extras=None):
1026 1027 1028 1029 1030 1031 1032
        """
        Evaluate markers for req against each extra that
        demanded it.

        Return False if the req has a marker and fails
        evaluation. Otherwise, return True.
        """
1033 1034
        extra_evals = (
            req.marker.evaluate({'extra': extra})
1035
            for extra in self.get(req, ()) + (extras or (None,))
1036
        )
1037
        return not req.marker or any(extra_evals)
1038 1039


1040
class Environment(object):
1041
    """Searchable snapshot of distributions on a search path"""
1042

Jason R. Coombs's avatar
Jason R. Coombs committed
1043 1044
    def __init__(self, search_path=None, platform=get_supported_platform(),
            python=PY_MAJOR):
1045
        """Snapshot distributions available on a search path
1046

1047 1048
        Any distributions found on `search_path` are added to the environment.
        `search_path` should be a sequence of ``sys.path`` items.  If not
1049 1050 1051 1052 1053
        supplied, ``sys.path`` is used.

        `platform` is an optional string specifying the name of the platform
        that platform-specific distributions must be compatible with.  If
        unspecified, it defaults to the current platform.  `python` is an
1054
        optional string naming the desired version of Python (e.g. ``'3.3'``);
1055
        it defaults to the current version.
1056

1057 1058 1059 1060
        You may explicitly set `platform` (and/or `python`) to ``None`` if you
        wish to map *all* distributions, not just those compatible with the
        running platform or Python version.
        """
1061
        self._distmap = {}
1062 1063 1064 1065 1066
        self.platform = platform
        self.python = python
        self.scan(search_path)

    def can_add(self, dist):
1067 1068 1069 1070 1071 1072
        """Is distribution `dist` acceptable for this environment?

        The distribution must match the platform and python version
        requirements specified when this environment was created, or False
        is returned.
        """
1073
        return (self.python is None or dist.py_version is None
1074
            or dist.py_version == self.python) \
1075
            and compatible_platforms(dist.platform, self.platform)
1076

1077 1078 1079
    def remove(self, dist):
        """Remove `dist` from the environment"""
        self._distmap[dist.key].remove(dist)
1080

1081
    def scan(self, search_path=None):
1082
        """Scan `search_path` for distributions usable in this environment
1083

1084
        Any distributions found are added to the environment.
1085
        `search_path` should be a sequence of ``sys.path`` items.  If not
1086 1087
        supplied, ``sys.path`` is used.  Only distributions conforming to
        the platform/python version defined at initialization are added.
1088 1089 1090
        """
        if search_path is None:
            search_path = sys.path
1091

1092
        for item in search_path:
1093
            for dist in find_distributions(item):
1094
                self.add(dist)
1095

1096
    def __getitem__(self, project_name):
1097
        """Return a newest-to-oldest list of distributions for `project_name`
1098

1099 1100 1101
        Uses case-insensitive `project_name` comparison, assuming all the
        project's distributions use their project's name converted to all
        lowercase as their key.
1102

1103
        """
1104
        distribution_key = project_name.lower()
1105
        return self._distmap.get(distribution_key, [])
1106

1107
    def add(self, dist):
1108 1109
        """Add `dist` if we ``can_add()`` it and it has not already been added
        """
1110
        if self.can_add(dist) and dist.has_version():
1111
            dists = self._distmap.setdefault(dist.key, [])
1112 1113
            if dist not in dists:
                dists.append(dist)
1114
                dists.sort(key=operator.attrgetter('hashcmp'), reverse=True)
1115

1116
    def best_match(self, req, working_set, installer=None, replace_conflicting=False):
1117
        """Find distribution best matching `req` and usable on `working_set`
1118

1119 1120 1121 1122 1123 1124 1125 1126 1127
        This calls the ``find(req)`` method of the `working_set` to see if a
        suitable distribution is already active.  (This may raise
        ``VersionConflict`` if an unsuitable version of the project is already
        active in the specified `working_set`.)  If a suitable distribution
        isn't active, this method returns the newest distribution in the
        environment that meets the ``Requirement`` in `req`.  If no suitable
        distribution is found, and `installer` is supplied, then the result of
        calling the environment's ``obtain(req, installer)`` method will be
        returned.
1128
        """
1129 1130 1131 1132 1133 1134
        try:
            dist = working_set.find(req)
        except VersionConflict:
            if not replace_conflicting:
                raise
            dist = None
1135 1136
        if dist is not None:
            return dist
1137
        for dist in self[req.key]:
1138 1139
            if dist in req:
                return dist
Jason R. Coombs's avatar
Jason R. Coombs committed
1140 1141
        # try to download/install
        return self.obtain(req, installer)
1142

1143
    def obtain(self, requirement, installer=None):
1144 1145 1146 1147 1148 1149 1150 1151
        """Obtain a distribution matching `requirement` (e.g. via download)

        Obtain a distro that matches requirement (e.g. via download).  In the
        base ``Environment`` class, this routine just returns
        ``installer(requirement)``, unless `installer` is None, in which case
        None is returned instead.  This method is a hook that allows subclasses
        to attempt other ways of obtaining a distribution before falling back
        to the `installer` argument."""
1152 1153
        if installer is not None:
            return installer(requirement)
1154

1155 1156 1157
    def __iter__(self):
        """Yield the unique project names of the available distributions"""
        for key in self._distmap.keys():
1158 1159
            if self[key]:
                yield key
1160

1161 1162
    def __iadd__(self, other):
        """In-place addition of a distribution or environment"""
1163
        if isinstance(other, Distribution):
1164
            self.add(other)
1165
        elif isinstance(other, Environment):
1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
            for project in other:
                for dist in other[project]:
                    self.add(dist)
        else:
            raise TypeError("Can't add %r to environment" % (other,))
        return self

    def __add__(self, other):
        """Add an environment or distribution to an environment"""
        new = self.__class__([], platform=None, python=None)
        for env in self, other:
            new += env
        return new


Jason R. Coombs's avatar
Jason R. Coombs committed
1181 1182
# XXX backward compatibility
AvailableDistributions = Environment
1183

1184

1185 1186
class ExtractionError(RuntimeError):
    """An error occurred extracting a resource
1187

1188
    The following attributes are available from instances of this exception:
1189

1190 1191
    manager
        The resource manager that raised this exception
1192

1193 1194
    cache_path
        The base directory for resource extraction
1195

1196 1197 1198
    original_error
        The exception instance that caused extraction to fail
    """
1199 1200


1201 1202 1203
class ResourceManager:
    """Manage resource extraction and packages"""
    extraction_path = None
1204

1205
    def __init__(self):
1206
        self.cached_files = {}
1207

1208 1209 1210
    def resource_exists(self, package_or_requirement, resource_name):
        """Does the named resource exist?"""
        return get_provider(package_or_requirement).has_resource(resource_name)
1211

1212 1213 1214 1215 1216
    def resource_isdir(self, package_or_requirement, resource_name):
        """Is the named resource an existing directory?"""
        return get_provider(package_or_requirement).resource_isdir(
            resource_name
        )
1217

1218
    def resource_filename(self, package_or_requirement, resource_name):
1219
        """Return a true filesystem path for specified resource"""
1220 1221
        return get_provider(package_or_requirement).get_resource_filename(
            self, resource_name
1222
        )
1223

1224
    def resource_stream(self, package_or_requirement, resource_name):
1225
        """Return a readable file-like object for specified resource"""
1226
        return get_provider(package_or_requirement).get_resource_stream(
1227 1228
            self, resource_name
        )
1229

1230
    def resource_string(self, package_or_requirement, resource_name):
1231
        """Return specified resource as a string"""
1232
        return get_provider(package_or_requirement).get_resource_string(
1233 1234
            self, resource_name
        )
1235

1236 1237 1238 1239 1240
    def resource_listdir(self, package_or_requirement, resource_name):
        """List the contents of the named resource directory"""
        return get_provider(package_or_requirement).resource_listdir(
            resource_name
        )
1241

1242 1243 1244 1245 1246
    def extraction_error(self):
        """Give an error message for problems extracting file(s)"""

        old_exc = sys.exc_info()[1]
        cache_path = self.extraction_path or get_default_cache()
1247

1248 1249
        tmpl = textwrap.dedent("""
            Can't extract file(s) to egg cache
1250

1251 1252
            The following error occurred while trying to extract file(s) to the Python egg
            cache:
1253

1254
              {old_exc}
1255

1256
            The Python egg cache directory is currently set to:
1257

1258
              {cache_path}
1259

1260 1261 1262 1263
            Perhaps your account does not have write access to this directory?  You can
            change the cache directory by setting the PYTHON_EGG_CACHE environment
            variable to point to an accessible directory.
            """).lstrip()
1264
        err = ExtractionError(tmpl.format(**locals()))
Jason R. Coombs's avatar
Jason R. Coombs committed
1265 1266
        err.manager = self
        err.cache_path = cache_path
1267 1268 1269
        err.original_error = old_exc
        raise err

1270 1271 1272 1273 1274 1275
    def get_cache_path(self, archive_name, names=()):
        """Return absolute location in cache for `archive_name` and `names`

        The parent directory of the resulting path will be created if it does
        not already exist.  `archive_name` should be the base filename of the
        enclosing egg (which may not be the name of the enclosing zipfile!),
1276
        including its ".egg" extension.  `names`, if provided, should be a
1277 1278 1279 1280 1281 1282
        sequence of path name parts "under" the egg's extraction location.

        This method should only be called by resource providers that need to
        obtain an extraction location, and only for names they intend to
        extract, as it tracks the generated names for possible cleanup later.
        """
1283
        extract_path = self.extraction_path or get_default_cache()
1284
        target_path = os.path.join(extract_path, archive_name + '-tmp', *names)
1285
        try:
1286
            _bypass_ensure_directory(target_path)
1287 1288
        except:
            self.extraction_error()
1289

1290 1291
        self._warn_unsafe_extraction_path(extract_path)

1292
        self.cached_files[target_path] = 1
1293 1294
        return target_path

1295
    @staticmethod
1296
    def _warn_unsafe_extraction_path(path):
1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310
        """
        If the default extraction path is overridden and set to an insecure
        location, such as /tmp, it opens up an opportunity for an attacker to
        replace an extracted file with an unauthorized payload. Warn the user
        if a known insecure location is used.

        See Distribute #375 for more details.
        """
        if os.name == 'nt' and not path.startswith(os.environ['windir']):
            # On Windows, permissions are generally restrictive by default
            #  and temp directories are not writable by other users, so
            #  bypass the warning.
            return
        mode = os.stat(path).st_mode
1311 1312 1313
        if mode & stat.S_IWOTH or mode & stat.S_IWGRP:
            msg = ("%s is writable by group/others and vulnerable to attack "
                "when "
1314 1315 1316 1317
                "used with get_resource_filename. Consider a more secure "
                "location (set with .set_extraction_path or the "
                "PYTHON_EGG_CACHE environment variable)." % path)
            warnings.warn(msg, UserWarning)
1318

1319 1320
    def postprocess(self, tempname, filename):
        """Perform any platform-specific postprocessing of `tempname`
1321 1322 1323 1324

        This is where Mac header rewrites should be done; other platforms don't
        have anything special they should do.

1325 1326
        Resource providers should call this method ONLY after successfully
        extracting a compressed resource.  They must NOT call it on resources
1327
        that are already in the filesystem.
1328 1329 1330 1331

        `tempname` is the current (temporary) name of the file, and `filename`
        is the name it will be renamed to by the caller after this routine
        returns.
1332
        """
1333 1334 1335

        if os.name == 'posix':
            # Make the resource executable
1336
            mode = ((os.stat(tempname).st_mode) | 0o555) & 0o7777
1337 1338
            os.chmod(tempname, mode)

1339 1340 1341
    def set_extraction_path(self, path):
        """Set the base path where resources will be extracted to, if needed.

1342 1343 1344 1345 1346
        If you do not call this routine before any extractions take place, the
        path defaults to the return value of ``get_default_cache()``.  (Which
        is based on the ``PYTHON_EGG_CACHE`` environment variable, with various
        platform-specific fallbacks.  See that routine's documentation for more
        details.)
1347

1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377
        Resources are extracted to subdirectories of this path based upon
        information given by the ``IResourceProvider``.  You may set this to a
        temporary directory, but then you must call ``cleanup_resources()`` to
        delete the extracted files when done.  There is no guarantee that
        ``cleanup_resources()`` will be able to remove all extracted files.

        (Note: you may not change the extraction path for a given resource
        manager once resources have been extracted, unless you first call
        ``cleanup_resources()``.)
        """
        if self.cached_files:
            raise ValueError(
                "Can't change extraction path, files already extracted"
            )

        self.extraction_path = path

    def cleanup_resources(self, force=False):
        """
        Delete all extracted resource files and directories, returning a list
        of the file and directory names that could not be successfully removed.
        This function does not have any concurrency protection, so it should
        generally only be called when the extraction path is a temporary
        directory exclusive to a single process.  This method is not
        automatically called; you must call it explicitly or register it as an
        ``atexit`` function if you wish to ensure cleanup of a temporary
        directory used for extractions.
        """
        # XXX

1378

1379 1380
def get_default_cache():
    """
1381 1382 1383 1384 1385 1386 1387 1388
    Return the ``PYTHON_EGG_CACHE`` environment variable
    or a platform-relevant user cache dir for an app
    named "Python-Eggs".
    """
    return (
        os.environ.get('PYTHON_EGG_CACHE')
        or appdirs.user_cache_dir(appname='Python-Eggs')
    )
1389

1390

1391 1392
def safe_name(name):
    """Convert an arbitrary string to a standard distribution name
1393

1394
    Any runs of non-alphanumeric/. characters are replaced with a single '-'.
1395
    """
1396
    return re.sub('[^A-Za-z0-9.]+', '-', name)
1397 1398


1399 1400
def safe_version(version):
    """
1401 1402 1403
    Convert an arbitrary string to a standard version string
    """
    try:
Jason R. Coombs's avatar
Jason R. Coombs committed
1404
        # normalize the version
1405 1406
        return str(packaging.version.Version(version))
    except packaging.version.InvalidVersion:
1407
        version = version.replace(' ', '.')
1408
        return re.sub('[^A-Za-z0-9.]+', '-', version)
1409

1410

1411 1412
def safe_extra(extra):
    """Convert an arbitrary string to a standard 'extra' name
1413

1414 1415 1416
    Any runs of non-alphanumeric characters are replaced with a single '_',
    and the result is always lowercased.
    """
1417
    return re.sub('[^A-Za-z0-9.-]+', '_', extra).lower()
1418 1419


1420 1421
def to_filename(name):
    """Convert a project or version name to its filename-escaped form
1422

1423 1424
    Any '-' characters are currently replaced with '_'.
    """
1425
    return name.replace('-', '_')
1426

1427

1428 1429 1430 1431 1432 1433 1434
def invalid_marker(text):
    """
    Validate text as a PEP 508 environment marker; return an exception
    if invalid or False otherwise.
    """
    try:
        evaluate_marker(text)
1435
    except SyntaxError as e:
1436 1437 1438 1439
        e.filename = None
        e.lineno = None
        return e
    return False
1440

1441

1442 1443 1444 1445
def evaluate_marker(text, extra=None):
    """
    Evaluate a PEP 508 environment marker.
    Return a boolean indicating the marker result in this environment.
1446
    Raise SyntaxError if marker is invalid.
1447

1448 1449
    This implementation uses the 'pyparsing' module.
    """
1450
    try:
1451
        marker = packaging.markers.Marker(text)
1452
        return marker.evaluate()
1453
    except packaging.markers.InvalidMarker as e:
1454
        raise SyntaxError(e)
1455 1456


1457 1458 1459 1460
class NullProvider:
    """Try to implement resources and metadata for arbitrary PEP 302 loaders"""

    egg_name = None
1461
    egg_info = None
1462
    loader = None
1463

1464
    def __init__(self, module):
1465
        self.loader = getattr(module, '__loader__', None)
1466
        self.module_path = os.path.dirname(getattr(module, '__file__', ''))
1467

1468
    def get_resource_filename(self, manager, resource_name):
1469
        return self._fn(self.module_path, resource_name)
1470

1471
    def get_resource_stream(self, manager, resource_name):
1472
        return io.BytesIO(self.get_resource_string(manager, resource_name))
1473

1474
    def get_resource_string(self, manager, resource_name):
1475
        return self._get(self._fn(self.module_path, resource_name))
1476

1477
    def has_resource(self, resource_name):
1478
        return self._has(self._fn(self.module_path, resource_name))
1479

1480
    def has_metadata(self, name):
1481
        return self.egg_info and self._has(self._fn(self.egg_info, name))
1482

1483 1484 1485 1486 1487
    def get_metadata(self, name):
        if not self.egg_info:
            return ""
        value = self._get(self._fn(self.egg_info, name))
        return value.decode('utf-8') if six.PY3 else value
1488

1489
    def get_metadata_lines(self, name):
1490 1491
        return yield_lines(self.get_metadata(name))

1492
    def resource_isdir(self, resource_name):
1493 1494
        return self._isdir(self._fn(self.module_path, resource_name))

1495 1496
    def metadata_isdir(self, name):
        return self.egg_info and self._isdir(self._fn(self.egg_info, name))
1497

1498 1499
    def resource_listdir(self, resource_name):
        return self._listdir(self._fn(self.module_path, resource_name))
1500

1501
    def metadata_listdir(self, name):
1502
        if self.egg_info:
1503
            return self._listdir(self._fn(self.egg_info, name))
1504 1505
        return []

1506
    def run_script(self, script_name, namespace):
1507
        script = 'scripts/' + script_name
1508 1509
        if not self.has_metadata(script):
            raise ResolutionError("No script named %r" % script_name)
1510 1511 1512
        script_text = self.get_metadata(script).replace('\r\n', '\n')
        script_text = script_text.replace('\r', '\n')
        script_filename = self._fn(self.egg_info, script)
1513
        namespace['__file__'] = script_filename
1514
        if os.path.exists(script_filename):
1515 1516 1517
            source = open(script_filename).read()
            code = compile(source, script_filename, 'exec')
            exec(code, namespace, namespace)
1518 1519 1520 1521 1522
        else:
            from linecache import cache
            cache[script_filename] = (
                len(script_text), 0, script_text.split('\n'), script_filename
            )
1523
            script_code = compile(script_text, script_filename, 'exec')
1524
            exec(script_code, namespace, namespace)
1525

1526
    def _has(self, path):
1527 1528 1529 1530
        raise NotImplementedError(
            "Can't perform this operation for unregistered loader type"
        )

1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541
    def _isdir(self, path):
        raise NotImplementedError(
            "Can't perform this operation for unregistered loader type"
        )

    def _listdir(self, path):
        raise NotImplementedError(
            "Can't perform this operation for unregistered loader type"
        )

    def _fn(self, base, resource_name):
1542 1543 1544
        if resource_name:
            return os.path.join(base, *resource_name.split('/'))
        return base
1545

1546
    def _get(self, path):
1547 1548 1549 1550 1551
        if hasattr(self.loader, 'get_data'):
            return self.loader.get_data(path)
        raise NotImplementedError(
            "Can't perform this operation for loaders without 'get_data()'"
        )
1552

1553

1554 1555 1556
register_loader_type(object, NullProvider)


1557 1558
class EggProvider(NullProvider):
    """Provider based on a virtual filesystem"""
1559

1560 1561
    def __init__(self, module):
        NullProvider.__init__(self, module)
1562
        self._setup_prefix()
1563

1564 1565 1566
    def _setup_prefix(self):
        # we assume here that our metadata may be nested inside a "basket"
        # of multiple eggs; that's why we use module_path instead of .archive
1567
        path = self.module_path
1568
        old = None
1569
        while path != old:
1570
            if _is_egg_path(path):
1571
                self.egg_name = os.path.basename(path)
1572
                self.egg_info = os.path.join(path, 'EGG-INFO')
1573
                self.egg_root = path
1574 1575 1576 1577
                break
            old = path
            path, base = os.path.split(path)

1578

1579 1580 1581 1582 1583 1584
class DefaultProvider(EggProvider):
    """Provides access to package resources in the filesystem"""

    def _has(self, path):
        return os.path.exists(path)

1585
    def _isdir(self, path):
1586
        return os.path.isdir(path)
1587

1588
    def _listdir(self, path):
1589
        return os.listdir(path)
1590

1591 1592 1593
    def get_resource_stream(self, manager, resource_name):
        return open(self._fn(self.module_path, resource_name), 'rb')

1594
    def _get(self, path):
Jason R. Coombs's avatar
Jason R. Coombs committed
1595
        with open(path, 'rb') as stream:
1596 1597
            return stream.read()

1598 1599
    @classmethod
    def _register(cls):
1600 1601 1602
        loader_cls = getattr(importlib_machinery, 'SourceFileLoader',
            type(None))
        register_loader_type(loader_cls, cls)
1603

1604

1605
DefaultProvider._register()
1606

1607

1608 1609
class EmptyProvider(NullProvider):
    """Provider that returns nothing for all requests"""
1610

1611 1612 1613
    _isdir = _has = lambda self, path: False
    _get = lambda self, path: ''
    _listdir = lambda self, path: []
Jason R. Coombs's avatar
Jason R. Coombs committed
1614
    module_path = None
1615

1616 1617
    def __init__(self):
        pass
1618

1619

1620
empty_provider = EmptyProvider()
1621 1622


1623
class ZipManifests(dict):
1624
    """
1625
    zip manifest builder
1626
    """
1627

1628 1629
    @classmethod
    def build(cls, path):
1630
        """
Jason R. Coombs's avatar
Jason R. Coombs committed
1631 1632
        Build a dictionary similar to the zipimport directory
        caches, except instead of tuples, store ZipInfo objects.
Jason R. Coombs's avatar
Jason R. Coombs committed
1633

1634 1635
        Use a platform-specific path separator (os.sep) for the path keys
        for compatibility with pypy on Windows.
1636 1637
        """
        with ContextualZipFile(path) as zfile:
1638 1639 1640 1641 1642 1643 1644 1645
            items = (
                (
                    name.replace('/', os.sep),
                    zfile.getinfo(name),
                )
                for name in zfile.namelist()
            )
            return dict(items)
1646

1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668
    load = build


class MemoizedZipManifests(ZipManifests):
    """
    Memoized zipfile manifests.
    """
    manifest_mod = collections.namedtuple('manifest_mod', 'manifest mtime')

    def load(self, path):
        """
        Load a manifest at path or return a suitable manifest already loaded.
        """
        path = os.path.normpath(path)
        mtime = os.stat(path).st_mtime

        if path not in self or self[path].mtime != mtime:
            manifest = self.build(path)
            self[path] = self.manifest_mod(manifest, mtime)

        return self[path].manifest

1669

1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680
class ContextualZipFile(zipfile.ZipFile):
    """
    Supplement ZipFile class to support context manager for Python 2.6
    """

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.close()

1681
    def __new__(cls, *args, **kwargs):
1682 1683 1684
        """
        Construct a ZipFile or ContextualZipFile as appropriate
        """
1685 1686
        if hasattr(zipfile.ZipFile, '__exit__'):
            return zipfile.ZipFile(*args, **kwargs)
1687
        return super(ContextualZipFile, cls).__new__(cls)
1688 1689


1690
class ZipProvider(EggProvider):
1691 1692 1693
    """Resource support for zips and eggs"""

    eagers = None
1694
    _zip_manifests = MemoizedZipManifests()
1695 1696

    def __init__(self, module):
1697
        EggProvider.__init__(self, module)
1698
        self.zip_pre = self.loader.archive + os.sep
1699

1700 1701 1702 1703 1704 1705
    def _zipinfo_name(self, fspath):
        # Convert a virtual filename (full path to file) into a zipfile subpath
        # usable with the zipimport directory cache for our target archive
        if fspath.startswith(self.zip_pre):
            return fspath[len(self.zip_pre):]
        raise AssertionError(
1706
            "%s is not a subpath of %s" % (fspath, self.zip_pre)
1707
        )
1708

1709
    def _parts(self, zip_path):
1710 1711
        # Convert a zipfile subpath into an egg-relative path part list.
        # pseudo-fs path
1712 1713 1714
        fspath = self.zip_pre + zip_path
        if fspath.startswith(self.egg_root + os.sep):
            return fspath[len(self.egg_root) + 1:].split(os.sep)
1715
        raise AssertionError(
1716
            "%s is not a subpath of %s" % (fspath, self.egg_root)
1717
        )
1718

1719 1720
    @property
    def zipinfo(self):
1721
        return self._zip_manifests.load(self.loader.archive)
1722

1723
    def get_resource_filename(self, manager, resource_name):
1724 1725 1726 1727
        if not self.egg_name:
            raise NotImplementedError(
                "resource_filename() only supported for .egg, not .zip"
            )
1728
        # no need to lock for extraction, since we use temp names
1729
        zip_path = self._resource_to_zip(resource_name)
1730
        eagers = self._get_eager_resources()
1731
        if '/'.join(self._parts(zip_path)) in eagers:
1732
            for name in eagers:
1733 1734 1735
                self._extract_resource(manager, self._eager_to_zip(name))
        return self._extract_resource(manager, zip_path)

1736 1737
    @staticmethod
    def _get_date_and_size(zip_stat):
1738
        size = zip_stat.file_size
Jason R. Coombs's avatar
Jason R. Coombs committed
1739 1740 1741
        # ymdhms+wday, yday, dst
        date_time = zip_stat.date_time + (0, 0, -1)
        # 1980 offset already done
1742 1743 1744
        timestamp = time.mktime(date_time)
        return timestamp, size

1745
    def _extract_resource(self, manager, zip_path):
1746

1747 1748 1749 1750 1751
        if zip_path in self._index():
            for name in self._index()[zip_path]:
                last = self._extract_resource(
                    manager, os.path.join(zip_path, name)
                )
Jason R. Coombs's avatar
Jason R. Coombs committed
1752 1753
            # return the extracted directory name
            return os.path.dirname(last)
1754

1755
        timestamp, size = self._get_date_and_size(self.zipinfo[zip_path])
1756

1757 1758 1759
        if not WRITE_SUPPORT:
            raise IOError('"os.rename" and "os.unlink" are not supported '
                          'on this platform')
1760
        try:
1761

1762 1763 1764 1765
            real_path = manager.get_cache_path(
                self.egg_name, self._parts(zip_path)
            )

1766
            if self._is_current(real_path, zip_path):
1767
                return real_path
1768

1769 1770 1771
            outf, tmpnam = _mkstemp(".$extract", dir=os.path.dirname(real_path))
            os.write(outf, self.loader.get_data(zip_path))
            os.close(outf)
1772
            utime(tmpnam, (timestamp, timestamp))
1773 1774 1775 1776
            manager.postprocess(tmpnam, real_path)

            try:
                rename(tmpnam, real_path)
1777 1778

            except os.error:
1779
                if os.path.isfile(real_path):
1780 1781 1782
                    if self._is_current(real_path, zip_path):
                        # the file became current since it was checked above,
                        #  so proceed.
1783
                        return real_path
Jason R. Coombs's avatar
Jason R. Coombs committed
1784
                    # Windows, del old file and retry
1785
                    elif os.name == 'nt':
1786 1787 1788 1789 1790 1791
                        unlink(real_path)
                        rename(tmpnam, real_path)
                        return real_path
                raise

        except os.error:
Jason R. Coombs's avatar
Jason R. Coombs committed
1792 1793
            # report a user-friendly error
            manager.extraction_error()
1794 1795

        return real_path
1796

1797 1798 1799 1800 1801 1802 1803 1804
    def _is_current(self, file_path, zip_path):
        """
        Return True if the file_path is current for this zip_path
        """
        timestamp, size = self._get_date_and_size(self.zipinfo[zip_path])
        if not os.path.isfile(file_path):
            return False
        stat = os.stat(file_path)
1805
        if stat.st_size != size or stat.st_mtime != timestamp:
1806 1807 1808
            return False
        # check that the contents match
        zip_contents = self.loader.get_data(zip_path)
1809 1810
        with open(file_path, 'rb') as f:
            file_contents = f.read()
1811
        return zip_contents == file_contents
1812

1813 1814 1815
    def _get_eager_resources(self):
        if self.eagers is None:
            eagers = []
1816
            for name in ('native_libs.txt', 'eager_resources.txt'):
1817 1818 1819 1820 1821
                if self.has_metadata(name):
                    eagers.extend(self.get_metadata_lines(name))
            self.eagers = eagers
        return self.eagers

1822 1823 1824 1825
    def _index(self):
        try:
            return self._dirindex
        except AttributeError:
1826
            ind = {}
1827 1828 1829
            for path in self.zipinfo:
                parts = path.split(os.sep)
                while parts:
1830
                    parent = os.sep.join(parts[:-1])
1831 1832 1833 1834 1835 1836 1837 1838
                    if parent in ind:
                        ind[parent].append(parts[-1])
                        break
                    else:
                        ind[parent] = [parts.pop()]
            self._dirindex = ind
            return ind

1839 1840 1841
    def _has(self, fspath):
        zip_path = self._zipinfo_name(fspath)
        return zip_path in self.zipinfo or zip_path in self._index()
1842

1843
    def _isdir(self, fspath):
1844
        return self._zipinfo_name(fspath) in self._index()
1845

1846
    def _listdir(self, fspath):
1847
        return list(self._index().get(self._zipinfo_name(fspath), ()))
1848

1849 1850
    def _eager_to_zip(self, resource_name):
        return self._zipinfo_name(self._fn(self.egg_root, resource_name))
1851

1852 1853
    def _resource_to_zip(self, resource_name):
        return self._zipinfo_name(self._fn(self.module_path, resource_name))
1854

1855

1856
register_loader_type(zipimport.zipimporter, ZipProvider)
1857 1858


1859 1860
class FileMetadata(EmptyProvider):
    """Metadata handler for standalone PKG-INFO files
1861

1862
    Usage::
1863

1864
        metadata = FileMetadata("/path/to/PKG-INFO")
1865

1866 1867 1868 1869
    This provider rejects all data and metadata requests except for PKG-INFO,
    which is treated as existing, and will be the contents of the file at
    the provided location.
    """
1870

1871
    def __init__(self, path):
1872
        self.path = path
1873

1874
    def has_metadata(self, name):
1875
        return name == 'PKG-INFO' and os.path.isfile(self.path)
1876

1877
    def get_metadata(self, name):
1878 1879 1880 1881 1882 1883 1884
        if name != 'PKG-INFO':
            raise KeyError("No metadata except PKG-INFO is available")

        with io.open(self.path, encoding='utf-8', errors="replace") as f:
            metadata = f.read()
        self._warn_on_replacement(metadata)
        return metadata
1885

1886
    def _warn_on_replacement(self, metadata):
1887 1888
        # Python 2.6 and 3.2 compat for: replacement_char = '�'
        replacement_char = b'\xef\xbf\xbd'.decode('utf-8')
1889 1890 1891 1892 1893
        if replacement_char in metadata:
            tmpl = "{self.path} could not be properly decoded in UTF-8"
            msg = tmpl.format(**locals())
            warnings.warn(msg)

1894
    def get_metadata_lines(self, name):
1895
        return yield_lines(self.get_metadata(name))
1896 1897


1898 1899
class PathMetadata(DefaultProvider):
    """Metadata provider for egg directories
1900

1901
    Usage::
1902

1903
        # Development eggs:
1904

1905 1906 1907 1908
        egg_info = "/path/to/PackageName.egg-info"
        base_dir = os.path.dirname(egg_info)
        metadata = PathMetadata(base_dir, egg_info)
        dist_name = os.path.splitext(os.path.basename(egg_info))[0]
1909
        dist = Distribution(basedir, project_name=dist_name, metadata=metadata)
1910

1911
        # Unpacked egg directories:
1912

1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928
        egg_path = "/path/to/PackageName-ver-pyver-etc.egg"
        metadata = PathMetadata(egg_path, os.path.join(egg_path,'EGG-INFO'))
        dist = Distribution.from_filename(egg_path, metadata=metadata)
    """

    def __init__(self, path, egg_info):
        self.module_path = path
        self.egg_info = egg_info


class EggMetadata(ZipProvider):
    """Metadata provider for .egg files"""

    def __init__(self, importer):
        """Create a metadata provider from a zipimporter"""

1929
        self.zip_pre = importer.archive + os.sep
1930
        self.loader = importer
1931 1932 1933 1934 1935
        if importer.prefix:
            self.module_path = os.path.join(importer.archive, importer.prefix)
        else:
            self.module_path = importer.archive
        self._setup_prefix()
1936

1937

1938
_declare_state('dict', _distribution_finders={})
1939

1940

1941 1942 1943 1944
def register_finder(importer_type, distribution_finder):
    """Register `distribution_finder` to find distributions in sys.path items

    `importer_type` is the type or class of a PEP 302 "Importer" (sys.path item
1945
    handler), and `distribution_finder` is a callable that, passed a path
1946 1947 1948 1949 1950
    item and the importer instance, yields ``Distribution`` instances found on
    that path item.  See ``pkg_resources.find_on_path`` for an example."""
    _distribution_finders[importer_type] = distribution_finder


1951
def find_distributions(path_item, only=False):
1952 1953 1954
    """Yield distributions accessible via `path_item`"""
    importer = get_importer(path_item)
    finder = _find_adapter(_distribution_finders, importer)
1955
    return finder(importer, path_item, only)
1956

1957

1958 1959 1960 1961 1962 1963 1964 1965
def find_eggs_in_zip(importer, path_item, only=False):
    """
    Find eggs in zip files; possibly multiple nested eggs.
    """
    if importer.archive.endswith('.whl'):
        # wheels are not supported with this finder
        # they don't have PKG-INFO metadata, and won't ever contain eggs
        return
1966 1967 1968
    metadata = EggMetadata(importer)
    if metadata.has_metadata('PKG-INFO'):
        yield Distribution.from_filename(path_item, metadata=metadata)
1969
    if only:
Jason R. Coombs's avatar
Jason R. Coombs committed
1970 1971
        # don't yield nested distros
        return
1972
    for subitem in metadata.resource_listdir('/'):
1973
        if _is_egg_path(subitem):
1974
            subpath = os.path.join(path_item, subitem)
1975
            for dist in find_eggs_in_zip(zipimport.zipimporter(subpath), subpath):
1976
                yield dist
1977 1978 1979 1980 1981 1982
        elif subitem.lower().endswith('.dist-info'):
            subpath = os.path.join(path_item, subitem)
            submeta = EggMetadata(zipimport.zipimporter(subpath))
            submeta.egg_info = subpath
            yield Distribution.from_location(path_item, subitem, submeta)

1983

1984

1985
register_finder(zipimport.zipimporter, find_eggs_in_zip)
1986

1987

1988
def find_nothing(importer, path_item, only=False):
1989
    return ()
1990 1991


1992
register_finder(object, find_nothing)
1993

1994

1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020
def _by_version_descending(names):
    """
    Given a list of filenames, return them in descending order
    by version number.

    >>> names = 'bar', 'foo', 'Python-2.7.10.egg', 'Python-2.7.2.egg'
    >>> _by_version_descending(names)
    ['Python-2.7.10.egg', 'Python-2.7.2.egg', 'foo', 'bar']
    >>> names = 'Setuptools-1.2.3b1.egg', 'Setuptools-1.2.3.egg'
    >>> _by_version_descending(names)
    ['Setuptools-1.2.3.egg', 'Setuptools-1.2.3b1.egg']
    >>> names = 'Setuptools-1.2.3b1.egg', 'Setuptools-1.2.3.post1.egg'
    >>> _by_version_descending(names)
    ['Setuptools-1.2.3.post1.egg', 'Setuptools-1.2.3b1.egg']
    """
    def _by_version(name):
        """
        Parse each component of the filename
        """
        name, ext = os.path.splitext(name)
        parts = itertools.chain(name.split('-'), [ext])
        return [packaging.version.parse(part) for part in parts]

    return sorted(names, key=_by_version, reverse=True)


2021
def find_on_path(importer, path_item, only=False):
2022
    """Yield distributions accessible on a sys.path directory"""
2023
    path_item = _normalize_cached(path_item)
2024

2025 2026 2027 2028
    if _is_unpacked_egg(path_item):
        yield Distribution.from_filename(
            path_item, metadata=PathMetadata(
                path_item, os.path.join(path_item, 'EGG-INFO')
2029
            )
2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075
        )
    else:
        try:
            entries = os.listdir(path_item)
        except (PermissionError, NotADirectoryError):
            return
        except OSError as e:
            # Ignore the directory if does not exist, not a directory or we
            # don't have permissions
            if (e.errno in (errno.ENOTDIR, errno.EACCES, errno.ENOENT)
                # Python 2 on Windows needs to be handled this way :(
               or hasattr(e, "winerror") and e.winerror == 267):
                return
            raise
        # scan for .egg and .egg-info in directory
        path_item_entries = _by_version_descending(entries)
        for entry in path_item_entries:
            lower = entry.lower()
            if lower.endswith('.egg-info') or lower.endswith('.dist-info'):
                fullpath = os.path.join(path_item, entry)
                if os.path.isdir(fullpath):
                    # egg-info directory, allow getting metadata
                    if len(os.listdir(fullpath)) == 0:
                        # Empty egg directory, skip.
                        continue
                    metadata = PathMetadata(path_item, fullpath)
                else:
                    metadata = FileMetadata(fullpath)
                yield Distribution.from_location(
                    path_item, entry, metadata, precedence=DEVELOP_DIST
                )
            elif not only and _is_egg_path(entry):
                dists = find_distributions(os.path.join(path_item, entry))
                for dist in dists:
                    yield dist
            elif not only and lower.endswith('.egg-link'):
                with open(os.path.join(path_item, entry)) as entry_file:
                    entry_lines = entry_file.readlines()
                for line in entry_lines:
                    if not line.strip():
                        continue
                    path = os.path.join(path_item, line.rstrip())
                    dists = find_distributions(path)
                    for item in dists:
                        yield item
                    break
2076 2077


2078
register_finder(pkgutil.ImpImporter, find_on_path)
2079

2080
if hasattr(importlib_machinery, 'FileFinder'):
2081
    register_finder(importlib_machinery.FileFinder, find_on_path)
2082

Tarek Ziade's avatar
Tarek Ziade committed
2083 2084 2085
_declare_state('dict', _namespace_handlers={})
_declare_state('dict', _namespace_packages={})

2086 2087 2088 2089 2090 2091 2092

def register_namespace_handler(importer_type, namespace_handler):
    """Register `namespace_handler` to declare namespace packages

    `importer_type` is the type or class of a PEP 302 "Importer" (sys.path item
    handler), and `namespace_handler` is a callable like this::

2093
        def namespace_handler(importer, path_entry, moduleName, module):
2094 2095 2096 2097 2098 2099 2100 2101 2102 2103
            # return a path_entry to use for child packages

    Namespace handlers are only called if the importer object has already
    agreed that it can handle the relevant path item, and they should only
    return a subpath if the module __path__ does not already contain an
    equivalent subpath.  For an example namespace handler, see
    ``pkg_resources.file_ns_handler``.
    """
    _namespace_handlers[importer_type] = namespace_handler

2104

2105 2106
def _handle_ns(packageName, path_item):
    """Ensure that named package includes a subpath of path_item (if needed)"""
2107

2108
    importer = get_importer(path_item)
PJ Eby's avatar
PJ Eby committed
2109 2110
    if importer is None:
        return None
2111 2112 2113
    loader = importer.find_module(packageName)
    if loader is None:
        return None
2114 2115
    module = sys.modules.get(packageName)
    if module is None:
2116
        module = sys.modules[packageName] = types.ModuleType(packageName)
Jason R. Coombs's avatar
Jason R. Coombs committed
2117 2118
        module.__path__ = []
        _set_parent_ns(packageName)
2119
    elif not hasattr(module, '__path__'):
2120
        raise TypeError("Not a package:", packageName)
PJ Eby's avatar
PJ Eby committed
2121
    handler = _find_adapter(_namespace_handlers, importer)
2122
    subpath = handler(importer, path_item, packageName, module)
2123
    if subpath is not None:
Jason R. Coombs's avatar
Jason R. Coombs committed
2124 2125 2126
        path = module.__path__
        path.append(subpath)
        loader.load_module(packageName)
2127
        _rebuild_mod_path(path, packageName, module)
2128 2129
    return subpath

2130

2131 2132 2133 2134 2135
def _rebuild_mod_path(orig_path, package_name, module):
    """
    Rebuild module.__path__ ensuring that all entries are ordered
    corresponding to their sys.path order
    """
2136
    sys_path = [_normalize_cached(p) for p in sys.path]
2137

2138 2139 2140 2141 2142 2143 2144 2145 2146
    def safe_sys_path_index(entry):
        """
        Workaround for #520 and #513.
        """
        try:
            return sys_path.index(entry)
        except ValueError:
            return float('inf')

2147
    def position_in_sys_path(path):
2148 2149 2150
        """
        Return the ordinal of the path based on its position in sys.path
        """
2151 2152 2153
        path_parts = path.split(os.sep)
        module_parts = package_name.count('.') + 1
        parts = path_parts[:-module_parts]
2154
        return safe_sys_path_index(_normalize_cached(os.sep.join(parts)))
2155

2156 2157 2158 2159 2160
    if not isinstance(orig_path, list):
        # Is this behavior useful when module.__path__ is not a list?
        return

    orig_path.sort(key=position_in_sys_path)
2161 2162
    module.__path__[:] = [_normalize_cached(p) for p in orig_path]

2163 2164 2165 2166

def declare_namespace(packageName):
    """Declare that package 'packageName' is a namespace package"""

2167
    _imp.acquire_lock()
2168 2169 2170 2171 2172 2173
    try:
        if packageName in _namespace_packages:
            return

        path, parent = sys.path, None
        if '.' in packageName:
2174
            parent = '.'.join(packageName.split('.')[:-1])
2175
            declare_namespace(parent)
2176 2177
            if parent not in _namespace_packages:
                __import__(parent)
2178 2179 2180 2181
            try:
                path = sys.modules[parent].__path__
            except AttributeError:
                raise TypeError("Not a package:", parent)
PJ Eby's avatar
PJ Eby committed
2182

2183 2184
        # Track what packages are namespaces, so when new path items are added,
        # they can be updated
2185 2186
        _namespace_packages.setdefault(parent, []).append(packageName)
        _namespace_packages.setdefault(packageName, [])
2187

2188 2189 2190 2191 2192
        for path_item in path:
            # Ensure all the parent's path items are reflected in the child,
            # if they apply
            _handle_ns(packageName, path_item)

2193
    finally:
2194
        _imp.release_lock()
2195

2196

2197 2198
def fixup_namespace_packages(path_item, parent=None):
    """Ensure that previously-declared namespace packages include path_item"""
2199
    _imp.acquire_lock()
2200
    try:
2201
        for package in _namespace_packages.get(parent, ()):
2202
            subpath = _handle_ns(package, path_item)
2203 2204
            if subpath:
                fixup_namespace_packages(subpath, package)
2205
    finally:
2206
        _imp.release_lock()
2207

2208

2209 2210 2211 2212
def file_ns_handler(importer, path_item, packageName, module):
    """Compute an ns-package subpath for a filesystem or zipfile importer"""

    subpath = os.path.join(path_item, packageName.split('.')[-1])
2213
    normalized = _normalize_cached(subpath)
2214
    for item in module.__path__:
2215
        if _normalize_cached(item) == normalized:
2216 2217 2218 2219 2220
            break
    else:
        # Only return the path if it's not already there
        return subpath

2221

2222 2223
register_namespace_handler(pkgutil.ImpImporter, file_ns_handler)
register_namespace_handler(zipimport.zipimporter, file_ns_handler)
2224

2225
if hasattr(importlib_machinery, 'FileFinder'):
2226
    register_namespace_handler(importlib_machinery.FileFinder, file_ns_handler)
2227

2228

PJ Eby's avatar
PJ Eby committed
2229 2230
def null_ns_handler(importer, path_item, packageName, module):
    return None
2231

2232

2233
register_namespace_handler(object, null_ns_handler)
2234 2235


2236 2237 2238
def normalize_path(filename):
    """Normalize a file/dir name for comparison purposes"""
    return os.path.normcase(os.path.realpath(filename))
2239

2240

2241
def _normalize_cached(filename, _cache={}):
2242 2243 2244 2245 2246
    try:
        return _cache[filename]
    except KeyError:
        _cache[filename] = result = normalize_path(filename)
        return result
2247

2248

2249 2250 2251 2252 2253 2254 2255 2256 2257
def _is_egg_path(path):
    """
    Determine if given path appears to be an egg.
    """
    return (
        path.lower().endswith('.egg')
    )


2258 2259 2260 2261 2262
def _is_unpacked_egg(path):
    """
    Determine if given path appears to be an unpacked egg.
    """
    return (
2263 2264
        _is_egg_path(path) and
        os.path.isfile(os.path.join(path, 'EGG-INFO', 'PKG-INFO'))
2265 2266
    )

2267

2268 2269 2270 2271 2272 2273
def _set_parent_ns(packageName):
    parts = packageName.split('.')
    name = parts.pop()
    if parts:
        parent = '.'.join(parts)
        setattr(sys.modules[parent], name, sys.modules[packageName])
2274 2275


2276
def yield_lines(strs):
2277
    """Yield non-empty/non-comment lines of a string or sequence"""
2278
    if isinstance(strs, six.string_types):
2279 2280
        for s in strs.splitlines():
            s = s.strip()
Jason R. Coombs's avatar
Jason R. Coombs committed
2281 2282
            # skip blank lines/comments
            if s and not s.startswith('#'):
2283 2284 2285 2286 2287 2288
                yield s
    else:
        for ss in strs:
            for s in yield_lines(ss):
                yield s

2289

Jason R. Coombs's avatar
Jason R. Coombs committed
2290
MODULE = re.compile(r"\w+(\.\w+)*$").match
2291
EGG_NAME = re.compile(
2292
    r"""
Jason R. Coombs's avatar
Jason R. Coombs committed
2293
    (?P<name>[^-]+) (
2294 2295 2296 2297 2298 2299 2300 2301
        -(?P<ver>[^-]+) (
            -py(?P<pyver>[^-]+) (
                -(?P<plat>.+)
            )?
        )?
    )?
    """,
    re.VERBOSE | re.IGNORECASE,
2302 2303
).match

2304

2305
class EntryPoint(object):
2306
    """Object representing an advertised importable object"""
2307

2308
    def __init__(self, name, module_name, attrs=(), extras=(), dist=None):
2309 2310 2311 2312 2313
        if not MODULE(module_name):
            raise ValueError("Invalid module name", module_name)
        self.name = name
        self.module_name = module_name
        self.attrs = tuple(attrs)
2314
        self.extras = Requirement.parse(("x[%s]" % ','.join(extras))).extras
2315
        self.dist = dist
2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327

    def __str__(self):
        s = "%s = %s" % (self.name, self.module_name)
        if self.attrs:
            s += ':' + '.'.join(self.attrs)
        if self.extras:
            s += ' [%s]' % ','.join(self.extras)
        return s

    def __repr__(self):
        return "EntryPoint.parse(%r)" % str(self)

2328 2329 2330 2331 2332
    def load(self, require=True, *args, **kwargs):
        """
        Require packages for this EntryPoint, then resolve it.
        """
        if not require or args or kwargs:
2333
            warnings.warn(
2334 2335
                "Parameters to load are deprecated.  Call .resolve and "
                ".require separately.",
2336
                DeprecationWarning,
2337
                stacklevel=2,
2338
            )
2339 2340 2341
        if require:
            self.require(*args, **kwargs)
        return self.resolve()
2342

2343 2344 2345 2346
    def resolve(self):
        """
        Resolve the entry point from its module and attrs.
        """
2347
        module = __import__(self.module_name, fromlist=['__name__'], level=0)
2348
        try:
2349
            return functools.reduce(getattr, self.attrs, module)
2350 2351
        except AttributeError as exc:
            raise ImportError(str(exc))
2352

2353
    def require(self, env=None, installer=None):
2354 2355
        if self.extras and not self.dist:
            raise UnknownExtra("Can't require() without a distribution", self)
2356 2357 2358 2359 2360 2361

        # Get the requirements for this entry point with all its extras and
        # then resolve them. We have to pass `extras` along when resolving so
        # that the working set knows what extras we want. Otherwise, for
        # dist-info distributions, the working set will assume that the
        # requirements for that extra are purely optional and skip over them.
2362
        reqs = self.dist.requires(self.extras)
2363
        items = working_set.resolve(reqs, env, installer, extras=self.extras)
2364
        list(map(working_set.add, items))
2365

2366
    pattern = re.compile(
2367
        r'\s*'
2368
        r'(?P<name>.+?)\s*'
2369 2370 2371 2372 2373 2374
        r'=\s*'
        r'(?P<module>[\w.]+)\s*'
        r'(:\s*(?P<attr>[\w.]+))?\s*'
        r'(?P<extras>\[.*\])?\s*$'
    )

2375
    @classmethod
2376
    def parse(cls, src, dist=None):
2377 2378 2379 2380
        """Parse a single entry point from string `src`

        Entry point syntax follows the form::

2381
            name = some.module:some.attr [extra1, extra2]
2382 2383 2384 2385

        The entry name and module name are required, but the ``:attrs`` and
        ``[extras]`` parts are optional
        """
2386 2387
        m = cls.pattern.match(src)
        if not m:
2388 2389
            msg = "EntryPoint must be in 'name=module:attrs [extras]' format"
            raise ValueError(msg, src)
2390 2391
        res = m.groupdict()
        extras = cls._parse_extras(res['extras'])
2392
        attrs = res['attr'].split('.') if res['attr'] else ()
2393 2394 2395 2396 2397 2398 2399 2400 2401 2402
        return cls(res['name'], res['module'], attrs, extras, dist)

    @classmethod
    def _parse_extras(cls, extras_spec):
        if not extras_spec:
            return ()
        req = Requirement.parse('x' + extras_spec)
        if req.specs:
            raise ValueError()
        return req.extras
2403

2404
    @classmethod
2405 2406 2407 2408
    def parse_group(cls, group, lines, dist=None):
        """Parse an entry point group"""
        if not MODULE(group):
            raise ValueError("Invalid group name", group)
2409
        this = {}
2410
        for line in yield_lines(lines):
2411
            ep = cls.parse(line, dist)
2412
            if ep.name in this:
2413
                raise ValueError("Duplicate entry point", group, ep.name)
2414
            this[ep.name] = ep
2415 2416
        return this

2417
    @classmethod
2418
    def parse_map(cls, data, dist=None):
2419
        """Parse a map of entry point groups"""
2420
        if isinstance(data, dict):
2421 2422 2423 2424
            data = data.items()
        else:
            data = split_sections(data)
        maps = {}
2425 2426 2427
        for group, lines in data:
            if group is None:
                if not lines:
2428
                    continue
2429 2430 2431 2432 2433
                raise ValueError("Entry points must be listed in groups")
            group = group.strip()
            if group in maps:
                raise ValueError("Duplicate group name", group)
            maps[group] = cls.parse_group(group, lines, dist)
2434 2435 2436
        return maps


2437 2438 2439
def _remove_md5_fragment(location):
    if not location:
        return ''
2440
    parsed = urllib.parse.urlparse(location)
2441
    if parsed[-1].startswith('md5='):
2442
        return urllib.parse.urlunparse(parsed[:-1] + ('',))
2443
    return location
2444 2445


2446
def _version_from_file(lines):
Jason R. Coombs's avatar
Jason R. Coombs committed
2447 2448 2449 2450
    """
    Given an iterable of lines from a Metadata file, return
    the value of the Version field, if present, or None otherwise.
    """
2451 2452
    is_version_line = lambda line: line.lower().startswith('version:')
    version_lines = filter(is_version_line, lines)
2453
    line = next(iter(version_lines), '')
2454
    _, _, value = line.partition(':')
2455
    return safe_version(value.strip()) or None
2456 2457


2458 2459
class Distribution(object):
    """Wrap an actual or potential sys.path entry w/metadata"""
2460
    PKG_INFO = 'PKG-INFO'
2461

Jason R. Coombs's avatar
Jason R. Coombs committed
2462 2463 2464
    def __init__(self, location=None, metadata=None, project_name=None,
            version=None, py_version=PY_MAJOR, platform=None,
            precedence=EGG_DIST):
2465
        self.project_name = safe_name(project_name or 'Unknown')
2466
        if version is not None:
2467
            self._version = safe_version(version)
2468
        self.py_version = py_version
2469
        self.platform = platform
2470
        self.location = location
2471
        self.precedence = precedence
2472
        self._provider = metadata or empty_provider
2473

2474
    @classmethod
2475
    def from_location(cls, location, basename, metadata=None, **kw):
2476
        project_name, version, py_version, platform = [None] * 4
2477
        basename, ext = os.path.splitext(basename)
2478
        if ext.lower() in _distributionImpl:
2479 2480
            cls = _distributionImpl[ext.lower()]

2481 2482
            match = EGG_NAME(basename)
            if match:
2483
                project_name, version, py_version, platform = match.group(
2484
                    'name', 'ver', 'pyver', 'plat'
2485 2486
                )
        return cls(
2487
            location, metadata, project_name=project_name, version=version,
2488
            py_version=py_version, platform=platform, **kw
2489
        )._reload_version()
2490

2491 2492
    def _reload_version(self):
        return self
2493

2494 2495 2496
    @property
    def hashcmp(self):
        return (
2497
            self.parsed_version,
2498 2499
            self.precedence,
            self.key,
2500
            _remove_md5_fragment(self.location),
2501 2502
            self.py_version or '',
            self.platform or '',
2503
        )
2504

Jason R. Coombs's avatar
Jason R. Coombs committed
2505 2506 2507
    def __hash__(self):
        return hash(self.hashcmp)

2508 2509
    def __lt__(self, other):
        return self.hashcmp < other.hashcmp
Jason R. Coombs's avatar
Jason R. Coombs committed
2510

2511 2512
    def __le__(self, other):
        return self.hashcmp <= other.hashcmp
Jason R. Coombs's avatar
Jason R. Coombs committed
2513

2514 2515
    def __gt__(self, other):
        return self.hashcmp > other.hashcmp
Jason R. Coombs's avatar
Jason R. Coombs committed
2516

2517 2518
    def __ge__(self, other):
        return self.hashcmp >= other.hashcmp
Jason R. Coombs's avatar
Jason R. Coombs committed
2519

2520 2521 2522 2523 2524
    def __eq__(self, other):
        if not isinstance(other, self.__class__):
            # It's not a Distribution, so they are not equal
            return False
        return self.hashcmp == other.hashcmp
Jason R. Coombs's avatar
Jason R. Coombs committed
2525

2526 2527
    def __ne__(self, other):
        return not self == other
2528

2529 2530 2531 2532
    # These properties have to be lazy so that we don't have to load any
    # metadata until/unless it's actually needed.  (i.e., some distributions
    # may not know their name or version without loading PKG-INFO)

2533
    @property
2534 2535 2536 2537
    def key(self):
        try:
            return self._key
        except AttributeError:
2538
            self._key = key = self.project_name.lower()
2539 2540
            return key

2541
    @property
2542
    def parsed_version(self):
2543 2544
        if not hasattr(self, "_parsed_version"):
            self._parsed_version = parse_version(self.version)
2545 2546 2547 2548

        return self._parsed_version

    def _warn_legacy_version(self):
2549 2550 2551 2552 2553
        LV = packaging.version.LegacyVersion
        is_legacy = isinstance(self._parsed_version, LV)
        if not is_legacy:
            return

2554
        # While an empty version is technically a legacy version and
2555 2556 2557 2558 2559 2560 2561 2562 2563
        # is not a valid PEP 440 version, it's also unlikely to
        # actually come from someone and instead it is more likely that
        # it comes from setuptools attempting to parse a filename and
        # including it in the list. So for that we'll gate this warning
        # on if the version is anything at all or not.
        if not self.version:
            return

        tmpl = textwrap.dedent("""
2564 2565
            '{project_name} ({version})' is being parsed as a legacy,
            non PEP 440,
2566 2567
            version. You may find odd behavior and sort order.
            In particular it will be sorted as less than 0.0. It
2568
            is recommended to migrate to PEP 440 compatible
2569 2570
            versions.
            """).strip().replace('\n', ' ')
2571

2572
        warnings.warn(tmpl.format(**vars(self)), PEP440Warning)
2573

2574
    @property
2575 2576 2577 2578
    def version(self):
        try:
            return self._version
        except AttributeError:
2579 2580
            version = _version_from_file(self._get_metadata(self.PKG_INFO))
            if version is None:
2581 2582
                tmpl = "Missing 'Version:' header and/or %s file"
                raise ValueError(tmpl % self.PKG_INFO, self)
2583
            return version
2584

2585
    @property
2586 2587 2588 2589 2590
    def _dep_map(self):
        try:
            return self.__dep_map
        except AttributeError:
            dm = self.__dep_map = {None: []}
2591
            for name in 'requires.txt', 'depends.txt':
2592
                for extra, reqs in split_sections(self._get_metadata(name)):
2593 2594
                    if extra:
                        if ':' in extra:
2595
                            extra, marker = extra.split(':', 1)
2596
                            if invalid_marker(marker):
Jason R. Coombs's avatar
Jason R. Coombs committed
2597
                                # XXX warn
2598
                                reqs = []
2599
                            elif not evaluate_marker(marker):
2600
                                reqs = []
2601
                        extra = safe_extra(extra) or None
2602
                    dm.setdefault(extra, []).extend(parse_requirements(reqs))
2603
            return dm
2604

2605
    def requires(self, extras=()):
2606
        """List of Requirements needed for this distro if `extras` are used"""
2607 2608
        dm = self._dep_map
        deps = []
2609
        deps.extend(dm.get(None, ()))
2610
        for ext in extras:
2611
            try:
2612
                deps.extend(dm[safe_extra(ext)])
2613
            except KeyError:
2614
                raise UnknownExtra(
2615 2616
                    "%s has no such extra feature %r" % (self, ext)
                )
2617
        return deps
2618

2619
    def _get_metadata(self, name):
2620 2621
        if self.has_metadata(name):
            for line in self.get_metadata_lines(name):
2622 2623
                yield line

2624
    def activate(self, path=None, replace=False):
2625
        """Ensure distribution is importable on `path` (default=sys.path)"""
2626 2627
        if path is None:
            path = sys.path
2628
        self.insert_on(path, replace=replace)
2629
        if path is sys.path:
2630
            fixup_namespace_packages(self.location)
2631 2632 2633
            for pkg in self._get_metadata('namespace_packages.txt'):
                if pkg in sys.modules:
                    declare_namespace(pkg)
2634 2635 2636 2637

    def egg_name(self):
        """Return what this distribution's standard .egg filename should be"""
        filename = "%s-%s-py%s" % (
2638
            to_filename(self.project_name), to_filename(self.version),
2639 2640 2641 2642
            self.py_version or PY_MAJOR
        )

        if self.platform:
2643
            filename += '-' + self.platform
2644 2645
        return filename

2646
    def __repr__(self):
2647
        if self.location:
2648
            return "%s (%s)" % (self, self.location)
2649 2650
        else:
            return str(self)
2651

2652
    def __str__(self):
2653 2654 2655 2656
        try:
            version = getattr(self, 'version', None)
        except ValueError:
            version = None
2657
        version = version or "[unknown version]"
2658
        return "%s %s" % (self.project_name, version)
2659

2660
    def __getattr__(self, attr):
2661 2662
        """Delegate all unrecognized public attributes to .metadata provider"""
        if attr.startswith('_'):
2663
            raise AttributeError(attr)
2664
        return getattr(self._provider, attr)
2665

2666
    @classmethod
2667
    def from_filename(cls, filename, metadata=None, **kw):
2668
        return cls.from_location(
2669 2670
            _normalize_cached(filename), os.path.basename(filename), metadata,
            **kw
2671
        )
2672

2673
    def as_requirement(self):
2674
        """Return a ``Requirement`` that matches this distribution exactly"""
2675
        if isinstance(self.parsed_version, packaging.version.Version):
2676 2677 2678 2679 2680
            spec = "%s==%s" % (self.project_name, self.parsed_version)
        else:
            spec = "%s===%s" % (self.project_name, self.parsed_version)

        return Requirement.parse(spec)
2681

2682 2683
    def load_entry_point(self, group, name):
        """Return the `name` entry point of `group` or raise ImportError"""
2684
        ep = self.get_entry_info(group, name)
2685
        if ep is None:
2686
            raise ImportError("Entry point %r not found" % ((group, name),))
2687 2688
        return ep.load()

2689 2690
    def get_entry_map(self, group=None):
        """Return the entry point map for `group`, or the full entry map"""
2691 2692 2693 2694
        try:
            ep_map = self._ep_map
        except AttributeError:
            ep_map = self._ep_map = EntryPoint.parse_map(
2695
                self._get_metadata('entry_points.txt'), self
2696
            )
2697
        if group is not None:
2698
            return ep_map.get(group, {})
2699 2700
        return ep_map

2701 2702 2703 2704
    def get_entry_info(self, group, name):
        """Return the EntryPoint object for `group`+`name`, or ``None``"""
        return self.get_entry_map(group).get(name)

2705
    def insert_on(self, path, loc=None, replace=False):
2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723
        """Ensure self.location is on path

        If replace=False (default):
            - If location is already in path anywhere, do nothing.
            - Else:
              - If it's an egg and its parent directory is on path,
                insert just ahead of the parent.
              - Else: add to the end of path.
        If replace=True:
            - If location is already on path anywhere (not eggs)
              or higher priority than its parent (eggs)
              do nothing.
            - Else:
              - If it's an egg and its parent directory is on path,
                insert just ahead of the parent,
                removing any lower-priority entries.
              - Else: add it to the front of path.
        """
2724

2725
        loc = loc or self.location
2726 2727 2728 2729 2730
        if not loc:
            return

        nloc = _normalize_cached(loc)
        bdir = os.path.dirname(nloc)
2731
        npath = [(p and _normalize_cached(p) or p) for p in path]
2732 2733

        for p, item in enumerate(npath):
2734
            if item == nloc:
2735 2736 2737 2738 2739
                if replace:
                    break
                else:
                    # don't modify path (even removing duplicates) if found and not replace
                    return
2740
            elif item == bdir and self.precedence == EGG_DIST:
2741
                # if it's an .egg, give it precedence over its directory
2742 2743 2744
                # UNLESS it's already been added to sys.path and replace=False
                if (not replace) and nloc in npath[p:]:
                    return
2745 2746
                if path is sys.path:
                    self.check_version_conflict()
2747 2748 2749 2750
                path.insert(p, loc)
                npath.insert(p, nloc)
                break
        else:
2751 2752
            if path is sys.path:
                self.check_version_conflict()
2753 2754 2755 2756
            if replace:
                path.insert(0, loc)
            else:
                path.append(loc)
2757 2758 2759
            return

        # p is the spot where we found or inserted loc; now remove duplicates
2760
        while True:
2761
            try:
2762
                np = npath.index(nloc, p + 1)
2763 2764 2765 2766
            except ValueError:
                break
            else:
                del npath[np], path[np]
Jason R. Coombs's avatar
Jason R. Coombs committed
2767 2768
                # ha!
                p = np
2769 2770 2771

        return

2772
    def check_version_conflict(self):
2773
        if self.key == 'setuptools':
Jason R. Coombs's avatar
Jason R. Coombs committed
2774 2775
            # ignore the inevitable setuptools self-conflicts  :(
            return
2776 2777

        nsp = dict.fromkeys(self._get_metadata('namespace_packages.txt'))
2778
        loc = normalize_path(self.location)
2779
        for modname in self._get_metadata('top_level.txt'):
2780
            if (modname not in sys.modules or modname in nsp
Jason R. Coombs's avatar
Jason R. Coombs committed
2781
                    or modname in _namespace_packages):
2782
                continue
2783 2784
            if modname in ('pkg_resources', 'setuptools', 'site'):
                continue
2785
            fn = getattr(sys.modules[modname], '__file__', None)
2786 2787
            if fn and (normalize_path(fn).startswith(loc) or
                       fn.startswith(self.location)):
2788
                continue
2789
            issue_warning(
2790
                "Module %s was already imported from %s, but %s is being added"
2791
                " to sys.path" % (modname, fn, self.location),
2792 2793
            )

2794 2795 2796 2797
    def has_version(self):
        try:
            self.version
        except ValueError:
2798
            issue_warning("Unbuilt egg for " + repr(self))
2799 2800
            return False
        return True
2801

2802
    def clone(self, **kw):
2803
        """Copy this distribution, substituting in any changed keyword args"""
2804 2805
        names = 'project_name version py_version platform location precedence'
        for attr in names.split():
2806
            kw.setdefault(attr, getattr(self, attr, None))
2807 2808 2809
        kw.setdefault('metadata', self._provider)
        return self.__class__(**kw)

2810
    @property
2811 2812 2813 2814
    def extras(self):
        return [dep for dep in self._dep_map if dep]


2815
class EggInfoDistribution(Distribution):
2816
    def _reload_version(self):
2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827
        """
        Packages installed by distutils (e.g. numpy or scipy),
        which uses an old safe_version, and so
        their version numbers can get mangled when
        converted to filenames (e.g., 1.11.0.dev0+2329eae to
        1.11.0.dev0_2329eae). These distributions will not be
        parsed properly
        downstream by Distribution and safe_version, so
        take an extra step and try to get the version number from
        the metadata file itself instead of the filename.
        """
2828
        md_version = _version_from_file(self._get_metadata(self.PKG_INFO))
2829 2830
        if md_version:
            self._version = md_version
2831
        return self
2832 2833


2834 2835 2836
class DistInfoDistribution(Distribution):
    """Wrap an actual or potential sys.path entry w/metadata, .dist-info style"""
    PKG_INFO = 'METADATA'
2837
    EQEQ = re.compile(r"([\(,])\s*(\d.*?)\s*([,\)])")
2838 2839 2840 2841 2842 2843 2844

    @property
    def _parsed_pkg_info(self):
        """Parse and cache metadata"""
        try:
            return self._pkg_info
        except AttributeError:
2845
            metadata = self.get_metadata(self.PKG_INFO)
Jason R. Coombs's avatar
Jason R. Coombs committed
2846
            self._pkg_info = email.parser.Parser().parsestr(metadata)
2847
            return self._pkg_info
2848

2849 2850 2851 2852 2853
    @property
    def _dep_map(self):
        try:
            return self.__dep_map
        except AttributeError:
Daniel Holth's avatar
Daniel Holth committed
2854 2855
            self.__dep_map = self._compute_dependencies()
            return self.__dep_map
2856

Daniel Holth's avatar
Daniel Holth committed
2857 2858 2859 2860 2861 2862
    def _compute_dependencies(self):
        """Recompute this distribution's dependencies."""
        dm = self.__dep_map = {None: []}

        reqs = []
        # Including any condition expressions
2863
        for req in self._parsed_pkg_info.get_all('Requires-Dist') or []:
2864
            reqs.extend(parse_requirements(req))
2865

Daniel Holth's avatar
Daniel Holth committed
2866 2867
        def reqs_for_extra(extra):
            for req in reqs:
2868
                if not req.marker or req.marker.evaluate({'extra': extra}):
Daniel Holth's avatar
Daniel Holth committed
2869 2870
                    yield req

2871
        common = frozenset(reqs_for_extra(None))
Daniel Holth's avatar
Daniel Holth committed
2872
        dm[None].extend(common)
2873

2874
        for extra in self._parsed_pkg_info.get_all('Provides-Extra') or []:
2875 2876
            s_extra = safe_extra(extra.strip())
            dm[s_extra] = list(frozenset(reqs_for_extra(extra)) - common)
Daniel Holth's avatar
Daniel Holth committed
2877 2878

        return dm
2879

2880

Jason R. Coombs's avatar
Jason R. Coombs committed
2881 2882
_distributionImpl = {
    '.egg': Distribution,
2883
    '.egg-info': EggInfoDistribution,
Jason R. Coombs's avatar
Jason R. Coombs committed
2884 2885
    '.dist-info': DistInfoDistribution,
    }
2886 2887


2888
def issue_warning(*args, **kw):
2889 2890 2891 2892 2893 2894 2895 2896 2897
    level = 1
    g = globals()
    try:
        # find the first stack frame that is *not* code in
        # the pkg_resources module, to use for the warning
        while sys._getframe(level).f_globals is g:
            level += 1
    except ValueError:
        pass
2898
    warnings.warn(stacklevel=level + 1, *args, **kw)
2899

2900

2901 2902 2903 2904 2905
class RequirementParseError(ValueError):
    def __str__(self):
        return ' '.join(self.args)


2906 2907 2908
def parse_requirements(strs):
    """Yield ``Requirement`` objects for each specification in `strs`

2909
    `strs` must be a string, or a (possibly-nested) iterable thereof.
2910 2911 2912
    """
    # create a steppable iterator, so we can handle \-continuations
    lines = iter(yield_lines(strs))
2913 2914

    for line in lines:
2915 2916 2917 2918 2919 2920 2921
        # Drop comments -- a hash without a space may be in a URL.
        if ' #' in line:
            line = line[:line.find(' #')]
        # If there is a line continuation, drop it, and append the next line.
        if line.endswith('\\'):
            line = line[:-2].strip()
            line += next(lines)
2922
        yield Requirement(line)
2923 2924


2925
class Requirement(packaging.requirements.Requirement):
2926
    def __init__(self, requirement_string):
2927
        """DO NOT CALL THIS UNDOCUMENTED METHOD; use Requirement.parse()!"""
2928
        try:
2929
            super(Requirement, self).__init__(requirement_string)
2930 2931
        except packaging.requirements.InvalidRequirement as e:
            raise RequirementParseError(str(e))
2932
        self.unsafe_name = self.name
2933 2934
        project_name = safe_name(self.name)
        self.project_name, self.key = project_name, project_name.lower()
2935
        self.specs = [
2936 2937
            (spec.operator, spec.version) for spec in self.specifier]
        self.extras = tuple(map(safe_extra, self.extras))
2938
        self.hashCmp = (
2939
            self.key,
2940
            self.specifier,
2941
            frozenset(self.extras),
2942
            str(self.marker) if self.marker else None,
2943 2944
        )
        self.__hash = hash(self.hashCmp)
2945

2946
    def __eq__(self, other):
2947 2948 2949 2950
        return (
            isinstance(other, Requirement) and
            self.hashCmp == other.hashCmp
        )
2951

2952 2953 2954
    def __ne__(self, other):
        return not self == other

2955 2956
    def __contains__(self, item):
        if isinstance(item, Distribution):
2957 2958
            if item.key != self.key:
                return False
2959 2960 2961 2962 2963 2964 2965

            item = item.version

        # Allow prereleases always in order to match the previous behavior of
        # this method. In the future this should be smarter and follow PEP 440
        # more accurately.
        return self.specifier.contains(item, prereleases=True)
2966

2967 2968 2969
    def __hash__(self):
        return self.__hash

2970 2971
    def __repr__(self): return "Requirement.parse(%r)" % str(self)

2972
    @staticmethod
2973
    def parse(s):
2974 2975
        req, = parse_requirements(s)
        return req
2976

2977

2978 2979 2980 2981 2982 2983 2984 2985
def _always_object(classes):
    """
    Ensure object appears in the mro even
    for old-style classes.
    """
    if object not in classes:
        return classes + (object,)
    return classes
2986

2987

2988
def _find_adapter(registry, ob):
2989
    """Return an adapter factory for `ob` from `registry`"""
2990 2991
    types = _always_object(inspect.getmro(getattr(ob, '__class__', type(ob))))
    for t in types:
2992 2993 2994 2995
        if t in registry:
            return registry[t]


2996
def ensure_directory(path):
2997
    """Ensure that the parent directory of `path` exists"""
2998
    dirname = os.path.dirname(path)
2999
    py31compat.makedirs(dirname, exist_ok=True)
3000

3001

Jason R. Coombs's avatar
Jason R. Coombs committed
3002
def _bypass_ensure_directory(path):
3003 3004 3005
    """Sandbox-bypassing version of ensure_directory()"""
    if not WRITE_SUPPORT:
        raise IOError('"os.mkdir" not supported on this platform.')
3006
    dirname, filename = split(path)
3007 3008
    if dirname and filename and not isdir(dirname):
        _bypass_ensure_directory(dirname)
Jason R. Coombs's avatar
Jason R. Coombs committed
3009
        mkdir(dirname, 0o755)
3010 3011


3012
def split_sections(s):
3013
    """Split a string or iterable thereof into (section, content) pairs
3014

3015
    Each ``section`` is a stripped version of the section header ("[section]")
3016 3017 3018 3019 3020 3021 3022 3023 3024
    and each ``content`` is a list of stripped lines excluding blank lines and
    comment-only lines.  If there are any such lines before the first section
    header, they're returned in a first ``section`` of ``None``.
    """
    section = None
    content = []
    for line in yield_lines(s):
        if line.startswith("["):
            if line.endswith("]"):
3025
                if section or content:
3026
                    yield section, content
3027
                section = line[1:-1].strip()
3028 3029 3030 3031 3032 3033 3034
                content = []
            else:
                raise ValueError("Invalid section heading", line)
        else:
            content.append(line)

    # wrap up last segment
3035 3036
    yield section, content

3037

3038
def _mkstemp(*args, **kw):
3039 3040
    old_open = os.open
    try:
Jason R. Coombs's avatar
Jason R. Coombs committed
3041 3042
        # temporarily bypass sandboxing
        os.open = os_open
3043
        return tempfile.mkstemp(*args, **kw)
3044
    finally:
Jason R. Coombs's avatar
Jason R. Coombs committed
3045 3046
        # and then put it back
        os.open = old_open
3047

3048

3049 3050 3051 3052 3053 3054 3055
# Silence the PEP440Warning by default, so that end users don't get hit by it
# randomly just because they use pkg_resources. We want to append the rule
# because we want earlier uses of filterwarnings to take precedence over this
# one.
warnings.filterwarnings("ignore", category=PEP440Warning, append=True)


3056 3057 3058 3059 3060 3061
# from jaraco.functools 1.3
def _call_aside(f, *args, **kwargs):
    f(*args, **kwargs)
    return f


3062 3063
@_call_aside
def _initialize(g=globals()):
3064 3065 3066
    "Set up global resource manager (deliberately not state-saved)"
    manager = ResourceManager()
    g['_manager'] = manager
3067 3068 3069 3070 3071
    g.update(
        (name, getattr(manager, name))
        for name in dir(manager)
        if not name.startswith('_')
    )
3072

3073

3074
@_call_aside
3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095
def _initialize_master_working_set():
    """
    Prepare the master working set and make the ``require()``
    API available.

    This function has explicit effects on the global state
    of pkg_resources. It is intended to be invoked once at
    the initialization of this module.

    Invocation by other packages is unsupported and done
    at their own risk.
    """
    working_set = WorkingSet._build_master()
    _declare_state('object', working_set=working_set)

    require = working_set.require
    iter_entry_points = working_set.iter_entry_points
    add_activation_listener = working_set.subscribe
    run_script = working_set.run_script
    # backward compatibility
    run_main = run_script
3096 3097 3098 3099
    # Activate all distributions already on sys.path with replace=False and
    # ensure that all distributions added to the working set in the future
    # (e.g. by calling ``require()``) will get activated as well,
    # with higher priority (replace=True).
3100
    tuple(
3101
        dist.activate(replace=False)
3102 3103
        for dist in working_set
    )
3104
    add_activation_listener(lambda dist: dist.activate(replace=True), existing=False)
3105
    working_set.entries = []
3106 3107 3108
    # match order
    list(map(working_set.add_entry, sys.path))
    globals().update(locals())