Merge pull request series from Steve Kowalik for updated and unified marker...

Merge pull request series from Steve Kowalik for updated and unified marker implementation. Fixes #122.

CHANGES.txt

@@ -6,6 +6,11 @@ CHANGES
 ----
 
 * Changelog now includes release dates and links to PEPs.
+* Pull Request #173: Replace dual PEP 345 _markerlib implementation
+  and PEP 426 implementation of environment marker support from
+  packaging 16.1 and PEP 508. Fixes Issue #122.
+  See also Pull Request #175, Pull Request #168, and
+  Pull Request #164.
 
 20.1.1
 ------

Makefile

@@ -4,5 +4,8 @@ empty:
 update-vendored:
 	rm -rf pkg_resources/_vendor/packaging*
 	rm -rf pkg_resources/_vendor/six*
+	rm -rf pkg_resources/_vendor/pyparsing*
 	python3 -m pip install -r pkg_resources/_vendor/vendored.txt -t pkg_resources/_vendor/
+	sed -i 's/ \(pyparsing\|six\)/ pkg_resources.extern.\1/' \
+		pkg_resources/_vendor/packaging/*.py
 	rm -rf pkg_resources/_vendor/*.{egg,dist}-info

_markerlib/__init__.py

-try:
-    import ast
-    from _markerlib.markers import default_environment, compile, interpret
-except ImportError:
-    if 'ast' in globals():
-        raise
-    def default_environment():
-        return {}
-    def compile(marker):
-        def marker_fn(environment=None, override=None):
-            # 'empty markers are True' heuristic won't install extra deps.
-            return not marker.strip()
-        marker_fn.__doc__ = marker
-        return marker_fn
-    def interpret(marker, environment=None, override=None):
-        return compile(marker)()

_markerlib/markers.py

-# -*- coding: utf-8 -*-
-"""Interpret PEP 345 environment markers.
-
-EXPR [in|==|!=|not in] EXPR [or|and] ...
-
-where EXPR belongs to any of those:
-
-    python_version = '%s.%s' % (sys.version_info[0], sys.version_info[1])
-    python_full_version = sys.version.split()[0]
-    os.name = os.name
-    sys.platform = sys.platform
-    platform.version = platform.version()
-    platform.machine = platform.machine()
-    platform.python_implementation = platform.python_implementation()
-    a free string, like '2.6', or 'win32'
-"""
-
-__all__ = ['default_environment', 'compile', 'interpret']
-
-import ast
-import os
-import platform
-import sys
-import weakref
-
-_builtin_compile = compile
-
-try:
-    from platform import python_implementation
-except ImportError:
-    if os.name == "java":
-        # Jython 2.5 has ast module, but not platform.python_implementation() function.
-        def python_implementation():
-            return "Jython"
-    else:
-        raise
-
-
-# restricted set of variables
-_VARS = {'sys.platform': sys.platform,
-         'python_version': '%s.%s' % sys.version_info[:2],
-         # FIXME parsing sys.platform is not reliable, but there is no other
-         # way to get e.g. 2.7.2+, and the PEP is defined with sys.version
-         'python_full_version': sys.version.split(' ', 1)[0],
-         'os.name': os.name,
-         'platform.version': platform.version(),
-         'platform.machine': platform.machine(),
-         'platform.python_implementation': python_implementation(),
-         'extra': None # wheel extension
-        }
-
-for var in list(_VARS.keys()):
-    if '.' in var:
-        _VARS[var.replace('.', '_')] = _VARS[var]
-
-def default_environment():
-    """Return copy of default PEP 385 globals dictionary."""
-    return dict(_VARS)
-
-class ASTWhitelist(ast.NodeTransformer):
-    def __init__(self, statement):
-        self.statement = statement # for error messages
-
-    ALLOWED = (ast.Compare, ast.BoolOp, ast.Attribute, ast.Name, ast.Load, ast.Str)
-    # Bool operations
-    ALLOWED += (ast.And, ast.Or)
-    # Comparison operations
-    ALLOWED += (ast.Eq, ast.Gt, ast.GtE, ast.In, ast.Is, ast.IsNot, ast.Lt, ast.LtE, ast.NotEq, ast.NotIn)
-
-    def visit(self, node):
-        """Ensure statement only contains allowed nodes."""
-        if not isinstance(node, self.ALLOWED):
-            raise SyntaxError('Not allowed in environment markers.\n%s\n%s' %
-                               (self.statement,
-                               (' ' * node.col_offset) + '^'))
-        return ast.NodeTransformer.visit(self, node)
-
-    def visit_Attribute(self, node):
-        """Flatten one level of attribute access."""
-        new_node = ast.Name("%s.%s" % (node.value.id, node.attr), node.ctx)
-        return ast.copy_location(new_node, node)
-
-def parse_marker(marker):
-    tree = ast.parse(marker, mode='eval')
-    new_tree = ASTWhitelist(marker).generic_visit(tree)
-    return new_tree
-
-def compile_marker(parsed_marker):
-    return _builtin_compile(parsed_marker, '<environment marker>', 'eval',
-                   dont_inherit=True)
-
-_cache = weakref.WeakValueDictionary()
-
-def compile(marker):
-    """Return compiled marker as a function accepting an environment dict."""
-    try:
-        return _cache[marker]
-    except KeyError:
-        pass
-    if not marker.strip():
-        def marker_fn(environment=None, override=None):
-            """"""
-            return True
-    else:
-        compiled_marker = compile_marker(parse_marker(marker))
-        def marker_fn(environment=None, override=None):
-            """override updates environment"""
-            if override is None:
-                override = {}
-            if environment is None:
-                environment = default_environment()
-            environment.update(override)
-            return eval(compiled_marker, environment)
-    marker_fn.__doc__ = marker
-    _cache[marker] = marker_fn
-    return _cache[marker]
-
-def interpret(marker, environment=None):
-    return compile(marker)(environment)

pkg_resources/_vendor/packaging/__about__.py

-# Copyright 2014 Donald Stufft
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
 from __future__ import absolute_import, division, print_function
 
 __all__ = [
@@ -22,10 +12,10 @@ __title__ = "packaging"
 __summary__ = "Core utilities for Python packages"
 __uri__ = "https://github.com/pypa/packaging"
 
-__version__ = "15.3"
+__version__ = "16.1"
 
-__author__ = "Donald Stufft"
+__author__ = "Donald Stufft and individual contributors"
 __email__ = "donald@stufft.io"
 
-__license__ = "Apache License, Version 2.0"
-__copyright__ = "Copyright 2014 %s" % __author__
+__license__ = "BSD or Apache License, Version 2.0"
+__copyright__ = "Copyright 2014-2016 %s" % __author__

pkg_resources/_vendor/packaging/__init__.py

-# Copyright 2014 Donald Stufft
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
 from __future__ import absolute_import, division, print_function
 
 from .__about__ import (

pkg_resources/_vendor/packaging/_compat.py

-# Copyright 2014 Donald Stufft
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
 from __future__ import absolute_import, division, print_function
 
 import sys

pkg_resources/_vendor/packaging/_structures.py

-# Copyright 2014 Donald Stufft
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
 from __future__ import absolute_import, division, print_function
 
 

pkg_resources/_vendor/packaging/markers.py

+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+from __future__ import absolute_import, division, print_function
+
+import operator
+import os
+import platform
+import sys
+
+from pkg_resources.extern.pyparsing import ParseException, ParseResults, stringStart, stringEnd
+from pkg_resources.extern.pyparsing import ZeroOrMore, Group, Forward, QuotedString
+from pkg_resources.extern.pyparsing import Literal as L  # noqa
+
+from ._compat import string_types
+from .specifiers import Specifier, InvalidSpecifier
+
+
+__all__ = [
+    "InvalidMarker", "UndefinedComparison", "UndefinedEnvironmentName",
+    "Marker", "default_environment",
+]
+
+
+class InvalidMarker(ValueError):
+    """
+    An invalid marker was found, users should refer to PEP 508.
+    """
+
+
+class UndefinedComparison(ValueError):
+    """
+    An invalid operation was attempted on a value that doesn't support it.
+    """
+
+
+class UndefinedEnvironmentName(ValueError):
+    """
+    A name was attempted to be used that does not exist inside of the
+    environment.
+    """
+
+
+class Node(object):
+
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return str(self.value)
+
+    def __repr__(self):
+        return "<{0}({1!r})>".format(self.__class__.__name__, str(self))
+
+
+class Variable(Node):
+    pass
+
+
+class Value(Node):
+    pass
+
+
+VARIABLE = (
+    L("implementation_version") |
+    L("platform_python_implementation") |
+    L("implementation_name") |
+    L("python_full_version") |
+    L("platform_release") |
+    L("platform_version") |
+    L("platform_machine") |
+    L("platform_system") |
+    L("python_version") |
+    L("sys_platform") |
+    L("os_name") |
+    L("extra")
+)
+VARIABLE.setParseAction(lambda s, l, t: Variable(t[0]))
+
+VERSION_CMP = (
+    L("===") |
+    L("==") |
+    L(">=") |
+    L("<=") |
+    L("!=") |
+    L("~=") |
+    L(">") |
+    L("<")
+)
+
+MARKER_OP = VERSION_CMP | L("not in") | L("in")
+
+MARKER_VALUE = QuotedString("'") | QuotedString('"')
+MARKER_VALUE.setParseAction(lambda s, l, t: Value(t[0]))
+
+BOOLOP = L("and") | L("or")
+
+MARKER_VAR = VARIABLE | MARKER_VALUE
+
+MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR)
+MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0]))
+
+LPAREN = L("(").suppress()
+RPAREN = L(")").suppress()
+
+MARKER_EXPR = Forward()
+MARKER_ATOM = MARKER_ITEM | Group(LPAREN + MARKER_EXPR + RPAREN)
+MARKER_EXPR << MARKER_ATOM + ZeroOrMore(BOOLOP + MARKER_EXPR)
+
+MARKER = stringStart + MARKER_EXPR + stringEnd
+
+
+def _coerce_parse_result(results):
+    if isinstance(results, ParseResults):
+        return [_coerce_parse_result(i) for i in results]
+    else:
+        return results
+
+
+def _format_marker(marker, first=True):
+    assert isinstance(marker, (list, tuple, string_types))
+
+    # Sometimes we have a structure like [[...]] which is a single item list
+    # where the single item is itself it's own list. In that case we want skip
+    # the rest of this function so that we don't get extraneous () on the
+    # outside.
+    if (isinstance(marker, list) and len(marker) == 1 and
+            isinstance(marker[0], (list, tuple))):
+        return _format_marker(marker[0])
+
+    if isinstance(marker, list):
+        inner = (_format_marker(m, first=False) for m in marker)
+        if first:
+            return " ".join(inner)
+        else:
+            return "(" + " ".join(inner) + ")"
+    elif isinstance(marker, tuple):
+        return '{0} {1} "{2}"'.format(*marker)
+    else:
+        return marker
+
+
+_operators = {
+    "in": lambda lhs, rhs: lhs in rhs,
+    "not in": lambda lhs, rhs: lhs not in rhs,
+    "<": operator.lt,
+    "<=": operator.le,
+    "==": operator.eq,
+    "!=": operator.ne,
+    ">=": operator.ge,
+    ">": operator.gt,
+}
+
+
+def _eval_op(lhs, op, rhs):
+    try:
+        spec = Specifier("".join([op, rhs]))
+    except InvalidSpecifier:
+        pass
+    else:
+        return spec.contains(lhs)
+
+    oper = _operators.get(op)
+    if oper is None:
+        raise UndefinedComparison(
+            "Undefined {0!r} on {1!r} and {2!r}.".format(op, lhs, rhs)
+        )
+
+    return oper(lhs, rhs)
+
+
+_undefined = object()
+
+
+def _get_env(environment, name):
+    value = environment.get(name, _undefined)
+
+    if value is _undefined:
+        raise UndefinedEnvironmentName(
+            "{0!r} does not exist in evaluation environment.".format(name)
+        )
+
+    return value
+
+
+def _evaluate_markers(markers, environment):
+    groups = [[]]
+
+    for marker in markers:
+        assert isinstance(marker, (list, tuple, string_types))
+
+        if isinstance(marker, list):
+            groups[-1].append(_evaluate_markers(marker, environment))
+        elif isinstance(marker, tuple):
+            lhs, op, rhs = marker
+
+            if isinstance(lhs, Variable):
+                lhs_value = _get_env(environment, lhs.value)
+                rhs_value = rhs.value
+            else:
+                lhs_value = lhs.value
+                rhs_value = _get_env(environment, rhs.value)
+
+            groups[-1].append(_eval_op(lhs_value, op, rhs_value))
+        else:
+            assert marker in ["and", "or"]
+            if marker == "or":
+                groups.append([])
+
+    return any(all(item) for item in groups)
+
+
+def format_full_version(info):
+    version = '{0.major}.{0.minor}.{0.micro}'.format(info)
+    kind = info.releaselevel
+    if kind != 'final':
+        version += kind[0] + str(info.serial)
+    return version
+
+
+def default_environment():
+    if hasattr(sys, 'implementation'):
+        iver = format_full_version(sys.implementation.version)
+        implementation_name = sys.implementation.name
+    else:
+        iver = '0'
+        implementation_name = ''
+
+    return {
+        "implementation_name": implementation_name,
+        "implementation_version": iver,
+        "os_name": os.name,
+        "platform_machine": platform.machine(),
+        "platform_release": platform.release(),
+        "platform_system": platform.system(),
+        "platform_version": platform.version(),
+        "python_full_version": platform.python_version(),
+        "platform_python_implementation": platform.python_implementation(),
+        "python_version": platform.python_version()[:3],
+        "sys_platform": sys.platform,
+    }
+
+
+class Marker(object):
+
+    def __init__(self, marker):
+        try:
+            self._markers = _coerce_parse_result(MARKER.parseString(marker))
+        except ParseException as e:
+            err_str = "Invalid marker: {0!r}, parse error at {1!r}".format(
+                marker, marker[e.loc:e.loc + 8])
+            raise InvalidMarker(err_str)
+
+    def __str__(self):
+        return _format_marker(self._markers)
+
+    def __repr__(self):
+        return "<Marker({0!r})>".format(str(self))
+
+    def evaluate(self, environment=None):
+        """Evaluate a marker.
+
+        Return the boolean from evaluating the given marker against the
+        environment. environment is an optional argument to override all or
+        part of the determined environment.
+
+        The environment is determined from the current Python process.
+        """
+        current_environment = default_environment()
+        if environment is not None:
+            current_environment.update(environment)
+
+        return _evaluate_markers(self._markers, current_environment)

pkg_resources/_vendor/packaging/requirements.py

+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+from __future__ import absolute_import, division, print_function
+
+import string
+import re
+
+from pkg_resources.extern.pyparsing import stringStart, stringEnd, originalTextFor, ParseException
+from pkg_resources.extern.pyparsing import ZeroOrMore, Word, Optional, Regex, Combine
+from pkg_resources.extern.pyparsing import Literal as L  # noqa
+from pkg_resources.extern.six.moves.urllib import parse as urlparse
+
+from .markers import MARKER_EXPR, Marker
+from .specifiers import LegacySpecifier, Specifier, SpecifierSet
+
+
+class InvalidRequirement(ValueError):
+    """
+    An invalid requirement was found, users should refer to PEP 508.
+    """
+
+
+ALPHANUM = Word(string.ascii_letters + string.digits)
+
+LBRACKET = L("[").suppress()
+RBRACKET = L("]").suppress()
+LPAREN = L("(").suppress()
+RPAREN = L(")").suppress()
+COMMA = L(",").suppress()
+SEMICOLON = L(";").suppress()
+AT = L("@").suppress()
+
+PUNCTUATION = Word("-_.")
+IDENTIFIER_END = ALPHANUM | (ZeroOrMore(PUNCTUATION) + ALPHANUM)
+IDENTIFIER = Combine(ALPHANUM + ZeroOrMore(IDENTIFIER_END))
+
+NAME = IDENTIFIER("name")
+EXTRA = IDENTIFIER
+
+URI = Regex(r'[^ ]+')("url")
+URL = (AT + URI)
+
+EXTRAS_LIST = EXTRA + ZeroOrMore(COMMA + EXTRA)
+EXTRAS = (LBRACKET + Optional(EXTRAS_LIST) + RBRACKET)("extras")
+
+VERSION_PEP440 = Regex(Specifier._regex_str, re.VERBOSE | re.IGNORECASE)
+VERSION_LEGACY = Regex(LegacySpecifier._regex_str, re.VERBOSE | re.IGNORECASE)
+
+VERSION_ONE = VERSION_PEP440 | VERSION_LEGACY
+VERSION_MANY = Combine(VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE),
+                       joinString=",")("_raw_spec")
+_VERSION_SPEC = Optional(((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY))
+_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or '')
+
+VERSION_SPEC = originalTextFor(_VERSION_SPEC)("specifier")
+VERSION_SPEC.setParseAction(lambda s, l, t: t[1])
+
+MARKER_EXPR = originalTextFor(MARKER_EXPR())("marker")
+MARKER_EXPR.setParseAction(
+    lambda s, l, t: Marker(s[t._original_start:t._original_end])
+)
+MARKER_SEPERATOR = SEMICOLON
+MARKER = MARKER_SEPERATOR + MARKER_EXPR
+
+VERSION_AND_MARKER = VERSION_SPEC + Optional(MARKER)
+URL_AND_MARKER = URL + Optional(MARKER)
+
+NAMED_REQUIREMENT = \
+    NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER)
+
+REQUIREMENT = stringStart + NAMED_REQUIREMENT + stringEnd
+
+
+class Requirement(object):
+    """Parse a requirement.
+
+    Parse a given requirement string into its parts, such as name, specifier,
+    URL, and extras. Raises InvalidRequirement on a badly-formed requirement
+    string.
+    """
+
+    # TODO: Can we test whether something is contained within a requirement?
+    #       If so how do we do that? Do we need to test against the _name_ of
+    #       the thing as well as the version? What about the markers?
+    # TODO: Can we normalize the name and extra name?
+
+    def __init__(self, requirement_string):
+        try:
+            req = REQUIREMENT.parseString(requirement_string)
+        except ParseException as e:
+            raise InvalidRequirement(
+                "Invalid requirement, parse error at \"{0!r}\"".format(
+                    requirement_string[e.loc:e.loc + 8]))
+
+        self.name = req.name
+        if req.url:
+            parsed_url = urlparse.urlparse(req.url)
+            if not (parsed_url.scheme and parsed_url.netloc) or (
+                    not parsed_url.scheme and not parsed_url.netloc):
+                raise InvalidRequirement("Invalid URL given")
+            self.url = req.url
+        else:
+            self.url = None
+        self.extras = set(req.extras.asList() if req.extras else [])
+        self.specifier = SpecifierSet(req.specifier)
+        self.marker = req.marker if req.marker else None
+
+    def __str__(self):
+        parts = [self.name]
+
+        if self.extras:
+            parts.append("[{0}]".format(",".join(sorted(self.extras))))
+
+        if self.specifier:
+            parts.append(str(self.specifier))
+
+        if self.url:
+            parts.append("@ {0}".format(self.url))
+
+        if self.marker:
+            parts.append("; {0}".format(self.marker))
+
+        return "".join(parts)
+
+    def __repr__(self):
+        return "<Requirement({0!r})>".format(str(self))

pkg_resources/_vendor/packaging/specifiers.py

-# Copyright 2014 Donald Stufft
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
 from __future__ import absolute_import, division, print_function
 
 import abc
@@ -204,8 +194,8 @@ class _IndividualSpecifier(BaseSpecifier):
                 # If our version is a prerelease, and we were not set to allow
                 # prereleases, then we'll store it for later incase nothing
                 # else matches this specifier.
-                if (parsed_version.is_prerelease
-                        and not (prereleases or self.prereleases)):
+                if (parsed_version.is_prerelease and not
+                        (prereleases or self.prereleases)):
                     found_prereleases.append(version)
                 # Either this is not a prerelease, or we should have been
                 # accepting prereleases from the begining.
@@ -223,10 +213,8 @@ class _IndividualSpecifier(BaseSpecifier):
 
 class LegacySpecifier(_IndividualSpecifier):
 
-    _regex = re.compile(
+    _regex_str = (
         r"""
-        ^
-        \s*
         (?P<operator>(==|!=|<=|>=|<|>))
         \s*
         (?P<version>
@@ -234,12 +222,12 @@ class LegacySpecifier(_IndividualSpecifier):
                    # is a "legacy" specifier and the version string can be just
                    # about anything.
         )
-        \s*
-        $
-        """,
-        re.VERBOSE | re.IGNORECASE,
+        """
     )
 
+    _regex = re.compile(
+        r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE)
+
     _operators = {
         "==": "equal",
         "!=": "not_equal",
@@ -284,10 +272,8 @@ def _require_version_compare(fn):
 
 class Specifier(_IndividualSpecifier):
 
-    _regex = re.compile(
+    _regex_str = (
         r"""
-        ^
-        \s*
         (?P<operator>(~=|==|!=|<=|>=|<|>|===))
         (?P<version>
             (?:
@@ -378,12 +364,12 @@ class Specifier(_IndividualSpecifier):
                 (?:[-_\.]?dev[-_\.]?[0-9]*)?          # dev release
             )
         )
-        \s*
-        $
-        """,
-        re.VERBOSE | re.IGNORECASE,
+        """
     )
 
+    _regex = re.compile(
+        r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE)
+
     _operators = {
         "~=": "compatible",
         "==": "equal",
@@ -409,8 +395,8 @@ class Specifier(_IndividualSpecifier):
         prefix = ".".join(
             list(
                 itertools.takewhile(
-                    lambda x: (not x.startswith("post")
-                               and not x.startswith("dev")),
+                    lambda x: (not x.startswith("post") and not
+                               x.startswith("dev")),
                     _version_split(spec),
                 )
             )[:-1]
@@ -419,13 +405,15 @@ class Specifier(_IndividualSpecifier):
         # Add the prefix notation to the end of our string
         prefix += ".*"
 
-        return (self._get_operator(">=")(prospective, spec)
-                and self._get_operator("==")(prospective, prefix))
+        return (self._get_operator(">=")(prospective, spec) and
+                self._get_operator("==")(prospective, prefix))
 
     @_require_version_compare
     def _compare_equal(self, prospective, spec):
         # We need special logic to handle prefix matching
         if spec.endswith(".*"):
+            # In the case of prefix matching we want to ignore local segment.
+            prospective = Version(prospective.public)
             # Split the spec out by dots, and pretend that there is an implicit
             # dot in between a release segment and a pre-release segment.
             spec = _version_split(spec[:-2])  # Remove the trailing .*
@@ -577,8 +565,8 @@ def _pad_version(left, right):
     right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right)))
 
     # Get the rest of our versions
-    left_split.append(left[len(left_split):])
-    right_split.append(left[len(right_split):])
+    left_split.append(left[len(left_split[0]):])
+    right_split.append(right[len(right_split[0]):])
 
     # Insert our padding
     left_split.insert(

pkg_resources/_vendor/packaging/version.py

-# Copyright 2014 Donald Stufft
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
 from __future__ import absolute_import, division, print_function
 
 import collections

pkg_resources/_vendor/vendored.txt

-packaging==15.3
+packaging==16.1
+pyparsing==2.0.6
 six==1.10.0

pkg_resources/api_tests.txt

@@ -338,88 +338,64 @@ Environment Markers
     >>> import os
 
     >>> print(im("sys_platform"))
-    Comparison or logical expression expected
+    Invalid marker: 'sys_platform', parse error at ''
 
     >>> print(im("sys_platform=="))
-    invalid syntax
+    Invalid marker: 'sys_platform==', parse error at ''
 
     >>> print(im("sys_platform=='win32'"))
     False
 
     >>> print(im("sys=='x'"))
-    Unknown name 'sys'
+    Invalid marker: "sys=='x'", parse error at "sys=='x'"
 
     >>> print(im("(extra)"))
-    Comparison or logical expression expected
+    Invalid marker: '(extra)', parse error at ')'
 
     >>> print(im("(extra"))
-    invalid syntax
+    Invalid marker: '(extra', parse error at ''
 
     >>> print(im("os.open('foo')=='y'"))
-    Language feature not supported in environment markers
+    Invalid marker: "os.open('foo')=='y'", parse error at 'os.open('
 
     >>> print(im("'x'=='y' and os.open('foo')=='y'"))   # no short-circuit!
-    Language feature not supported in environment markers
+    Invalid marker: "'x'=='y' and os.open('foo')=='y'", parse error at 'and os.o'
 
     >>> print(im("'x'=='x' or os.open('foo')=='y'"))   # no short-circuit!
-    Language feature not supported in environment markers
+    Invalid marker: "'x'=='x' or os.open('foo')=='y'", parse error at 'or os.op'
 
     >>> print(im("'x' < 'y' < 'z'"))
-    Chained comparison not allowed in environment markers
+    Invalid marker: "'x' < 'y' < 'z'", parse error at "< 'z'"
 
     >>> print(im("r'x'=='x'"))
-    Only plain strings allowed in environment markers
+    Invalid marker: "r'x'=='x'", parse error at "r'x'=='x"
 
     >>> print(im("'''x'''=='x'"))
-    Only plain strings allowed in environment markers
+    Invalid marker: "'''x'''=='x'", parse error at "'x'''=='"
 
     >>> print(im('"""x"""=="x"'))
-    Only plain strings allowed in environment markers
+    Invalid marker: '"""x"""=="x"', parse error at '"x"""=="'
 
-    >>> print(im(r"'x\n'=='x'"))
-    Only plain strings allowed in environment markers
+    >>> print(im(r"x\n=='x'"))
+    Invalid marker: "x\\n=='x'", parse error at "x\\n=='x'"
 
     >>> print(im("os.open=='y'"))
-    Language feature not supported in environment markers
-
-    >>> em('"x"=="x"')
-    True
-
-    >>> em('"x"=="y"')
-    False
-
-    >>> em('"x"=="y" and "x"=="x"')
-    False
-
-    >>> em('"x"=="y" or "x"=="x"')
-    True
-
-    >>> em('"x"=="y" and "x"=="q" or "z"=="z"')
-    True
-
-    >>> em('"x"=="y" and ("x"=="q" or "z"=="z")')
-    False
-
-    >>> em('"x"=="y" and "z"=="z" or "x"=="q"')
-    False
-
-    >>> em('"x"=="x" and "z"=="z" or "x"=="q"')
-    True
+    Invalid marker: "os.open=='y'", parse error at 'os.open='
 
     >>> em("sys_platform=='win32'") == (sys.platform=='win32')
     True
 
-    >>> em("'x' in 'yx'")
-    True
-
-    >>> em("'yx' in 'x'")
-    False
-
     >>> em("python_version >= '2.6'")
     True
 
     >>> em("python_version > '2.5'")
     True
 
+    >>> im("implementation_name=='cpython'")
+    False
+
     >>> im("platform_python_implementation=='CPython'")
     False
+
+    >>> im("implementation_version=='3.5.1'")
+    False

pkg_resources/extern/__init__.py

@@ -67,5 +67,5 @@ class VendorImporter:
         if self not in sys.meta_path:
             sys.meta_path.append(self)
 
-names = 'packaging', 'six'
+names = 'packaging', 'pyparsing', 'six'
 VendorImporter(__name__, names).install()

pkg_resources/tests/test_markers.py

 try:
-	import unittest.mock as mock
+    import unittest.mock as mock
 except ImportError:
-	import mock
+    import mock
 
 from pkg_resources import evaluate_marker
 
-
-@mock.patch.dict('pkg_resources.MarkerEvaluation.values',
-	python_full_version=mock.Mock(return_value='2.7.10'))
-def test_lexicographic_ordering():
-	"""
-	Although one might like 2.7.10 to be greater than 2.7.3,
-	the marker spec only supports lexicographic ordering.
-	"""
-	assert evaluate_marker("python_full_version > '2.7.3'") is False
+@mock.patch('platform.python_version', return_value='2.7.10')
+def test_ordering(python_version_mock):
+    assert evaluate_marker("python_full_version > '2.7.3'") is True

pkg_resources/tests/test_resources.py

@@ -384,8 +384,8 @@ class TestRequirements:
         r1 = Requirement.parse("Twisted[foo,bar]>=1.2")
         r2 = Requirement.parse("Twisted[bar,FOO]>=1.2")
         assert r1 == r2
-        assert r1.extras == ("foo","bar")
-        assert r2.extras == ("bar","foo")  # extras are normalized
+        assert set(r1.extras) == set(("foo", "bar"))
+        assert set(r2.extras) == set(("foo", "bar"))
         assert hash(r1) == hash(r2)
         assert (
             hash(r1)

setuptools/tests/test_dist_info.py

@@ -28,7 +28,6 @@ class TestDistInfo:
         assert versioned.version == '2.718' # from filename
         assert unversioned.version == '0.3' # from METADATA
 
-    @pytest.mark.importorskip('ast')
     def test_conditional_dependencies(self):
         specs = 'splort==4', 'quux>=1.1'
         requires = list(map(pkg_resources.Requirement.parse, specs))

setuptools/tests/test_markerlib.py

-import os
-
-import pytest
-
-
-class TestMarkerlib:
-
-    @pytest.mark.importorskip('ast')
-    def test_markers(self):
-        from _markerlib import interpret, default_environment, compile
-
-        os_name = os.name
-
-        assert interpret("")
-
-        assert interpret("os.name != 'buuuu'")
-        assert interpret("os_name != 'buuuu'")
-        assert interpret("python_version > '1.0'")
-        assert interpret("python_version < '5.0'")
-        assert interpret("python_version <= '5.0'")
-        assert interpret("python_version >= '1.0'")
-        assert interpret("'%s' in os.name" % os_name)
-        assert interpret("'%s' in os_name" % os_name)
-        assert interpret("'buuuu' not in os.name")
-
-        assert not interpret("os.name == 'buuuu'")
-        assert not interpret("os_name == 'buuuu'")
-        assert not interpret("python_version < '1.0'")
-        assert not interpret("python_version > '5.0'")
-        assert not interpret("python_version >= '5.0'")
-        assert not interpret("python_version <= '1.0'")
-        assert not interpret("'%s' not in os.name" % os_name)
-        assert not interpret("'buuuu' in os.name and python_version >= '5.0'")
-        assert not interpret("'buuuu' in os_name and python_version >= '5.0'")
-
-        environment = default_environment()
-        environment['extra'] = 'test'
-        assert interpret("extra == 'test'", environment)
-        assert not interpret("extra == 'doc'", environment)
-
-        def raises_nameError():
-            try:
-                interpret("python.version == '42'")
-            except NameError:
-                pass
-            else:
-                raise Exception("Expected NameError")
-
-        raises_nameError()
-
-        def raises_syntaxError():
-            try:
-                interpret("(x for x in (4,))")
-            except SyntaxError:
-                pass
-            else:
-                raise Exception("Expected SyntaxError")
-
-        raises_syntaxError()
-
-        statement = "python_version == '5'"
-        assert compile(statement).__doc__ == statement
-

setuptools/version.py

-__version__ = '20.1.2'
+__version__ = '19.3'