Merge pull request #1088 from gevent/dnspython

Add a DNSpython resolver

.gitignore

@@ -13,8 +13,8 @@ src/gevent/libev/corecext.c
 src/gevent/libev/corecext.h
 src/gevent/libev/_corecffi.c
 src/gevent/libev/_corecffi.o
-src/gevent/ares.c
-src/gevent/ares.h
+src/gevent/resolver/cares.c
+
 Makefile.ext
 MANIFEST
 *_flymake.py

CHANGES.rst

@@ -32,6 +32,9 @@
 - Travis CI tests on Python 3.7.0a4 and PyPy 2.7 5.10.0 and PyPy 3.5
   5.10.1.
 
+- Add the ``dnspython`` resolver as a lightweight alternative to
+  c-ares. See :pr:`1088`.
+
 1.3a1 (2018-01-27)
 ==================
 

Makefile

@@ -14,7 +14,7 @@ export LC_ALL=C.UTF-8
 
 clean:
 	rm -f src/gevent/libev/corecext.c src/gevent/libev/corecext.h
-	rm -f src/gevent/ares.c src/gevent/ares.h
+	rm -f src/gevent/resolver/cares.c src/gevent/resolver/cares.h
 	rm -f src/gevent/_semaphore.c src/gevent/_semaphore.h
 	rm -f src/gevent/local.c src/gevent/local.h
 	rm -f src/gevent/*.so src/gevent/*.pyd src/gevent/libev/*.so src/gevent/libuv/*.so src/gevent/libev/*.pyd src/gevent/libuv/*.pyd
@@ -33,7 +33,7 @@ doc:
 	cd doc && PYTHONPATH=.. make html
 
 whitespace:
-	! find . -not -path "*.pem" -not -path "./.eggs/*" -not -path "./src/greentest/htmlcov/*" -not -path "./src/greentest/.coverage.*" -not -path "./.tox/*" -not -path "*/__pycache__/*" -not -path "*.so" -not -path "*.pyc" -not -path "./.git/*" -not -path "./build/*"  -not -path "./src/gevent/libev/*" -not -path "./src/gevent.egg-info/*" -not -path "./dist/*" -not -path "./.DS_Store" -not -path "./deps/*" -not -path "./src/gevent/libev/corecext.*.[ch]" -not -path "./src/gevent/ares.*" -not -path "./doc/_build/*" -not -path "./doc/mytheme/static/*" -type f | xargs egrep -l " $$"
+	! find . -not -path "*.pem" -not -path "./.eggs/*" -not -path "./src/greentest/htmlcov/*" -not -path "./src/greentest/.coverage.*" -not -path "./.tox/*" -not -path "*/__pycache__/*" -not -path "*.so" -not -path "*.pyc" -not -path "./.git/*" -not -path "./build/*"  -not -path "./src/gevent/libev/*" -not -path "./src/gevent.egg-info/*" -not -path "./dist/*" -not -path "./.DS_Store" -not -path "./deps/*" -not -path "./src/gevent/libev/corecext.*.[ch]" -not -path "./src/gevent/resolver/cares.*" -not -path "./doc/_build/*" -not -path "./doc/mytheme/static/*" -type f | xargs egrep -l " $$"
 
 prospector:
 	which prospector
@@ -66,6 +66,9 @@ alltest: basictest
 	${PYTHON} scripts/travis.py fold_start ares "Running c-ares tests"
 	cd src/greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 ${PYTHON} testrunner.py --config known_failures.py --ignore tests_that_dont_use_resolver.txt --quiet
 	${PYTHON} scripts/travis.py fold_end ares
+	${PYTHON} scripts/travis.py fold_start dnspython "Running dnspython tests"
+	cd src/greentest && GEVENT_RESOLVER=dnspython ${PYTHON} testrunner.py --config known_failures.py --ignore tests_that_dont_use_resolver.txt,tests_that_dont_monkeypatch.txt --quiet
+	${PYTHON} scripts/travis.py fold_end dnspython
 # In the past, we included all test files that had a reference to 'subprocess'' somewhere in their
 # text. The monkey-patched stdlib tests were specifically included here.
 # However, we now always also test on AppVeyor (Windows) which only has GEVENT_FILE=thread,

_setupares.py

@@ -79,13 +79,13 @@ def configure_ares(bext, ext):
         os.chdir(cwd)
 
 
-ARES = Extension(name='gevent.ares',
-                 sources=['src/gevent/ares.pyx'],
-                 include_dirs=['src/gevent'] + [dep_abspath('c-ares')] if CARES_EMBED else [],
+ARES = Extension(name='gevent.resolver.cares',
+                 sources=['src/gevent/resolver/cares.pyx'],
+                 include_dirs=['src/gevent/resolver'] + [dep_abspath('c-ares')] if CARES_EMBED else [],
                  libraries=list(LIBRARIES),
                  define_macros=list(DEFINE_MACROS),
-                 depends=glob_many('src/gevent/dnshelper.c',
-                                   'src/gevent/cares_*.[ch]'))
+                 depends=glob_many('src/gevent/resolver/dnshelper.c',
+                                   'src/gevent/resolver/cares_*.[ch]'))
 
 ARES.optional = not RUNNING_ON_CI
 

dev-requirements.txt

@@ -15,6 +15,8 @@ coveralls>=1.0
 # See version requirements in setup.py
 cffi
 futures
+dnspython
+idna
 # Makes tests faster
 psutil
 # For viewing README.rst (restview --long-description),

doc/dns.rst

@@ -11,7 +11,7 @@ through the :attr:`resolver attribute <gevent.hub.Hub.resolver>` of the
 :mod:`gevent.socket` module.
 
 A resolver implements the 5 standandard functions from the
-:mod:`socket` module for resolving hostnames:
+:mod:`socket` module for resolving hostnames and addresses:
 
 * :func:`socket.gethostbyname`
 * :func:`socket.gethostbyname_ex`
@@ -22,19 +22,25 @@ A resolver implements the 5 standandard functions from the
 Configuration
 =============
 
-gevent includes three implementations of resolvers, and applications
+gevent includes four implementations of resolvers, and applications
 can provide their own implementation. By default, gevent uses
-:class:`gevent.resolver_thread.Resolver`.
+:class:`gevent.resolver.thread.Resolver`.
 
 Configuration can be done through the ``GEVENT_RESOLVER`` environment
-variable. Specify ``ares``, ``thread``, or ``block`` to use the
-:class:`gevent.resolver_ares.Resolver`,
-:class:`gevent.resolver_thread.Resolver`, or
-:class:`gevent.socket.BlockingResolver`, respectively, or set it to
+variable. Specify ``ares``, ``thread``, ``dnspython``, or ``block`` to use the
+:class:`gevent.resolver.ares.Resolver`,
+:class:`gevent.resolver.thread.Resolver`, or
+:class:`gevent.resolver.dnspython.Resolver`, or
+:class:`gevent.resolver.blocking.Resolver`, respectively, or set it to
 the fully-qualified name of an implementation of the standard
 functions.
 
+Please see the documentation for each resolver class to understand the
+relative performance and correctness tradeoffs.
+
 .. toctree::
 
-   gevent.resolver_thread
-   gevent.resolver_ares
+   gevent.resolver.thread
+   gevent.resolver.ares
+   gevent.resolver.dnspython
+   gevent.resolver.blocking

doc/gevent.resolver.blocking.rst

+=============================================================
+ :mod:`gevent.resolver.blocking` -- Non-cooperative resolver
+=============================================================
+
+.. automodule:: gevent.resolver.blocking
+    :members:

doc/gevent.socket.rst

@@ -2,6 +2,8 @@
  :mod:`gevent.socket` -- Cooperative low-level networking interface
 ====================================================================
 
+.. module:: gevent.socket
+
 This module provides socket operations and some related functions. The
 API of the functions and classes matches the API of the corresponding
 items in the standard :mod:`socket` module exactly, but the
@@ -48,6 +50,10 @@ Beyond the basic standard library interface, ``gevent.socket``
 provides some extensions. These are identical and shared by all
 versions of Python.
 
+.. versionchanged:: 1.3a2
+   The undocumented class ``BlockingResolver`` has been documented
+   and moved to :class:`gevent.resolver.blocking.Resolver`.
+
 Waiting
 -------
 

setup.py

@@ -169,6 +169,12 @@ def run_setup(ext_modules, run_make):
         cmdclass=dict(build_ext=ConfiguringBuildExt),
         install_requires=install_requires,
         setup_requires=setup_requires,
+        extras_require={
+            'dnspython': [
+                'dnspython',
+                'idna',
+            ],
+        },
         # It's always safe to pass the CFFI keyword, even if
         # cffi is not installed: it's just ignored in that case.
         cffi_modules=cffi_modules,

src/gevent/_socketcommon.py

@@ -238,20 +238,6 @@ def cancel_wait(watcher, error=cancel_wait_ex):
     get_hub().cancel_wait(watcher, error)
 
 
-class BlockingResolver(object):
-
-    def __init__(self, hub=None):
-        pass
-
-    def close(self):
-        pass
-
-    for method in ['gethostbyname',
-                   'gethostbyname_ex',
-                   'getaddrinfo',
-                   'gethostbyaddr',
-                   'getnameinfo']:
-        locals()[method] = staticmethod(getattr(_socket, method))
 
 
 def gethostbyname(hostname):

src/gevent/ares.py

+"""Backwards compatibility alias for :mod:`gevent.resolver.cares`.
+
+.. deprecated:: 1.3
+   Use :mod:`gevent.resolver.cares`
+"""
+
+from gevent.resolver.cares import * # pylint:disable=wildcard-import,unused-wildcard-import
+import gevent.resolver.cares as _cares
+__all__ = _cares.__all__
+del _cares

src/gevent/hub.py

@@ -450,9 +450,12 @@ def resolver_config(default, envvar):
     return [_resolvers.get(x, x) for x in result]
 
 
-_resolvers = {'ares': 'gevent.resolver_ares.Resolver',
-              'thread': 'gevent.resolver_thread.Resolver',
-              'block': 'gevent.socket.BlockingResolver'}
+_resolvers = {
+    'ares': 'gevent.resolver.ares.Resolver',
+    'thread': 'gevent.resolver.thread.Resolver',
+    'block': 'gevent.resolver.blocking.Resolver',
+    'dnspython': 'gevent.resolver.dnspython.Resolver',
+}
 
 
 _DEFAULT_LOOP_CLASS = 'gevent.core.loop'
@@ -494,9 +497,12 @@ class Hub(RawGreenlet):
     if loop_class == [_DEFAULT_LOOP_CLASS]:
         loop_class = [_import(loop_class)]
 
-    resolver_class = ['gevent.resolver_thread.Resolver',
-                      'gevent.resolver_ares.Resolver',
-                      'gevent.socket.BlockingResolver']
+    resolver_class = [
+        'gevent.resolver.thread.Resolver',
+        'gevent.resolver.dnspython.Resolver',
+        'gevent.resolver.ares.Resolver',
+        'gevent.resolver.blacking.Resolver',
+    ]
     #: The class or callable object, or the name of a factory function or class,
     #: that will be used to create :attr:`resolver`. By default, configured according to
     #: :doc:`dns`. If a list, a list of objects in preference order.

src/gevent/resolver/__init__.py

+# Copyright (c) 2018  gevent contributors. See LICENSE for details.
+
+from _socket import gaierror
+from _socket import error
+from _socket import getservbyname
+from _socket import getaddrinfo
+
+from gevent._compat import string_types
+from gevent._compat import integer_types
+
+from gevent.socket import SOCK_STREAM
+from gevent.socket import SOCK_DGRAM
+from gevent.socket import SOL_TCP
+from gevent.socket import AI_CANONNAME
+from gevent.socket import EAI_SERVICE
+from gevent.socket import AF_INET
+from gevent.socket import AI_PASSIVE
+
+
+def _lookup_port(port, socktype):
+    # pylint:disable=too-many-branches
+    socktypes = []
+    if isinstance(port, string_types):
+        try:
+            port = int(port)
+        except ValueError:
+            try:
+                if socktype == 0:
+                    origport = port
+                    try:
+                        port = getservbyname(port, 'tcp')
+                        socktypes.append(SOCK_STREAM)
+                    except error:
+                        port = getservbyname(port, 'udp')
+                        socktypes.append(SOCK_DGRAM)
+                    else:
+                        try:
+                            if port == getservbyname(origport, 'udp'):
+                                socktypes.append(SOCK_DGRAM)
+                        except error:
+                            pass
+                elif socktype == SOCK_STREAM:
+                    port = getservbyname(port, 'tcp')
+                elif socktype == SOCK_DGRAM:
+                    port = getservbyname(port, 'udp')
+                else:
+                    raise gaierror(EAI_SERVICE, 'Servname not supported for ai_socktype')
+            except error as ex:
+                if 'not found' in str(ex):
+                    raise gaierror(EAI_SERVICE, 'Servname not supported for ai_socktype')
+                else:
+                    raise gaierror(str(ex))
+            except UnicodeEncodeError:
+                raise error('Int or String expected', port)
+    elif port is None:
+        port = 0
+    elif isinstance(port, integer_types):
+        pass
+    else:
+        raise error('Int or String expected', port, type(port))
+    port = int(port % 65536)
+    if not socktypes and socktype:
+        socktypes.append(socktype)
+    return port, socktypes
+
+
+def _resolve_special(hostname, family):
+    if hostname == '':
+        result = getaddrinfo(None, 0, family, SOCK_DGRAM, 0, AI_PASSIVE)
+        if len(result) != 1:
+            raise error('wildcard resolved to multiple address')
+        return result[0][4][0]
+    return hostname
+
+
+class AbstractResolver(object):
+
+    def gethostbyname(self, hostname, family=AF_INET):
+        hostname = _resolve_special(hostname, family)
+        return self.gethostbyname_ex(hostname, family)[-1][0]
+
+    def gethostbyname_ex(self, hostname, family=AF_INET):
+        aliases = []
+        addresses = []
+        tuples = self.getaddrinfo(hostname, 0, family,
+                                  SOCK_STREAM,
+                                  SOL_TCP, AI_CANONNAME)
+        canonical = tuples[0][3]
+        for item in tuples:
+            addresses.append(item[4][0])
+        # XXX we just ignore aliases
+        return (canonical, aliases, addresses)
+
+    def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
+        raise NotImplementedError()

src/gevent/resolver/ares.py

+# Copyright (c) 2011-2015 Denis Bilenko. See LICENSE for details.
+"""
+c-ares based hostname resolver.
+"""
+from __future__ import absolute_import, print_function, division
+import os
+import sys
+
+from _socket import getaddrinfo
+from _socket import gaierror
+from _socket import error
+
+from gevent._compat import string_types
+from gevent._compat import text_type
+
+from gevent._compat import reraise
+from gevent._compat import PY3
+
+from gevent.hub import Waiter
+from gevent.hub import get_hub
+
+from gevent.socket import AF_UNSPEC
+from gevent.socket import AF_INET
+from gevent.socket import AF_INET6
+from gevent.socket import SOCK_STREAM
+from gevent.socket import SOCK_DGRAM
+from gevent.socket import SOCK_RAW
+from gevent.socket import AI_NUMERICHOST
+
+from .cares import channel, InvalidIP # pylint:disable=import-error,no-name-in-module
+from . import _lookup_port as lookup_port
+from . import _resolve_special
+from . import AbstractResolver
+
+__all__ = ['Resolver']
+
+
+class Resolver(AbstractResolver):
+    """
+    Implementation of the resolver API using the `c-ares`_ library.
+
+    This implementation uses the c-ares library to handle name
+    resolution. c-ares is natively asynchronous at the socket level
+    and so integrates well into gevent's event loop.
+
+    In comparison to :class:`gevent.resolver_thread.Resolver` (which
+    delegates to the native system resolver), the implementation is
+    much more complex. In addition, there have been reports of it not
+    properly honoring certain system configurations (for example, the
+    order in which IPv4 and IPv6 results are returned may not match
+    the threaded resolver). However, because it does not use threads,
+    it may scale better for applications that make many lookups.
+
+    There are some known differences from the system resolver:
+
+    - ``gethostbyname_ex`` and ``gethostbyaddr`` may return different
+      for the ``aliaslist`` tuple member. (Sometimes the same,
+      sometimes in a different order, sometimes a different alias
+      altogether.)
+    - ``gethostbyname_ex`` may return the ``ipaddrlist`` in a different order.
+    - ``getaddrinfo`` does not return ``SOCK_RAW`` results.
+    - ``getaddrinfo`` may return results in a different order.
+    - Handling of ``.local`` (mDNS) names may be different, even if they are listed in
+      the hosts file.
+    - c-ares will not resolve ``broadcasthost``, even if listed in the hosts file.
+    - This implementation may raise ``gaierror(4)`` where the system implementation would raise
+      ``herror(1)``.
+    - The results for ``localhost`` may be different. In particular, some system
+      resolvers will return more results from ``getaddrinfo`` than c-ares does,
+      such as SOCK_DGRAM results, and c-ares may report more ips on a multi-homed
+      host.
+
+    .. caution:: This module is considered extremely experimental on PyPy, and
+       due to its implementation in cython, it may be slower. It may also lead to
+       interpreter crashes.
+
+    .. _c-ares: http://c-ares.haxx.se
+    """
+
+    ares_class = channel
+
+    def __init__(self, hub=None, use_environ=True, **kwargs):
+        if hub is None:
+            hub = get_hub()
+        self.hub = hub
+        if use_environ:
+            for key in os.environ:
+                if key.startswith('GEVENTARES_'):
+                    name = key[11:].lower()
+                    if name:
+                        value = os.environ[key]
+                        kwargs.setdefault(name, value)
+        self.ares = self.ares_class(hub.loop, **kwargs)
+        self.pid = os.getpid()
+        self.params = kwargs
+        self.fork_watcher = hub.loop.fork(ref=False)
+        self.fork_watcher.start(self._on_fork)
+
+    def __repr__(self):
+        return '<gevent.resolver_ares.Resolver at 0x%x ares=%r>' % (id(self), self.ares)
+
+    def _on_fork(self):
+        # NOTE: See comment in gevent.hub.reinit.
+        pid = os.getpid()
+        if pid != self.pid:
+            self.hub.loop.run_callback(self.ares.destroy)
+            self.ares = self.ares_class(self.hub.loop, **self.params)
+            self.pid = pid
+
+    def close(self):
+        if self.ares is not None:
+            self.hub.loop.run_callback(self.ares.destroy)
+            self.ares = None
+        self.fork_watcher.stop()
+
+    def gethostbyname(self, hostname, family=AF_INET):
+        hostname = _resolve_special(hostname, family)
+        return self.gethostbyname_ex(hostname, family)[-1][0]
+
+    def gethostbyname_ex(self, hostname, family=AF_INET):
+        if PY3:
+            if isinstance(hostname, str):
+                hostname = hostname.encode('idna')
+            elif not isinstance(hostname, (bytes, bytearray)):
+                raise TypeError('Expected es(idna), not %s' % type(hostname).__name__)
+        else:
+            if isinstance(hostname, text_type):
+                hostname = hostname.encode('ascii')
+            elif not isinstance(hostname, str):
+                raise TypeError('Expected string, not %s' % type(hostname).__name__)
+
+        while True:
+            ares = self.ares
+            try:
+                waiter = Waiter(self.hub)
+                ares.gethostbyname(waiter, hostname, family)
+                result = waiter.get()
+                if not result[-1]:
+                    raise gaierror(-5, 'No address associated with hostname')
+                return result
+            except gaierror:
+                if ares is self.ares:
+                    if hostname == b'255.255.255.255':
+                        # The stdlib handles this case in 2.7 and 3.x, but ares does not.
+                        # It is tested by test_socket.py in 3.4.
+                        # HACK: So hardcode the expected return.
+                        return ('255.255.255.255', [], ['255.255.255.255'])
+                    raise
+                # "self.ares is not ares" means channel was destroyed (because we were forked)
+
+    def _lookup_port(self, port, socktype):
+        return lookup_port(port, socktype)
+
+    def _getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
+        # pylint:disable=too-many-locals,too-many-branches
+        if isinstance(host, text_type):
+            host = host.encode('idna')
+        elif not isinstance(host, str) or (flags & AI_NUMERICHOST):
+            # this handles cases which do not require network access
+            # 1) host is None
+            # 2) host is of an invalid type
+            # 3) AI_NUMERICHOST flag is set
+            return getaddrinfo(host, port, family, socktype, proto, flags)
+            # we also call _socket.getaddrinfo below if family is not one of AF_*
+
+        port, socktypes = self._lookup_port(port, socktype)
+
+        socktype_proto = [(SOCK_STREAM, 6), (SOCK_DGRAM, 17), (SOCK_RAW, 0)]
+        if socktypes:
+            socktype_proto = [(x, y) for (x, y) in socktype_proto if x in socktypes]
+        if proto:
+            socktype_proto = [(x, y) for (x, y) in socktype_proto if proto == y]
+
+        ares = self.ares
+
+        if family == AF_UNSPEC:
+            ares_values = Values(self.hub, 2)
+            ares.gethostbyname(ares_values, host, AF_INET)
+            ares.gethostbyname(ares_values, host, AF_INET6)
+        elif family == AF_INET:
+            ares_values = Values(self.hub, 1)
+            ares.gethostbyname(ares_values, host, AF_INET)
+        elif family == AF_INET6:
+            ares_values = Values(self.hub, 1)
+            ares.gethostbyname(ares_values, host, AF_INET6)
+        else:
+            raise gaierror(5, 'ai_family not supported: %r' % (family, ))
+
+        values = ares_values.get()
+        if len(values) == 2 and values[0] == values[1]:
+            values.pop()
+
+        result = []
+        result4 = []
+        result6 = []
+
+        for addrs in values:
+            if addrs.family == AF_INET:
+                for addr in addrs[-1]:
+                    sockaddr = (addr, port)
+                    for socktype4, proto4 in socktype_proto:
+                        result4.append((AF_INET, socktype4, proto4, '', sockaddr))
+            elif addrs.family == AF_INET6:
+                for addr in addrs[-1]:
+                    if addr == '::1':
+                        dest = result
+                    else:
+                        dest = result6
+                    sockaddr = (addr, port, 0, 0)
+                    for socktype6, proto6 in socktype_proto:
+                        dest.append((AF_INET6, socktype6, proto6, '', sockaddr))
+
+        # As of 2016, some platforms return IPV6 first and some do IPV4 first,
+        # and some might even allow configuration of which is which. For backwards
+        # compatibility with earlier releases (but not necessarily resolver_thread!)
+        # we return 4 first. See https://github.com/gevent/gevent/issues/815 for more.
+        result += result4 + result6
+
+        if not result:
+            raise gaierror(-5, 'No address associated with hostname')
+
+        return result
+
+    def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
+        while True:
+            ares = self.ares
+            try:
+                return self._getaddrinfo(host, port, family, socktype, proto, flags)
+            except gaierror:
+                if ares is self.ares:
+                    raise
+
+    def _gethostbyaddr(self, ip_address):
+        if PY3:
+            if isinstance(ip_address, str):
+                ip_address = ip_address.encode('idna')
+            elif not isinstance(ip_address, (bytes, bytearray)):
+                raise TypeError('Expected es(idna), not %s' % type(ip_address).__name__)
+        else:
+            if isinstance(ip_address, text_type):
+                ip_address = ip_address.encode('ascii')
+            elif not isinstance(ip_address, str):
+                raise TypeError('Expected string, not %s' % type(ip_address).__name__)
+
+        waiter = Waiter(self.hub)
+        try:
+            self.ares.gethostbyaddr(waiter, ip_address)
+            return waiter.get()
+        except InvalidIP:
+            result = self._getaddrinfo(ip_address, None, family=AF_UNSPEC, socktype=SOCK_DGRAM)
+            if not result:
+                raise
+            _ip_address = result[0][-1][0]
+            if isinstance(_ip_address, text_type):
+                _ip_address = _ip_address.encode('ascii')
+            if _ip_address == ip_address:
+                raise
+            waiter.clear()
+            self.ares.gethostbyaddr(waiter, _ip_address)
+            return waiter.get()
+
+    def gethostbyaddr(self, ip_address):
+        ip_address = _resolve_special(ip_address, AF_UNSPEC)
+        while True:
+            ares = self.ares
+            try:
+                return self._gethostbyaddr(ip_address)
+            except gaierror:
+                if ares is self.ares:
+                    raise
+
+    def _getnameinfo(self, sockaddr, flags):
+        if not isinstance(flags, int):
+            raise TypeError('an integer is required')
+        if not isinstance(sockaddr, tuple):
+            raise TypeError('getnameinfo() argument 1 must be a tuple')
+
+        address = sockaddr[0]
+        if not PY3 and isinstance(address, text_type):
+            address = address.encode('ascii')
+
+        if not isinstance(address, string_types):
+            raise TypeError('sockaddr[0] must be a string, not %s' % type(address).__name__)
+
+        port = sockaddr[1]
+        if not isinstance(port, int):
+            raise TypeError('port must be an integer, not %s' % type(port))
+
+        waiter = Waiter(self.hub)
+        result = self._getaddrinfo(address, str(sockaddr[1]), family=AF_UNSPEC, socktype=SOCK_DGRAM)
+        if not result:
+            reraise(*sys.exc_info())
+        elif len(result) != 1:
+            raise error('sockaddr resolved to multiple addresses')
+        family, _socktype, _proto, _name, address = result[0]
+
+        if family == AF_INET:
+            if len(sockaddr) != 2:
+                raise error("IPv4 sockaddr must be 2 tuple")
+        elif family == AF_INET6:
+            address = address[:2] + sockaddr[2:]
+
+        self.ares.getnameinfo(waiter, address, flags)
+        node, service = waiter.get()
+
+        if service is None:
+            if PY3:
+                # ares docs: "If the query did not complete
+                # successfully, or one of the values was not
+                # requested, node or service will be NULL ". Python 2
+                # allows that for the service, but Python 3 raises
+                # an error. This is tested by test_socket in py 3.4
+                err = gaierror('nodename nor servname provided, or not known')
+                err.errno = 8
+                raise err
+            service = '0'
+        return node, service
+
+    def getnameinfo(self, sockaddr, flags):
+        while True:
+            ares = self.ares
+            try:
+                return self._getnameinfo(sockaddr, flags)
+            except gaierror:
+                if ares is self.ares:
+                    raise
+
+
+class Values(object):
+    # helper to collect multiple values; ignore errors unless nothing has succeeded
+    # QQQ could probably be moved somewhere - hub.py?
+
+    __slots__ = ['count', 'values', 'error', 'waiter']
+
+    def __init__(self, hub, count):
+        self.count = count
+        self.values = []
+        self.error = None
+        self.waiter = Waiter(hub)
+
+    def __call__(self, source):
+        self.count -= 1
+        if source.exception is None:
+            self.values.append(source.value)
+        else:
+            self.error = source.exception
+        if self.count <= 0:
+            self.waiter.switch()
+
+    def get(self):
+        self.waiter.get()
+        if self.values:
+            return self.values
+        else:
+            assert error is not None
+            raise self.error # pylint:disable=raising-bad-type

src/gevent/resolver/blocking.py

+# Copyright (c) 2018  gevent contributors. See LICENSE for details.
+
+import _socket
+
+class Resolver(object):
+    """
+    A resolver that directly uses the system's resolver functions.
+
+    .. caution::
+
+        This resolver is *not* cooperative.
+
+    This resolver has the lowest overhead of any resolver and
+    typically approaches the speed of the unmodified :mod:`socket`
+    functions. However, it is not cooperative, so if name resolution
+    blocks, the entire thread and all its greenlets will be blocked.
+
+    This can be useful during debugging, or it may be a good choice if
+    your operating system provides a good caching resolver (such as
+    macOS's Directory Services) that is usually very fast and
+    functionally non-blocking.
+
+    .. versionchanged:: 1.3a2
+       This was previously undocumented and existed in :mod:`gevent.socket`.
+
+    """
+
+    def __init__(self, hub=None):
+        pass
+
+    def close(self):
+        pass
+
+    for method in (
+            'gethostbyname',
+            'gethostbyname_ex',
+            'getaddrinfo',
+            'gethostbyaddr',
+            'getnameinfo'
+    ):
+        locals()[method] = staticmethod(getattr(_socket, method))

src/gevent/resolver/dnspython.py

+# Copyright (c) 2018  gevent contributors. See LICENSE for details.
+
+import _socket
+from _socket import AI_NUMERICHOST
+from _socket import error
+from _socket import NI_NUMERICSERV
+
+import socket
+
+from . import AbstractResolver
+
+from dns import resolver
+import dns
+
+
+__all__ = [
+    'Resolver',
+]
+
+# This is a copy of resolver._getaddrinfo with the crucial change that it
+# doesn't have a bare except:, because that breaks Timeout and KeyboardInterrupt
+# See https://github.com/rthalley/dnspython/pull/300
+def _getaddrinfo(host=None, service=None, family=socket.AF_UNSPEC, socktype=0,
+                 proto=0, flags=0):
+    # pylint:disable=too-many-locals,broad-except,too-many-statements
+    # pylint:disable=too-many-branches
+    # pylint:disable=redefined-argument-from-local
+    if flags & (socket.AI_ADDRCONFIG | socket.AI_V4MAPPED) != 0:
+        raise NotImplementedError
+    if host is None and service is None:
+        raise socket.gaierror(socket.EAI_NONAME)
+    v6addrs = []
+    v4addrs = []
+    canonical_name = None
+    try:
+        # Is host None or a V6 address literal?
+        if host is None:
+            canonical_name = 'localhost'
+            if flags & socket.AI_PASSIVE != 0:
+                v6addrs.append('::')
+                v4addrs.append('0.0.0.0')
+            else:
+                v6addrs.append('::1')
+                v4addrs.append('127.0.0.1')
+        else:
+            parts = host.split('%')
+            if len(parts) == 2:
+                ahost = parts[0]
+            else:
+                ahost = host
+            addr = dns.ipv6.inet_aton(ahost)
+            v6addrs.append(host)
+            canonical_name = host
+    except Exception:
+        try:
+            # Is it a V4 address literal?
+            addr = dns.ipv4.inet_aton(host)
+            v4addrs.append(host)
+            canonical_name = host
+        except Exception:
+            if flags & socket.AI_NUMERICHOST == 0:
+                try:
+                    if family == socket.AF_INET6 or family == socket.AF_UNSPEC:
+                        v6 = resolver._resolver.query(host, dns.rdatatype.AAAA,
+                                                      raise_on_no_answer=False)
+                        # Note that setting host ensures we query the same name
+                        # for A as we did for AAAA.
+                        host = v6.qname
+                        canonical_name = v6.canonical_name.to_text(True)
+                        if v6.rrset is not None:
+                            for rdata in v6.rrset:
+                                v6addrs.append(rdata.address)
+                    if family == socket.AF_INET or family == socket.AF_UNSPEC:
+                        v4 = resolver._resolver.query(host, dns.rdatatype.A,
+                                                      raise_on_no_answer=False)
+                        host = v4.qname
+                        canonical_name = v4.canonical_name.to_text(True)
+                        if v4.rrset is not None:
+                            for rdata in v4.rrset:
+                                v4addrs.append(rdata.address)
+                except dns.resolver.NXDOMAIN:
+                    raise socket.gaierror(socket.EAI_NONAME)
+                except Exception:
+                    raise socket.gaierror(socket.EAI_SYSTEM)
+    port = None
+    try:
+        # Is it a port literal?
+        if service is None:
+            port = 0
+        else:
+            port = int(service)
+    except Exception:
+        if flags & socket.AI_NUMERICSERV == 0:
+            try:
+                port = socket.getservbyname(service)
+            except Exception:
+                pass
+    if port is None:
+        raise socket.gaierror(socket.EAI_NONAME)
+    tuples = []
+    if socktype == 0:
+        socktypes = [socket.SOCK_DGRAM, socket.SOCK_STREAM]
+    else:
+        socktypes = [socktype]
+    if flags & socket.AI_CANONNAME != 0:
+        cname = canonical_name
+    else:
+        cname = ''
+    if family == socket.AF_INET6 or family == socket.AF_UNSPEC:
+        for addr in v6addrs:
+            for socktype in socktypes:
+                for proto in resolver._protocols_for_socktype[socktype]:
+                    tuples.append((socket.AF_INET6, socktype, proto,
+                                   cname, (addr, port, 0, 0)))
+    if family == socket.AF_INET or family == socket.AF_UNSPEC:
+        for addr in v4addrs:
+            for socktype in socktypes:
+                for proto in resolver._protocols_for_socktype[socktype]:
+                    tuples.append((socket.AF_INET, socktype, proto,
+                                   cname, (addr, port)))
+    if len(tuples) == 0: # pylint:disable=len-as-condition
+        raise socket.gaierror(socket.EAI_NONAME)
+    return tuples
+
+
+class Resolver(AbstractResolver):
+    """
+    An *experimental* resolver that uses `dnspython`_.
+
+    This is typically slower than the default threaded resolver
+    (unless there's a cache hit, in which case it can be much faster).
+    It is usually much faster than the c-ares resolver. It tends to
+    scale well as more concurrent resolutions are attempted.
+
+    Under Python 2, if the ``idna`` package is installed, this
+    resolver can resolve Unicode host names that the system resolver
+    cannot.
+
+    This uses thread locks and sockets, so it only functions if the
+    system has been monkey-patched. Otherwise it will raise a
+    ``ValueError``.
+
+    This uses dnspython's default resolver object. This object has
+    several useful attributes that can be used to adjust the behaviour
+    of the DNS system; in particular, the ``cache`` attribute could be
+    set to an instance of :class:`dns.resolver.Cache` or
+    :class:`dns.resolver.LRUCache` (by default a ``LRUCache`` is
+    used), and ``nameservers`` controls which nameservers to talk to,
+    and ``lifetime`` configures a timeout for each individual query.
+
+    .. caution::
+
+        This completely ignores the contents of ``/etc/hosts``, but it is
+        configured by ``/etc/resolv.conf`` (on Unix) or the registry (on
+        Windows). There are some measures in place to be able to resolve
+        ``localhost`` related names and addresses through the system resolver.
+
+    .. caution::
+
+        This resolver is experimental. It may be removed or modified in
+        the future. As always, feedback is welcome.
+
+    .. versionadded:: 1.3a2
+
+    .. _dnspython: http://www.dnspython.org
+    """
+
+    def __init__(self, hub=None): # pylint: disable=unused-argument
+        from gevent import monkey
+        if not all(monkey.is_module_patched(m) for m in ['threading', 'socket', 'select']):
+            raise ValueError("Can only be used when monkey-patched")
+        if resolver._resolver is None:
+            resolver._resolver = resolver.get_default_resolver()
+            # Add a default cache
+            resolver._resolver.cache = resolver.LRUCache()
+        if resolver._getaddrinfo is not _getaddrinfo:
+            resolver._getaddrinfo = _getaddrinfo
+
+    def close(self):
+        pass
+
+    def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
+        if ((host == u'localhost' or host == b'localhost')
+                or not isinstance(host, str) or (flags & AI_NUMERICHOST)):
+            # this handles cases which do not require network access
+            # 1) host is None
+            # 2) host is of an invalid type
+            # 3) AI_NUMERICHOST flag is set
+            return _socket.getaddrinfo(host, port, family, socktype, proto, flags)
+
+        return _getaddrinfo(host, port, family, socktype, proto, flags)
+
+    def getnameinfo(self, sockaddr, flags):
+        if (sockaddr
+                and isinstance(sockaddr, (list, tuple))
+                and sockaddr[0] in ('::1', '127.0.0.1', 'localhost')):
+            return _socket.getnameinfo(sockaddr, flags)
+        try:
+            return resolver._getnameinfo(sockaddr, flags)
+        except error:
+            if not flags:
+                # dnspython doesn't like getting ports it can't resolve.
+                # We have one test, test__socket_dns.py:Test_getnameinfo_geventorg.test_port_zero
+                # that does this. We conservatively fix it here; this could be expanded later.
+                return resolver._getnameinfo(sockaddr, NI_NUMERICSERV)
+
+    def gethostbyaddr(self, ip_address):
+        if ip_address in (u'127.0.0.1', u'::1',
+                          b'127.0.0.1', b'::1',
+                          'localhost'):
+            return _socket.gethostbyaddr(ip_address)
+        return resolver._gethostbyaddr(ip_address)

src/gevent/resolver/thread.py

+# Copyright (c) 2012-2015 Denis Bilenko. See LICENSE for details.
+"""
+Native thread-based hostname resolver.
+"""
+import _socket
+
+from gevent.hub import get_hub
+
+
+__all__ = ['Resolver']
+
+
+# trigger import of encodings.idna to avoid https://github.com/gevent/gevent/issues/349
+u'foo'.encode('idna')
+
+
+class Resolver(object):
+    """
+    Implementation of the resolver API using native threads and native resolution
+    functions.
+
+    Using the native resolution mechanisms ensures the highest
+    compatibility with what a non-gevent program would return
+    including good support for platform specific configuration
+    mechanisms. The use of native (non-greenlet) threads ensures that
+    a caller doesn't block other greenlets.
+
+    This implementation also has the benefit of being very simple in comparison to
+    :class:`gevent.resolver_ares.Resolver`.
+
+    .. tip::
+
+        Most users find this resolver to be quite reliable in a
+        properly monkey-patched environment. However, there have been
+        some reports of long delays, slow performance or even hangs,
+        particularly in long-lived programs that make many, many DNS
+        requests. If you suspect that may be happening to you, try the
+        ares resolver (and submit a bug report).
+    """
+    def __init__(self, hub=None):
+        if hub is None:
+            hub = get_hub()
+        self.pool = hub.threadpool
+        if _socket.gaierror not in hub.NOT_ERROR:
+            # Do not cause lookup failures to get printed by the default
+            # error handler. This can be very noisy.
+            hub.NOT_ERROR += (_socket.gaierror, _socket.herror)
+
+    def __repr__(self):
+        return '<gevent.resolver_thread.Resolver at 0x%x pool=%r>' % (id(self), self.pool)
+
+    def close(self):
+        pass
+
+    # from briefly reading socketmodule.c, it seems that all of the functions
+    # below are thread-safe in Python, even if they are not thread-safe in C.
+
+    def gethostbyname(self, *args):
+        return self.pool.apply(_socket.gethostbyname, args)
+
+    def gethostbyname_ex(self, *args):
+        return self.pool.apply(_socket.gethostbyname_ex, args)
+
+    def getaddrinfo(self, *args, **kwargs):
+        return self.pool.apply(_socket.getaddrinfo, args, kwargs)
+
+    def gethostbyaddr(self, *args, **kwargs):
+        return self.pool.apply(_socket.gethostbyaddr, args, kwargs)
+
+    def getnameinfo(self, *args, **kwargs):
+        return self.pool.apply(_socket.getnameinfo, args, kwargs)

src/gevent/resolver_ares.py

-# Copyright (c) 2011-2015 Denis Bilenko. See LICENSE for details.
-"""
-c-ares based hostname resolver.
-"""
-from __future__ import absolute_import
-import os
-import sys
-from _socket import getservbyname, getaddrinfo, gaierror, error
-from gevent.hub import Waiter, get_hub
-from gevent._compat import string_types, text_type, integer_types, reraise, PY3
-from gevent.socket import AF_UNSPEC, AF_INET, AF_INET6, SOCK_STREAM, SOCK_DGRAM, SOCK_RAW, AI_NUMERICHOST, EAI_SERVICE, AI_PASSIVE
-from gevent.ares import channel, InvalidIP # pylint:disable=import-error,no-name-in-module
-
-
-__all__ = ['Resolver']
-
-
-class Resolver(object):
-    """
-    Implementation of the resolver API using the `c-ares`_ library.
-
-    This implementation uses the c-ares library to handle name
-    resolution. c-ares is natively asynchronous at the socket level
-    and so integrates well into gevent's event loop.
-
-    In comparison to :class:`gevent.resolver_thread.Resolver` (which
-    delegates to the native system resolver), the implementation is
-    much more complex. In addition, there have been reports of it not
-    properly honoring certain system configurations (for example, the
-    order in which IPv4 and IPv6 results are returned may not match
-    the threaded resolver). However, because it does not use threads,
-    it may scale better for applications that make many lookups.
-
-    There are some known differences from the system resolver:
-
-    - ``gethostbyname_ex`` and ``gethostbyaddr`` may return different
-      for the ``aliaslist`` tuple member. (Sometimes the same,
-      sometimes in a different order, sometimes a different alias
-      altogether.)
-    - ``gethostbyname_ex`` may return the ``ipaddrlist`` in a different order.
-    - ``getaddrinfo`` does not return ``SOCK_RAW`` results.
-    - ``getaddrinfo`` may return results in a different order.
-    - Handling of ``.local`` (mDNS) names may be different, even if they are listed in
-      the hosts file.
-    - c-ares will not resolve ``broadcasthost``, even if listed in the hosts file.
-    - This implementation may raise ``gaierror(4)`` where the system implementation would raise
-      ``herror(1)``.
-    - The results for ``localhost`` may be different. In particular, some system
-      resolvers will return more results from ``getaddrinfo`` than c-ares does,
-      such as SOCK_DGRAM results, and c-ares may report more ips on a multi-homed
-      host.
-
-    .. caution:: This module is considered extremely experimental on PyPy, and
-       due to its implementation in cython, it may be slower. It may also lead to
-       interpreter crashes.
-
-    .. _c-ares: http://c-ares.haxx.se
-    """
-
-    ares_class = channel
-
-    def __init__(self, hub=None, use_environ=True, **kwargs):
-        if hub is None:
-            hub = get_hub()
-        self.hub = hub
-        if use_environ:
-            for key in os.environ:
-                if key.startswith('GEVENTARES_'):
-                    name = key[11:].lower()
-                    if name:
-                        value = os.environ[key]
-                        kwargs.setdefault(name, value)
-        self.ares = self.ares_class(hub.loop, **kwargs)
-        self.pid = os.getpid()
-        self.params = kwargs
-        self.fork_watcher = hub.loop.fork(ref=False)
-        self.fork_watcher.start(self._on_fork)
-
-    def __repr__(self):
-        return '<gevent.resolver_ares.Resolver at 0x%x ares=%r>' % (id(self), self.ares)
-
-    def _on_fork(self):
-        # NOTE: See comment in gevent.hub.reinit.
-        pid = os.getpid()
-        if pid != self.pid:
-            self.hub.loop.run_callback(self.ares.destroy)
-            self.ares = self.ares_class(self.hub.loop, **self.params)
-            self.pid = pid
-
-    def close(self):
-        if self.ares is not None:
-            self.hub.loop.run_callback(self.ares.destroy)
-            self.ares = None
-        self.fork_watcher.stop()
-
-    def gethostbyname(self, hostname, family=AF_INET):
-        hostname = _resolve_special(hostname, family)
-        return self.gethostbyname_ex(hostname, family)[-1][0]
-
-    def gethostbyname_ex(self, hostname, family=AF_INET):
-        if PY3:
-            if isinstance(hostname, str):
-                hostname = hostname.encode('idna')
-            elif not isinstance(hostname, (bytes, bytearray)):
-                raise TypeError('Expected es(idna), not %s' % type(hostname).__name__)
-        else:
-            if isinstance(hostname, text_type):
-                hostname = hostname.encode('ascii')
-            elif not isinstance(hostname, str):
-                raise TypeError('Expected string, not %s' % type(hostname).__name__)
-
-        while True:
-            ares = self.ares
-            try:
-                waiter = Waiter(self.hub)
-                ares.gethostbyname(waiter, hostname, family)
-                result = waiter.get()
-                if not result[-1]:
-                    raise gaierror(-5, 'No address associated with hostname')
-                return result
-            except gaierror:
-                if ares is self.ares:
-                    if hostname == b'255.255.255.255':
-                        # The stdlib handles this case in 2.7 and 3.x, but ares does not.
-                        # It is tested by test_socket.py in 3.4.
-                        # HACK: So hardcode the expected return.
-                        return ('255.255.255.255', [], ['255.255.255.255'])
-                    raise
-                # "self.ares is not ares" means channel was destroyed (because we were forked)
-
-    def _lookup_port(self, port, socktype):
-        # pylint:disable=too-many-branches
-        socktypes = []
-        if isinstance(port, string_types):
-            try:
-                port = int(port)
-            except ValueError:
-                try:
-                    if socktype == 0:
-                        origport = port
-                        try:
-                            port = getservbyname(port, 'tcp')
-                            socktypes.append(SOCK_STREAM)
-                        except error:
-                            port = getservbyname(port, 'udp')
-                            socktypes.append(SOCK_DGRAM)
-                        else:
-                            try:
-                                if port == getservbyname(origport, 'udp'):
-                                    socktypes.append(SOCK_DGRAM)
-                            except error:
-                                pass
-                    elif socktype == SOCK_STREAM:
-                        port = getservbyname(port, 'tcp')
-                    elif socktype == SOCK_DGRAM:
-                        port = getservbyname(port, 'udp')
-                    else:
-                        raise gaierror(EAI_SERVICE, 'Servname not supported for ai_socktype')
-                except error as ex:
-                    if 'not found' in str(ex):
-                        raise gaierror(EAI_SERVICE, 'Servname not supported for ai_socktype')
-                    else:
-                        raise gaierror(str(ex))
-                except UnicodeEncodeError:
-                    raise error('Int or String expected')
-        elif port is None:
-            port = 0
-        elif isinstance(port, integer_types):
-            pass
-        else:
-            raise error('Int or String expected', port, type(port))
-        port = int(port % 65536)
-        if not socktypes and socktype:
-            socktypes.append(socktype)
-        return port, socktypes
-
-    def _getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
-        # pylint:disable=too-many-locals,too-many-branches
-        if isinstance(host, text_type):
-            host = host.encode('idna')
-        elif not isinstance(host, str) or (flags & AI_NUMERICHOST):
-            # this handles cases which do not require network access
-            # 1) host is None
-            # 2) host is of an invalid type
-            # 3) AI_NUMERICHOST flag is set
-            return getaddrinfo(host, port, family, socktype, proto, flags)
-            # we also call _socket.getaddrinfo below if family is not one of AF_*
-
-        port, socktypes = self._lookup_port(port, socktype)
-
-        socktype_proto = [(SOCK_STREAM, 6), (SOCK_DGRAM, 17), (SOCK_RAW, 0)]
-        if socktypes:
-            socktype_proto = [(x, y) for (x, y) in socktype_proto if x in socktypes]
-        if proto:
-            socktype_proto = [(x, y) for (x, y) in socktype_proto if proto == y]
-
-        ares = self.ares
-
-        if family == AF_UNSPEC:
-            ares_values = Values(self.hub, 2)
-            ares.gethostbyname(ares_values, host, AF_INET)
-            ares.gethostbyname(ares_values, host, AF_INET6)
-        elif family == AF_INET:
-            ares_values = Values(self.hub, 1)
-            ares.gethostbyname(ares_values, host, AF_INET)
-        elif family == AF_INET6:
-            ares_values = Values(self.hub, 1)
-            ares.gethostbyname(ares_values, host, AF_INET6)
-        else:
-            raise gaierror(5, 'ai_family not supported: %r' % (family, ))
-
-        values = ares_values.get()
-        if len(values) == 2 and values[0] == values[1]:
-            values.pop()
-
-        result = []
-        result4 = []
-        result6 = []
-
-        for addrs in values:
-            if addrs.family == AF_INET:
-                for addr in addrs[-1]:
-                    sockaddr = (addr, port)
-                    for socktype4, proto4 in socktype_proto:
-                        result4.append((AF_INET, socktype4, proto4, '', sockaddr))
-            elif addrs.family == AF_INET6:
-                for addr in addrs[-1]:
-                    if addr == '::1':
-                        dest = result
-                    else:
-                        dest = result6
-                    sockaddr = (addr, port, 0, 0)
-                    for socktype6, proto6 in socktype_proto:
-                        dest.append((AF_INET6, socktype6, proto6, '', sockaddr))
-
-        # As of 2016, some platforms return IPV6 first and some do IPV4 first,
-        # and some might even allow configuration of which is which. For backwards
-        # compatibility with earlier releases (but not necessarily resolver_thread!)
-        # we return 4 first. See https://github.com/gevent/gevent/issues/815 for more.
-        result += result4 + result6
-
-        if not result:
-            raise gaierror(-5, 'No address associated with hostname')
-
-        return result
-
-    def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
-        while True:
-            ares = self.ares
-            try:
-                return self._getaddrinfo(host, port, family, socktype, proto, flags)
-            except gaierror:
-                if ares is self.ares:
-                    raise
-
-    def _gethostbyaddr(self, ip_address):
-        if PY3:
-            if isinstance(ip_address, str):
-                ip_address = ip_address.encode('idna')
-            elif not isinstance(ip_address, (bytes, bytearray)):
-                raise TypeError('Expected es(idna), not %s' % type(ip_address).__name__)
-        else:
-            if isinstance(ip_address, text_type):
-                ip_address = ip_address.encode('ascii')
-            elif not isinstance(ip_address, str):
-                raise TypeError('Expected string, not %s' % type(ip_address).__name__)
-
-        waiter = Waiter(self.hub)
-        try:
-            self.ares.gethostbyaddr(waiter, ip_address)
-            return waiter.get()
-        except InvalidIP:
-            result = self._getaddrinfo(ip_address, None, family=AF_UNSPEC, socktype=SOCK_DGRAM)
-            if not result:
-                raise
-            _ip_address = result[0][-1][0]
-            if isinstance(_ip_address, text_type):
-                _ip_address = _ip_address.encode('ascii')
-            if _ip_address == ip_address:
-                raise
-            waiter.clear()
-            self.ares.gethostbyaddr(waiter, _ip_address)
-            return waiter.get()
-
-    def gethostbyaddr(self, ip_address):
-        ip_address = _resolve_special(ip_address, AF_UNSPEC)
-        while True:
-            ares = self.ares
-            try:
-                return self._gethostbyaddr(ip_address)
-            except gaierror:
-                if ares is self.ares:
-                    raise
-
-    def _getnameinfo(self, sockaddr, flags):
-        if not isinstance(flags, int):
-            raise TypeError('an integer is required')
-        if not isinstance(sockaddr, tuple):
-            raise TypeError('getnameinfo() argument 1 must be a tuple')
-
-        address = sockaddr[0]
-        if not PY3 and isinstance(address, text_type):
-            address = address.encode('ascii')
-
-        if not isinstance(address, string_types):
-            raise TypeError('sockaddr[0] must be a string, not %s' % type(address).__name__)
-
-        port = sockaddr[1]
-        if not isinstance(port, int):
-            raise TypeError('port must be an integer, not %s' % type(port))
-
-        waiter = Waiter(self.hub)
-        result = self._getaddrinfo(address, str(sockaddr[1]), family=AF_UNSPEC, socktype=SOCK_DGRAM)
-        if not result:
-            reraise(*sys.exc_info())
-        elif len(result) != 1:
-            raise error('sockaddr resolved to multiple addresses')
-        family, _socktype, _proto, _name, address = result[0]
-
-        if family == AF_INET:
-            if len(sockaddr) != 2:
-                raise error("IPv4 sockaddr must be 2 tuple")
-        elif family == AF_INET6:
-            address = address[:2] + sockaddr[2:]
-
-        self.ares.getnameinfo(waiter, address, flags)
-        node, service = waiter.get()
-
-        if service is None:
-            if PY3:
-                # ares docs: "If the query did not complete
-                # successfully, or one of the values was not
-                # requested, node or service will be NULL ". Python 2
-                # allows that for the service, but Python 3 raises
-                # an error. This is tested by test_socket in py 3.4
-                err = gaierror('nodename nor servname provided, or not known')
-                err.errno = 8
-                raise err
-            service = '0'
-        return node, service
-
-    def getnameinfo(self, sockaddr, flags):
-        while True:
-            ares = self.ares
-            try:
-                return self._getnameinfo(sockaddr, flags)
-            except gaierror:
-                if ares is self.ares:
-                    raise
-
-
-class Values(object):
-    # helper to collect multiple values; ignore errors unless nothing has succeeded
-    # QQQ could probably be moved somewhere - hub.py?
-
-    __slots__ = ['count', 'values', 'error', 'waiter']
-
-    def __init__(self, hub, count):
-        self.count = count
-        self.values = []
-        self.error = None
-        self.waiter = Waiter(hub)
-
-    def __call__(self, source):
-        self.count -= 1
-        if source.exception is None:
-            self.values.append(source.value)
-        else:
-            self.error = source.exception
-        if self.count <= 0:
-            self.waiter.switch()
-
-    def get(self):
-        self.waiter.get()
-        if self.values:
-            return self.values
-        else:
-            assert error is not None
-            raise self.error # pylint:disable=raising-bad-type
+"""Backwards compatibility alias for :mod:`gevent.resolver.ares`.
 
+.. deprecated:: 1.3
+   Use :mod:`gevent.resolver.ares`
+"""
 
-def _resolve_special(hostname, family):
-    if hostname == '':
-        result = getaddrinfo(None, 0, family, SOCK_DGRAM, 0, AI_PASSIVE)
-        if len(result) != 1:
-            raise error('wildcard resolved to multiple address')
-        return result[0][4][0]
-    return hostname
+from gevent.resolver.ares import * # pylint:disable=wildcard-import,unused-wildcard-import
+import gevent.resolver.ares as _ares
+__all__ = _ares.__all__
+del _ares

src/gevent/resolver_thread.py

-# Copyright (c) 2012-2015 Denis Bilenko. See LICENSE for details.
-"""
-Native thread-based hostname resolver.
-"""
-import _socket
-
-from gevent.hub import get_hub
-
-
-__all__ = ['Resolver']
-
-
-# trigger import of encodings.idna to avoid https://github.com/gevent/gevent/issues/349
-u'foo'.encode('idna')
-
-
-class Resolver(object):
-    """
-    Implementation of the resolver API using native threads and native resolution
-    functions.
-
-    Using the native resolution mechanisms ensures the highest
-    compatibility with what a non-gevent program would return
-    including good support for platform specific configuration
-    mechanisms. The use of native (non-greenlet) threads ensures that
-    a caller doesn't block other greenlets.
+"""Backwards compatibility alias for :mod:`gevent.resolver.thread`.
 
-    This implementation also has the benefit of being very simple in comparison to
-    :class:`gevent.resolver_ares.Resolver`.
-
-    .. tip::
-
-        Most users find this resolver to be quite reliable in a
-        properly monkey-patched environment. However, there have been
-        some reports of long delays, slow performance or even hangs,
-        particularly in long-lived programs that make many, many DNS
-        requests. If you suspect that may be happening to you, try the
-        ares resolver (and submit a bug report).
-    """
-    def __init__(self, hub=None):
-        if hub is None:
-            hub = get_hub()
-        self.pool = hub.threadpool
-        if _socket.gaierror not in hub.NOT_ERROR:
-            # Do not cause lookup failures to get printed by the default
-            # error handler. This can be very noisy.
-            hub.NOT_ERROR += (_socket.gaierror, _socket.herror)
-
-    def __repr__(self):
-        return '<gevent.resolver_thread.Resolver at 0x%x pool=%r>' % (id(self), self.pool)
-
-    def close(self):
-        pass
-
-    # from briefly reading socketmodule.c, it seems that all of the functions
-    # below are thread-safe in Python, even if they are not thread-safe in C.
-
-    def gethostbyname(self, *args):
-        return self.pool.apply(_socket.gethostbyname, args)
-
-    def gethostbyname_ex(self, *args):
-        return self.pool.apply(_socket.gethostbyname_ex, args)
-
-    def getaddrinfo(self, *args, **kwargs):
-        return self.pool.apply(_socket.getaddrinfo, args, kwargs)
-
-    def gethostbyaddr(self, *args, **kwargs):
-        return self.pool.apply(_socket.gethostbyaddr, args, kwargs)
+.. deprecated:: 1.3
+   Use :mod:`gevent.resolver.cares`
+"""
 
-    def getnameinfo(self, *args, **kwargs):
-        return self.pool.apply(_socket.getnameinfo, args, kwargs)
+from gevent.resolver.thread import * # pylint:disable=wildcard-import,unused-wildcard-import
+import gevent.resolver.thread as _thread
+__all__ = _thread.__all__
+del _thread

src/greentest/greentest/patched_tests_setup.py

@@ -15,7 +15,7 @@ import re
 
 from greentest.sysinfo import RUNNING_ON_APPVEYOR as APPVEYOR
 from greentest.sysinfo import RUNNING_ON_TRAVIS as TRAVIS
-from greentest.sysinfo import RESOLVER_ARES as ARES
+from greentest.sysinfo import RESOLVER_NOT_SYSTEM as ARES
 from greentest.sysinfo import RUN_COVERAGE
 
 

src/greentest/greentest/sysinfo.py

@@ -126,3 +126,6 @@ CONN_ABORTED_ERRORS.append(ECONNRESET)
 CONN_ABORTED_ERRORS = frozenset(CONN_ABORTED_ERRORS)
 
 RESOLVER_ARES = os.getenv('GEVENT_RESOLVER') == 'ares'
+RESOLVER_DNSPYTHON = os.getenv('GEVENT_RESOLVER') == 'dnspython'
+
+RESOLVER_NOT_SYSTEM = RESOLVER_ARES or RESOLVER_DNSPYTHON

src/greentest/greentest/testrunner.py

@@ -141,7 +141,10 @@ def run_many(tests, expected=(), failfast=False, quiet=False):
 
 def discover(tests=None, ignore=(), coverage=False):
     if isinstance(ignore, six.string_types):
-        ignore = set(load_list_from_file(ignore))
+        ignore_files = ignore.split(',')
+        ignore = set()
+        for f in ignore_files:
+            ignore.update(set(load_list_from_file(f)))
 
     ignore = set(ignore or ())
     if coverage:

src/greentest/known_failures.py

@@ -10,7 +10,7 @@ from greentest.sysinfo import RUNNING_ON_APPVEYOR as APPVEYOR
 from greentest.sysinfo import RUNNING_ON_TRAVIS as TRAVIS
 from greentest.sysinfo import RUN_LEAKCHECKS as LEAKTEST
 from greentest.sysinfo import RUN_COVERAGE as COVERAGE
-from greentest.sysinfo import RESOLVER_ARES
+from greentest.sysinfo import RESOLVER_NOT_SYSTEM
 
 from greentest.sysinfo import PYPY
 from greentest.sysinfo import PY3
@@ -145,7 +145,7 @@ if PYPY:
         'FLAKY test__backdoor.py',
     ]
 
-    if RESOLVER_ARES:
+    if RESOLVER_NOT_SYSTEM:
 
         FAILING_TESTS += [
             # A few errors and differences:

src/greentest/test__socket_dns.py

 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
+import gevent
+from gevent import monkey
+
+if ['gevent.resolver.dnspython.Resolver'] == gevent.get_hub().resolver_class:
+    # dnspython requires monkey-patching
+    monkey.patch_all()
+
+import os
 import re
 import greentest
 import unittest
 import socket
 from time import time
-import gevent
+import traceback
 import gevent.socket as gevent_socket
 from greentest.util import log
 from greentest import six
@@ -19,13 +27,15 @@ log('Resolver: %s', resolver)
 if getattr(resolver, 'pool', None) is not None:
     resolver.pool.size = 1
 
+from greentest.sysinfo import RESOLVER_NOT_SYSTEM
+from greentest.sysinfo import RESOLVER_DNSPYTHON
+from greentest.sysinfo import PY2
 
-RESOLVER_IS_ARES = 'ares' in gevent.get_hub().resolver_class.__module__
 
 assert gevent_socket.gaierror is socket.gaierror
 assert gevent_socket.error is socket.error
 
-DEBUG = False
+DEBUG = os.getenv('GEVENT_DEBUG', '') == 'trace'
 
 
 def _run(function, *args):
@@ -34,6 +44,8 @@ def _run(function, *args):
         assert not isinstance(result, BaseException), repr(result)
         return result
     except Exception as ex:
+        if DEBUG:
+            traceback.print_exc()
         return ex
 
 
@@ -146,8 +158,11 @@ def relaxed_is_equal(a, b):
         return True
     if isinstance(a, six.string_types):
         return compare_relaxed(a, b)
+    try:
         if len(a) != len(b):
             return False
+    except TypeError:
+        return False
     if contains_5tuples(a) and contains_5tuples(b):
         # getaddrinfo results
         a = sorted(a)
@@ -219,7 +234,7 @@ class TestCase(greentest.TestCase):
 
     def _test(self, func, *args):
         gevent_func = getattr(gevent_socket, func)
-        real_func = getattr(socket, func)
+        real_func = monkey.get_original('socket', func)
         real_result, time_real = run(real_func, *args)
         gevent_result, time_gevent = run(gevent_func, *args)
         if not DEBUG and self.should_log_results(real_result, gevent_result):
@@ -251,7 +266,7 @@ class TestCase(greentest.TestCase):
         # can be in different orders if we're hitting different servers,
         # or using the native and ares resolvers due to load-balancing techniques.
         # We sort them.
-        if not RESOLVER_IS_ARES or isinstance(result, BaseException):
+        if not RESOLVER_NOT_SYSTEM or isinstance(result, BaseException):
             return result
         # result[1].sort() # we wind up discarding this
 
@@ -275,7 +290,7 @@ class TestCase(greentest.TestCase):
         return (result[0], [], ips)
 
     def _normalize_result_getaddrinfo(self, result):
-        if not RESOLVER_IS_ARES:
+        if not RESOLVER_NOT_SYSTEM:
             return result
         # On Python 3, the builtin resolver can return SOCK_RAW results, but
         # c-ares doesn't do that. So we remove those if we find them.
@@ -286,7 +301,7 @@ class TestCase(greentest.TestCase):
         return result
 
     def _normalize_result_gethostbyaddr(self, result):
-        if not RESOLVER_IS_ARES:
+        if not RESOLVER_NOT_SYSTEM:
             return result
 
         if isinstance(result, tuple):
@@ -316,7 +331,7 @@ class TestCase(greentest.TestCase):
         # If we're using the ares resolver, allow the real resolver to generate an
         # error that the ares resolver actually gets an answer to.
 
-        if (RESOLVER_IS_ARES
+        if (RESOLVER_NOT_SYSTEM
             and isinstance(real_result, errors)
             and not isinstance(gevent_result, errors)):
             return
@@ -334,6 +349,8 @@ add(TestTypeError, None)
 add(TestTypeError, 25)
 
 
+@unittest.skipIf(RESOLVER_DNSPYTHON,
+                 "This commonly needs /etc/hosts to function, and dnspython doesn't do that.")
 class TestHostname(TestCase):
     pass
 
@@ -348,13 +365,13 @@ class TestLocalhost(TestCase):
     # anymore.
 
     def _normalize_result_getaddrinfo(self, result):
-        if RESOLVER_IS_ARES:
+        if RESOLVER_NOT_SYSTEM:
             # We see that some impls (OS X) return extra results
             # like DGRAM that ares does not.
             return ()
         return super(TestLocalhost, self)._normalize_result_getaddrinfo(result)
 
-    if greentest.RUNNING_ON_TRAVIS and greentest.PY2 and RESOLVER_IS_ARES:
+    if greentest.RUNNING_ON_TRAVIS and greentest.PY2 and RESOLVER_NOT_SYSTEM:
         def _normalize_result_gethostbyaddr(self, result):
             # Beginning in November 2017 after an upgrade to Travis,
             # we started seeing ares return ::1 for localhost, but
@@ -398,8 +415,8 @@ if not greentest.RUNNING_ON_TRAVIS:
 class TestBroadcast(TestCase):
     switch_expected = False
 
-    if RESOLVER_IS_ARES:
-        # ares raises errors for broadcasthost/255.255.255.255
+    if RESOLVER_NOT_SYSTEM:
+        # ares and dnspython raises errors for broadcasthost/255.255.255.255
 
         @unittest.skip('ares raises errors for broadcasthost/255.255.255.255')
         def test__broadcast__gethostbyaddr(self):
@@ -410,6 +427,7 @@ class TestBroadcast(TestCase):
 add(TestBroadcast, '<broadcast>')
 
 
+@unittest.skipIf(RESOLVER_DNSPYTHON, "/etc/hosts completely ignored under dnspython")
 class TestEtcHosts(TestCase):
     pass
 
@@ -420,7 +438,7 @@ except IOError:
     etc_hosts = ''
 
 for ip, host in re.findall(r'^\s*(\d+\.\d+\.\d+\.\d+)\s+([^\s]+)', etc_hosts, re.M)[:10]:
-    if (RESOLVER_IS_ARES
+    if (RESOLVER_NOT_SYSTEM
         and (host.endswith('local') # ignore bonjour, ares can't find them
              # ignore common aliases that ares can't find
              or ip == '255.255.255.255'
@@ -521,6 +539,8 @@ class Test_getaddrinfo(TestCase):
     def test2(self):
         return self._test_getaddrinfo(TestGeventOrg.HOSTNAME, 53, socket.AF_INET, socket.SOCK_DGRAM, 17)
 
+    @unittest.skipIf(RESOLVER_DNSPYTHON,
+                     "dnspython only returns some of the possibilities")
     def test3(self):
         return self._test_getaddrinfo('google.com', 'http', socket.AF_INET6)
 
@@ -528,11 +548,15 @@ class Test_getaddrinfo(TestCase):
 class TestInternational(TestCase):
     pass
 
-add(TestInternational, u'президент.рф', 'russian')
+if not (PY2 and RESOLVER_DNSPYTHON):
+    # dns python can actually resolve these: it uses
+    # the 2008 version of idna encoding, whereas on Python 2,
+    # with the default resolver, it tries to encode to ascii and
+    # raises a UnicodeEncodeError. So we get different results.
+    add(TestInternational, u'президент.рф', 'russian')
 add(TestInternational, u'президент.рф'.encode('idna'), 'idna')
 
 
-
 class TestInterrupted_gethostbyname(greentest.GenericWaitTestCase):
 
     # There are refs to a Waiter in the C code that don't go
@@ -573,7 +597,6 @@ class TestInterrupted_gethostbyname(greentest.GenericWaitTestCase):
         try:
             gevent.get_hub().threadpool.join()
         except Exception: # pylint:disable=broad-except
-            import traceback
             traceback.print_exc()
 
 
@@ -604,7 +627,6 @@ add(TestBadIP, '1.2.3.400')
 class Test_getnameinfo_127001(TestCase):
 
     def test(self):
-        assert gevent_socket.getnameinfo is not socket.getnameinfo
         self._test('getnameinfo', ('127.0.0.1', 80), 0)
 
     def test_DGRAM(self):
@@ -658,6 +680,10 @@ class TestInvalidPort(TestCase):
     def test3(self):
         self._test('getnameinfo', ('www.gevent.org', 'x'), 0)
 
+    @unittest.skipIf(RESOLVER_DNSPYTHON,
+                     "System resolvers do funny things with this: macOS raises gaierror, "
+                     "Travis CI returns (readthedocs.org, '0'). It's hard to match that exactly. "
+                     "dnspython raises OverflowError.")
     def test4(self):
         self._test('getnameinfo', ('www.gevent.org', 65536), 0)
 

src/greentest/test__socket_dns6.py

@@ -2,9 +2,12 @@
 # -*- coding: utf-8 -*-
 import greentest
 import socket
-from test__socket_dns import TestCase, add, RESOLVER_IS_ARES
+from test__socket_dns import TestCase, add
 
-if not greentest.RUNNING_ON_CI:
+from greentest.sysinfo import RESOLVER_NOT_SYSTEM
+from greentest.sysinfo import RESOLVER_DNSPYTHON
+
+if not greentest.RUNNING_ON_CI and not RESOLVER_DNSPYTHON:
 
 
     # We can't control the DNS servers we use there
@@ -46,7 +49,7 @@ if not greentest.RUNNING_ON_CI:
         host = 'ipv6.google.com'
 
         def _normalize_result_getnameinfo(self, result):
-            if greentest.RUNNING_ON_CI and RESOLVER_IS_ARES:
+            if greentest.RUNNING_ON_CI and RESOLVER_NOT_SYSTEM:
                 # Disabled, there are multiple possibilities
                 # and we can get different ones, rarely.
                 return ()

src/greentest/tests_that_dont_monkeypatch.txt

+test___example_servers.py
+test__backdoor.py
+test__example_echoserver.py
+test__example_udp_client.py
+test__getaddrinfo_import.py
+test__example_portforwarder.py
+test__pywsgi.py
+test__server.py
+test__server_pywsgi.py
+test__socket_close.py
+test__socket_dns6.py
+test__socket_errors.py
+test__socket_send_memoryview.py
+test__socket_timeout.py
+test__examples.py