Merge pull request #1088 from gevent/dnspython

Add a DNSpython resolver

.gitignore

@@ -13,8 +13,8 @@ src/gevent/libev/corecext.c
 src/gevent/libev/corecext.h
 src/gevent/libev/_corecffi.c
 src/gevent/libev/_corecffi.o
-src/gevent/ares.c
-src/gevent/ares.h
+src/gevent/resolver/cares.c
+
 Makefile.ext
 MANIFEST
 *_flymake.py

CHANGES.rst

@@ -32,6 +32,9 @@
 - Travis CI tests on Python 3.7.0a4 and PyPy 2.7 5.10.0 and PyPy 3.5
   5.10.1.
 
+- Add the ``dnspython`` resolver as a lightweight alternative to
+  c-ares. See :pr:`1088`.
+
 1.3a1 (2018-01-27)
 ==================
 

Makefile

@@ -14,7 +14,7 @@ export LC_ALL=C.UTF-8
 
 clean:
 	rm -f src/gevent/libev/corecext.c src/gevent/libev/corecext.h
-	rm -f src/gevent/ares.c src/gevent/ares.h
+	rm -f src/gevent/resolver/cares.c src/gevent/resolver/cares.h
 	rm -f src/gevent/_semaphore.c src/gevent/_semaphore.h
 	rm -f src/gevent/local.c src/gevent/local.h
 	rm -f src/gevent/*.so src/gevent/*.pyd src/gevent/libev/*.so src/gevent/libuv/*.so src/gevent/libev/*.pyd src/gevent/libuv/*.pyd
@@ -33,7 +33,7 @@ doc:
 	cd doc && PYTHONPATH=.. make html
 
 whitespace:
-	! find . -not -path "*.pem" -not -path "./.eggs/*" -not -path "./src/greentest/htmlcov/*" -not -path "./src/greentest/.coverage.*" -not -path "./.tox/*" -not -path "*/__pycache__/*" -not -path "*.so" -not -path "*.pyc" -not -path "./.git/*" -not -path "./build/*"  -not -path "./src/gevent/libev/*" -not -path "./src/gevent.egg-info/*" -not -path "./dist/*" -not -path "./.DS_Store" -not -path "./deps/*" -not -path "./src/gevent/libev/corecext.*.[ch]" -not -path "./src/gevent/ares.*" -not -path "./doc/_build/*" -not -path "./doc/mytheme/static/*" -type f | xargs egrep -l " $$"
+	! find . -not -path "*.pem" -not -path "./.eggs/*" -not -path "./src/greentest/htmlcov/*" -not -path "./src/greentest/.coverage.*" -not -path "./.tox/*" -not -path "*/__pycache__/*" -not -path "*.so" -not -path "*.pyc" -not -path "./.git/*" -not -path "./build/*"  -not -path "./src/gevent/libev/*" -not -path "./src/gevent.egg-info/*" -not -path "./dist/*" -not -path "./.DS_Store" -not -path "./deps/*" -not -path "./src/gevent/libev/corecext.*.[ch]" -not -path "./src/gevent/resolver/cares.*" -not -path "./doc/_build/*" -not -path "./doc/mytheme/static/*" -type f | xargs egrep -l " $$"
 
 prospector:
 	which prospector
@@ -66,6 +66,9 @@ alltest: basictest
 	${PYTHON} scripts/travis.py fold_start ares "Running c-ares tests"
 	cd src/greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 ${PYTHON} testrunner.py --config known_failures.py --ignore tests_that_dont_use_resolver.txt --quiet
 	${PYTHON} scripts/travis.py fold_end ares
+	${PYTHON} scripts/travis.py fold_start dnspython "Running dnspython tests"
+	cd src/greentest && GEVENT_RESOLVER=dnspython ${PYTHON} testrunner.py --config known_failures.py --ignore tests_that_dont_use_resolver.txt,tests_that_dont_monkeypatch.txt --quiet
+	${PYTHON} scripts/travis.py fold_end dnspython
 # In the past, we included all test files that had a reference to 'subprocess'' somewhere in their
 # text. The monkey-patched stdlib tests were specifically included here.
 # However, we now always also test on AppVeyor (Windows) which only has GEVENT_FILE=thread,

_setupares.py

@@ -79,13 +79,13 @@ def configure_ares(bext, ext):
         os.chdir(cwd)
 
 
-ARES = Extension(name='gevent.ares',
-                 sources=['src/gevent/ares.pyx'],
-                 include_dirs=['src/gevent'] + [dep_abspath('c-ares')] if CARES_EMBED else [],
+ARES = Extension(name='gevent.resolver.cares',
+                 sources=['src/gevent/resolver/cares.pyx'],
+                 include_dirs=['src/gevent/resolver'] + [dep_abspath('c-ares')] if CARES_EMBED else [],
                  libraries=list(LIBRARIES),
                  define_macros=list(DEFINE_MACROS),
-                 depends=glob_many('src/gevent/dnshelper.c',
-                                   'src/gevent/cares_*.[ch]'))
+                 depends=glob_many('src/gevent/resolver/dnshelper.c',
+                                   'src/gevent/resolver/cares_*.[ch]'))
 
 ARES.optional = not RUNNING_ON_CI
 

dev-requirements.txt

@@ -15,6 +15,8 @@ coveralls>=1.0
 # See version requirements in setup.py
 cffi
 futures
+dnspython
+idna
 # Makes tests faster
 psutil
 # For viewing README.rst (restview --long-description),

doc/dns.rst

@@ -11,7 +11,7 @@ through the :attr:`resolver attribute <gevent.hub.Hub.resolver>` of the
 :mod:`gevent.socket` module.
 
 A resolver implements the 5 standandard functions from the
-:mod:`socket` module for resolving hostnames:
+:mod:`socket` module for resolving hostnames and addresses:
 
 * :func:`socket.gethostbyname`
 * :func:`socket.gethostbyname_ex`
@@ -22,19 +22,25 @@ A resolver implements the 5 standandard functions from the
 Configuration
 =============
 
-gevent includes three implementations of resolvers, and applications
+gevent includes four implementations of resolvers, and applications
 can provide their own implementation. By default, gevent uses
-:class:`gevent.resolver_thread.Resolver`.
+:class:`gevent.resolver.thread.Resolver`.
 
 Configuration can be done through the ``GEVENT_RESOLVER`` environment
-variable. Specify ``ares``, ``thread``, or ``block`` to use the
-:class:`gevent.resolver_ares.Resolver`,
-:class:`gevent.resolver_thread.Resolver`, or
-:class:`gevent.socket.BlockingResolver`, respectively, or set it to
+variable. Specify ``ares``, ``thread``, ``dnspython``, or ``block`` to use the
+:class:`gevent.resolver.ares.Resolver`,
+:class:`gevent.resolver.thread.Resolver`, or
+:class:`gevent.resolver.dnspython.Resolver`, or
+:class:`gevent.resolver.blocking.Resolver`, respectively, or set it to
 the fully-qualified name of an implementation of the standard
 functions.
 
+Please see the documentation for each resolver class to understand the
+relative performance and correctness tradeoffs.
+
 .. toctree::
 
-   gevent.resolver_thread
-   gevent.resolver_ares
+   gevent.resolver.thread
+   gevent.resolver.ares
+   gevent.resolver.dnspython
+   gevent.resolver.blocking

doc/gevent.resolver.blocking.rst

+=============================================================
+ :mod:`gevent.resolver.blocking` -- Non-cooperative resolver
+=============================================================
+
+.. automodule:: gevent.resolver.blocking
+    :members:

doc/gevent.socket.rst

@@ -2,6 +2,8 @@
  :mod:`gevent.socket` -- Cooperative low-level networking interface
 ====================================================================
 
+.. module:: gevent.socket
+
 This module provides socket operations and some related functions. The
 API of the functions and classes matches the API of the corresponding
 items in the standard :mod:`socket` module exactly, but the
@@ -48,6 +50,10 @@ Beyond the basic standard library interface, ``gevent.socket``
 provides some extensions. These are identical and shared by all
 versions of Python.
 
+.. versionchanged:: 1.3a2
+   The undocumented class ``BlockingResolver`` has been documented
+   and moved to :class:`gevent.resolver.blocking.Resolver`.
+
 Waiting
 -------
 

setup.py

@@ -169,6 +169,12 @@ def run_setup(ext_modules, run_make):
         cmdclass=dict(build_ext=ConfiguringBuildExt),
         install_requires=install_requires,
         setup_requires=setup_requires,
+        extras_require={
+            'dnspython': [
+                'dnspython',
+                'idna',
+            ],
+        },
         # It's always safe to pass the CFFI keyword, even if
         # cffi is not installed: it's just ignored in that case.
         cffi_modules=cffi_modules,

src/gevent/_socketcommon.py

@@ -238,20 +238,6 @@ def cancel_wait(watcher, error=cancel_wait_ex):
     get_hub().cancel_wait(watcher, error)
 
 
-class BlockingResolver(object):
-
-    def __init__(self, hub=None):
-        pass
-
-    def close(self):
-        pass
-
-    for method in ['gethostbyname',
-                   'gethostbyname_ex',
-                   'getaddrinfo',
-                   'gethostbyaddr',
-                   'getnameinfo']:
-        locals()[method] = staticmethod(getattr(_socket, method))
 
 
 def gethostbyname(hostname):

src/gevent/ares.py

+"""Backwards compatibility alias for :mod:`gevent.resolver.cares`.
+
+.. deprecated:: 1.3
+   Use :mod:`gevent.resolver.cares`
+"""
+
+from gevent.resolver.cares import * # pylint:disable=wildcard-import,unused-wildcard-import
+import gevent.resolver.cares as _cares
+__all__ = _cares.__all__
+del _cares

src/gevent/hub.py

@@ -450,9 +450,12 @@ def resolver_config(default, envvar):
     return [_resolvers.get(x, x) for x in result]
 
 
-_resolvers = {'ares': 'gevent.resolver_ares.Resolver',
-              'thread': 'gevent.resolver_thread.Resolver',
-              'block': 'gevent.socket.BlockingResolver'}
+_resolvers = {
+    'ares': 'gevent.resolver.ares.Resolver',
+    'thread': 'gevent.resolver.thread.Resolver',
+    'block': 'gevent.resolver.blocking.Resolver',
+    'dnspython': 'gevent.resolver.dnspython.Resolver',
+}
 
 
 _DEFAULT_LOOP_CLASS = 'gevent.core.loop'
@@ -494,9 +497,12 @@ class Hub(RawGreenlet):
     if loop_class == [_DEFAULT_LOOP_CLASS]:
         loop_class = [_import(loop_class)]
 
-    resolver_class = ['gevent.resolver_thread.Resolver',
-                      'gevent.resolver_ares.Resolver',
-                      'gevent.socket.BlockingResolver']
+    resolver_class = [
+        'gevent.resolver.thread.Resolver',
+        'gevent.resolver.dnspython.Resolver',
+        'gevent.resolver.ares.Resolver',
+        'gevent.resolver.blacking.Resolver',
+    ]
     #: The class or callable object, or the name of a factory function or class,
     #: that will be used to create :attr:`resolver`. By default, configured according to
     #: :doc:`dns`. If a list, a list of objects in preference order.

src/gevent/resolver/__init__.py

+# Copyright (c) 2018  gevent contributors. See LICENSE for details.
+
+from _socket import gaierror
+from _socket import error
+from _socket import getservbyname
+from _socket import getaddrinfo
+
+from gevent._compat import string_types
+from gevent._compat import integer_types
+
+from gevent.socket import SOCK_STREAM
+from gevent.socket import SOCK_DGRAM
+from gevent.socket import SOL_TCP
+from gevent.socket import AI_CANONNAME
+from gevent.socket import EAI_SERVICE
+from gevent.socket import AF_INET
+from gevent.socket import AI_PASSIVE
+
+
+def _lookup_port(port, socktype):
+    # pylint:disable=too-many-branches
+    socktypes = []
+    if isinstance(port, string_types):
+        try:
+            port = int(port)
+        except ValueError:
+            try:
+                if socktype == 0:
+                    origport = port
+                    try:
+                        port = getservbyname(port, 'tcp')
+                        socktypes.append(SOCK_STREAM)
+                    except error:
+                        port = getservbyname(port, 'udp')
+                        socktypes.append(SOCK_DGRAM)
+                    else:
+                        try:
+                            if port == getservbyname(origport, 'udp'):
+                                socktypes.append(SOCK_DGRAM)
+                        except error:
+                            pass
+                elif socktype == SOCK_STREAM:
+                    port = getservbyname(port, 'tcp')
+                elif socktype == SOCK_DGRAM:
+                    port = getservbyname(port, 'udp')
+                else:
+                    raise gaierror(EAI_SERVICE, 'Servname not supported for ai_socktype')
+            except error as ex:
+                if 'not found' in str(ex):
+                    raise gaierror(EAI_SERVICE, 'Servname not supported for ai_socktype')
+                else:
+                    raise gaierror(str(ex))
+            except UnicodeEncodeError:
+                raise error('Int or String expected', port)
+    elif port is None:
+        port = 0
+    elif isinstance(port, integer_types):
+        pass
+    else:
+        raise error('Int or String expected', port, type(port))
+    port = int(port % 65536)
+    if not socktypes and socktype:
+        socktypes.append(socktype)
+    return port, socktypes
+
+
+def _resolve_special(hostname, family):
+    if hostname == '':
+        result = getaddrinfo(None, 0, family, SOCK_DGRAM, 0, AI_PASSIVE)
+        if len(result) != 1:
+            raise error('wildcard resolved to multiple address')
+        return result[0][4][0]
+    return hostname
+
+
+class AbstractResolver(object):
+
+    def gethostbyname(self, hostname, family=AF_INET):
+        hostname = _resolve_special(hostname, family)
+        return self.gethostbyname_ex(hostname, family)[-1][0]
+
+    def gethostbyname_ex(self, hostname, family=AF_INET):
+        aliases = []
+        addresses = []
+        tuples = self.getaddrinfo(hostname, 0, family,
+                                  SOCK_STREAM,
+                                  SOL_TCP, AI_CANONNAME)
+        canonical = tuples[0][3]
+        for item in tuples:
+            addresses.append(item[4][0])
+        # XXX we just ignore aliases
+        return (canonical, aliases, addresses)
+
+    def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
+        raise NotImplementedError()

src/gevent/resolver/blocking.py

+# Copyright (c) 2018  gevent contributors. See LICENSE for details.
+
+import _socket
+
+class Resolver(object):
+    """
+    A resolver that directly uses the system's resolver functions.
+
+    .. caution::
+
+        This resolver is *not* cooperative.
+
+    This resolver has the lowest overhead of any resolver and
+    typically approaches the speed of the unmodified :mod:`socket`
+    functions. However, it is not cooperative, so if name resolution
+    blocks, the entire thread and all its greenlets will be blocked.
+
+    This can be useful during debugging, or it may be a good choice if
+    your operating system provides a good caching resolver (such as
+    macOS's Directory Services) that is usually very fast and
+    functionally non-blocking.
+
+    .. versionchanged:: 1.3a2
+       This was previously undocumented and existed in :mod:`gevent.socket`.
+
+    """
+
+    def __init__(self, hub=None):
+        pass
+
+    def close(self):
+        pass
+
+    for method in (
+            'gethostbyname',
+            'gethostbyname_ex',
+            'getaddrinfo',
+            'gethostbyaddr',
+            'getnameinfo'
+    ):
+        locals()[method] = staticmethod(getattr(_socket, method))

src/gevent/resolver/dnspython.py

+# Copyright (c) 2018  gevent contributors. See LICENSE for details.
+
+import _socket
+from _socket import AI_NUMERICHOST
+from _socket import error
+from _socket import NI_NUMERICSERV
+
+import socket
+
+from . import AbstractResolver
+
+from dns import resolver
+import dns
+
+
+__all__ = [
+    'Resolver',
+]
+
+# This is a copy of resolver._getaddrinfo with the crucial change that it
+# doesn't have a bare except:, because that breaks Timeout and KeyboardInterrupt
+# See https://github.com/rthalley/dnspython/pull/300
+def _getaddrinfo(host=None, service=None, family=socket.AF_UNSPEC, socktype=0,
+                 proto=0, flags=0):
+    # pylint:disable=too-many-locals,broad-except,too-many-statements
+    # pylint:disable=too-many-branches
+    # pylint:disable=redefined-argument-from-local
+    if flags & (socket.AI_ADDRCONFIG | socket.AI_V4MAPPED) != 0:
+        raise NotImplementedError
+    if host is None and service is None:
+        raise socket.gaierror(socket.EAI_NONAME)
+    v6addrs = []
+    v4addrs = []
+    canonical_name = None
+    try:
+        # Is host None or a V6 address literal?
+        if host is None:
+            canonical_name = 'localhost'
+            if flags & socket.AI_PASSIVE != 0:
+                v6addrs.append('::')
+                v4addrs.append('0.0.0.0')
+            else:
+                v6addrs.append('::1')
+                v4addrs.append('127.0.0.1')
+        else:
+            parts = host.split('%')
+            if len(parts) == 2:
+                ahost = parts[0]
+            else:
+                ahost = host
+            addr = dns.ipv6.inet_aton(ahost)
+            v6addrs.append(host)
+            canonical_name = host
+    except Exception:
+        try:
+            # Is it a V4 address literal?
+            addr = dns.ipv4.inet_aton(host)
+            v4addrs.append(host)
+            canonical_name = host
+        except Exception:
+            if flags & socket.AI_NUMERICHOST == 0:
+                try:
+                    if family == socket.AF_INET6 or family == socket.AF_UNSPEC:
+                        v6 = resolver._resolver.query(host, dns.rdatatype.AAAA,
+                                                      raise_on_no_answer=False)
+                        # Note that setting host ensures we query the same name
+                        # for A as we did for AAAA.
+                        host = v6.qname
+                        canonical_name = v6.canonical_name.to_text(True)
+                        if v6.rrset is not None:
+                            for rdata in v6.rrset:
+                                v6addrs.append(rdata.address)
+                    if family == socket.AF_INET or family == socket.AF_UNSPEC:
+                        v4 = resolver._resolver.query(host, dns.rdatatype.A,
+                                                      raise_on_no_answer=False)
+                        host = v4.qname
+                        canonical_name = v4.canonical_name.to_text(True)
+                        if v4.rrset is not None:
+                            for rdata in v4.rrset:
+                                v4addrs.append(rdata.address)
+                except dns.resolver.NXDOMAIN:
+                    raise socket.gaierror(socket.EAI_NONAME)
+                except Exception:
+                    raise socket.gaierror(socket.EAI_SYSTEM)
+    port = None
+    try:
+        # Is it a port literal?
+        if service is None:
+            port = 0
+        else:
+            port = int(service)
+    except Exception:
+        if flags & socket.AI_NUMERICSERV == 0:
+            try:
+                port = socket.getservbyname(service)
+            except Exception:
+                pass
+    if port is None:
+        raise socket.gaierror(socket.EAI_NONAME)
+    tuples = []
+    if socktype == 0:
+        socktypes = [socket.SOCK_DGRAM, socket.SOCK_STREAM]
+    else:
+        socktypes = [socktype]
+    if flags & socket.AI_CANONNAME != 0:
+        cname = canonical_name
+    else:
+        cname = ''
+    if family == socket.AF_INET6 or family == socket.AF_UNSPEC:
+        for addr in v6addrs:
+            for socktype in socktypes:
+                for proto in resolver._protocols_for_socktype[socktype]:
+                    tuples.append((socket.AF_INET6, socktype, proto,
+                                   cname, (addr, port, 0, 0)))
+    if family == socket.AF_INET or family == socket.AF_UNSPEC:
+        for addr in v4addrs:
+            for socktype in socktypes:
+                for proto in resolver._protocols_for_socktype[socktype]:
+                    tuples.append((socket.AF_INET, socktype, proto,
+                                   cname, (addr, port)))
+    if len(tuples) == 0: # pylint:disable=len-as-condition
+        raise socket.gaierror(socket.EAI_NONAME)
+    return tuples
+
+
+class Resolver(AbstractResolver):
+    """
+    An *experimental* resolver that uses `dnspython`_.
+
+    This is typically slower than the default threaded resolver
+    (unless there's a cache hit, in which case it can be much faster).
+    It is usually much faster than the c-ares resolver. It tends to
+    scale well as more concurrent resolutions are attempted.
+
+    Under Python 2, if the ``idna`` package is installed, this
+    resolver can resolve Unicode host names that the system resolver
+    cannot.
+
+    This uses thread locks and sockets, so it only functions if the
+    system has been monkey-patched. Otherwise it will raise a
+    ``ValueError``.
+
+    This uses dnspython's default resolver object. This object has
+    several useful attributes that can be used to adjust the behaviour
+    of the DNS system; in particular, the ``cache`` attribute could be
+    set to an instance of :class:`dns.resolver.Cache` or
+    :class:`dns.resolver.LRUCache` (by default a ``LRUCache`` is
+    used), and ``nameservers`` controls which nameservers to talk to,
+    and ``lifetime`` configures a timeout for each individual query.
+
+    .. caution::
+
+        This completely ignores the contents of ``/etc/hosts``, but it is
+        configured by ``/etc/resolv.conf`` (on Unix) or the registry (on
+        Windows). There are some measures in place to be able to resolve
+        ``localhost`` related names and addresses through the system resolver.
+
+    .. caution::
+
+        This resolver is experimental. It may be removed or modified in
+        the future. As always, feedback is welcome.
+
+    .. versionadded:: 1.3a2
+
+    .. _dnspython: http://www.dnspython.org
+    """
+
+    def __init__(self, hub=None): # pylint: disable=unused-argument
+        from gevent import monkey
+        if not all(monkey.is_module_patched(m) for m in ['threading', 'socket', 'select']):
+            raise ValueError("Can only be used when monkey-patched")
+        if resolver._resolver is None:
+            resolver._resolver = resolver.get_default_resolver()
+            # Add a default cache
+            resolver._resolver.cache = resolver.LRUCache()
+        if resolver._getaddrinfo is not _getaddrinfo:
+            resolver._getaddrinfo = _getaddrinfo
+
+    def close(self):
+        pass
+
+    def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
+        if ((host == u'localhost' or host == b'localhost')
+                or not isinstance(host, str) or (flags & AI_NUMERICHOST)):
+            # this handles cases which do not require network access
+            # 1) host is None
+            # 2) host is of an invalid type
+            # 3) AI_NUMERICHOST flag is set
+            return _socket.getaddrinfo(host, port, family, socktype, proto, flags)
+
+        return _getaddrinfo(host, port, family, socktype, proto, flags)
+
+    def getnameinfo(self, sockaddr, flags):
+        if (sockaddr
+                and isinstance(sockaddr, (list, tuple))
+                and sockaddr[0] in ('::1', '127.0.0.1', 'localhost')):
+            return _socket.getnameinfo(sockaddr, flags)
+        try:
+            return resolver._getnameinfo(sockaddr, flags)
+        except error:
+            if not flags:
+                # dnspython doesn't like getting ports it can't resolve.
+                # We have one test, test__socket_dns.py:Test_getnameinfo_geventorg.test_port_zero
+                # that does this. We conservatively fix it here; this could be expanded later.
+                return resolver._getnameinfo(sockaddr, NI_NUMERICSERV)
+
+    def gethostbyaddr(self, ip_address):
+        if ip_address in (u'127.0.0.1', u'::1',
+                          b'127.0.0.1', b'::1',
+                          'localhost'):
+            return _socket.gethostbyaddr(ip_address)
+        return resolver._gethostbyaddr(ip_address)

src/gevent/resolver/thread.py

+# Copyright (c) 2012-2015 Denis Bilenko. See LICENSE for details.
+"""
+Native thread-based hostname resolver.
+"""
+import _socket
+
+from gevent.hub import get_hub
+
+
+__all__ = ['Resolver']
+
+
+# trigger import of encodings.idna to avoid https://github.com/gevent/gevent/issues/349
+u'foo'.encode('idna')
+
+
+class Resolver(object):
+    """
+    Implementation of the resolver API using native threads and native resolution
+    functions.
+
+    Using the native resolution mechanisms ensures the highest
+    compatibility with what a non-gevent program would return
+    including good support for platform specific configuration
+    mechanisms. The use of native (non-greenlet) threads ensures that
+    a caller doesn't block other greenlets.
+
+    This implementation also has the benefit of being very simple in comparison to
+    :class:`gevent.resolver_ares.Resolver`.
+
+    .. tip::
+
+        Most users find this resolver to be quite reliable in a
+        properly monkey-patched environment. However, there have been
+        some reports of long delays, slow performance or even hangs,
+        particularly in long-lived programs that make many, many DNS
+        requests. If you suspect that may be happening to you, try the
+        ares resolver (and submit a bug report).
+    """
+    def __init__(self, hub=None):
+        if hub is None:
+            hub = get_hub()
+        self.pool = hub.threadpool
+        if _socket.gaierror not in hub.NOT_ERROR:
+            # Do not cause lookup failures to get printed by the default
+            # error handler. This can be very noisy.
+            hub.NOT_ERROR += (_socket.gaierror, _socket.herror)
+
+    def __repr__(self):
+        return '<gevent.resolver_thread.Resolver at 0x%x pool=%r>' % (id(self), self.pool)
+
+    def close(self):
+        pass
+
+    # from briefly reading socketmodule.c, it seems that all of the functions
+    # below are thread-safe in Python, even if they are not thread-safe in C.
+
+    def gethostbyname(self, *args):
+        return self.pool.apply(_socket.gethostbyname, args)
+
+    def gethostbyname_ex(self, *args):
+        return self.pool.apply(_socket.gethostbyname_ex, args)
+
+    def getaddrinfo(self, *args, **kwargs):
+        return self.pool.apply(_socket.getaddrinfo, args, kwargs)
+
+    def gethostbyaddr(self, *args, **kwargs):
+        return self.pool.apply(_socket.gethostbyaddr, args, kwargs)
+
+    def getnameinfo(self, *args, **kwargs):
+        return self.pool.apply(_socket.getnameinfo, args, kwargs)

src/gevent/resolver_thread.py

-# Copyright (c) 2012-2015 Denis Bilenko. See LICENSE for details.
-"""
-Native thread-based hostname resolver.
-"""
-import _socket
-
-from gevent.hub import get_hub
-
-
-__all__ = ['Resolver']
-
-
-# trigger import of encodings.idna to avoid https://github.com/gevent/gevent/issues/349
-u'foo'.encode('idna')
-
-
-class Resolver(object):
-    """
-    Implementation of the resolver API using native threads and native resolution
-    functions.
-
-    Using the native resolution mechanisms ensures the highest
-    compatibility with what a non-gevent program would return
-    including good support for platform specific configuration
-    mechanisms. The use of native (non-greenlet) threads ensures that
-    a caller doesn't block other greenlets.
+"""Backwards compatibility alias for :mod:`gevent.resolver.thread`.
 
-    This implementation also has the benefit of being very simple in comparison to
-    :class:`gevent.resolver_ares.Resolver`.
-
-    .. tip::
-
-        Most users find this resolver to be quite reliable in a
-        properly monkey-patched environment. However, there have been
-        some reports of long delays, slow performance or even hangs,
-        particularly in long-lived programs that make many, many DNS
-        requests. If you suspect that may be happening to you, try the
-        ares resolver (and submit a bug report).
-    """
-    def __init__(self, hub=None):
-        if hub is None:
-            hub = get_hub()
-        self.pool = hub.threadpool
-        if _socket.gaierror not in hub.NOT_ERROR:
-            # Do not cause lookup failures to get printed by the default
-            # error handler. This can be very noisy.
-            hub.NOT_ERROR += (_socket.gaierror, _socket.herror)
-
-    def __repr__(self):
-        return '<gevent.resolver_thread.Resolver at 0x%x pool=%r>' % (id(self), self.pool)
-
-    def close(self):
-        pass
-
-    # from briefly reading socketmodule.c, it seems that all of the functions
-    # below are thread-safe in Python, even if they are not thread-safe in C.
-
-    def gethostbyname(self, *args):
-        return self.pool.apply(_socket.gethostbyname, args)
-
-    def gethostbyname_ex(self, *args):
-        return self.pool.apply(_socket.gethostbyname_ex, args)
-
-    def getaddrinfo(self, *args, **kwargs):
-        return self.pool.apply(_socket.getaddrinfo, args, kwargs)
-
-    def gethostbyaddr(self, *args, **kwargs):
-        return self.pool.apply(_socket.gethostbyaddr, args, kwargs)
+.. deprecated:: 1.3
+   Use :mod:`gevent.resolver.cares`
+"""
 
-    def getnameinfo(self, *args, **kwargs):
-        return self.pool.apply(_socket.getnameinfo, args, kwargs)
+from gevent.resolver.thread import * # pylint:disable=wildcard-import,unused-wildcard-import
+import gevent.resolver.thread as _thread
+__all__ = _thread.__all__
+del _thread

src/greentest/greentest/patched_tests_setup.py

@@ -15,7 +15,7 @@ import re
 
 from greentest.sysinfo import RUNNING_ON_APPVEYOR as APPVEYOR
 from greentest.sysinfo import RUNNING_ON_TRAVIS as TRAVIS
-from greentest.sysinfo import RESOLVER_ARES as ARES
+from greentest.sysinfo import RESOLVER_NOT_SYSTEM as ARES
 from greentest.sysinfo import RUN_COVERAGE
 
 

src/greentest/greentest/sysinfo.py

@@ -126,3 +126,6 @@ CONN_ABORTED_ERRORS.append(ECONNRESET)
 CONN_ABORTED_ERRORS = frozenset(CONN_ABORTED_ERRORS)
 
 RESOLVER_ARES = os.getenv('GEVENT_RESOLVER') == 'ares'
+RESOLVER_DNSPYTHON = os.getenv('GEVENT_RESOLVER') == 'dnspython'
+
+RESOLVER_NOT_SYSTEM = RESOLVER_ARES or RESOLVER_DNSPYTHON

src/greentest/greentest/testrunner.py

@@ -141,7 +141,10 @@ def run_many(tests, expected=(), failfast=False, quiet=False):
 
 def discover(tests=None, ignore=(), coverage=False):
     if isinstance(ignore, six.string_types):
-        ignore = set(load_list_from_file(ignore))
+        ignore_files = ignore.split(',')
+        ignore = set()
+        for f in ignore_files:
+            ignore.update(set(load_list_from_file(f)))
 
     ignore = set(ignore or ())
     if coverage:

src/greentest/known_failures.py

@@ -10,7 +10,7 @@ from greentest.sysinfo import RUNNING_ON_APPVEYOR as APPVEYOR
 from greentest.sysinfo import RUNNING_ON_TRAVIS as TRAVIS
 from greentest.sysinfo import RUN_LEAKCHECKS as LEAKTEST
 from greentest.sysinfo import RUN_COVERAGE as COVERAGE
-from greentest.sysinfo import RESOLVER_ARES
+from greentest.sysinfo import RESOLVER_NOT_SYSTEM
 
 from greentest.sysinfo import PYPY
 from greentest.sysinfo import PY3
@@ -145,7 +145,7 @@ if PYPY:
         'FLAKY test__backdoor.py',
     ]
 
-    if RESOLVER_ARES:
+    if RESOLVER_NOT_SYSTEM:
 
         FAILING_TESTS += [
             # A few errors and differences:

src/greentest/test__socket_dns.py

 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
+import gevent
+from gevent import monkey
+
+if ['gevent.resolver.dnspython.Resolver'] == gevent.get_hub().resolver_class:
+    # dnspython requires monkey-patching
+    monkey.patch_all()
+
+import os
 import re
 import greentest
 import unittest
 import socket
 from time import time
-import gevent
+import traceback
 import gevent.socket as gevent_socket
 from greentest.util import log
 from greentest import six
@@ -19,13 +27,15 @@ log('Resolver: %s', resolver)
 if getattr(resolver, 'pool', None) is not None:
     resolver.pool.size = 1
 
+from greentest.sysinfo import RESOLVER_NOT_SYSTEM
+from greentest.sysinfo import RESOLVER_DNSPYTHON
+from greentest.sysinfo import PY2
 
-RESOLVER_IS_ARES = 'ares' in gevent.get_hub().resolver_class.__module__
 
 assert gevent_socket.gaierror is socket.gaierror
 assert gevent_socket.error is socket.error
 
-DEBUG = False
+DEBUG = os.getenv('GEVENT_DEBUG', '') == 'trace'
 
 
 def _run(function, *args):
@@ -34,6 +44,8 @@ def _run(function, *args):
         assert not isinstance(result, BaseException), repr(result)
         return result
     except Exception as ex:
+        if DEBUG:
+            traceback.print_exc()
         return ex
 
 
@@ -146,7 +158,10 @@ def relaxed_is_equal(a, b):
         return True
     if isinstance(a, six.string_types):
         return compare_relaxed(a, b)
-    if len(a) != len(b):
+    try:
+        if len(a) != len(b):
+            return False
+    except TypeError:
         return False
     if contains_5tuples(a) and contains_5tuples(b):
         # getaddrinfo results
@@ -219,7 +234,7 @@ class TestCase(greentest.TestCase):
 
     def _test(self, func, *args):
         gevent_func = getattr(gevent_socket, func)
-        real_func = getattr(socket, func)
+        real_func = monkey.get_original('socket', func)
         real_result, time_real = run(real_func, *args)
         gevent_result, time_gevent = run(gevent_func, *args)
         if not DEBUG and self.should_log_results(real_result, gevent_result):
@@ -251,7 +266,7 @@ class TestCase(greentest.TestCase):
         # can be in different orders if we're hitting different servers,
         # or using the native and ares resolvers due to load-balancing techniques.
         # We sort them.
-        if not RESOLVER_IS_ARES or isinstance(result, BaseException):
+        if not RESOLVER_NOT_SYSTEM or isinstance(result, BaseException):
             return result
         # result[1].sort() # we wind up discarding this
 
@@ -275,7 +290,7 @@ class TestCase(greentest.TestCase):
         return (result[0], [], ips)
 
     def _normalize_result_getaddrinfo(self, result):
-        if not RESOLVER_IS_ARES:
+        if not RESOLVER_NOT_SYSTEM:
             return result
         # On Python 3, the builtin resolver can return SOCK_RAW results, but
         # c-ares doesn't do that. So we remove those if we find them.
@@ -286,7 +301,7 @@ class TestCase(greentest.TestCase):
         return result
 
     def _normalize_result_gethostbyaddr(self, result):
-        if not RESOLVER_IS_ARES:
+        if not RESOLVER_NOT_SYSTEM:
             return result
 
         if isinstance(result, tuple):
@@ -316,7 +331,7 @@ class TestCase(greentest.TestCase):
         # If we're using the ares resolver, allow the real resolver to generate an
         # error that the ares resolver actually gets an answer to.
 
-        if (RESOLVER_IS_ARES
+        if (RESOLVER_NOT_SYSTEM
             and isinstance(real_result, errors)
             and not isinstance(gevent_result, errors)):
             return
@@ -334,6 +349,8 @@ add(TestTypeError, None)
 add(TestTypeError, 25)
 
 
+@unittest.skipIf(RESOLVER_DNSPYTHON,
+                 "This commonly needs /etc/hosts to function, and dnspython doesn't do that.")
 class TestHostname(TestCase):
     pass
 
@@ -348,13 +365,13 @@ class TestLocalhost(TestCase):
     # anymore.
 
     def _normalize_result_getaddrinfo(self, result):
-        if RESOLVER_IS_ARES:
+        if RESOLVER_NOT_SYSTEM:
             # We see that some impls (OS X) return extra results
             # like DGRAM that ares does not.
             return ()
         return super(TestLocalhost, self)._normalize_result_getaddrinfo(result)
 
-    if greentest.RUNNING_ON_TRAVIS and greentest.PY2 and RESOLVER_IS_ARES:
+    if greentest.RUNNING_ON_TRAVIS and greentest.PY2 and RESOLVER_NOT_SYSTEM:
         def _normalize_result_gethostbyaddr(self, result):
             # Beginning in November 2017 after an upgrade to Travis,
             # we started seeing ares return ::1 for localhost, but
@@ -398,8 +415,8 @@ if not greentest.RUNNING_ON_TRAVIS:
 class TestBroadcast(TestCase):
     switch_expected = False
 
-    if RESOLVER_IS_ARES:
-        # ares raises errors for broadcasthost/255.255.255.255
+    if RESOLVER_NOT_SYSTEM:
+        # ares and dnspython raises errors for broadcasthost/255.255.255.255
 
         @unittest.skip('ares raises errors for broadcasthost/255.255.255.255')
         def test__broadcast__gethostbyaddr(self):
@@ -410,6 +427,7 @@ class TestBroadcast(TestCase):
 add(TestBroadcast, '<broadcast>')
 
 
+@unittest.skipIf(RESOLVER_DNSPYTHON, "/etc/hosts completely ignored under dnspython")
 class TestEtcHosts(TestCase):
     pass
 
@@ -420,7 +438,7 @@ except IOError:
     etc_hosts = ''
 
 for ip, host in re.findall(r'^\s*(\d+\.\d+\.\d+\.\d+)\s+([^\s]+)', etc_hosts, re.M)[:10]:
-    if (RESOLVER_IS_ARES
+    if (RESOLVER_NOT_SYSTEM
         and (host.endswith('local') # ignore bonjour, ares can't find them
              # ignore common aliases that ares can't find
              or ip == '255.255.255.255'
@@ -521,6 +539,8 @@ class Test_getaddrinfo(TestCase):
     def test2(self):
         return self._test_getaddrinfo(TestGeventOrg.HOSTNAME, 53, socket.AF_INET, socket.SOCK_DGRAM, 17)
 
+    @unittest.skipIf(RESOLVER_DNSPYTHON,
+                     "dnspython only returns some of the possibilities")
     def test3(self):
         return self._test_getaddrinfo('google.com', 'http', socket.AF_INET6)
 
@@ -528,11 +548,15 @@ class Test_getaddrinfo(TestCase):
 class TestInternational(TestCase):
     pass
 
-add(TestInternational, u'президент.рф', 'russian')
+if not (PY2 and RESOLVER_DNSPYTHON):
+    # dns python can actually resolve these: it uses
+    # the 2008 version of idna encoding, whereas on Python 2,
+    # with the default resolver, it tries to encode to ascii and
+    # raises a UnicodeEncodeError. So we get different results.
+    add(TestInternational, u'президент.рф', 'russian')
 add(TestInternational, u'президент.рф'.encode('idna'), 'idna')
 
 
-
 class TestInterrupted_gethostbyname(greentest.GenericWaitTestCase):
 
     # There are refs to a Waiter in the C code that don't go
@@ -573,7 +597,6 @@ class TestInterrupted_gethostbyname(greentest.GenericWaitTestCase):
         try:
             gevent.get_hub().threadpool.join()
         except Exception: # pylint:disable=broad-except
-            import traceback
             traceback.print_exc()
 
 
@@ -604,7 +627,6 @@ add(TestBadIP, '1.2.3.400')
 class Test_getnameinfo_127001(TestCase):
 
     def test(self):
-        assert gevent_socket.getnameinfo is not socket.getnameinfo
         self._test('getnameinfo', ('127.0.0.1', 80), 0)
 
     def test_DGRAM(self):
@@ -658,6 +680,10 @@ class TestInvalidPort(TestCase):
     def test3(self):
         self._test('getnameinfo', ('www.gevent.org', 'x'), 0)
 
+    @unittest.skipIf(RESOLVER_DNSPYTHON,
+                     "System resolvers do funny things with this: macOS raises gaierror, "
+                     "Travis CI returns (readthedocs.org, '0'). It's hard to match that exactly. "
+                     "dnspython raises OverflowError.")
     def test4(self):
         self._test('getnameinfo', ('www.gevent.org', 65536), 0)
 

src/greentest/test__socket_dns6.py

@@ -2,9 +2,12 @@
 # -*- coding: utf-8 -*-
 import greentest
 import socket
-from test__socket_dns import TestCase, add, RESOLVER_IS_ARES
+from test__socket_dns import TestCase, add
 
-if not greentest.RUNNING_ON_CI:
+from greentest.sysinfo import RESOLVER_NOT_SYSTEM
+from greentest.sysinfo import RESOLVER_DNSPYTHON
+
+if not greentest.RUNNING_ON_CI and not RESOLVER_DNSPYTHON:
 
 
     # We can't control the DNS servers we use there
@@ -46,7 +49,7 @@ if not greentest.RUNNING_ON_CI:
         host = 'ipv6.google.com'
 
         def _normalize_result_getnameinfo(self, result):
-            if greentest.RUNNING_ON_CI and RESOLVER_IS_ARES:
+            if greentest.RUNNING_ON_CI and RESOLVER_NOT_SYSTEM:
                 # Disabled, there are multiple possibilities
                 # and we can get different ones, rarely.
                 return ()

src/greentest/tests_that_dont_monkeypatch.txt

+test___example_servers.py
+test__backdoor.py
+test__example_echoserver.py
+test__example_udp_client.py
+test__getaddrinfo_import.py
+test__example_portforwarder.py
+test__pywsgi.py
+test__server.py
+test__server_pywsgi.py
+test__socket_close.py
+test__socket_dns6.py
+test__socket_errors.py
+test__socket_send_memoryview.py
+test__socket_timeout.py
+test__examples.py