Merge pull request #1594 from gevent/issue1012

Use `ares_getaddrinfo`

_setupares.py

@@ -95,7 +95,6 @@ ARES = Extension(
     libraries=list(LIBRARIES),
     define_macros=list(DEFINE_MACROS),
     depends=glob_many(
-        'src/gevent/resolver/dnshelper.c',
         'src/gevent/resolver/cares_*.[ch]')
 )
 

docs/changes/1012.bugfix

+Use ``ares_getaddrinfo`` instead of a manual lookup.
+
+This requires c-ares 1.16.0.
+
+Note that this may change the results, in particular their order.
+
+As part of this, certain parts of the c-ares extension were adapted to
+use modern Cython idioms.
+
+A few minor errors and discrepancies were fixed as well, such as
+``gethostbyaddr(''localhost')`` working on Python 3 and failing on
+Python 2. The DNSpython resolver now raises the expected TypeError in
+more cases instead of an AttributeError.

src/gevent/libev/_corecffi_source.c

@@ -76,7 +76,9 @@ static void gevent_zero_prepare(struct ev_prepare* handle)
 
 static void gevent_set_ev_alloc()
 {
-    ev_set_allocator(gevent_realloc);
+    void* (*ptr)(void*, long);
+    ptr = (void*(*)(void*, long))&gevent_realloc;
+    ev_set_allocator(ptr);
 }
 
 #ifdef __clang__

src/gevent/resolver/_addresses.py

@@ -142,9 +142,9 @@ def _ipv6_inet_aton(text,
 
 
 def _is_addr(host, parse=_ipv4_inet_aton):
-    if not host:
+    if not host or not isinstance(host, hostname_types):
         return False
-    assert isinstance(host, hostname_types), repr(host)
+
     try:
         parse(host)
     except AddressSyntaxError:
@@ -158,7 +158,7 @@ is_ipv4_addr = _is_addr
 
 def is_ipv6_addr(host):
     # Return True if host is a valid IPv6 address
-    if host:
+    if host and isinstance(host, hostname_types):
         s = '%' if isinstance(host, str) else b'%'
         host = host.split(s, 1)[0]
     return _is_addr(host, _ipv6_inet_aton)

src/gevent/resolver/ares.py

@@ -4,16 +4,14 @@ c-ares based hostname resolver.
 """
 from __future__ import absolute_import, print_function, division
 import os
-import sys
 
-from _socket import getaddrinfo
+from _socket import getaddrinfo as native_getaddrinfo
 from _socket import gaierror
 from _socket import error
 
 from gevent._compat import string_types
 from gevent._compat import text_type
-
-from gevent._compat import reraise
+from gevent._compat import integer_types
 from gevent._compat import PY3
 
 from gevent.hub import Waiter
@@ -22,9 +20,10 @@ from gevent.hub import get_hub
 from gevent.socket import AF_UNSPEC
 from gevent.socket import AF_INET
 from gevent.socket import AF_INET6
-from gevent.socket import SOCK_STREAM
 from gevent.socket import SOCK_DGRAM
-from gevent.socket import SOCK_RAW
+from gevent.socket import SOCK_STREAM
+from gevent.socket import SOL_TCP
+from gevent.socket import SOL_UDP
 from gevent.socket import AI_NUMERICHOST
 
 from gevent._config import config
@@ -83,6 +82,17 @@ class Resolver(AbstractResolver):
       results, and c-ares may report more ips on a multi-homed
       host.
 
+    - The system implementation may return some names fully qualified, where
+      this implementation returns only the host name. This appears to be
+      the case only with entries found in ``/etc/hosts``.
+
+    - c-ares supports a limited set of flags for ``getnameinfo`` and
+      ``getaddrinfo``; unknown flags are ignored. System-specific flags
+      such as ``AI_V4MAPPED_CFG`` are not supported.
+
+    - ``getaddrinfo`` may return canonical names even without the ``AI_CANONNAME``
+      being set.
+
     .. caution::
 
         This module is considered extremely experimental on PyPy, and
@@ -95,10 +105,14 @@ class Resolver(AbstractResolver):
        resolved <https://github.com/c-ares/c-ares/issues/196>`_ or even
        sent to the DNS server.
 
+    .. versionchanged:: NEXT
+       ``getaddrinfo`` is now implemented using the native c-ares function
+       from c-ares 1.16 or newer.
+
     .. _c-ares: http://c-ares.haxx.se
     """
 
-    ares_class = channel
+    cares_class = channel
 
     def __init__(self, hub=None, use_environ=True, **kwargs):
         if hub is None:
@@ -110,27 +124,27 @@ class Resolver(AbstractResolver):
                     value = setting.get()
                     if value is not None:
                         kwargs.setdefault(setting.kwarg_name, value)
-        self.ares = self.ares_class(hub.loop, **kwargs)
+        self.cares = self.cares_class(hub.loop, **kwargs)
         self.pid = os.getpid()
         self.params = kwargs
         self.fork_watcher = hub.loop.fork(ref=False)
         self.fork_watcher.start(self._on_fork)
 
     def __repr__(self):
-        return '<gevent.resolver_ares.Resolver at 0x%x ares=%r>' % (id(self), self.ares)
+        return '<gevent.resolver_ares.Resolver at 0x%x ares=%r>' % (id(self), self.cares)
 
     def _on_fork(self):
         # NOTE: See comment in gevent.hub.reinit.
         pid = os.getpid()
         if pid != self.pid:
-            self.hub.loop.run_callback(self.ares.destroy)
-            self.ares = self.ares_class(self.hub.loop, **self.params)
+            self.hub.loop.run_callback(self.cares.destroy)
+            self.cares = self.cares_class(self.hub.loop, **self.params)
             self.pid = pid
 
     def close(self):
-        if self.ares is not None:
-            self.hub.loop.run_callback(self.ares.destroy)
-            self.ares = None
+        if self.cares is not None:
+            self.hub.loop.run_callback(self.cares.destroy)
+            self.cares = None
         self.fork_watcher.stop()
 
     def gethostbyname(self, hostname, family=AF_INET):
@@ -150,7 +164,7 @@ class Resolver(AbstractResolver):
                 raise TypeError('Expected string, not %s' % type(hostname).__name__)
 
         while True:
-            ares = self.ares
+            ares = self.cares
             try:
                 waiter = Waiter(self.hub)
                 ares.gethostbyname(waiter, hostname, family)
@@ -159,95 +173,114 @@ class Resolver(AbstractResolver):
                     raise gaierror(-5, 'No address associated with hostname')
                 return result
             except gaierror:
-                if ares is self.ares:
+                if ares is self.cares:
                     if hostname == b'255.255.255.255':
                         # The stdlib handles this case in 2.7 and 3.x, but ares does not.
                         # It is tested by test_socket.py in 3.4.
                         # HACK: So hardcode the expected return.
                         return ('255.255.255.255', [], ['255.255.255.255'])
                     raise
-                # "self.ares is not ares" means channel was destroyed (because we were forked)
+                # "self.cares is not ares" means channel was destroyed (because we were forked)
 
     def _lookup_port(self, port, socktype):
         return lookup_port(port, socktype)
 
-    def _getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
+    def _getaddrinfo(
+            self, host, port,
+            family=0, socktype=0, proto=0, flags=0,
+            fill_in_type_proto=True
+    ):
+        """
+        Returns a list ``(family, socktype, proto, canonname, sockaddr)``
+
+        :raises gaierror: If no results are found.
+        """
         # pylint:disable=too-many-locals,too-many-branches
         if isinstance(host, text_type):
             host = host.encode('idna')
-        elif not isinstance(host, str) or (flags & AI_NUMERICHOST):
+        if not isinstance(host, bytes) or (flags & AI_NUMERICHOST) or host in (
+                b'localhost', b'ip6-localhost'):
+            # XXX: Now that we're using ares_getaddrinfo, how much of this is still
+            # necessary?
             # this handles cases which do not require network access
             # 1) host is None
             # 2) host is of an invalid type
             # 3) AI_NUMERICHOST flag is set
-            return getaddrinfo(host, port, family, socktype, proto, flags)
-            # we also call _socket.getaddrinfo below if family is not one of AF_*
-
-        port, socktypes = self._lookup_port(port, socktype)
-
-        socktype_proto = [(SOCK_STREAM, 6), (SOCK_DGRAM, 17), (SOCK_RAW, 0)]
-        if socktypes:
-            socktype_proto = [(x, y) for (x, y) in socktype_proto if x in socktypes]
-        if proto:
-            socktype_proto = [(x, y) for (x, y) in socktype_proto if proto == y]
-
-        ares = self.ares
-
-        if family == AF_UNSPEC:
-            ares_values = Values(self.hub, 2)
-            ares.gethostbyname(ares_values, host, AF_INET)
-            ares.gethostbyname(ares_values, host, AF_INET6)
-        elif family == AF_INET:
-            ares_values = Values(self.hub, 1)
-            ares.gethostbyname(ares_values, host, AF_INET)
-        elif family == AF_INET6:
-            ares_values = Values(self.hub, 1)
-            ares.gethostbyname(ares_values, host, AF_INET6)
+            # 4) It's a well-known alias. TODO: This is special casing that we don't
+            #    really want to do. It's here because it resolves a discrepancy with the system
+            #    resolvers caught by test cases. In gevent 20.4.0, this only worked correctly on
+            #    Python 3 and not Python 2, by accident.
+            return native_getaddrinfo(host, port, family, socktype, proto, flags)
+
+
+        if isinstance(port, text_type):
+            port = port.encode('ascii')
+        elif isinstance(port, integer_types):
+            if port == 0:
+                port = None
             else:
-            raise gaierror(5, 'ai_family not supported: %r' % (family, ))
-
-        values = ares_values.get()
-        if len(values) == 2 and values[0] == values[1]:
-            values.pop()
-
-        result = []
-        result4 = []
-        result6 = []
-
-        for addrs in values:
-            if addrs.family == AF_INET:
-                for addr in addrs[-1]:
-                    sockaddr = (addr, port)
-                    for socktype4, proto4 in socktype_proto:
-                        result4.append((AF_INET, socktype4, proto4, '', sockaddr))
-            elif addrs.family == AF_INET6:
-                for addr in addrs[-1]:
-                    if addr == '::1':
-                        dest = result
-                    else:
-                        dest = result6
-                    sockaddr = (addr, port, 0, 0)
-                    for socktype6, proto6 in socktype_proto:
-                        dest.append((AF_INET6, socktype6, proto6, '', sockaddr))
+                port = str(port).encode('ascii')
 
-        # As of 2016, some platforms return IPV6 first and some do IPV4 first,
-        # and some might even allow configuration of which is which. For backwards
-        # compatibility with earlier releases (but not necessarily resolver_thread!)
-        # we return 4 first. See https://github.com/gevent/gevent/issues/815 for more.
-        result += result4 + result6
+        waiter = Waiter(self.hub)
+        self.cares.getaddrinfo(
+            waiter,
+            host,
+            port,
+            family,
+            socktype,
+            proto,
+            flags,
+        )
+        # Result is a list of:
+        # (family, socktype, proto, canonname, sockaddr)
+        # Where sockaddr depends on family; for INET it is
+        # (address, port)
+        # and INET6 is
+        # (address, port, flow info, scope id)
+        result = waiter.get()
 
         if not result:
             raise gaierror(-5, 'No address associated with hostname')
 
+        if fill_in_type_proto:
+            # c-ares 1.16 DOES NOT fill in socktype or proto in the results,
+            # ever. It's at least supposed to do that if they were given as
+            # hints, but it doesn't (https://github.com/c-ares/c-ares/issues/317)
+            # Sigh.
+            if socktype:
+                hard_type_proto = [
+                    (socktype, SOL_TCP if socktype == SOCK_STREAM else SOL_UDP),
+                ]
+            elif proto:
+                hard_type_proto = [
+                    (SOCK_STREAM if proto == SOL_TCP else SOCK_DGRAM, proto),
+                ]
+            else:
+                hard_type_proto = [
+                    (SOCK_STREAM, SOL_TCP),
+                    (SOCK_DGRAM, SOL_UDP),
+                ]
+
+            result = [
+                (rfamily,
+                 hard_type if not rtype else rtype,
+                 hard_proto if not rproto else rproto,
+                 rcanon,
+                 raddr)
+                for rfamily, rtype, rproto, rcanon, raddr
+                in result
+                for hard_type, hard_proto
+                in hard_type_proto
+            ]
         return result
 
     def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
         while True:
-            ares = self.ares
+            ares = self.cares
             try:
                 return self._getaddrinfo(host, port, family, socktype, proto, flags)
             except gaierror:
-                if ares is self.ares:
+                if ares is self.cares:
                     raise
 
     def _gethostbyaddr(self, ip_address):
@@ -264,7 +297,7 @@ class Resolver(AbstractResolver):
 
         waiter = Waiter(self.hub)
         try:
-            self.ares.gethostbyaddr(waiter, ip_address)
+            self.cares.gethostbyaddr(waiter, ip_address)
             return waiter.get()
         except InvalidIP:
             result = self._getaddrinfo(ip_address, None, family=AF_UNSPEC, socktype=SOCK_DGRAM)
@@ -276,21 +309,21 @@ class Resolver(AbstractResolver):
             if _ip_address == ip_address:
                 raise
             waiter.clear()
-            self.ares.gethostbyaddr(waiter, _ip_address)
+            self.cares.gethostbyaddr(waiter, _ip_address)
             return waiter.get()
 
     def gethostbyaddr(self, ip_address):
         ip_address = _resolve_special(ip_address, AF_UNSPEC)
         while True:
-            ares = self.ares
+            ares = self.cares
             try:
                 return self._gethostbyaddr(ip_address)
             except gaierror:
-                if ares is self.ares:
+                if ares is self.cares:
                     raise
 
     def _getnameinfo(self, sockaddr, flags):
-        if not isinstance(flags, int):
+        if not isinstance(flags, integer_types):
             raise TypeError('an integer is required')
         if not isinstance(sockaddr, tuple):
             raise TypeError('getnameinfo() argument 1 must be a tuple')
@@ -303,15 +336,20 @@ class Resolver(AbstractResolver):
             raise TypeError('sockaddr[0] must be a string, not %s' % type(address).__name__)
 
         port = sockaddr[1]
-        if not isinstance(port, int):
+        if not isinstance(port, integer_types):
             raise TypeError('port must be an integer, not %s' % type(port))
 
-        waiter = Waiter(self.hub)
-        result = self._getaddrinfo(address, str(sockaddr[1]), family=AF_UNSPEC, socktype=SOCK_DGRAM)
-        if not result:
-            reraise(*sys.exc_info())
-        elif len(result) != 1:
+        if len(sockaddr) > 2:
+            # Must be IPv6: (host, port, [flowinfo, [scopeid]])
+            flowinfo = sockaddr[2]
+            if flowinfo > 0xfffff:
+                raise OverflowError("getnameinfo(): flowinfo must be 0-1048575.")
+
+        result = self._getaddrinfo(address, port,
+                                   family=AF_UNSPEC, socktype=SOCK_DGRAM, fill_in_type_proto=False)
+        if len(result) != 1:
             raise error('sockaddr resolved to multiple addresses')
+
         family, _socktype, _proto, _name, address = result[0]
 
         if family == AF_INET:
@@ -320,11 +358,11 @@ class Resolver(AbstractResolver):
         elif family == AF_INET6:
             address = address[:2] + sockaddr[2:]
 
-        self.ares.getnameinfo(waiter, address, flags)
+        waiter = Waiter(self.hub)
+        self.cares.getnameinfo(waiter, address, flags)
         node, service = waiter.get()
 
-        if service is None:
-            if PY3:
+        if service is None and PY3:
             # ares docs: "If the query did not complete
             # successfully, or one of the values was not
             # requested, node or service will be NULL ". Python 2
@@ -333,43 +371,14 @@ class Resolver(AbstractResolver):
             err = gaierror('nodename nor servname provided, or not known')
             err.errno = 8
             raise err
-            service = '0'
-        return node, service
+
+        return node, service or '0'
 
     def getnameinfo(self, sockaddr, flags):
         while True:
-            ares = self.ares
+            ares = self.cares
             try:
                 return self._getnameinfo(sockaddr, flags)
             except gaierror:
-                if ares is self.ares:
+                if ares is self.cares:
                     raise
-
-
-class Values(object):
-    # helper to collect multiple values; ignore errors unless nothing has succeeded
-    # QQQ could probably be moved somewhere - hub.py?
-
-    __slots__ = ['count', 'values', 'error', 'waiter']
-
-    def __init__(self, hub, count):
-        self.count = count
-        self.values = []
-        self.error = None
-        self.waiter = Waiter(hub)
-
-    def __call__(self, source):
-        self.count -= 1
-        if source.exception is None:
-            self.values.append(source.value)
-        else:
-            self.error = source.exception
-        if self.count <= 0:
-            self.waiter.switch(None)
-
-    def get(self):
-        self.waiter.get()
-        if self.values:
-            return self.values
-        assert error is not None
-        raise self.error # pylint:disable=raising-bad-type

src/gevent/resolver/cares.pyx

@@ -3,43 +3,61 @@
 # seems to be buggy (at least for the `result` class) and produces code that
 # can't compile ("local variable 'result' referenced before assignment").
 # See https://github.com/cython/cython/issues/1786
-# cython: auto_pickle=False
+# cython: auto_pickle=False,language_level=3str
 cimport libcares as cares
 import sys
 
+from cpython.version cimport PY_MAJOR_VERSION
+from cpython.tuple cimport PyTuple_Check
+from cpython.getargs cimport PyArg_ParseTuple
 from cpython.ref cimport Py_INCREF
 from cpython.ref cimport Py_DECREF
 from cpython.mem cimport PyMem_Malloc
 from cpython.mem cimport PyMem_Free
+from libc.string cimport memset
 
+import _socket
 from _socket import gaierror
 
 
 __all__ = ['channel']
 
-cdef object string_types
-cdef object text_type
+cdef tuple string_types
+cdef type text_type
 
-if sys.version_info[0] >= 3:
+if PY_MAJOR_VERSION >= 3:
     string_types = str,
     text_type = str
 else:
     string_types = __builtins__.basestring,
     text_type = __builtins__.unicode
 
-TIMEOUT = 1
+DEF TIMEOUT = 1
 
 DEF EV_READ = 1
 DEF EV_WRITE = 2
 
+cdef extern from *:
+    """
+#ifdef CARES_EMBED
+#include "ares_setup.h"
+#endif
 
-cdef extern from "dnshelper.c":
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+    """
+
+cdef extern from "ares.h":
     int AF_INET
     int AF_INET6
+    int INET6_ADDRSTRLEN
 
     struct hostent:
         char* h_name
         int h_addrtype
+        char** h_aliases
+        char** h_addr_list
 
     struct sockaddr_t "sockaddr":
         pass
@@ -47,112 +65,68 @@ cdef extern from "dnshelper.c":
     struct ares_channeldata:
         pass
 
-    object parse_h_name(hostent*)
-    object parse_h_aliases(hostent*)
-    object parse_h_addr_list(hostent*)
-    void* create_object_from_hostent(void*)
+    struct in_addr:
+        unsigned int s_addr
+
+    struct sockaddr_in:
+        int sin_family
+        int sin_port
+        in_addr sin_addr
+
+    struct in6_addr:
+        char s6_addr[16]
 
-    # this imports _socket lazily
-    object PyUnicode_FromString(char*)
-    int PyTuple_Check(object)
-    int PyArg_ParseTuple(object, char*, ...) except 0
     struct sockaddr_in6:
-        pass
-    int gevent_make_sockaddr(char* hostp, int port, int flowinfo, int scope_id, sockaddr_in6* sa6)
-
-
-    void memset(void*, int, int)
-
-
-ARES_SUCCESS = cares.ARES_SUCCESS
-ARES_ENODATA = cares.ARES_ENODATA
-ARES_EFORMERR = cares.ARES_EFORMERR
-ARES_ESERVFAIL = cares.ARES_ESERVFAIL
-ARES_ENOTFOUND = cares.ARES_ENOTFOUND
-ARES_ENOTIMP = cares.ARES_ENOTIMP
-ARES_EREFUSED = cares.ARES_EREFUSED
-ARES_EBADQUERY = cares.ARES_EBADQUERY
-ARES_EBADNAME = cares.ARES_EBADNAME
-ARES_EBADFAMILY = cares.ARES_EBADFAMILY
-ARES_EBADRESP = cares.ARES_EBADRESP
-ARES_ECONNREFUSED = cares.ARES_ECONNREFUSED
-ARES_ETIMEOUT = cares.ARES_ETIMEOUT
-ARES_EOF = cares.ARES_EOF
-ARES_EFILE = cares.ARES_EFILE
-ARES_ENOMEM = cares.ARES_ENOMEM
-ARES_EDESTRUCTION = cares.ARES_EDESTRUCTION
-ARES_EBADSTR = cares.ARES_EBADSTR
-ARES_EBADFLAGS = cares.ARES_EBADFLAGS
-ARES_ENONAME = cares.ARES_ENONAME
-ARES_EBADHINTS = cares.ARES_EBADHINTS
-ARES_ENOTINITIALIZED = cares.ARES_ENOTINITIALIZED
-ARES_ELOADIPHLPAPI = cares.ARES_ELOADIPHLPAPI
-ARES_EADDRGETNETWORKPARAMS = cares.ARES_EADDRGETNETWORKPARAMS
-ARES_ECANCELLED = cares.ARES_ECANCELLED
-
-ARES_FLAG_USEVC = cares.ARES_FLAG_USEVC
-ARES_FLAG_PRIMARY = cares.ARES_FLAG_PRIMARY
-ARES_FLAG_IGNTC = cares.ARES_FLAG_IGNTC
-ARES_FLAG_NORECURSE = cares.ARES_FLAG_NORECURSE
-ARES_FLAG_STAYOPEN = cares.ARES_FLAG_STAYOPEN
-ARES_FLAG_NOSEARCH = cares.ARES_FLAG_NOSEARCH
-ARES_FLAG_NOALIASES = cares.ARES_FLAG_NOALIASES
-ARES_FLAG_NOCHECKRESP = cares.ARES_FLAG_NOCHECKRESP
+        int sin6_family
+        int sin6_port
+        unsigned int sin6_flowinfo
+        in6_addr sin6_addr
+        unsigned int sin6_scope_id
+
+
+    unsigned int htons(unsigned int hostshort)
+    unsigned int ntohs(unsigned int hostshort)
+    unsigned int htonl(unsigned int hostlong)
+    unsigned int ntohl(unsigned int hostlong)
+
+cdef int AI_NUMERICSERV = _socket.AI_NUMERICSERV
+cdef int AI_CANONNAME = _socket.AI_CANONNAME
+cdef int NI_NUMERICHOST = _socket.NI_NUMERICHOST
+cdef int NI_NUMERICSERV = _socket.NI_NUMERICSERV
+cdef int NI_NOFQDN = _socket.NI_NOFQDN
+cdef int NI_NAMEREQD = _socket.NI_NAMEREQD
+cdef int NI_DGRAM = _socket.NI_DGRAM
 
 
 _ares_errors = dict([
     (cares.ARES_SUCCESS, 'ARES_SUCCESS'),
-                (cares.ARES_ENODATA, 'ARES_ENODATA'),
-                (cares.ARES_EFORMERR, 'ARES_EFORMERR'),
-                (cares.ARES_ESERVFAIL, 'ARES_ESERVFAIL'),
-                (cares.ARES_ENOTFOUND, 'ARES_ENOTFOUND'),
-                (cares.ARES_ENOTIMP, 'ARES_ENOTIMP'),
-                (cares.ARES_EREFUSED, 'ARES_EREFUSED'),
-                (cares.ARES_EBADQUERY, 'ARES_EBADQUERY'),
-                (cares.ARES_EBADNAME, 'ARES_EBADNAME'),
+
+    (cares.ARES_EADDRGETNETWORKPARAMS, 'ARES_EADDRGETNETWORKPARAMS'),
     (cares.ARES_EBADFAMILY, 'ARES_EBADFAMILY'),
+    (cares.ARES_EBADFLAGS, 'ARES_EBADFLAGS'),
+    (cares.ARES_EBADHINTS, 'ARES_EBADHINTS'),
+    (cares.ARES_EBADNAME, 'ARES_EBADNAME'),
+    (cares.ARES_EBADQUERY, 'ARES_EBADQUERY'),
     (cares.ARES_EBADRESP, 'ARES_EBADRESP'),
+    (cares.ARES_EBADSTR, 'ARES_EBADSTR'),
+    (cares.ARES_ECANCELLED, 'ARES_ECANCELLED'),
     (cares.ARES_ECONNREFUSED, 'ARES_ECONNREFUSED'),
-                (cares.ARES_ETIMEOUT, 'ARES_ETIMEOUT'),
-                (cares.ARES_EOF, 'ARES_EOF'),
+    (cares.ARES_EDESTRUCTION, 'ARES_EDESTRUCTION'),
     (cares.ARES_EFILE, 'ARES_EFILE'),
+    (cares.ARES_EFORMERR, 'ARES_EFORMERR'),
+    (cares.ARES_ELOADIPHLPAPI, 'ARES_ELOADIPHLPAPI'),
+    (cares.ARES_ENODATA, 'ARES_ENODATA'),
     (cares.ARES_ENOMEM, 'ARES_ENOMEM'),
-                (cares.ARES_EDESTRUCTION, 'ARES_EDESTRUCTION'),
-                (cares.ARES_EBADSTR, 'ARES_EBADSTR'),
-                (cares.ARES_EBADFLAGS, 'ARES_EBADFLAGS'),
     (cares.ARES_ENONAME, 'ARES_ENONAME'),
-                (cares.ARES_EBADHINTS, 'ARES_EBADHINTS'),
+    (cares.ARES_ENOTFOUND, 'ARES_ENOTFOUND'),
+    (cares.ARES_ENOTIMP, 'ARES_ENOTIMP'),
     (cares.ARES_ENOTINITIALIZED, 'ARES_ENOTINITIALIZED'),
-                (cares.ARES_ELOADIPHLPAPI, 'ARES_ELOADIPHLPAPI'),
-                (cares.ARES_EADDRGETNETWORKPARAMS, 'ARES_EADDRGETNETWORKPARAMS'),
-                (cares.ARES_ECANCELLED, 'ARES_ECANCELLED')])
-
-
-# maps c-ares flag to _socket module flag
-_cares_flag_map = None
-
-
-cdef _prepare_cares_flag_map():
-    global _cares_flag_map
-    import _socket
-    _cares_flag_map = [
-        (getattr(_socket, 'NI_NUMERICHOST', 1), cares.ARES_NI_NUMERICHOST),
-        (getattr(_socket, 'NI_NUMERICSERV', 2), cares.ARES_NI_NUMERICSERV),
-        (getattr(_socket, 'NI_NOFQDN', 4), cares.ARES_NI_NOFQDN),
-        (getattr(_socket, 'NI_NAMEREQD', 8), cares.ARES_NI_NAMEREQD),
-        (getattr(_socket, 'NI_DGRAM', 16), cares.ARES_NI_DGRAM)]
-
-
-cpdef _convert_cares_flags(int flags, int default=cares.ARES_NI_LOOKUPHOST|cares.ARES_NI_LOOKUPSERVICE):
-    if _cares_flag_map is None:
-        _prepare_cares_flag_map()
-    for socket_flag, cares_flag in _cares_flag_map:
-        if socket_flag & flags:
-            default |= cares_flag
-            flags &= ~socket_flag
-        if not flags:
-            return default
-    raise gaierror(-1, "Bad value for ai_flags: 0x%x" % flags)
+    (cares.ARES_EOF, 'ARES_EOF'),
+    (cares.ARES_EREFUSED, 'ARES_EREFUSED'),
+    (cares.ARES_ESERVICE, 'ARES_ESERVICE'),
+    (cares.ARES_ESERVFAIL, 'ARES_ESERVFAIL'),
+    (cares.ARES_ETIMEOUT, 'ARES_ETIMEOUT'),
+])
 
 
 cpdef strerror(code):
@@ -170,7 +144,7 @@ cdef void gevent_sock_state_callback(void *data, int s, int read, int write):
     ch._sock_state_callback(s, read, write)
 
 
-cdef class result:
+cdef class Result(object):
     cdef public object value
     cdef public object exception
 
@@ -207,6 +181,42 @@ class ares_host_result(tuple):
         return (self.family, tuple(self))
 
 
+cdef list _parse_h_aliases(hostent* host):
+    cdef list result = []
+    cdef char** aliases = host.h_aliases
+
+    if not aliases or not aliases[0]:
+        return result
+
+    while aliases[0]: # *aliases
+        # The old C version of this excluded an alias if
+        # it matched the host name. I don't think the stdlib does that?
+        result.append(_as_str(aliases[0]))
+        aliases += 1
+    return result
+
+
+cdef list _parse_h_addr_list(hostent* host):
+    cdef list result = []
+    cdef char** addr_list = host.h_addr_list
+    cdef int addr_type = host.h_addrtype
+    # INET6_ADDRSTRLEN is 46, but we can't use that named constant
+    # here; cython doesn't like it.
+    cdef char tmpbuf[46]
+
+    if not addr_list or not addr_list[0]:
+        return result
+
+    while addr_list[0]:
+        if not cares.ares_inet_ntop(host.h_addrtype, addr_list[0], tmpbuf, INET6_ADDRSTRLEN):
+            raise _socket.error("Failed in ares_inet_ntop")
+
+        result.append(_as_str(tmpbuf))
+        addr_list += 1
+
+    return result
+
+
 cdef void gevent_ares_host_callback(void *arg, int status, int timeouts, hostent* host):
     cdef channel channel
     cdef object callback
@@ -215,18 +225,30 @@ cdef void gevent_ares_host_callback(void *arg, int status, int timeouts, hostent
     cdef object host_result
     try:
         if status or not host:
-            callback(result(None, gaierror(status, strerror(status))))
+            callback(Result(None, gaierror(status, strerror(status))))
         else:
             try:
-                host_result = ares_host_result(host.h_addrtype, (parse_h_name(host), parse_h_aliases(host), parse_h_addr_list(host)))
+                host_result = ares_host_result(host.h_addrtype,
+                                               (_as_str(host.h_name),
+                                                _parse_h_aliases(host),
+                                                _parse_h_addr_list(host)))
             except:
-                callback(result(None, sys.exc_info()[1]))
+                callback(Result(None, sys.exc_info()[1]))
             else:
-                callback(result(host_result))
+                callback(Result(host_result))
     except:
         channel.loop.handle_error(callback, *sys.exc_info())
 
 
+cdef object _as_str(const char* val):
+    if not val:
+        return None
+
+    if PY_MAJOR_VERSION < 3:
+        return <bytes>val
+    return val.decode('utf-8')
+
+
 cdef void gevent_ares_nameinfo_callback(void *arg, int status, int timeouts, char *c_node, char *c_service):
     cdef channel channel
     cdef object callback
@@ -236,27 +258,39 @@ cdef void gevent_ares_nameinfo_callback(void *arg, int status, int timeouts, cha
     cdef object service
     try:
         if status:
-            callback(result(None, gaierror(status, strerror(status))))
-        else:
-            if c_node:
-                node = PyUnicode_FromString(c_node)
-            else:
-                node = None
-            if c_service:
-                service = PyUnicode_FromString(c_service)
+            callback(Result(None, gaierror(status, strerror(status))))
         else:
-                service = None
-            callback(result((node, service)))
+            node = _as_str(c_node)
+            service = _as_str(c_service)
+            callback(Result((node, service)))
     except:
         channel.loop.handle_error(callback, *sys.exc_info())
 
 
-cdef class channel:
 
-    cdef public object loop
+cdef int _make_sockaddr(const char* hostp, int port, int flowinfo, int scope_id, sockaddr_in6* sa6):
+    if cares.ares_inet_pton(AF_INET, hostp, &(<sockaddr_in*>sa6).sin_addr.s_addr) > 0:
+        (<sockaddr_in*>sa6).sin_family = AF_INET
+        (<sockaddr_in*>sa6).sin_port = htons(port)
+        return sizeof(sockaddr_in)
+
+    if cares.ares_inet_pton(AF_INET6, hostp, &(sa6.sin6_addr).s6_addr) > 0:
+        sa6.sin6_family = AF_INET6
+        sa6.sin6_port = htons(port)
+        sa6.sin6_flowinfo = flowinfo
+        sa6.sin6_scope_id = scope_id
+        return sizeof(sockaddr_in6);
+
+    return -1;
+
+
+cdef class channel:
     cdef ares_channeldata* channel
-    cdef public dict _watchers
-    cdef public object _timer
+
+    cdef readonly object loop
+
+    cdef dict _watchers
+    cdef object _timer
 
     def __init__(self, object loop, flags=None, timeout=None, tries=None, ndots=None,
                  udp_port=None, tcp_port=None, servers=None):
@@ -264,26 +298,34 @@ cdef class channel:
         cdef cares.ares_options options
         memset(&options, 0, sizeof(cares.ares_options))
         cdef int optmask = cares.ARES_OPT_SOCK_STATE_CB
+
         options.sock_state_cb = <void*>gevent_sock_state_callback
         options.sock_state_cb_data = <void*>self
+
         if flags is not None:
             options.flags = int(flags)
             optmask |= cares.ARES_OPT_FLAGS
+
         if timeout is not None:
             options.timeout = int(float(timeout) * 1000)
             optmask |= cares.ARES_OPT_TIMEOUTMS
+
         if tries is not None:
             options.tries = int(tries)
             optmask |= cares.ARES_OPT_TRIES
+
         if ndots is not None:
             options.ndots = int(ndots)
             optmask |= cares.ARES_OPT_NDOTS
+
         if udp_port is not None:
             options.udp_port = int(udp_port)
             optmask |= cares.ARES_OPT_UDP_PORT
+
         if tcp_port is not None:
             options.tcp_port = int(tcp_port)
             optmask |= cares.ARES_OPT_TCP_PORT
+
         cdef int result = cares.ares_library_init(cares.ARES_LIB_INIT_ALL)  # ARES_LIB_INIT_WIN32 -DUSE_WINSOCK?
         if result:
             raise gaierror(result, strerror(result))
@@ -413,7 +455,8 @@ cdef class channel:
         # note that for file lookups still AF_INET can be returned for AF_INET6 request
         cdef object arg = (self, callback)
         Py_INCREF(arg)
-        cares.ares_gethostbyname(self.channel, name, family, <void*>gevent_ares_host_callback, <void*>arg)
+        cares.ares_gethostbyname(self.channel, name, family,
+                                 <void*>gevent_ares_host_callback, <void*>arg)
 
     def gethostbyaddr(self, object callback, char* addr):
         if not self.channel:
@@ -447,7 +490,7 @@ cdef class channel:
         PyArg_ParseTuple(sockaddr, "si|ii", &hostp, &port, &flowinfo, &scope_id)
         if port < 0 or port > 65535:
             raise gaierror(-8, 'Invalid value for port: %r' % port)
-        cdef int length = gevent_make_sockaddr(hostp, port, flowinfo, scope_id, &sa6)
+        cdef int length = _make_sockaddr(hostp, port, flowinfo, scope_id, &sa6)
         if length <= 0:
             raise InvalidIP(repr(hostp))
         cdef object arg = (self, callback)
@@ -455,10 +498,146 @@ cdef class channel:
         cdef sockaddr_t* x = <sockaddr_t*>&sa6
         cares.ares_getnameinfo(self.channel, x, length, flags, <void*>gevent_ares_nameinfo_callback, <void*>arg)
 
+    @staticmethod
+    cdef int _convert_cares_ni_flags(int flags):
+        cdef int cares_flags = cares.ARES_NI_LOOKUPHOST | cares.ARES_NI_LOOKUPSERVICE
+        if flags & NI_NUMERICHOST:
+            cares_flags |= cares.ARES_NI_NUMERICHOST
+        if flags & NI_NUMERICSERV:
+            cares_flags |= cares.ARES_NI_NUMERICSERV
+        if flags & NI_NOFQDN:
+            cares_flags |= cares.ARES_NI_NOFQDN
+        if flags & NI_NAMEREQD:
+            cares_flags |= cares.ARES_NI_NAMEREQD
+        if flags & NI_DGRAM:
+            cares_flags |= cares.ARES_NI_DGRAM
+        return cares_flags
+
     def getnameinfo(self, object callback, tuple sockaddr, int flags):
-        try:
-            flags = _convert_cares_flags(flags)
-        except gaierror:
-            # The stdlib just ignores bad flags
-            flags = 0
+        flags = channel._convert_cares_ni_flags(flags)
         return self._getnameinfo(callback, sockaddr, flags)
+
+    @staticmethod
+    cdef int _convert_cares_ai_flags(int flags):
+        # c-ares supports a limited set of flags.
+        # We always want NOSORT, because that implies that
+        # c-ares will not connect to resolved addresses.
+        cdef int cares_flags = cares.ARES_AI_NOSORT
+        if flags & AI_CANONNAME:
+            cares_flags |= cares.ARES_AI_CANONNAME
+        if flags & AI_NUMERICSERV:
+            cares_flags |= cares.ARES_AI_NUMERICSERV
+        return cares_flags
+
+    @staticmethod
+    cdef void _getaddrinfo_cb(void *arg,
+                              int status,
+                              int timeouts,
+                              cares.ares_addrinfo* result):
+        cdef cares.ares_addrinfo_node* nodes
+        cdef cares.ares_addrinfo_cname* cnames
+        cdef sockaddr_in* sadr4
+        cdef sockaddr_in6* sadr6
+        cdef object canonname = ''
+
+        cdef channel channel
+        cdef object callback
+        # INET6_ADDRSTRLEN is 46, but we can't use that named constant
+        # here; cython doesn't like it.
+        cdef char tmpbuf[46]
+
+        channel, callback = <tuple>arg
+        Py_DECREF(<tuple>arg)
+
+
+        # Result is a list of:
+        # (family, socktype, proto, canonname, sockaddr)
+        # Where sockaddr depends on family; for INET it is
+        # (address, port)
+        # and INET6 is
+        # (address, port, flow info, scope id)
+        # TODO: Check the canonnames.
+        addrs = []
+        try:
+            if status != cares.ARES_SUCCESS:
+                callback(Result(None, gaierror(status, strerror(status))))
+                return
+            if result.cnames:
+                # These tend to come in pairs:
+                #
+                # alias: www.gevent.org name: python-gevent.readthedocs.org
+                # alias: python-gevent.readthedocs.org name: readthedocs.io
+                #
+                # The standard library returns the last name so we do too.
+
+                cnames = result.cnames
+                while cnames:
+                    canonname = _as_str(cnames.name)
+                    cnames = cnames.next
+
+            nodes = result.nodes
+            while nodes:
+                if nodes.ai_family == AF_INET:
+                    sadr4 = <sockaddr_in*>nodes.ai_addr
+                    cares.ares_inet_ntop(nodes.ai_family, &sadr4.sin_addr, tmpbuf,
+                                            INET6_ADDRSTRLEN)
+                    sockaddr = (
+                        _as_str(tmpbuf),
+                        ntohs(sadr4.sin_port),
+                    )
+                elif nodes.ai_family == AF_INET6:
+                    sadr6 = <sockaddr_in6*>nodes.ai_addr
+                    cares.ares_inet_ntop(nodes.ai_family, &sadr6.sin6_addr, tmpbuf,
+                                            INET6_ADDRSTRLEN)
+
+                    sockaddr = (
+                        _as_str(tmpbuf),
+                        ntohs(sadr6.sin6_port),
+                        sadr6.sin6_flowinfo,
+                        sadr6.sin6_scope_id,
+                    )
+                addrs.append((
+                    nodes.ai_family,
+                    nodes.ai_socktype,
+                    nodes.ai_protocol,
+                    canonname,
+                    sockaddr,
+                ))
+                nodes = nodes.ai_next
+
+            callback(Result(addrs, None))
+        except:
+            channel.loop.handle_error(callback, *sys.exc_info())
+        finally:
+            if result:
+                cares.ares_freeaddrinfo(result)
+
+    def getaddrinfo(self,
+                    object callback,
+                    const char* name,
+                    object service, # AKA port
+                    int family=0,
+                    int type=0,
+                    int proto=0,
+                    int flags=0):
+        if not self.channel:
+            raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed')
+
+        cdef cares.ares_addrinfo_hints hints
+        memset(&hints, 0, sizeof(cares.ares_addrinfo_hints))
+
+        hints.ai_flags = channel._convert_cares_ai_flags(flags)
+        hints.ai_family = family
+        hints.ai_socktype = type
+        hints.ai_protocol = proto
+        cdef object arg = (self, callback)
+        Py_INCREF(arg)
+
+        cares.ares_getaddrinfo(
+            self.channel,
+            name,
+            NULL if service is None else <char*>service,
+            &hints,
+            <void*>channel._getaddrinfo_cb,
+            <void*>arg
+        )

src/gevent/resolver/cares_ntop.h

-#ifdef CARES_EMBED
-#include "ares_setup.h"
-#include "ares.h"
-#else
-#include <arpa/inet.h>
-#define ares_inet_ntop(w,x,y,z) inet_ntop(w,x,y,z)
-#endif

src/gevent/resolver/cares_pton.h

-#ifdef CARES_EMBED
-#include "ares_setup.h"
-#include "ares_inet_net_pton.h"
-#else
-#include <arpa/inet.h>
-#define ares_inet_pton(x,y,z) inet_pton(x,y,z)
-#define ares_inet_net_pton(w,x,y,z) inet_net_pton(w,x,y,z)
-#endif

src/gevent/resolver/dnshelper.c

-/* Copyright (c) 2011 Denis Bilenko. See LICENSE for details. */
-#include "Python.h"
-#ifdef CARES_EMBED
-#include "ares_setup.h"
-#endif
-
-#ifdef HAVE_NETDB_H
-#include <netdb.h>
-#endif
-
-#include "ares.h"
-
-#include "cares_ntop.h"
-#include "cares_pton.h"
-
-#if PY_MAJOR_VERSION >= 3
-#define PY3K
-#define GPyNative_FromString PyUnicode_FromString
-#else
-#define GPyNative_FromString PyString_FromString
-#endif
-
-
-static PyObject* _socket_error = 0;
-
-static PyObject*
-get_socket_object(PyObject** pobject, const char* name)
-{
-    if (!*pobject) {
-        PyObject* _socket;
-        _socket = PyImport_ImportModule("_socket");
-        if (_socket) {
-            *pobject = PyObject_GetAttrString(_socket, name);
-            if (!*pobject) {
-                PyErr_WriteUnraisable(Py_None);
-            }
-            Py_DECREF(_socket);
-        }
-        else {
-            PyErr_WriteUnraisable(Py_None);
-        }
-        if (!*pobject) {
-            *pobject = PyExc_IOError;
-        }
-    }
-    return *pobject;
-}
-
-
-static int
-gevent_append_addr(PyObject* list, int family, void* src, char* tmpbuf, size_t tmpsize) {
-    int status = -1;
-    PyObject* tmp;
-    if (ares_inet_ntop(family, src, tmpbuf, tmpsize)) {
-        tmp = GPyNative_FromString(tmpbuf);
-        if (tmp) {
-            status = PyList_Append(list, tmp);
-            Py_DECREF(tmp);
-        }
-    }
-    return status;
-}
-
-
-static PyObject*
-parse_h_name(struct hostent *h)
-{
-    return GPyNative_FromString(h->h_name);
-}
-
-
-static PyObject*
-parse_h_aliases(struct hostent *h)
-{
-    char **pch;
-    PyObject *result = NULL;
-    PyObject *tmp;
-
-    result = PyList_New(0);
-
-    if (result && h->h_aliases) {
-        for (pch = h->h_aliases; *pch != NULL; pch++) {
-            if (*pch != h->h_name && strcmp(*pch, h->h_name)) {
-                int status;
-                tmp = GPyNative_FromString(*pch);
-                if (tmp == NULL) {
-                    break;
-                }
-
-                status = PyList_Append(result, tmp);
-                Py_DECREF(tmp);
-
-                if (status) {
-                    break;
-                }
-            }
-        }
-    }
-
-    return result;
-}
-
-
-static PyObject *
-parse_h_addr_list(struct hostent *h)
-{
-    char **pch;
-    PyObject *result = NULL;
-
-    result = PyList_New(0);
-
-    if (result) {
-        switch (h->h_addrtype) {
-            case AF_INET:
-            {
-                char tmpbuf[sizeof "255.255.255.255"];
-                for (pch = h->h_addr_list; *pch != NULL; pch++) {
-                    if (gevent_append_addr(result, AF_INET, *pch, tmpbuf, sizeof(tmpbuf))) {
-                        break;
-                    }
-                }
-                break;
-            }
-            case AF_INET6:
-            {
-                char tmpbuf[sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")];
-                for (pch = h->h_addr_list; *pch != NULL; pch++) {
-                    if (gevent_append_addr(result, AF_INET6, *pch, tmpbuf, sizeof(tmpbuf))) {
-                        break;
-                    }
-                }
-                break;
-            }
-            default:
-                PyErr_SetString(get_socket_object(&_socket_error, "error"), "unsupported address family");
-                Py_DECREF(result);
-                result = NULL;
-        }
-    }
-
-    return result;
-}
-
-
-static int
-gevent_make_sockaddr(char* hostp, int port, int flowinfo, int scope_id, struct sockaddr_in6* sa6) {
-    if ( ares_inet_pton(AF_INET, hostp, &((struct sockaddr_in*)sa6)->sin_addr.s_addr) > 0 ) {
-        ((struct sockaddr_in*)sa6)->sin_family = AF_INET;
-        ((struct sockaddr_in*)sa6)->sin_port = htons(port);
-        return sizeof(struct sockaddr_in);
-    }
-    else if ( ares_inet_pton(AF_INET6, hostp, &sa6->sin6_addr.s6_addr) > 0 ) {
-        sa6->sin6_family = AF_INET6;
-        sa6->sin6_port = htons(port);
-        sa6->sin6_flowinfo = flowinfo;
-        sa6->sin6_scope_id = scope_id;
-        return sizeof(struct sockaddr_in6);
-    }
-    return -1;
-}

src/gevent/resolver/dnspython.py

@@ -393,13 +393,17 @@ class Resolver(AbstractResolver):
         aliases = self._resolver.hosts_resolver.getaliases(hostname)
         net_resolver = self._resolver.network_resolver
         rdtype = _family_to_rdtype(family)
-        while True:
+        while 1:
             try:
                 ans = net_resolver.query(hostname, dns.rdatatype.CNAME, rdtype)
             except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.resolver.NoNameservers):
                 break
             except dTimeout:
                 break
+            except AttributeError as ex:
+                if hostname is None or isinstance(hostname, int):
+                    raise TypeError(ex)
+                raise
             else:
                 aliases.extend(str(rr.target) for rr in ans.rrset)
                 hostname = ans[0].target

src/gevent/resolver/libcares.pxd

 cdef extern from "ares.h":
+    # These two are defined in <sys/socket.h> and <netdb.h>, respectively,
+    # on POSIX. On Windows, they are in <winsock2.h>. "ares.h" winds up
+    # indirectly including both of those.
+    struct sockaddr:
+        pass
+    struct hostent:
+        pass
+
     struct ares_options:
         int flags
         void* sock_state_cb
@@ -36,30 +44,31 @@ cdef extern from "ares.h":
     int ARES_SOCKET_BAD
 
     int ARES_SUCCESS
-    int ARES_ENODATA
-    int ARES_EFORMERR
-    int ARES_ESERVFAIL
-    int ARES_ENOTFOUND
-    int ARES_ENOTIMP
-    int ARES_EREFUSED
-    int ARES_EBADQUERY
-    int ARES_EBADNAME
+    int ARES_EADDRGETNETWORKPARAMS
     int ARES_EBADFAMILY
+    int ARES_EBADFLAGS
+    int ARES_EBADHINTS
+    int ARES_EBADNAME
+    int ARES_EBADQUERY
     int ARES_EBADRESP
+    int ARES_EBADSTR
+    int ARES_ECANCELLED
     int ARES_ECONNREFUSED
-    int ARES_ETIMEOUT
-    int ARES_EOF
+    int ARES_EDESTRUCTION
     int ARES_EFILE
+    int ARES_EFORMERR
+    int ARES_ELOADIPHLPAPI
+    int ARES_ENODATA
     int ARES_ENOMEM
-    int ARES_EDESTRUCTION
-    int ARES_EBADSTR
-    int ARES_EBADFLAGS
     int ARES_ENONAME
-    int ARES_EBADHINTS
+    int ARES_ENOTFOUND
+    int ARES_ENOTIMP
     int ARES_ENOTINITIALIZED
-    int ARES_ELOADIPHLPAPI
-    int ARES_EADDRGETNETWORKPARAMS
-    int ARES_ECANCELLED
+    int ARES_EOF
+    int ARES_EREFUSED
+    int ARES_ESERVFAIL
+    int ARES_ESERVICE
+    int ARES_ETIMEOUT
 
     int ARES_NI_NOFQDN
     int ARES_NI_NUMERICHOST
@@ -74,6 +83,7 @@ cdef extern from "ares.h":
     int ARES_NI_LOOKUPHOST
     int ARES_NI_LOOKUPSERVICE
 
+    ctypedef int ares_socklen_t
 
     int ares_library_init(int flags)
     void ares_library_cleanup()
@@ -87,6 +97,11 @@ cdef extern from "ares.h":
     void ares_cancel(void* channel)
     void ares_getnameinfo(void* channel, void* sa, int salen, int flags, void* callback, void *arg)
 
+    # Added in 1.10
+    int ares_inet_pton(int af, const char *src, void *dst)
+    const char* ares_inet_ntop(int af, const void *src, char *dst, ares_socklen_t size);
+
+
     struct in_addr:
         pass
 
@@ -104,6 +119,45 @@ cdef extern from "ares.h":
 
     int ares_set_servers(void* channel, ares_addr_node *servers)
 
+    # Added in 1.16
+    int ARES_AI_NOSORT
+    int ARES_AI_ENVHOSTS
+    int ARES_AI_CANONNAME
+    int ARES_AI_NUMERICSERV
+
+    struct ares_addrinfo_hints:
+        int ai_flags
+        int ai_family
+        int ai_socktype
+        int ai_protocol
+
+    struct ares_addrinfo_node:
+        int ai_ttl
+        int ai_flags
+        int ai_family
+        int ai_socktype
+        int ai_protocol
+        ares_socklen_t ai_addrlen
+        sockaddr *ai_addr
+        ares_addrinfo_node *ai_next
+
+    struct ares_addrinfo_cname:
+        int ttl
+        char *alias
+        char *name
+        ares_addrinfo_cname *next
+
+    struct ares_addrinfo:
+        ares_addrinfo_cname *cnames
+        ares_addrinfo_node  *nodes
+
+    void ares_getaddrinfo(
+        void* channel,
+        const char *name,
+        const char* service,
+        const ares_addrinfo_hints *hints,
+        #ares_addrinfo_callback callback,
+        void* callback,
+        void *arg)
 
-cdef extern from "cares_pton.h":
-    int ares_inet_pton(int af, char *src, void *dst)
+    void ares_freeaddrinfo(ares_addrinfo *ai)

src/gevent/tests/test__socket_dns.py

@@ -14,8 +14,7 @@ import traceback
 
 import gevent.socket as gevent_socket
 import gevent.testing as greentest
-#from gevent.testing.util import log
-#from gevent.testing.util import debug
+
 from gevent.testing import util
 from gevent.testing import six
 from gevent.testing.six import xrange
@@ -31,6 +30,7 @@ if getattr(resolver, 'pool', None) is not None:
 
 from gevent.testing.sysinfo import RESOLVER_NOT_SYSTEM
 from gevent.testing.sysinfo import RESOLVER_DNSPYTHON
+from gevent.testing.sysinfo import RESOLVER_ARES
 from gevent.testing.sysinfo import PY2
 import gevent.testing.timing
 
@@ -38,63 +38,8 @@ import gevent.testing.timing
 assert gevent_socket.gaierror is socket.gaierror
 assert gevent_socket.error is socket.error
 
-TRACE = not util.QUIET and os.getenv('GEVENT_DEBUG', '') == 'trace'
-
-def trace(message, *args, **kwargs):
-    if TRACE:
-        util.debug(message, *args, **kwargs)
-
-def _run(function, *args):
-    try:
-        result = function(*args)
-        assert not isinstance(result, BaseException), repr(result)
-        return result
-    except Exception as ex:
-        if TRACE:
-            traceback.print_exc()
-        return ex
-
-
-def format_call(function, args):
-    args = repr(args)
-    if args.endswith(',)'):
-        args = args[:-2] + ')'
-    try:
-        module = function.__module__.replace('gevent._socketcommon', 'gevent')
-        name = function.__name__
-        return '%s:%s%s' % (module, name, args)
-    except AttributeError:
-        return function + args
-
-
-def trace_fresult(result, seconds):
-    if not TRACE:
-        return
-    if isinstance(result, Exception):
-        msg = '  -=>  raised %r' % (result, )
-    else:
-        msg = '  -=>  returned %r' % (result, )
-    time_ms = ' %.2fms' % (seconds * 1000.0, )
-    space = 80 - len(msg) - len(time_ms)
-    if space > 0:
-        space = ' ' * space
-    else:
-        space = ''
-    util.debug(msg + space + time_ms)
-
 
-def run(function, *args):
-    trace(format_call(function, args))
-    delta = time()
-    result = _run(function, *args)
-    delta = time() - delta
-    trace_fresult(result, delta)
-    return result, delta
-
-
-def trace_call(result, runtime, function, *args):
-    util.debug(format_call(function, args))
-    trace_fresult(result, runtime)
+RUN_ALL_HOST_TESTS = os.getenv('GEVENTTEST_RUN_ALL_ETC_HOST_TESTS', '')
 
 
 def compare_relaxed(a, b):
@@ -200,13 +145,13 @@ def add(klass, hostname, name=None,
     test1.__name__ = 'test_%s_getaddrinfo' % name
     _setattr(klass, test1.__name__, test1)
 
-    def test2(self):
+    def test_gethostbyname(self):
         x = hostname() if call else hostname
         ipaddr = self._test('gethostbyname', x)
         if not isinstance(ipaddr, Exception):
             self._test('gethostbyaddr', ipaddr)
-    test2.__name__ = 'test_%s_gethostbyname' % name
-    _setattr(klass, test2.__name__, test2)
+    test_gethostbyname.__name__ = 'test_%s_gethostbyname' % name
+    _setattr(klass, test_gethostbyname.__name__, test_gethostbyname)
 
     def test3(self):
         x = hostname() if call else hostname
@@ -228,11 +173,74 @@ def add(klass, hostname, name=None,
 
 @skipWithoutExternalNetwork("Tries to resolve and compare hostnames/addrinfo")
 class TestCase(greentest.TestCase):
-
+    maxDiff = None
     __timeout__ = 30
     switch_expected = None
+
+    TRACE = not util.QUIET and os.getenv('GEVENT_DEBUG', '') == 'trace'
     verbose_dns = TRACE
 
+    def trace(self, message, *args, **kwargs):
+        if self.TRACE:
+            util.debug(message, *args, **kwargs)
+
+    # Things that the stdlib should never raise and neither should we;
+    # these indicate bugs in our code and we want to raise them.
+    REAL_ERRORS = (AttributeError, ValueError, NameError)
+
+    def __run_resolver(self, function, args):
+        try:
+            result = function(*args)
+            assert not isinstance(result, BaseException), repr(result)
+            return result
+        except self.REAL_ERRORS:
+            raise
+        except Exception as ex:
+            if self.TRACE:
+                traceback.print_exc()
+            return ex
+
+    def __trace_call(self, result, runtime, function, *args):
+        util.debug(self.__format_call(function, args))
+        self.__trace_fresult(result, runtime)
+
+    def __format_call(self, function, args):
+        args = repr(args)
+        if args.endswith(',)'):
+            args = args[:-2] + ')'
+        try:
+            module = function.__module__.replace('gevent._socketcommon', 'gevent')
+            name = function.__name__
+            return '%s:%s%s' % (module, name, args)
+        except AttributeError:
+            return function + args
+
+    def __trace_fresult(self, result, seconds):
+        if isinstance(result, Exception):
+            msg = '  -=>  raised %r' % (result, )
+        else:
+            msg = '  -=>  returned %r' % (result, )
+        time_ms = ' %.2fms' % (seconds * 1000.0, )
+        space = 80 - len(msg) - len(time_ms)
+        if space > 0:
+            space = ' ' * space
+        else:
+            space = ''
+        util.debug(msg + space + time_ms)
+
+    if not TRACE:
+        def run_resolver(self, function, func_args):
+            now = time()
+            return self.__run_resolver(function, func_args), time() - now
+    else:
+        def run_resolver(self, function, func_args):
+            self.trace(self.__format_call(function, func_args))
+            delta = time()
+            result = self.__run_resolver(function, func_args)
+            delta = time() - delta
+            self.__trace_fresult(result, delta)
+            return result, delta
+
     def setUp(self):
         super(TestCase, self).setUp()
         if not self.verbose_dns:
@@ -256,19 +264,17 @@ class TestCase(greentest.TestCase):
             return type(result1) is not type(result2)
         return repr(result1) != repr(result2)
 
-    def _test(self, func, *args):
-        gevent_func = getattr(gevent_socket, func)
-        real_func = monkey.get_original('socket', func)
-        real_result, time_real = run(real_func, *args)
-        gevent_result, time_gevent = run(gevent_func, *args)
-        if util.QUIET and self.should_log_results(real_result, gevent_result):
-            util.log('')
-            trace_call(real_result, time_real, real_func, *args)
-            trace_call(gevent_result, time_gevent, gevent_func, *args)
-        self.assertEqualResults(real_result, gevent_result, func)
+    def _test(self, func_name, *args):
+        gevent_func = getattr(gevent_socket, func_name)
+        real_func = monkey.get_original('socket', func_name)
 
-        if self.verbose_dns and time_gevent > time_real + 0.01 and time_gevent > 0.02:
-            msg = 'gevent:%s%s took %dms versus %dms stdlib' % (func, args, time_gevent * 1000.0, time_real * 1000.0)
+        tester = getattr(self, '_run_test_' + func_name, self._run_test_generic)
+        result = tester(func_name, real_func, gevent_func, args)
+        _real_result, time_real, gevent_result, time_gevent = result
+
+        if self.verbose_dns and time_gevent > time_real + 0.02 and time_gevent > 0.03:
+            msg = 'gevent:%s%s took %dms versus %dms stdlib' % (
+                func_name, args, time_gevent * 1000.0, time_real * 1000.0)
 
             if time_gevent > time_real + 1:
                 word = 'VERY'
@@ -279,6 +285,16 @@ class TestCase(greentest.TestCase):
 
         return gevent_result
 
+    def _run_test_generic(self, func_name, real_func, gevent_func, func_args):
+        real_result, time_real = self.run_resolver(real_func, func_args)
+        gevent_result, time_gevent = self.run_resolver(gevent_func, func_args)
+        if util.QUIET and self.should_log_results(real_result, gevent_result):
+            util.log('')
+            self.__trace_call(real_result, time_real, real_func, func_args)
+            self.__trace_call(gevent_result, time_gevent, gevent_func, func_args)
+        self.assertEqualResults(real_result, gevent_result, func_name)
+        return real_result, time_real, gevent_result, time_gevent
+
     def _normalize_result(self, result, func_name):
         norm_name = '_normalize_result_' + func_name
         if hasattr(self, norm_name):
@@ -314,24 +330,38 @@ class TestCase(greentest.TestCase):
         # On some systems, the hostname can get caps
         return (result[0].lower(), [], ips)
 
-    def _normalize_result_getaddrinfo(self, result):
+    IGNORE_CANONICAL_NAME = RESOLVER_ARES # It tends to return them even when not asked for
     if not RESOLVER_NOT_SYSTEM:
+        def _normalize_result_getaddrinfo(self, result):
             return result
+    else:
+        def _normalize_result_getaddrinfo(self, result):
             # On Python 3, the builtin resolver can return SOCK_RAW results, but
             # c-ares doesn't do that. So we remove those if we find them.
             if hasattr(socket, 'SOCK_RAW') and isinstance(result, list):
                 result = [x for x in result if x[1] != socket.SOCK_RAW]
+            if self.IGNORE_CANONICAL_NAME:
+                result = [
+                    (family, kind, proto, '', addr)
+                    for family, kind, proto, _, addr
+                    in result
+                ]
             if isinstance(result, list):
                 result.sort()
             return result
 
+    def _normalize_result_getnameinfo(self, result):
+        return result
+
+    NORMALIZE_GHBA_IGNORE_ALIAS = False
     def _normalize_result_gethostbyaddr(self, result):
         if not RESOLVER_NOT_SYSTEM:
             return result
 
-        if isinstance(result, tuple):
+        if self.NORMALIZE_GHBA_IGNORE_ALIAS and isinstance(result, tuple):
             # On some systems, a random alias is found in the aliaslist
-            # by the system resolver, but not by cares and vice versa. We deem the aliaslist
+            # by the system resolver, but not by cares and vice versa. This is *probably* only the
+            # case for localhost or things otherwise in /etc/hosts. We deem the aliaslist
             # unimportant and discard it.
             return (result[0], [], result[2])
         return result
@@ -379,7 +409,27 @@ add(TestTypeError, 25)
 
 
 class TestHostname(TestCase):
-    pass
+    NORMALIZE_GHBA_IGNORE_ALIAS = True
+
+    def _ares_normalize_name(self, result):
+        if RESOLVER_ARES and isinstance(result, tuple):
+            # The system resolver can return the FQDN, in the first result,
+            # when given certain configurations. But c-ares
+            # does not.
+            name = result[0]
+            name = name.split('.', 1)[0]
+            result = (name,) + result[1:]
+        return result
+    def _normalize_result_gethostbyaddr(self, result):
+        result = TestCase._normalize_result_gethostbyaddr(self, result)
+        return self._ares_normalize_name(result)
+
+    def _normalize_result_getnameinfo(self, result):
+        result = TestCase._normalize_result_getnameinfo(self, result)
+        if PY2:
+            # Not sure why we only saw this on Python 2
+            result = self._ares_normalize_name(result)
+        return result
 
 add(
     TestHostname,
@@ -405,6 +455,7 @@ class TestLocalhost(TestCase):
             return ()
         return super(TestLocalhost, self)._normalize_result_getaddrinfo(result)
 
+    NORMALIZE_GHBA_IGNORE_ALIAS = True
     if greentest.RUNNING_ON_TRAVIS and greentest.PY2 and RESOLVER_NOT_SYSTEM:
         def _normalize_result_gethostbyaddr(self, result):
             # Beginning in November 2017 after an upgrade to Travis,
@@ -443,7 +494,7 @@ add(Test1234, '1.2.3.4')
 
 
 class Test127001(TestCase):
-    pass
+    NORMALIZE_GHBA_IGNORE_ALIAS = True
 
 add(
     Test127001, '127.0.0.1',
@@ -504,7 +555,7 @@ class TestEtcHosts(TestCase):
         hf = SanitizedHostsFile(os.path.join(os.path.dirname(__file__),
                                              'hosts_file.txt'))
         all_etc_hosts = sorted(hf.iter_all_host_addr_pairs())
-        if len(all_etc_hosts) > cls.MAX_HOSTS and util.QUIET:
+        if len(all_etc_hosts) > cls.MAX_HOSTS and not RUN_ALL_HOST_TESTS:
             all_etc_hosts = all_etc_hosts[:cls.MAX_HOSTS]
 
         for host, ip in all_etc_hosts:
@@ -518,14 +569,46 @@ TestEtcHosts.populate_tests()
 
 
 class TestGeventOrg(TestCase):
-
+    # For this test to work correctly, it needs to resolve to
+    # an address with a single A record; round-robin DNS and multiple A records
+    # may mess it up (subsequent requests---and we always make two---may return
+    # unequal results). We used to use gevent.org, but that now has multiple A records;
+    # trying www.gevent.org which is a CNAME to readthedocs.org then worked, but it became
+    # an alias for python-gevent.readthedocs.org, which is an alias for readthedocs.io,
+    # and which also has multiple addresses. So we run the resolver twice to try to get
+    # the different answers, if needed.
     HOSTNAME = 'www.gevent.org'
 
-# For this test to work correctly, it needs to resolve to
-# an address with a single A record; round-robin DNS and multiple A records
-# may mess it up (subsequent requests---and we always make two---may return
-# unequal results). We used to use gevent.org, but that now has multiple A records;
-# trying www.gevent.org which is a CNAME to readthedocs.org.
+
+    if RESOLVER_NOT_SYSTEM:
+        def _normalize_result_gethostbyname(self, result):
+            if result == '104.17.33.82':
+                result = '104.17.32.82'
+            return result
+
+        def _normalize_result_gethostbyname_ex(self, result):
+            result = super(TestGeventOrg, self)._normalize_result_gethostbyname_ex(result)
+            if result[0] == 'python-gevent.readthedocs.org':
+                result = ('readthedocs.io', ) + result[1:]
+            return result
+
+    def test_AI_CANONNAME(self):
+        self.IGNORE_CANONICAL_NAME = False
+        result = self._test('getaddrinfo',
+                            # host
+                            TestGeventOrg.HOSTNAME,
+                            # port
+                            None,
+                            # family
+                            socket.AF_INET,
+                            # type
+                            0,
+                            # proto
+                            0,
+                            # flags
+                            socket.AI_CANONNAME)
+        self.assertEqual(result[0][3], 'readthedocs.io')
+
 add(TestGeventOrg, TestGeventOrg.HOSTNAME)
 
 
@@ -537,21 +620,11 @@ class TestFamily(TestCase):
             cls._result = socket.getaddrinfo(TestGeventOrg.HOSTNAME, None)
         return cls._result
 
-    @unittest.skip(
-        "In April 2020, the system resolvers started returning INET6 answers on macOS and Travis "
-        "whereas gevent only returns INET (presumably the RTD configuration changed). "
-    )
     def test_inet(self):
-        self.assertEqualResults(
-            self.getresult(),
-            gevent_socket.getaddrinfo(TestGeventOrg.HOSTNAME, None, socket.AF_INET),
-            'getaddrinfo')
+        self._test('getaddrinfo', TestGeventOrg.HOSTNAME, None, socket.AF_INET)
 
     def test_unspec(self):
-        self.assertEqualResults(
-            self.getresult(),
-            gevent_socket.getaddrinfo(TestGeventOrg.HOSTNAME, None, socket.AF_UNSPEC),
-            'getaddrinfo')
+        self._test('getaddrinfo', TestGeventOrg.HOSTNAME, None, socket.AF_UNSPEC)
 
     def test_badvalue(self):
         self._test('getaddrinfo', TestGeventOrg.HOSTNAME, None, 255)
@@ -612,7 +685,10 @@ class Test_getaddrinfo(TestCase):
         self.assertIs(af, socket.AF_INET)
 
 class TestInternational(TestCase):
-    pass
+    if PY2:
+        # We expect these to raise UnicodeEncodeError, which is a
+        # subclass of ValueError
+        REAL_ERRORS = set(TestCase.REAL_ERRORS) - {ValueError,}
 
 # dns python can actually resolve these: it uses
 # the 2008 version of idna encoding, whereas on Python 2,

src/gevent/tests/test__socket_dns6.py

@@ -36,7 +36,7 @@ from gevent.testing.sysinfo import RESOLVER_DNSPYTHON
 # by default we skip the tests everywhere else.
 
 class Test6(TestCase):
-
+    NORMALIZE_GHBA_IGNORE_ALIAS = True
     # host that only has AAAA record
     host = 'aaaa.test-ipv6.com'