Skip to content

G
gevent
Commit 5acba203 authored by Jason Madden's avatar Jason Madden Committed by GitHub
Browse files
Options

Merge pull request #1594 from gevent/issue1012 

Use `ares_getaddrinfo`
parents adba05d0 f9ad8e95
Hide whitespace changes
Inline Side-by-side
Showing with 732 additions and 571 deletions
_setupares.py
View file @ 5acba203
......@@ -95,7 +95,6 @@ ARES = Extension(
libraries=list(LIBRARIES),
define_macros=list(DEFINE_MACROS),
depends=glob_many(
'src/gevent/resolver/dnshelper.c',
'src/gevent/resolver/cares_*.[ch]')
)
......
docs/changes/1012.bugfix 0 → 100644
View file @ 5acba203
Use ``ares_getaddrinfo`` instead of a manual lookup.
This requires c-ares 1.16.0.
Note that this may change the results, in particular their order.
As part of this, certain parts of the c-ares extension were adapted to
use modern Cython idioms.
A few minor errors and discrepancies were fixed as well, such as
``gethostbyaddr(''localhost')`` working on Python 3 and failing on
Python 2. The DNSpython resolver now raises the expected TypeError in
more cases instead of an AttributeError.
src/gevent/libev/_corecffi_source.c
View file @ 5acba203
......@@ -76,7 +76,9 @@ static void gevent_zero_prepare(struct ev_prepare* handle)
static void gevent_set_ev_alloc()
{
ev_set_allocator(gevent_realloc);
void* (*ptr)(void*, long);
ptr = (void*(*)(void*, long))&gevent_realloc;
ev_set_allocator(ptr);
}
#ifdef __clang__
......
src/gevent/resolver/_addresses.py
View file @ 5acba203
......@@ -142,9 +142,9 @@ def _ipv6_inet_aton(text,
def _is_addr(host, parse=_ipv4_inet_aton):
if not host:
if not host or not isinstance(host, hostname_types):
return False
assert isinstance(host, hostname_types), repr(host)
try:
parse(host)
except AddressSyntaxError:
......@@ -158,7 +158,7 @@ is_ipv4_addr = _is_addr
def is_ipv6_addr(host):
# Return True if host is a valid IPv6 address
if host:
if host and isinstance(host, hostname_types):
s = '%' if isinstance(host, str) else b'%'
host = host.split(s, 1)[0]
return _is_addr(host, _ipv6_inet_aton)
src/gevent/resolver/ares.py
View file @ 5acba203
......@@ -4,16 +4,14 @@ c-ares based hostname resolver.
"""
from __future__ import absolute_import, print_function, division
import os
import sys
from _socket import getaddrinfo
from _socket import getaddrinfo as native_getaddrinfo
from _socket import gaierror
from _socket import error
from gevent._compat import string_types
from gevent._compat import text_type
from gevent._compat import reraise
from gevent._compat import integer_types
from gevent._compat import PY3
from gevent.hub import Waiter
......@@ -22,9 +20,10 @@ from gevent.hub import get_hub
from gevent.socket import AF_UNSPEC
from gevent.socket import AF_INET
from gevent.socket import AF_INET6
from gevent.socket import SOCK_STREAM
from gevent.socket import SOCK_DGRAM
from gevent.socket import SOCK_RAW
from gevent.socket import SOCK_STREAM
from gevent.socket import SOL_TCP
from gevent.socket import SOL_UDP
from gevent.socket import AI_NUMERICHOST
from gevent._config import config
......@@ -83,6 +82,17 @@ class Resolver(AbstractResolver):
results, and c-ares may report more ips on a multi-homed
host.
- The system implementation may return some names fully qualified, where
this implementation returns only the host name. This appears to be
the case only with entries found in ``/etc/hosts``.
- c-ares supports a limited set of flags for ``getnameinfo`` and
``getaddrinfo``; unknown flags are ignored. System-specific flags
such as ``AI_V4MAPPED_CFG`` are not supported.
- ``getaddrinfo`` may return canonical names even without the ``AI_CANONNAME``
being set.
.. caution::
This module is considered extremely experimental on PyPy, and
......@@ -95,10 +105,14 @@ class Resolver(AbstractResolver):
resolved <https://github.com/c-ares/c-ares/issues/196>`_ or even
sent to the DNS server.
.. versionchanged:: NEXT
``getaddrinfo`` is now implemented using the native c-ares function
from c-ares 1.16 or newer.
.. _c-ares: http://c-ares.haxx.se
"""
ares_class = channel
cares_class = channel
def __init__(self, hub=None, use_environ=True, **kwargs):
if hub is None:
......@@ -110,27 +124,27 @@ class Resolver(AbstractResolver):
value = setting.get()
if value is not None:
kwargs.setdefault(setting.kwarg_name, value)
self.ares = self.ares_class(hub.loop, **kwargs)
self.cares = self.cares_class(hub.loop, **kwargs)
self.pid = os.getpid()
self.params = kwargs
self.fork_watcher = hub.loop.fork(ref=False)
self.fork_watcher.start(self._on_fork)
def __repr__(self):
return '<gevent.resolver_ares.Resolver at 0x%x ares=%r>' % (id(self), self.ares)
return '<gevent.resolver_ares.Resolver at 0x%x ares=%r>' % (id(self), self.cares)
def _on_fork(self):
# NOTE: See comment in gevent.hub.reinit.
pid = os.getpid()
if pid != self.pid:
self.hub.loop.run_callback(self.ares.destroy)
self.ares = self.ares_class(self.hub.loop, **self.params)
self.hub.loop.run_callback(self.cares.destroy)
self.cares = self.cares_class(self.hub.loop, **self.params)
self.pid = pid
def close(self):
if self.ares is not None:
self.hub.loop.run_callback(self.ares.destroy)
self.ares = None
if self.cares is not None:
self.hub.loop.run_callback(self.cares.destroy)
self.cares = None
self.fork_watcher.stop()
def gethostbyname(self, hostname, family=AF_INET):
......@@ -150,7 +164,7 @@ class Resolver(AbstractResolver):
raise TypeError('Expected string, not %s' % type(hostname).__name__)
while True:
ares = self.ares
ares = self.cares
try:
waiter = Waiter(self.hub)
ares.gethostbyname(waiter, hostname, family)
......@@ -159,95 +173,114 @@ class Resolver(AbstractResolver):
raise gaierror(-5, 'No address associated with hostname')
return result
except gaierror:
if ares is self.ares:
if ares is self.cares:
if hostname == b'255.255.255.255':
# The stdlib handles this case in 2.7 and 3.x, but ares does not.
# It is tested by test_socket.py in 3.4.
# HACK: So hardcode the expected return.
return ('255.255.255.255', [], ['255.255.255.255'])
raise
# "self.ares is not ares" means channel was destroyed (because we were forked)
# "self.cares is not ares" means channel was destroyed (because we were forked)
def _lookup_port(self, port, socktype):
return lookup_port(port, socktype)
def _getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
def _getaddrinfo(
self, host, port,
family=0, socktype=0, proto=0, flags=0,
fill_in_type_proto=True
):
"""
Returns a list ``(family, socktype, proto, canonname, sockaddr)``
:raises gaierror: If no results are found.
"""
# pylint:disable=too-many-locals,too-many-branches
if isinstance(host, text_type):
host = host.encode('idna')
elif not isinstance(host, str) or (flags & AI_NUMERICHOST):
if not isinstance(host, bytes) or (flags & AI_NUMERICHOST) or host in (
b'localhost', b'ip6-localhost'):
# XXX: Now that we're using ares_getaddrinfo, how much of this is still
# necessary?
# this handles cases which do not require network access
# 1) host is None
# 2) host is of an invalid type
# 3) AI_NUMERICHOST flag is set
return getaddrinfo(host, port, family, socktype, proto, flags)
# we also call _socket.getaddrinfo below if family is not one of AF_*
port, socktypes = self._lookup_port(port, socktype)
socktype_proto = [(SOCK_STREAM, 6), (SOCK_DGRAM, 17), (SOCK_RAW, 0)]
if socktypes:
socktype_proto = [(x, y) for (x, y) in socktype_proto if x in socktypes]
if proto:
socktype_proto = [(x, y) for (x, y) in socktype_proto if proto == y]
ares = self.ares
if family == AF_UNSPEC:
ares_values = Values(self.hub, 2)
ares.gethostbyname(ares_values, host, AF_INET)
ares.gethostbyname(ares_values, host, AF_INET6)
elif family == AF_INET:
ares_values = Values(self.hub, 1)
ares.gethostbyname(ares_values, host, AF_INET)
elif family == AF_INET6:
ares_values = Values(self.hub, 1)
ares.gethostbyname(ares_values, host, AF_INET6)
else:
raise gaierror(5, 'ai_family not supported: %r' % (family, ))
values = ares_values.get()
if len(values) == 2 and values[0] == values[1]:
values.pop()
result = []
result4 = []
result6 = []
for addrs in values:
if addrs.family == AF_INET:
for addr in addrs[-1]:
sockaddr = (addr, port)
for socktype4, proto4 in socktype_proto:
result4.append((AF_INET, socktype4, proto4, '', sockaddr))
elif addrs.family == AF_INET6:
for addr in addrs[-1]:
if addr == '::1':
dest = result
else:
dest = result6
sockaddr = (addr, port, 0, 0)
for socktype6, proto6 in socktype_proto:
dest.append((AF_INET6, socktype6, proto6, '', sockaddr))
# As of 2016, some platforms return IPV6 first and some do IPV4 first,
# and some might even allow configuration of which is which. For backwards
# compatibility with earlier releases (but not necessarily resolver_thread!)
# we return 4 first. See https://github.com/gevent/gevent/issues/815 for more.
result += result4 + result6
# 4) It's a well-known alias. TODO: This is special casing that we don't
# really want to do. It's here because it resolves a discrepancy with the system
# resolvers caught by test cases. In gevent 20.4.0, this only worked correctly on
# Python 3 and not Python 2, by accident.
return native_getaddrinfo(host, port, family, socktype, proto, flags)
if isinstance(port, text_type):
port = port.encode('ascii')
elif isinstance(port, integer_types):
if port == 0:
port = None
else:
port = str(port).encode('ascii')
waiter = Waiter(self.hub)
self.cares.getaddrinfo(
waiter,
host,
port,
family,
socktype,
proto,
flags,
)
# Result is a list of:
# (family, socktype, proto, canonname, sockaddr)
# Where sockaddr depends on family; for INET it is
# (address, port)
# and INET6 is
# (address, port, flow info, scope id)
result = waiter.get()
if not result:
raise gaierror(-5, 'No address associated with hostname')
if fill_in_type_proto:
# c-ares 1.16 DOES NOT fill in socktype or proto in the results,
# ever. It's at least supposed to do that if they were given as
# hints, but it doesn't (https://github.com/c-ares/c-ares/issues/317)
# Sigh.
if socktype:
hard_type_proto = [
(socktype, SOL_TCP if socktype == SOCK_STREAM else SOL_UDP),
]
elif proto:
hard_type_proto = [
(SOCK_STREAM if proto == SOL_TCP else SOCK_DGRAM, proto),
]
else:
hard_type_proto = [
(SOCK_STREAM, SOL_TCP),
(SOCK_DGRAM, SOL_UDP),
]
result = [
(rfamily,
hard_type if not rtype else rtype,
hard_proto if not rproto else rproto,
rcanon,
raddr)
for rfamily, rtype, rproto, rcanon, raddr
in result
for hard_type, hard_proto
in hard_type_proto
]
return result
def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
while True:
ares = self.ares
ares = self.cares
try:
return self._getaddrinfo(host, port, family, socktype, proto, flags)
except gaierror:
if ares is self.ares:
if ares is self.cares:
raise
def _gethostbyaddr(self, ip_address):
......@@ -264,7 +297,7 @@ class Resolver(AbstractResolver):
waiter = Waiter(self.hub)
try:
self.ares.gethostbyaddr(waiter, ip_address)
self.cares.gethostbyaddr(waiter, ip_address)
return waiter.get()
except InvalidIP:
result = self._getaddrinfo(ip_address, None, family=AF_UNSPEC, socktype=SOCK_DGRAM)
......@@ -276,21 +309,21 @@ class Resolver(AbstractResolver):
if _ip_address == ip_address:
raise
waiter.clear()
self.ares.gethostbyaddr(waiter, _ip_address)
self.cares.gethostbyaddr(waiter, _ip_address)
return waiter.get()
def gethostbyaddr(self, ip_address):
ip_address = _resolve_special(ip_address, AF_UNSPEC)
while True:
ares = self.ares
ares = self.cares
try:
return self._gethostbyaddr(ip_address)
except gaierror:
if ares is self.ares:
if ares is self.cares:
raise
def _getnameinfo(self, sockaddr, flags):
if not isinstance(flags, int):
if not isinstance(flags, integer_types):
raise TypeError('an integer is required')
if not isinstance(sockaddr, tuple):
raise TypeError('getnameinfo() argument 1 must be a tuple')
......@@ -303,15 +336,20 @@ class Resolver(AbstractResolver):
raise TypeError('sockaddr[0] must be a string, not %s' % type(address).__name__)
port = sockaddr[1]
if not isinstance(port, int):
if not isinstance(port, integer_types):
raise TypeError('port must be an integer, not %s' % type(port))
waiter = Waiter(self.hub)
result = self._getaddrinfo(address, str(sockaddr[1]), family=AF_UNSPEC, socktype=SOCK_DGRAM)
if not result:
reraise(*sys.exc_info())
elif len(result) != 1:
if len(sockaddr) > 2:
# Must be IPv6: (host, port, [flowinfo, [scopeid]])
flowinfo = sockaddr[2]
if flowinfo > 0xfffff:
raise OverflowError("getnameinfo(): flowinfo must be 0-1048575.")
result = self._getaddrinfo(address, port,
family=AF_UNSPEC, socktype=SOCK_DGRAM, fill_in_type_proto=False)
if len(result) != 1:
raise error('sockaddr resolved to multiple addresses')
family, _socktype, _proto, _name, address = result[0]
if family == AF_INET:
......@@ -320,56 +358,27 @@ class Resolver(AbstractResolver):
elif family == AF_INET6:
address = address[:2] + sockaddr[2:]
self.ares.getnameinfo(waiter, address, flags)
waiter = Waiter(self.hub)
self.cares.getnameinfo(waiter, address, flags)
node, service = waiter.get()
if service is None:
if PY3:
# ares docs: "If the query did not complete
# successfully, or one of the values was not
# requested, node or service will be NULL ". Python 2
# allows that for the service, but Python 3 raises
# an error. This is tested by test_socket in py 3.4
err = gaierror('nodename nor servname provided, or not known')
err.errno = 8
raise err
service = '0'
return node, service
if service is None and PY3:
# ares docs: "If the query did not complete
# successfully, or one of the values was not
# requested, node or service will be NULL ". Python 2
# allows that for the service, but Python 3 raises
# an error. This is tested by test_socket in py 3.4
err = gaierror('nodename nor servname provided, or not known')
err.errno = 8
raise err
return node, service or '0'
def getnameinfo(self, sockaddr, flags):
while True:
ares = self.ares
ares = self.cares
try:
return self._getnameinfo(sockaddr, flags)
except gaierror:
if ares is self.ares:
if ares is self.cares:
raise
class Values(object):
# helper to collect multiple values; ignore errors unless nothing has succeeded
# QQQ could probably be moved somewhere - hub.py?
__slots__ = ['count', 'values', 'error', 'waiter']
def __init__(self, hub, count):
self.count = count
self.values = []
self.error = None
self.waiter = Waiter(hub)
def __call__(self, source):
self.count -= 1
if source.exception is None:
self.values.append(source.value)
else:
self.error = source.exception
if self.count <= 0:
self.waiter.switch(None)
def get(self):
self.waiter.get()
if self.values:
return self.values
assert error is not None
raise self.error # pylint:disable=raising-bad-type
src/gevent/resolver/cares.pyx
View file @ 5acba203
......@@ -3,43 +3,61 @@
# seems to be buggy (at least for the `result` class) and produces code that
# can't compile ("local variable 'result' referenced before assignment").
# See https://github.com/cython/cython/issues/1786
# cython: auto_pickle=False
# cython: auto_pickle=False,language_level=3str
cimport libcares as cares
import sys
from cpython.version cimport PY_MAJOR_VERSION
from cpython.tuple cimport PyTuple_Check
from cpython.getargs cimport PyArg_ParseTuple
from cpython.ref cimport Py_INCREF
from cpython.ref cimport Py_DECREF
from cpython.mem cimport PyMem_Malloc
from cpython.mem cimport PyMem_Free
from libc.string cimport memset
import _socket
from _socket import gaierror
__all__ = ['channel']
cdef object string_types
cdef object text_type
cdef tuple string_types
cdef type text_type
if sys.version_info[0] >= 3:
if PY_MAJOR_VERSION >= 3:
string_types = str,
text_type = str
else:
string_types = __builtins__.basestring,
text_type = __builtins__.unicode
TIMEOUT = 1
DEF TIMEOUT = 1
DEF EV_READ = 1
DEF EV_WRITE = 2
cdef extern from *:
"""
#ifdef CARES_EMBED
#include "ares_setup.h"
#endif
cdef extern from "dnshelper.c":
#ifdef HAVE_NETDB_H
#include <netdb.h>
#endif
"""
cdef extern from "ares.h":
int AF_INET
int AF_INET6
int INET6_ADDRSTRLEN
struct hostent:
char* h_name
int h_addrtype
char** h_aliases
char** h_addr_list
struct sockaddr_t "sockaddr":
pass
......@@ -47,112 +65,68 @@ cdef extern from "dnshelper.c":
struct ares_channeldata:
pass
object parse_h_name(hostent*)
object parse_h_aliases(hostent*)
object parse_h_addr_list(hostent*)
void* create_object_from_hostent(void*)
struct in_addr:
unsigned int s_addr
struct sockaddr_in:
int sin_family
int sin_port
in_addr sin_addr
struct in6_addr:
char s6_addr[16]
# this imports _socket lazily
object PyUnicode_FromString(char*)
int PyTuple_Check(object)
int PyArg_ParseTuple(object, char*, ...) except 0
struct sockaddr_in6:
pass
int gevent_make_sockaddr(char* hostp, int port, int flowinfo, int scope_id, sockaddr_in6* sa6)
void memset(void*, int, int)
ARES_SUCCESS = cares.ARES_SUCCESS
ARES_ENODATA = cares.ARES_ENODATA
ARES_EFORMERR = cares.ARES_EFORMERR
ARES_ESERVFAIL = cares.ARES_ESERVFAIL
ARES_ENOTFOUND = cares.ARES_ENOTFOUND
ARES_ENOTIMP = cares.ARES_ENOTIMP
ARES_EREFUSED = cares.ARES_EREFUSED
ARES_EBADQUERY = cares.ARES_EBADQUERY
ARES_EBADNAME = cares.ARES_EBADNAME
ARES_EBADFAMILY = cares.ARES_EBADFAMILY
ARES_EBADRESP = cares.ARES_EBADRESP
ARES_ECONNREFUSED = cares.ARES_ECONNREFUSED
ARES_ETIMEOUT = cares.ARES_ETIMEOUT
ARES_EOF = cares.ARES_EOF
ARES_EFILE = cares.ARES_EFILE
ARES_ENOMEM = cares.ARES_ENOMEM
ARES_EDESTRUCTION = cares.ARES_EDESTRUCTION
ARES_EBADSTR = cares.ARES_EBADSTR
ARES_EBADFLAGS = cares.ARES_EBADFLAGS
ARES_ENONAME = cares.ARES_ENONAME
ARES_EBADHINTS = cares.ARES_EBADHINTS
ARES_ENOTINITIALIZED = cares.ARES_ENOTINITIALIZED
ARES_ELOADIPHLPAPI = cares.ARES_ELOADIPHLPAPI
ARES_EADDRGETNETWORKPARAMS = cares.ARES_EADDRGETNETWORKPARAMS
ARES_ECANCELLED = cares.ARES_ECANCELLED
ARES_FLAG_USEVC = cares.ARES_FLAG_USEVC
ARES_FLAG_PRIMARY = cares.ARES_FLAG_PRIMARY
ARES_FLAG_IGNTC = cares.ARES_FLAG_IGNTC
ARES_FLAG_NORECURSE = cares.ARES_FLAG_NORECURSE
ARES_FLAG_STAYOPEN = cares.ARES_FLAG_STAYOPEN
ARES_FLAG_NOSEARCH = cares.ARES_FLAG_NOSEARCH
ARES_FLAG_NOALIASES = cares.ARES_FLAG_NOALIASES
ARES_FLAG_NOCHECKRESP = cares.ARES_FLAG_NOCHECKRESP
int sin6_family
int sin6_port
unsigned int sin6_flowinfo
in6_addr sin6_addr
unsigned int sin6_scope_id
unsigned int htons(unsigned int hostshort)
unsigned int ntohs(unsigned int hostshort)
unsigned int htonl(unsigned int hostlong)
unsigned int ntohl(unsigned int hostlong)
cdef int AI_NUMERICSERV = _socket.AI_NUMERICSERV
cdef int AI_CANONNAME = _socket.AI_CANONNAME
cdef int NI_NUMERICHOST = _socket.NI_NUMERICHOST
cdef int NI_NUMERICSERV = _socket.NI_NUMERICSERV
cdef int NI_NOFQDN = _socket.NI_NOFQDN
cdef int NI_NAMEREQD = _socket.NI_NAMEREQD
cdef int NI_DGRAM = _socket.NI_DGRAM
_ares_errors = dict([
(cares.ARES_SUCCESS, 'ARES_SUCCESS'),
(cares.ARES_ENODATA, 'ARES_ENODATA'),
(cares.ARES_EFORMERR, 'ARES_EFORMERR'),
(cares.ARES_ESERVFAIL, 'ARES_ESERVFAIL'),
(cares.ARES_ENOTFOUND, 'ARES_ENOTFOUND'),
(cares.ARES_ENOTIMP, 'ARES_ENOTIMP'),
(cares.ARES_EREFUSED, 'ARES_EREFUSED'),
(cares.ARES_EBADQUERY, 'ARES_EBADQUERY'),
(cares.ARES_EBADNAME, 'ARES_EBADNAME'),
(cares.ARES_EBADFAMILY, 'ARES_EBADFAMILY'),
(cares.ARES_EBADRESP, 'ARES_EBADRESP'),
(cares.ARES_ECONNREFUSED, 'ARES_ECONNREFUSED'),
(cares.ARES_ETIMEOUT, 'ARES_ETIMEOUT'),
(cares.ARES_EOF, 'ARES_EOF'),
(cares.ARES_EFILE, 'ARES_EFILE'),
(cares.ARES_ENOMEM, 'ARES_ENOMEM'),
(cares.ARES_EDESTRUCTION, 'ARES_EDESTRUCTION'),
(cares.ARES_EBADSTR, 'ARES_EBADSTR'),
(cares.ARES_EBADFLAGS, 'ARES_EBADFLAGS'),
(cares.ARES_ENONAME, 'ARES_ENONAME'),
(cares.ARES_EBADHINTS, 'ARES_EBADHINTS'),
(cares.ARES_ENOTINITIALIZED, 'ARES_ENOTINITIALIZED'),
(cares.ARES_ELOADIPHLPAPI, 'ARES_ELOADIPHLPAPI'),
(cares.ARES_EADDRGETNETWORKPARAMS, 'ARES_EADDRGETNETWORKPARAMS'),
(cares.ARES_ECANCELLED, 'ARES_ECANCELLED')])
# maps c-ares flag to _socket module flag
_cares_flag_map = None
cdef _prepare_cares_flag_map():
global _cares_flag_map
import _socket
_cares_flag_map = [
(getattr(_socket, 'NI_NUMERICHOST', 1), cares.ARES_NI_NUMERICHOST),
(getattr(_socket, 'NI_NUMERICSERV', 2), cares.ARES_NI_NUMERICSERV),
(getattr(_socket, 'NI_NOFQDN', 4), cares.ARES_NI_NOFQDN),
(getattr(_socket, 'NI_NAMEREQD', 8), cares.ARES_NI_NAMEREQD),
(getattr(_socket, 'NI_DGRAM', 16), cares.ARES_NI_DGRAM)]
cpdef _convert_cares_flags(int flags, int default=cares.ARES_NI_LOOKUPHOST|cares.ARES_NI_LOOKUPSERVICE):
if _cares_flag_map is None:
_prepare_cares_flag_map()
for socket_flag, cares_flag in _cares_flag_map:
if socket_flag & flags:
default |= cares_flag
flags &= ~socket_flag
if not flags:
return default
raise gaierror(-1, "Bad value for ai_flags: 0x%x" % flags)
(cares.ARES_SUCCESS, 'ARES_SUCCESS'),
(cares.ARES_EADDRGETNETWORKPARAMS, 'ARES_EADDRGETNETWORKPARAMS'),
(cares.ARES_EBADFAMILY, 'ARES_EBADFAMILY'),
(cares.ARES_EBADFLAGS, 'ARES_EBADFLAGS'),
(cares.ARES_EBADHINTS, 'ARES_EBADHINTS'),
(cares.ARES_EBADNAME, 'ARES_EBADNAME'),
(cares.ARES_EBADQUERY, 'ARES_EBADQUERY'),
(cares.ARES_EBADRESP, 'ARES_EBADRESP'),
(cares.ARES_EBADSTR, 'ARES_EBADSTR'),
(cares.ARES_ECANCELLED, 'ARES_ECANCELLED'),
(cares.ARES_ECONNREFUSED, 'ARES_ECONNREFUSED'),
(cares.ARES_EDESTRUCTION, 'ARES_EDESTRUCTION'),
(cares.ARES_EFILE, 'ARES_EFILE'),
(cares.ARES_EFORMERR, 'ARES_EFORMERR'),
(cares.ARES_ELOADIPHLPAPI, 'ARES_ELOADIPHLPAPI'),
(cares.ARES_ENODATA, 'ARES_ENODATA'),
(cares.ARES_ENOMEM, 'ARES_ENOMEM'),
(cares.ARES_ENONAME, 'ARES_ENONAME'),
(cares.ARES_ENOTFOUND, 'ARES_ENOTFOUND'),
(cares.ARES_ENOTIMP, 'ARES_ENOTIMP'),
(cares.ARES_ENOTINITIALIZED, 'ARES_ENOTINITIALIZED'),
(cares.ARES_EOF, 'ARES_EOF'),
(cares.ARES_EREFUSED, 'ARES_EREFUSED'),
(cares.ARES_ESERVICE, 'ARES_ESERVICE'),
(cares.ARES_ESERVFAIL, 'ARES_ESERVFAIL'),
(cares.ARES_ETIMEOUT, 'ARES_ETIMEOUT'),
])
cpdef strerror(code):
......@@ -170,7 +144,7 @@ cdef void gevent_sock_state_callback(void *data, int s, int read, int write):
ch._sock_state_callback(s, read, write)
cdef class result:
cdef class Result(object):
cdef public object value
cdef public object exception
......@@ -207,6 +181,42 @@ class ares_host_result(tuple):
return (self.family, tuple(self))
cdef list _parse_h_aliases(hostent* host):
cdef list result = []
cdef char** aliases = host.h_aliases
if not aliases or not aliases[0]:
return result
while aliases[0]: # *aliases
# The old C version of this excluded an alias if
# it matched the host name. I don't think the stdlib does that?
result.append(_as_str(aliases[0]))
aliases += 1
return result
cdef list _parse_h_addr_list(hostent* host):
cdef list result = []
cdef char** addr_list = host.h_addr_list
cdef int addr_type = host.h_addrtype
# INET6_ADDRSTRLEN is 46, but we can't use that named constant
# here; cython doesn't like it.
cdef char tmpbuf[46]
if not addr_list or not addr_list[0]:
return result
while addr_list[0]:
if not cares.ares_inet_ntop(host.h_addrtype, addr_list[0], tmpbuf, INET6_ADDRSTRLEN):
raise _socket.error("Failed in ares_inet_ntop")
result.append(_as_str(tmpbuf))
addr_list += 1
return result
cdef void gevent_ares_host_callback(void *arg, int status, int timeouts, hostent* host):
cdef channel channel
cdef object callback
......@@ -215,18 +225,30 @@ cdef void gevent_ares_host_callback(void *arg, int status, int timeouts, hostent
cdef object host_result
try:
if status or not host:
callback(result(None, gaierror(status, strerror(status))))
callback(Result(None, gaierror(status, strerror(status))))
else:
try:
host_result = ares_host_result(host.h_addrtype, (parse_h_name(host), parse_h_aliases(host), parse_h_addr_list(host)))
host_result = ares_host_result(host.h_addrtype,
(_as_str(host.h_name),
_parse_h_aliases(host),
_parse_h_addr_list(host)))
except:
callback(result(None, sys.exc_info()[1]))
callback(Result(None, sys.exc_info()[1]))
else:
callback(result(host_result))
callback(Result(host_result))
except:
channel.loop.handle_error(callback, *sys.exc_info())
cdef object _as_str(const char* val):
if not val:
return None
if PY_MAJOR_VERSION < 3:
return <bytes>val
return val.decode('utf-8')
cdef void gevent_ares_nameinfo_callback(void *arg, int status, int timeouts, char *c_node, char *c_service):
cdef channel channel
cdef object callback
......@@ -236,27 +258,39 @@ cdef void gevent_ares_nameinfo_callback(void *arg, int status, int timeouts, cha
cdef object service
try:
if status:
callback(result(None, gaierror(status, strerror(status))))
callback(Result(None, gaierror(status, strerror(status))))
else:
if c_node:
node = PyUnicode_FromString(c_node)
else:
node = None
if c_service:
service = PyUnicode_FromString(c_service)
else:
service = None
callback(result((node, service)))
node = _as_str(c_node)
service = _as_str(c_service)
callback(Result((node, service)))
except:
channel.loop.handle_error(callback, *sys.exc_info())
cdef class channel:
cdef public object loop
cdef int _make_sockaddr(const char* hostp, int port, int flowinfo, int scope_id, sockaddr_in6* sa6):
if cares.ares_inet_pton(AF_INET, hostp, &(<sockaddr_in*>sa6).sin_addr.s_addr) > 0:
(<sockaddr_in*>sa6).sin_family = AF_INET
(<sockaddr_in*>sa6).sin_port = htons(port)
return sizeof(sockaddr_in)
if cares.ares_inet_pton(AF_INET6, hostp, &(sa6.sin6_addr).s6_addr) > 0:
sa6.sin6_family = AF_INET6
sa6.sin6_port = htons(port)
sa6.sin6_flowinfo = flowinfo
sa6.sin6_scope_id = scope_id
return sizeof(sockaddr_in6);
return -1;
cdef class channel:
cdef ares_channeldata* channel
cdef public dict _watchers
cdef public object _timer
cdef readonly object loop
cdef dict _watchers
cdef object _timer
def __init__(self, object loop, flags=None, timeout=None, tries=None, ndots=None,
udp_port=None, tcp_port=None, servers=None):
......@@ -264,26 +298,34 @@ cdef class channel:
cdef cares.ares_options options
memset(&options, 0, sizeof(cares.ares_options))
cdef int optmask = cares.ARES_OPT_SOCK_STATE_CB
options.sock_state_cb = <void*>gevent_sock_state_callback
options.sock_state_cb_data = <void*>self
if flags is not None:
options.flags = int(flags)
optmask |= cares.ARES_OPT_FLAGS
if timeout is not None:
options.timeout = int(float(timeout) * 1000)
optmask |= cares.ARES_OPT_TIMEOUTMS
if tries is not None:
options.tries = int(tries)
optmask |= cares.ARES_OPT_TRIES
if ndots is not None:
options.ndots = int(ndots)
optmask |= cares.ARES_OPT_NDOTS
if udp_port is not None:
options.udp_port = int(udp_port)
optmask |= cares.ARES_OPT_UDP_PORT
if tcp_port is not None:
options.tcp_port = int(tcp_port)
optmask |= cares.ARES_OPT_TCP_PORT
cdef int result = cares.ares_library_init(cares.ARES_LIB_INIT_ALL) # ARES_LIB_INIT_WIN32 -DUSE_WINSOCK?
if result:
raise gaierror(result, strerror(result))
......@@ -413,7 +455,8 @@ cdef class channel:
# note that for file lookups still AF_INET can be returned for AF_INET6 request
cdef object arg = (self, callback)
Py_INCREF(arg)
cares.ares_gethostbyname(self.channel, name, family, <void*>gevent_ares_host_callback, <void*>arg)
cares.ares_gethostbyname(self.channel, name, family,
<void*>gevent_ares_host_callback, <void*>arg)
def gethostbyaddr(self, object callback, char* addr):
if not self.channel:
......@@ -447,7 +490,7 @@ cdef class channel:
PyArg_ParseTuple(sockaddr, "si|ii", &hostp, &port, &flowinfo, &scope_id)
if port < 0 or port > 65535:
raise gaierror(-8, 'Invalid value for port: %r' % port)
cdef int length = gevent_make_sockaddr(hostp, port, flowinfo, scope_id, &sa6)
cdef int length = _make_sockaddr(hostp, port, flowinfo, scope_id, &sa6)
if length <= 0:
raise InvalidIP(repr(hostp))
cdef object arg = (self, callback)
......@@ -455,10 +498,146 @@ cdef class channel:
cdef sockaddr_t* x = <sockaddr_t*>&sa6
cares.ares_getnameinfo(self.channel, x, length, flags, <void*>gevent_ares_nameinfo_callback, <void*>arg)
@staticmethod
cdef int _convert_cares_ni_flags(int flags):
cdef int cares_flags = cares.ARES_NI_LOOKUPHOST | cares.ARES_NI_LOOKUPSERVICE
if flags & NI_NUMERICHOST:
cares_flags |= cares.ARES_NI_NUMERICHOST
if flags & NI_NUMERICSERV:
cares_flags |= cares.ARES_NI_NUMERICSERV
if flags & NI_NOFQDN:
cares_flags |= cares.ARES_NI_NOFQDN
if flags & NI_NAMEREQD:
cares_flags |= cares.ARES_NI_NAMEREQD
if flags & NI_DGRAM:
cares_flags |= cares.ARES_NI_DGRAM
return cares_flags
def getnameinfo(self, object callback, tuple sockaddr, int flags):
try:
flags = _convert_cares_flags(flags)
except gaierror:
# The stdlib just ignores bad flags
flags = 0
flags = channel._convert_cares_ni_flags(flags)
return self._getnameinfo(callback, sockaddr, flags)
@staticmethod
cdef int _convert_cares_ai_flags(int flags):
# c-ares supports a limited set of flags.
# We always want NOSORT, because that implies that
# c-ares will not connect to resolved addresses.
cdef int cares_flags = cares.ARES_AI_NOSORT
if flags & AI_CANONNAME:
cares_flags |= cares.ARES_AI_CANONNAME
if flags & AI_NUMERICSERV:
cares_flags |= cares.ARES_AI_NUMERICSERV
return cares_flags
@staticmethod
cdef void _getaddrinfo_cb(void *arg,
int status,
int timeouts,
cares.ares_addrinfo* result):
cdef cares.ares_addrinfo_node* nodes
cdef cares.ares_addrinfo_cname* cnames
cdef sockaddr_in* sadr4
cdef sockaddr_in6* sadr6
cdef object canonname = ''
cdef channel channel
cdef object callback
# INET6_ADDRSTRLEN is 46, but we can't use that named constant
# here; cython doesn't like it.
cdef char tmpbuf[46]
channel, callback = <tuple>arg
Py_DECREF(<tuple>arg)
# Result is a list of:
# (family, socktype, proto, canonname, sockaddr)
# Where sockaddr depends on family; for INET it is
# (address, port)
# and INET6 is
# (address, port, flow info, scope id)
# TODO: Check the canonnames.
addrs = []
try:
if status != cares.ARES_SUCCESS:
callback(Result(None, gaierror(status, strerror(status))))
return
if result.cnames:
# These tend to come in pairs:
#
# alias: www.gevent.org name: python-gevent.readthedocs.org
# alias: python-gevent.readthedocs.org name: readthedocs.io
#
# The standard library returns the last name so we do too.
cnames = result.cnames
while cnames:
canonname = _as_str(cnames.name)
cnames = cnames.next
nodes = result.nodes
while nodes:
if nodes.ai_family == AF_INET:
sadr4 = <sockaddr_in*>nodes.ai_addr
cares.ares_inet_ntop(nodes.ai_family, &sadr4.sin_addr, tmpbuf,
INET6_ADDRSTRLEN)
sockaddr = (
_as_str(tmpbuf),
ntohs(sadr4.sin_port),
)
elif nodes.ai_family == AF_INET6:
sadr6 = <sockaddr_in6*>nodes.ai_addr
cares.ares_inet_ntop(nodes.ai_family, &sadr6.sin6_addr, tmpbuf,
INET6_ADDRSTRLEN)
sockaddr = (
_as_str(tmpbuf),
ntohs(sadr6.sin6_port),
sadr6.sin6_flowinfo,
sadr6.sin6_scope_id,
)
addrs.append((
nodes.ai_family,
nodes.ai_socktype,
nodes.ai_protocol,
canonname,
sockaddr,
))
nodes = nodes.ai_next
callback(Result(addrs, None))
except:
channel.loop.handle_error(callback, *sys.exc_info())
finally:
if result:
cares.ares_freeaddrinfo(result)
def getaddrinfo(self,
object callback,
const char* name,
object service, # AKA port
int family=0,
int type=0,
int proto=0,
int flags=0):
if not self.channel:
raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed')
cdef cares.ares_addrinfo_hints hints
memset(&hints, 0, sizeof(cares.ares_addrinfo_hints))
hints.ai_flags = channel._convert_cares_ai_flags(flags)
hints.ai_family = family
hints.ai_socktype = type
hints.ai_protocol = proto
cdef object arg = (self, callback)
Py_INCREF(arg)
cares.ares_getaddrinfo(
self.channel,
name,
NULL if service is None else <char*>service,
&hints,
<void*>channel._getaddrinfo_cb,
<void*>arg
)
src/gevent/resolver/cares_ntop.h deleted 100644 → 0
View file @ adba05d0
#ifdef CARES_EMBED
#include "ares_setup.h"
#include "ares.h"
#else
#include <arpa/inet.h>
#define ares_inet_ntop(w,x,y,z) inet_ntop(w,x,y,z)
#endif
src/gevent/resolver/cares_pton.h deleted 100644 → 0
View file @ adba05d0
#ifdef CARES_EMBED
#include "ares_setup.h"
#include "ares_inet_net_pton.h"
#else
#include <arpa/inet.h>
#define ares_inet_pton(x,y,z) inet_pton(x,y,z)
#define ares_inet_net_pton(w,x,y,z) inet_net_pton(w,x,y,z)
#endif
src/gevent/resolver/dnshelper.c deleted 100644 → 0
View file @ adba05d0
/* Copyright (c) 2011 Denis Bilenko. See LICENSE for details. */
#include "Python.h"
#ifdef CARES_EMBED
#include "ares_setup.h"
#endif
#ifdef HAVE_NETDB_H
#include <netdb.h>
#endif
#include "ares.h"
#include "cares_ntop.h"
#include "cares_pton.h"
#if PY_MAJOR_VERSION >= 3
#define PY3K
#define GPyNative_FromString PyUnicode_FromString
#else
#define GPyNative_FromString PyString_FromString
#endif
static PyObject* _socket_error = 0;
static PyObject*
get_socket_object(PyObject** pobject, const char* name)
{
if (!*pobject) {
PyObject* _socket;
_socket = PyImport_ImportModule("_socket");
if (_socket) {
*pobject = PyObject_GetAttrString(_socket, name);
if (!*pobject) {
PyErr_WriteUnraisable(Py_None);
}
Py_DECREF(_socket);
}
else {
PyErr_WriteUnraisable(Py_None);
}
if (!*pobject) {
*pobject = PyExc_IOError;
}
}
return *pobject;
}
static int
gevent_append_addr(PyObject* list, int family, void* src, char* tmpbuf, size_t tmpsize) {
int status = -1;
PyObject* tmp;
if (ares_inet_ntop(family, src, tmpbuf, tmpsize)) {
tmp = GPyNative_FromString(tmpbuf);
if (tmp) {
status = PyList_Append(list, tmp);
Py_DECREF(tmp);
}
}
return status;
}
static PyObject*
parse_h_name(struct hostent *h)
{
return GPyNative_FromString(h->h_name);
}
static PyObject*
parse_h_aliases(struct hostent *h)
{
char **pch;
PyObject *result = NULL;
PyObject *tmp;
result = PyList_New(0);
if (result && h->h_aliases) {
for (pch = h->h_aliases; *pch != NULL; pch++) {
if (*pch != h->h_name && strcmp(*pch, h->h_name)) {
int status;
tmp = GPyNative_FromString(*pch);
if (tmp == NULL) {
break;
}
status = PyList_Append(result, tmp);
Py_DECREF(tmp);
if (status) {
break;
}
}
}
}
return result;
}
static PyObject *
parse_h_addr_list(struct hostent *h)
{
char **pch;
PyObject *result = NULL;
result = PyList_New(0);
if (result) {
switch (h->h_addrtype) {
case AF_INET:
{
char tmpbuf[sizeof "255.255.255.255"];
for (pch = h->h_addr_list; *pch != NULL; pch++) {
if (gevent_append_addr(result, AF_INET, *pch, tmpbuf, sizeof(tmpbuf))) {
break;
}
}
break;
}
case AF_INET6:
{
char tmpbuf[sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")];
for (pch = h->h_addr_list; *pch != NULL; pch++) {
if (gevent_append_addr(result, AF_INET6, *pch, tmpbuf, sizeof(tmpbuf))) {
break;
}
}
break;
}
default:
PyErr_SetString(get_socket_object(&_socket_error, "error"), "unsupported address family");
Py_DECREF(result);
result = NULL;
}
}
return result;
}
static int
gevent_make_sockaddr(char* hostp, int port, int flowinfo, int scope_id, struct sockaddr_in6* sa6) {
if ( ares_inet_pton(AF_INET, hostp, &((struct sockaddr_in*)sa6)->sin_addr.s_addr) > 0 ) {
((struct sockaddr_in*)sa6)->sin_family = AF_INET;
((struct sockaddr_in*)sa6)->sin_port = htons(port);
return sizeof(struct sockaddr_in);
}
else if ( ares_inet_pton(AF_INET6, hostp, &sa6->sin6_addr.s6_addr) > 0 ) {
sa6->sin6_family = AF_INET6;
sa6->sin6_port = htons(port);
sa6->sin6_flowinfo = flowinfo;
sa6->sin6_scope_id = scope_id;
return sizeof(struct sockaddr_in6);
}
return -1;
}
src/gevent/resolver/dnspython.py
View file @ 5acba203
......@@ -393,13 +393,17 @@ class Resolver(AbstractResolver):
aliases = self._resolver.hosts_resolver.getaliases(hostname)
net_resolver = self._resolver.network_resolver
rdtype = _family_to_rdtype(family)
while True:
while 1:
try:
ans = net_resolver.query(hostname, dns.rdatatype.CNAME, rdtype)
except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.resolver.NoNameservers):
break
except dTimeout:
break
except AttributeError as ex:
if hostname is None or isinstance(hostname, int):
raise TypeError(ex)
raise
else:
aliases.extend(str(rr.target) for rr in ans.rrset)
hostname = ans[0].target
......
src/gevent/resolver/libcares.pxd
View file @ 5acba203
cdef extern from "ares.h":
# These two are defined in <sys/socket.h> and <netdb.h>, respectively,
# on POSIX. On Windows, they are in <winsock2.h>. "ares.h" winds up
# indirectly including both of those.
struct sockaddr:
pass
struct hostent:
pass
struct ares_options:
int flags
void* sock_state_cb
......@@ -36,30 +44,31 @@ cdef extern from "ares.h":
int ARES_SOCKET_BAD
int ARES_SUCCESS
int ARES_ENODATA
int ARES_EFORMERR
int ARES_ESERVFAIL
int ARES_ENOTFOUND
int ARES_ENOTIMP
int ARES_EREFUSED
int ARES_EBADQUERY
int ARES_EBADNAME
int ARES_EADDRGETNETWORKPARAMS
int ARES_EBADFAMILY
int ARES_EBADFLAGS
int ARES_EBADHINTS
int ARES_EBADNAME
int ARES_EBADQUERY
int ARES_EBADRESP
int ARES_EBADSTR
int ARES_ECANCELLED
int ARES_ECONNREFUSED
int ARES_ETIMEOUT
int ARES_EOF
int ARES_EDESTRUCTION
int ARES_EFILE
int ARES_EFORMERR
int ARES_ELOADIPHLPAPI
int ARES_ENODATA
int ARES_ENOMEM
int ARES_EDESTRUCTION
int ARES_EBADSTR
int ARES_EBADFLAGS
int ARES_ENONAME
int ARES_EBADHINTS
int ARES_ENOTFOUND
int ARES_ENOTIMP
int ARES_ENOTINITIALIZED
int ARES_ELOADIPHLPAPI
int ARES_EADDRGETNETWORKPARAMS
int ARES_ECANCELLED
int ARES_EOF
int ARES_EREFUSED
int ARES_ESERVFAIL
int ARES_ESERVICE
int ARES_ETIMEOUT
int ARES_NI_NOFQDN
int ARES_NI_NUMERICHOST
......@@ -74,6 +83,7 @@ cdef extern from "ares.h":
int ARES_NI_LOOKUPHOST
int ARES_NI_LOOKUPSERVICE
ctypedef int ares_socklen_t
int ares_library_init(int flags)
void ares_library_cleanup()
......@@ -87,6 +97,11 @@ cdef extern from "ares.h":
void ares_cancel(void* channel)
void ares_getnameinfo(void* channel, void* sa, int salen, int flags, void* callback, void *arg)
# Added in 1.10
int ares_inet_pton(int af, const char *src, void *dst)
const char* ares_inet_ntop(int af, const void *src, char *dst, ares_socklen_t size);
struct in_addr:
pass
......@@ -104,6 +119,45 @@ cdef extern from "ares.h":
int ares_set_servers(void* channel, ares_addr_node *servers)
# Added in 1.16
int ARES_AI_NOSORT
int ARES_AI_ENVHOSTS
int ARES_AI_CANONNAME
int ARES_AI_NUMERICSERV
struct ares_addrinfo_hints:
int ai_flags
int ai_family
int ai_socktype
int ai_protocol
struct ares_addrinfo_node:
int ai_ttl
int ai_flags
int ai_family
int ai_socktype
int ai_protocol
ares_socklen_t ai_addrlen
sockaddr *ai_addr
ares_addrinfo_node *ai_next
struct ares_addrinfo_cname:
int ttl
char *alias
char *name
ares_addrinfo_cname *next
struct ares_addrinfo:
ares_addrinfo_cname *cnames
ares_addrinfo_node *nodes
void ares_getaddrinfo(
void* channel,
const char *name,
const char* service,
const ares_addrinfo_hints *hints,
#ares_addrinfo_callback callback,
void* callback,
void *arg)
cdef extern from "cares_pton.h":
int ares_inet_pton(int af, char *src, void *dst)
void ares_freeaddrinfo(ares_addrinfo *ai)
src/gevent/tests/test__socket_dns.py
View file @ 5acba203
......@@ -14,8 +14,7 @@ import traceback
import gevent.socket as gevent_socket
import gevent.testing as greentest
#from gevent.testing.util import log
#from gevent.testing.util import debug
from gevent.testing import util
from gevent.testing import six
from gevent.testing.six import xrange
......@@ -31,6 +30,7 @@ if getattr(resolver, 'pool', None) is not None:
from gevent.testing.sysinfo import RESOLVER_NOT_SYSTEM
from gevent.testing.sysinfo import RESOLVER_DNSPYTHON
from gevent.testing.sysinfo import RESOLVER_ARES
from gevent.testing.sysinfo import PY2
import gevent.testing.timing
......@@ -38,63 +38,8 @@ import gevent.testing.timing
assert gevent_socket.gaierror is socket.gaierror
assert gevent_socket.error is socket.error
TRACE = not util.QUIET and os.getenv('GEVENT_DEBUG', '') == 'trace'
def trace(message, *args, **kwargs):
if TRACE:
util.debug(message, *args, **kwargs)
def _run(function, *args):
try:
result = function(*args)
assert not isinstance(result, BaseException), repr(result)
return result
except Exception as ex:
if TRACE:
traceback.print_exc()
return ex
def format_call(function, args):
args = repr(args)
if args.endswith(',)'):
args = args[:-2] + ')'
try:
module = function.__module__.replace('gevent._socketcommon', 'gevent')
name = function.__name__
return '%s:%s%s' % (module, name, args)
except AttributeError:
return function + args
def trace_fresult(result, seconds):
if not TRACE:
return
if isinstance(result, Exception):
msg = ' -=> raised %r' % (result, )
else:
msg = ' -=> returned %r' % (result, )
time_ms = ' %.2fms' % (seconds * 1000.0, )
space = 80 - len(msg) - len(time_ms)
if space > 0:
space = ' ' * space
else:
space = ''
util.debug(msg + space + time_ms)
def run(function, *args):
trace(format_call(function, args))
delta = time()
result = _run(function, *args)
delta = time() - delta
trace_fresult(result, delta)
return result, delta
def trace_call(result, runtime, function, *args):
util.debug(format_call(function, args))
trace_fresult(result, runtime)
RUN_ALL_HOST_TESTS = os.getenv('GEVENTTEST_RUN_ALL_ETC_HOST_TESTS', '')
def compare_relaxed(a, b):
......@@ -200,13 +145,13 @@ def add(klass, hostname, name=None,
test1.__name__ = 'test_%s_getaddrinfo' % name
_setattr(klass, test1.__name__, test1)
def test2(self):
def test_gethostbyname(self):
x = hostname() if call else hostname
ipaddr = self._test('gethostbyname', x)
if not isinstance(ipaddr, Exception):
self._test('gethostbyaddr', ipaddr)
test2.__name__ = 'test_%s_gethostbyname' % name
_setattr(klass, test2.__name__, test2)
test_gethostbyname.__name__ = 'test_%s_gethostbyname' % name
_setattr(klass, test_gethostbyname.__name__, test_gethostbyname)
def test3(self):
x = hostname() if call else hostname
......@@ -228,11 +173,74 @@ def add(klass, hostname, name=None,
@skipWithoutExternalNetwork("Tries to resolve and compare hostnames/addrinfo")
class TestCase(greentest.TestCase):
maxDiff = None
__timeout__ = 30
switch_expected = None
TRACE = not util.QUIET and os.getenv('GEVENT_DEBUG', '') == 'trace'
verbose_dns = TRACE
def trace(self, message, *args, **kwargs):
if self.TRACE:
util.debug(message, *args, **kwargs)
# Things that the stdlib should never raise and neither should we;
# these indicate bugs in our code and we want to raise them.
REAL_ERRORS = (AttributeError, ValueError, NameError)
def __run_resolver(self, function, args):
try:
result = function(*args)
assert not isinstance(result, BaseException), repr(result)
return result
except self.REAL_ERRORS:
raise
except Exception as ex:
if self.TRACE:
traceback.print_exc()
return ex
def __trace_call(self, result, runtime, function, *args):
util.debug(self.__format_call(function, args))
self.__trace_fresult(result, runtime)
def __format_call(self, function, args):
args = repr(args)
if args.endswith(',)'):
args = args[:-2] + ')'
try:
module = function.__module__.replace('gevent._socketcommon', 'gevent')
name = function.__name__
return '%s:%s%s' % (module, name, args)
except AttributeError:
return function + args
def __trace_fresult(self, result, seconds):
if isinstance(result, Exception):
msg = ' -=> raised %r' % (result, )
else:
msg = ' -=> returned %r' % (result, )
time_ms = ' %.2fms' % (seconds * 1000.0, )
space = 80 - len(msg) - len(time_ms)
if space > 0:
space = ' ' * space
else:
space = ''
util.debug(msg + space + time_ms)
if not TRACE:
def run_resolver(self, function, func_args):
now = time()
return self.__run_resolver(function, func_args), time() - now
else:
def run_resolver(self, function, func_args):
self.trace(self.__format_call(function, func_args))
delta = time()
result = self.__run_resolver(function, func_args)
delta = time() - delta
self.__trace_fresult(result, delta)
return result, delta
def setUp(self):
super(TestCase, self).setUp()
if not self.verbose_dns:
......@@ -256,19 +264,17 @@ class TestCase(greentest.TestCase):
return type(result1) is not type(result2)
return repr(result1) != repr(result2)
def _test(self, func, *args):
gevent_func = getattr(gevent_socket, func)
real_func = monkey.get_original('socket', func)
real_result, time_real = run(real_func, *args)
gevent_result, time_gevent = run(gevent_func, *args)
if util.QUIET and self.should_log_results(real_result, gevent_result):
util.log('')
trace_call(real_result, time_real, real_func, *args)
trace_call(gevent_result, time_gevent, gevent_func, *args)
self.assertEqualResults(real_result, gevent_result, func)
def _test(self, func_name, *args):
gevent_func = getattr(gevent_socket, func_name)
real_func = monkey.get_original('socket', func_name)
if self.verbose_dns and time_gevent > time_real + 0.01 and time_gevent > 0.02:
msg = 'gevent:%s%s took %dms versus %dms stdlib' % (func, args, time_gevent * 1000.0, time_real * 1000.0)
tester = getattr(self, '_run_test_' + func_name, self._run_test_generic)
result = tester(func_name, real_func, gevent_func, args)
_real_result, time_real, gevent_result, time_gevent = result
if self.verbose_dns and time_gevent > time_real + 0.02 and time_gevent > 0.03:
msg = 'gevent:%s%s took %dms versus %dms stdlib' % (
func_name, args, time_gevent * 1000.0, time_real * 1000.0)
if time_gevent > time_real + 1:
word = 'VERY'
......@@ -279,6 +285,16 @@ class TestCase(greentest.TestCase):
return gevent_result
def _run_test_generic(self, func_name, real_func, gevent_func, func_args):
real_result, time_real = self.run_resolver(real_func, func_args)
gevent_result, time_gevent = self.run_resolver(gevent_func, func_args)
if util.QUIET and self.should_log_results(real_result, gevent_result):
util.log('')
self.__trace_call(real_result, time_real, real_func, func_args)
self.__trace_call(gevent_result, time_gevent, gevent_func, func_args)
self.assertEqualResults(real_result, gevent_result, func_name)
return real_result, time_real, gevent_result, time_gevent
def _normalize_result(self, result, func_name):
norm_name = '_normalize_result_' + func_name
if hasattr(self, norm_name):
......@@ -314,24 +330,38 @@ class TestCase(greentest.TestCase):
# On some systems, the hostname can get caps
return (result[0].lower(), [], ips)
def _normalize_result_getaddrinfo(self, result):
if not RESOLVER_NOT_SYSTEM:
IGNORE_CANONICAL_NAME = RESOLVER_ARES # It tends to return them even when not asked for
if not RESOLVER_NOT_SYSTEM:
def _normalize_result_getaddrinfo(self, result):
return result
# On Python 3, the builtin resolver can return SOCK_RAW results, but
# c-ares doesn't do that. So we remove those if we find them.
if hasattr(socket, 'SOCK_RAW') and isinstance(result, list):
result = [x for x in result if x[1] != socket.SOCK_RAW]
if isinstance(result, list):
result.sort()
else:
def _normalize_result_getaddrinfo(self, result):
# On Python 3, the builtin resolver can return SOCK_RAW results, but
# c-ares doesn't do that. So we remove those if we find them.
if hasattr(socket, 'SOCK_RAW') and isinstance(result, list):
result = [x for x in result if x[1] != socket.SOCK_RAW]
if self.IGNORE_CANONICAL_NAME:
result = [
(family, kind, proto, '', addr)
for family, kind, proto, _, addr
in result
]
if isinstance(result, list):
result.sort()
return result
def _normalize_result_getnameinfo(self, result):
return result
NORMALIZE_GHBA_IGNORE_ALIAS = False
def _normalize_result_gethostbyaddr(self, result):
if not RESOLVER_NOT_SYSTEM:
return result
if isinstance(result, tuple):
if self.NORMALIZE_GHBA_IGNORE_ALIAS and isinstance(result, tuple):
# On some systems, a random alias is found in the aliaslist
# by the system resolver, but not by cares and vice versa. We deem the aliaslist
# by the system resolver, but not by cares and vice versa. This is *probably* only the
# case for localhost or things otherwise in /etc/hosts. We deem the aliaslist
# unimportant and discard it.
return (result[0], [], result[2])
return result
......@@ -379,7 +409,27 @@ add(TestTypeError, 25)
class TestHostname(TestCase):
pass
NORMALIZE_GHBA_IGNORE_ALIAS = True
def _ares_normalize_name(self, result):
if RESOLVER_ARES and isinstance(result, tuple):
# The system resolver can return the FQDN, in the first result,
# when given certain configurations. But c-ares
# does not.
name = result[0]
name = name.split('.', 1)[0]
result = (name,) + result[1:]
return result
def _normalize_result_gethostbyaddr(self, result):
result = TestCase._normalize_result_gethostbyaddr(self, result)
return self._ares_normalize_name(result)
def _normalize_result_getnameinfo(self, result):
result = TestCase._normalize_result_getnameinfo(self, result)
if PY2:
# Not sure why we only saw this on Python 2
result = self._ares_normalize_name(result)
return result
add(
TestHostname,
......@@ -405,6 +455,7 @@ class TestLocalhost(TestCase):
return ()
return super(TestLocalhost, self)._normalize_result_getaddrinfo(result)
NORMALIZE_GHBA_IGNORE_ALIAS = True
if greentest.RUNNING_ON_TRAVIS and greentest.PY2 and RESOLVER_NOT_SYSTEM:
def _normalize_result_gethostbyaddr(self, result):
# Beginning in November 2017 after an upgrade to Travis,
......@@ -443,7 +494,7 @@ add(Test1234, '1.2.3.4')
class Test127001(TestCase):
pass
NORMALIZE_GHBA_IGNORE_ALIAS = True
add(
Test127001, '127.0.0.1',
......@@ -504,7 +555,7 @@ class TestEtcHosts(TestCase):
hf = SanitizedHostsFile(os.path.join(os.path.dirname(__file__),
'hosts_file.txt'))
all_etc_hosts = sorted(hf.iter_all_host_addr_pairs())
if len(all_etc_hosts) > cls.MAX_HOSTS and util.QUIET:
if len(all_etc_hosts) > cls.MAX_HOSTS and not RUN_ALL_HOST_TESTS:
all_etc_hosts = all_etc_hosts[:cls.MAX_HOSTS]
for host, ip in all_etc_hosts:
......@@ -518,14 +569,46 @@ TestEtcHosts.populate_tests()
class TestGeventOrg(TestCase):
# For this test to work correctly, it needs to resolve to
# an address with a single A record; round-robin DNS and multiple A records
# may mess it up (subsequent requests---and we always make two---may return
# unequal results). We used to use gevent.org, but that now has multiple A records;
# trying www.gevent.org which is a CNAME to readthedocs.org then worked, but it became
# an alias for python-gevent.readthedocs.org, which is an alias for readthedocs.io,
# and which also has multiple addresses. So we run the resolver twice to try to get
# the different answers, if needed.
HOSTNAME = 'www.gevent.org'
# For this test to work correctly, it needs to resolve to
# an address with a single A record; round-robin DNS and multiple A records
# may mess it up (subsequent requests---and we always make two---may return
# unequal results). We used to use gevent.org, but that now has multiple A records;
# trying www.gevent.org which is a CNAME to readthedocs.org.
if RESOLVER_NOT_SYSTEM:
def _normalize_result_gethostbyname(self, result):
if result == '104.17.33.82':
result = '104.17.32.82'
return result
def _normalize_result_gethostbyname_ex(self, result):
result = super(TestGeventOrg, self)._normalize_result_gethostbyname_ex(result)
if result[0] == 'python-gevent.readthedocs.org':
result = ('readthedocs.io', ) + result[1:]
return result
def test_AI_CANONNAME(self):
self.IGNORE_CANONICAL_NAME = False
result = self._test('getaddrinfo',
# host
TestGeventOrg.HOSTNAME,
# port
None,
# family
socket.AF_INET,
# type
0,
# proto
0,
# flags
socket.AI_CANONNAME)
self.assertEqual(result[0][3], 'readthedocs.io')
add(TestGeventOrg, TestGeventOrg.HOSTNAME)
......@@ -537,21 +620,11 @@ class TestFamily(TestCase):
cls._result = socket.getaddrinfo(TestGeventOrg.HOSTNAME, None)
return cls._result
@unittest.skip(
"In April 2020, the system resolvers started returning INET6 answers on macOS and Travis "
"whereas gevent only returns INET (presumably the RTD configuration changed). "
)
def test_inet(self):
self.assertEqualResults(
self.getresult(),
gevent_socket.getaddrinfo(TestGeventOrg.HOSTNAME, None, socket.AF_INET),
'getaddrinfo')
self._test('getaddrinfo', TestGeventOrg.HOSTNAME, None, socket.AF_INET)
def test_unspec(self):
self.assertEqualResults(
self.getresult(),
gevent_socket.getaddrinfo(TestGeventOrg.HOSTNAME, None, socket.AF_UNSPEC),
'getaddrinfo')
self._test('getaddrinfo', TestGeventOrg.HOSTNAME, None, socket.AF_UNSPEC)
def test_badvalue(self):
self._test('getaddrinfo', TestGeventOrg.HOSTNAME, None, 255)
......@@ -612,7 +685,10 @@ class Test_getaddrinfo(TestCase):
self.assertIs(af, socket.AF_INET)
class TestInternational(TestCase):
pass
if PY2:
# We expect these to raise UnicodeEncodeError, which is a
# subclass of ValueError
REAL_ERRORS = set(TestCase.REAL_ERRORS) - {ValueError,}
# dns python can actually resolve these: it uses
# the 2008 version of idna encoding, whereas on Python 2,
......
src/gevent/tests/test__socket_dns6.py
View file @ 5acba203
......@@ -36,7 +36,7 @@ from gevent.testing.sysinfo import RESOLVER_DNSPYTHON
# by default we skip the tests everywhere else.
class Test6(TestCase):
NORMALIZE_GHBA_IGNORE_ALIAS = True
# host that only has AAAA record
host = 'aaaa.test-ipv6.com'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7