Skip to content

G
gevent
Commit 1c3dd450 authored by Jason Madden's avatar Jason Madden Committed by GitHub
Browse files
Options

Merge pull request #1596 from gevent/issue1012 

Better exception handling for alternate resolvers
parents 5acba203 86c7bd41
Hide whitespace changes
Inline Side-by-side
Showing with 691 additions and 319 deletions
docs/changes/1459.bugfix 0 → 100644
View file @ 1c3dd450
The c-ares and DNSPython resolvers now raise exceptions much more
consistently with the standard resolver. Types and errnos are
substantially more likely to match what the standard library produces.
Depending on the system and configuration, results may not match
exactly, at least with DNSPython. There are still some rare cases
where the system resolver can raise ``herror`` but DNSPython will
raise ``gaierror`` or vice versa. There doesn't seem to be a
deterministic way to account for this. On PyPy, ``getnameinfo`` can
produce results when CPython raises ``socket.error``, and gevent's
DNSPython resolver also raises ``socket.error``.
In addition, several other small discrepancies were addressed,
including handling of localhost and broadcast host names.
src/gevent/_compat.py
View file @ 1c3dd450
......@@ -43,6 +43,8 @@ else:
native_path_types = string_types
thread_mod_name = 'thread'
hostname_types = tuple(set(string_types + (bytearray, bytes)))
def NativeStrIO():
import io
return io.BytesIO() if str is bytes else io.StringIO()
......
src/gevent/resolver/__init__.py
View file @ 1c3dd450
# Copyright (c) 2018 gevent contributors. See LICENSE for details.
from _socket import gaierror
from _socket import error
from _socket import getservbyname
from _socket import getaddrinfo
from _socket import SOCK_STREAM
from _socket import SOCK_DGRAM
from _socket import SOL_TCP
from _socket import AI_CANONNAME
from _socket import EAI_SERVICE
import _socket
from _socket import AF_INET
from _socket import AF_UNSPEC
from _socket import AI_CANONNAME
from _socket import AI_PASSIVE
from _socket import AI_NUMERICHOST
from _socket import EAI_NONAME
from _socket import EAI_SERVICE
from _socket import SOCK_DGRAM
from _socket import SOCK_STREAM
from _socket import SOL_TCP
from _socket import error
from _socket import gaierror
from _socket import getaddrinfo as native_getaddrinfo
from _socket import getnameinfo as native_getnameinfo
from _socket import gethostbyaddr as native_gethostbyaddr
from _socket import gethostbyname as native_gethostbyname
from _socket import gethostbyname_ex as native_gethostbyname_ex
from _socket import getservbyname as native_getservbyname
from gevent._compat import string_types
from gevent._compat import text_type
from gevent._compat import hostname_types
from gevent._compat import integer_types
from gevent._compat import PY3
from gevent._compat import PYPY
from gevent._compat import MAC
from gevent.resolver._addresses import is_ipv6_addr
# Nothing public here.
__all__ = ()
......@@ -34,21 +48,21 @@ def _lookup_port(port, socktype):
if socktype == 0:
origport = port
try:
port = getservbyname(port, 'tcp')
port = native_getservbyname(port, 'tcp')
socktypes.append(SOCK_STREAM)
except error:
port = getservbyname(port, 'udp')
port = native_getservbyname(port, 'udp')
socktypes.append(SOCK_DGRAM)
else:
try:
if port == getservbyname(origport, 'udp'):
if port == native_getservbyname(origport, 'udp'):
socktypes.append(SOCK_DGRAM)
except error:
pass
elif socktype == SOCK_STREAM:
port = getservbyname(port, 'tcp')
port = native_getservbyname(port, 'tcp')
elif socktype == SOCK_DGRAM:
port = getservbyname(port, 'udp')
port = native_getservbyname(port, 'udp')
else:
raise gaierror(EAI_SERVICE, 'Servname not supported for ai_socktype')
except error as ex:
......@@ -68,14 +82,14 @@ def _lookup_port(port, socktype):
socktypes.append(socktype)
return port, socktypes
hostname_types = tuple(set(string_types + (bytearray, bytes)))
def _resolve_special(hostname, family):
if not isinstance(hostname, hostname_types):
raise TypeError("argument 1 must be str, bytes or bytearray, not %s" % (type(hostname),))
if hostname == '':
result = getaddrinfo(None, 0, family, SOCK_DGRAM, 0, AI_PASSIVE)
if hostname in (u'', b''):
result = native_getaddrinfo(None, 0, family, SOCK_DGRAM, 0, AI_PASSIVE)
if len(result) != 1:
raise error('wildcard resolved to multiple address')
return result[0][4][0]
......@@ -84,14 +98,84 @@ def _resolve_special(hostname, family):
class AbstractResolver(object):
HOSTNAME_ENCODING = 'idna' if PY3 else 'ascii'
_LOCAL_HOSTNAMES = (
b'localhost',
b'ip6-localhost',
b'::1',
b'127.0.0.1',
)
_LOCAL_AND_BROADCAST_HOSTNAMES = _LOCAL_HOSTNAMES + (
b'255.255.255.255',
b'<broadcast>',
)
EAI_NONAME_MSG = (
'nodename nor servname provided, or not known'
if MAC else
'Name or service not known'
)
EAI_FAMILY_MSG = (
'ai_family not supported'
)
_KNOWN_ADDR_FAMILIES = {
v
for k, v in vars(_socket).items()
if k.startswith('AF_')
}
_KNOWN_SOCKTYPES = {
v
for k, v in vars(_socket).items()
if k.startswith('SOCK_')
and k not in ('SOCK_CLOEXEC', 'SOCK_MAX_SIZE')
}
@staticmethod
def fixup_gaierror(func):
import functools
@functools.wraps(func)
def resolve(self, *args, **kwargs):
try:
return func(self, *args, **kwargs)
except gaierror as ex:
if ex.args[0] == EAI_NONAME and len(ex.args) == 1:
# dnspython doesn't set an error message
ex.args = (EAI_NONAME, self.EAI_NONAME_MSG)
ex.errno = EAI_NONAME
raise
return resolve
def _hostname_to_bytes(self, hostname):
if isinstance(hostname, text_type):
hostname = hostname.encode(self.HOSTNAME_ENCODING)
elif not isinstance(hostname, (bytes, bytearray)):
raise TypeError('Expected str, bytes or bytearray, not %s' % type(hostname).__name__)
return bytes(hostname)
def gethostbyname(self, hostname, family=AF_INET):
# The native ``gethostbyname`` and ``gethostbyname_ex`` have some different
# behaviour with special names. Notably, ``gethostbyname`` will handle
# both "<broadcast>" and "255.255.255.255", while ``gethostbyname_ex`` refuses to
# handle those; they result in different errors, too. So we can't
# pass those throgh.
hostname = self._hostname_to_bytes(hostname)
if hostname in self._LOCAL_AND_BROADCAST_HOSTNAMES:
return native_gethostbyname(hostname)
hostname = _resolve_special(hostname, family)
return self.gethostbyname_ex(hostname, family)[-1][0]
def gethostbyname_ex(self, hostname, family=AF_INET):
aliases = self._getaliases(hostname, family)
def _gethostbyname_ex(self, hostname_bytes, family):
"""Raise an ``herror`` or a ``gaierror``."""
aliases = self._getaliases(hostname_bytes, family)
addresses = []
tuples = self.getaddrinfo(hostname, 0, family,
tuples = self.getaddrinfo(hostname_bytes, 0, family,
SOCK_STREAM,
SOL_TCP, AI_CANONNAME)
canonical = tuples[0][3]
......@@ -100,9 +184,94 @@ class AbstractResolver(object):
# XXX we just ignore aliases
return (canonical, aliases, addresses)
def gethostbyname_ex(self, hostname, family=AF_INET):
hostname = self._hostname_to_bytes(hostname)
if hostname in self._LOCAL_AND_BROADCAST_HOSTNAMES:
# The broadcast specials aren't handled here, but they may produce
# special errors that are hard to replicate across all systems.
return native_gethostbyname_ex(hostname)
return self._gethostbyname_ex(hostname, family)
def _getaddrinfo(self, host_bytes, port, family, socktype, proto, flags):
raise NotImplementedError
def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
raise NotImplementedError()
host = self._hostname_to_bytes(host) if host is not None else None
if (
not isinstance(host, bytes) # 1, 2
or (flags & AI_NUMERICHOST) # 3
or host in self._LOCAL_HOSTNAMES # 4
or (is_ipv6_addr(host) and host.startswith(b'fe80')) # 5
):
# This handles cases which do not require network access
# 1) host is None
# 2) host is of an invalid type
# 3) AI_NUMERICHOST flag is set
# 4) It's a well-known alias. TODO: This is special casing for c-ares that we don't
# really want to do. It's here because it resolves a discrepancy with the system
# resolvers caught by test cases. In gevent 20.4.0, this only worked correctly on
# Python 3 and not Python 2, by accident.
# 5) host is a link-local ipv6; dnspython returns the wrong
# scope-id for those.
return native_getaddrinfo(host, port, family, socktype, proto, flags)
return self._getaddrinfo(host, port, family, socktype, proto, flags)
def _getaliases(self, hostname, family):
# pylint:disable=unused-argument
return []
def _gethostbyaddr(self, ip_address_bytes):
"""Raises herror."""
raise NotImplementedError
def gethostbyaddr(self, ip_address):
ip_address = _resolve_special(ip_address, AF_UNSPEC)
ip_address = self._hostname_to_bytes(ip_address)
if ip_address in self._LOCAL_AND_BROADCAST_HOSTNAMES:
return native_gethostbyaddr(ip_address)
return self._gethostbyaddr(ip_address)
def _getnameinfo(self, address_bytes, port, sockaddr, flags):
raise NotImplementedError
def getnameinfo(self, sockaddr, flags):
if not isinstance(flags, integer_types):
raise TypeError('an integer is required')
if not isinstance(sockaddr, tuple):
raise TypeError('getnameinfo() argument 1 must be a tuple')
address = sockaddr[0]
address = self._hostname_to_bytes(sockaddr[0])
if address in self._LOCAL_AND_BROADCAST_HOSTNAMES:
return native_getnameinfo(sockaddr, flags)
port = sockaddr[1]
if not isinstance(port, integer_types):
raise TypeError('port must be an integer, not %s' % type(port))
if not PYPY and port >= 65536:
# System resolvers do different things with an
# out-of-bound port; macOS CPython 3.8 raises ``gaierror: [Errno 8]
# nodename nor servname provided, or not known``, while
# manylinux CPython 2.7 appears to ignore it and raises ``error:
# sockaddr resolved to multiple addresses``. TravisCI, at least ot
# one point, successfully resolved www.gevent.org to ``(readthedocs.org, '0')``.
# But c-ares 1.16 would raise ``gaierror(25, 'ARES_ESERVICE: unknown')``.
# Doing this appears to get the expected results on CPython
port = 0
if PYPY and (port < 0 or port >= 65536):
# PyPy seems to always be strict about that and produce the same results
# on all platforms.
raise OverflowError("port must be 0-65535.")
if len(sockaddr) > 2:
# Must be IPv6: (host, port, [flowinfo, [scopeid]])
flowinfo = sockaddr[2]
if flowinfo > 0xfffff:
raise OverflowError("getnameinfo(): flowinfo must be 0-1048575.")
return self._getnameinfo(address, port, sockaddr, flags)
src/gevent/resolver/ares.py
View file @ 1c3dd450
......@@ -5,11 +5,11 @@ c-ares based hostname resolver.
from __future__ import absolute_import, print_function, division
import os
from _socket import getaddrinfo as native_getaddrinfo
from _socket import gaierror
from _socket import herror
from _socket import error
from _socket import EAI_NONAME
from gevent._compat import string_types
from gevent._compat import text_type
from gevent._compat import integer_types
from gevent._compat import PY3
......@@ -24,14 +24,13 @@ from gevent.socket import SOCK_DGRAM
from gevent.socket import SOCK_STREAM
from gevent.socket import SOL_TCP
from gevent.socket import SOL_UDP
from gevent.socket import AI_NUMERICHOST
from gevent._config import config
from gevent._config import AresSettingMixin
from .cares import channel, InvalidIP # pylint:disable=import-error,no-name-in-module
from . import _lookup_port as lookup_port
from . import _resolve_special
from . import AbstractResolver
__all__ = ['Resolver']
......@@ -71,10 +70,12 @@ class Resolver(AbstractResolver):
if they are listed in the hosts file.
- c-ares will not resolve ``broadcasthost``, even if listed in
the hosts file.
the hosts file prior to 2020-04-30.
- This implementation may raise ``gaierror(4)`` where the
system implementation would raise ``herror(1)``.
system implementation would raise ``herror(1)`` or vice versa,
with different error numbers. However, after 2020-04-30, this should be
much reduced.
- The results for ``localhost`` may be different. In
particular, some system resolvers will return more results
......@@ -109,6 +110,12 @@ class Resolver(AbstractResolver):
``getaddrinfo`` is now implemented using the native c-ares function
from c-ares 1.16 or newer.
.. versionchanged:: NEXT
Now ``herror`` and ``gaierror`` are raised more consistently with
the standard library resolver, and have more consistent errno values.
Handling of localhost and broadcast names is now more consistent.
.. _c-ares: http://c-ares.haxx.se
"""
......@@ -147,45 +154,31 @@ class Resolver(AbstractResolver):
self.cares = None
self.fork_watcher.stop()
def gethostbyname(self, hostname, family=AF_INET):
hostname = _resolve_special(hostname, family)
return self.gethostbyname_ex(hostname, family)[-1][0]
def gethostbyname_ex(self, hostname, family=AF_INET):
if PY3:
if isinstance(hostname, str):
hostname = hostname.encode('idna')
elif not isinstance(hostname, (bytes, bytearray)):
raise TypeError('Expected es(idna), not %s' % type(hostname).__name__)
else:
if isinstance(hostname, text_type):
hostname = hostname.encode('ascii')
elif not isinstance(hostname, str):
raise TypeError('Expected string, not %s' % type(hostname).__name__)
def _gethostbyname_ex(self, hostname_bytes, family):
while True:
ares = self.cares
try:
waiter = Waiter(self.hub)
ares.gethostbyname(waiter, hostname, family)
ares.gethostbyname(waiter, hostname_bytes, family)
result = waiter.get()
if not result[-1]:
raise gaierror(-5, 'No address associated with hostname')
raise herror(EAI_NONAME, self.EAI_NONAME_MSG)
return result
except gaierror:
except herror as ex:
if ares is self.cares:
if hostname == b'255.255.255.255':
# The stdlib handles this case in 2.7 and 3.x, but ares does not.
# It is tested by test_socket.py in 3.4.
# HACK: So hardcode the expected return.
return ('255.255.255.255', [], ['255.255.255.255'])
if ex.args[0] == 1:
# Somewhere along the line, the internal
# implementation of gethostbyname_ex changed to invoke
# getaddrinfo() as a first pass, much like we do for ``getnameinfo()``;
# this means it raises a different error for not-found hosts.
raise gaierror(EAI_NONAME, self.EAI_NONAME_MSG)
raise
# "self.cares is not ares" means channel was destroyed (because we were forked)
def _lookup_port(self, port, socktype):
return lookup_port(port, socktype)
def _getaddrinfo(
def __getaddrinfo(
self, host, port,
family=0, socktype=0, proto=0, flags=0,
fill_in_type_proto=True
......@@ -198,19 +191,6 @@ class Resolver(AbstractResolver):
# pylint:disable=too-many-locals,too-many-branches
if isinstance(host, text_type):
host = host.encode('idna')
if not isinstance(host, bytes) or (flags & AI_NUMERICHOST) or host in (
b'localhost', b'ip6-localhost'):
# XXX: Now that we're using ares_getaddrinfo, how much of this is still
# necessary?
# this handles cases which do not require network access
# 1) host is None
# 2) host is of an invalid type
# 3) AI_NUMERICHOST flag is set
# 4) It's a well-known alias. TODO: This is special casing that we don't
# really want to do. It's here because it resolves a discrepancy with the system
# resolvers caught by test cases. In gevent 20.4.0, this only worked correctly on
# Python 3 and not Python 2, by accident.
return native_getaddrinfo(host, port, family, socktype, proto, flags)
if isinstance(port, text_type):
......@@ -240,13 +220,14 @@ class Resolver(AbstractResolver):
result = waiter.get()
if not result:
raise gaierror(-5, 'No address associated with hostname')
raise gaierror(EAI_NONAME, self.EAI_NONAME_MSG)
if fill_in_type_proto:
# c-ares 1.16 DOES NOT fill in socktype or proto in the results,
# ever. It's at least supposed to do that if they were given as
# hints, but it doesn't (https://github.com/c-ares/c-ares/issues/317)
# Sigh.
# The SOL_* constants are another (older?) name for IPPROTO_*
if socktype:
hard_type_proto = [
(socktype, SOL_TCP if socktype == SOCK_STREAM else SOL_UDP),
......@@ -274,33 +255,24 @@ class Resolver(AbstractResolver):
]
return result
def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
def _getaddrinfo(self, host_bytes, port, family, socktype, proto, flags):
while True:
ares = self.cares
try:
return self._getaddrinfo(host, port, family, socktype, proto, flags)
return self.__getaddrinfo(host_bytes, port, family, socktype, proto, flags)
except gaierror:
if ares is self.cares:
raise
def _gethostbyaddr(self, ip_address):
if PY3:
if isinstance(ip_address, str):
ip_address = ip_address.encode('idna')
elif not isinstance(ip_address, (bytes, bytearray)):
raise TypeError('Expected es(idna), not %s' % type(ip_address).__name__)
else:
if isinstance(ip_address, text_type):
ip_address = ip_address.encode('ascii')
elif not isinstance(ip_address, str):
raise TypeError('Expected string, not %s' % type(ip_address).__name__)
def __gethostbyaddr(self, ip_address):
waiter = Waiter(self.hub)
try:
self.cares.gethostbyaddr(waiter, ip_address)
return waiter.get()
except InvalidIP:
result = self._getaddrinfo(ip_address, None, family=AF_UNSPEC, socktype=SOCK_DGRAM)
result = self._getaddrinfo(ip_address, None,
family=AF_UNSPEC, socktype=SOCK_DGRAM,
proto=0, flags=0)
if not result:
raise
_ip_address = result[0][-1][0]
......@@ -312,41 +284,21 @@ class Resolver(AbstractResolver):
self.cares.gethostbyaddr(waiter, _ip_address)
return waiter.get()
def gethostbyaddr(self, ip_address):
ip_address = _resolve_special(ip_address, AF_UNSPEC)
def _gethostbyaddr(self, ip_address_bytes):
while True:
ares = self.cares
try:
return self._gethostbyaddr(ip_address)
except gaierror:
return self.__gethostbyaddr(ip_address_bytes)
except herror:
if ares is self.cares:
raise
def _getnameinfo(self, sockaddr, flags):
if not isinstance(flags, integer_types):
raise TypeError('an integer is required')
if not isinstance(sockaddr, tuple):
raise TypeError('getnameinfo() argument 1 must be a tuple')
address = sockaddr[0]
if not PY3 and isinstance(address, text_type):
address = address.encode('ascii')
if not isinstance(address, string_types):
raise TypeError('sockaddr[0] must be a string, not %s' % type(address).__name__)
port = sockaddr[1]
if not isinstance(port, integer_types):
raise TypeError('port must be an integer, not %s' % type(port))
if len(sockaddr) > 2:
# Must be IPv6: (host, port, [flowinfo, [scopeid]])
flowinfo = sockaddr[2]
if flowinfo > 0xfffff:
raise OverflowError("getnameinfo(): flowinfo must be 0-1048575.")
result = self._getaddrinfo(address, port,
family=AF_UNSPEC, socktype=SOCK_DGRAM, fill_in_type_proto=False)
def __getnameinfo(self, hostname, port, sockaddr, flags):
result = self.__getaddrinfo(
hostname, port,
family=AF_UNSPEC, socktype=SOCK_DGRAM,
proto=0, flags=0,
fill_in_type_proto=False)
if len(result) != 1:
raise error('sockaddr resolved to multiple addresses')
......@@ -368,17 +320,20 @@ class Resolver(AbstractResolver):
# requested, node or service will be NULL ". Python 2
# allows that for the service, but Python 3 raises
# an error. This is tested by test_socket in py 3.4
err = gaierror('nodename nor servname provided, or not known')
err.errno = 8
err = gaierror(EAI_NONAME, self.EAI_NONAME_MSG)
err.errno = EAI_NONAME
raise err
return node, service or '0'
def getnameinfo(self, sockaddr, flags):
def _getnameinfo(self, address_bytes, port, sockaddr, flags):
while True:
ares = self.cares
try:
return self._getnameinfo(sockaddr, flags)
return self.__getnameinfo(address_bytes, port, sockaddr, flags)
except gaierror:
if ares is self.cares:
raise
# # Things that need proper error handling
# gethostbyaddr = AbstractResolver.convert_gaierror_to_herror(AbstractResolver.gethostbyaddr)
src/gevent/resolver/cares.pyx
View file @ 1c3dd450
......@@ -16,8 +16,11 @@ from cpython.mem cimport PyMem_Malloc
from cpython.mem cimport PyMem_Free
from libc.string cimport memset
from gevent._compat import MAC
import _socket
from _socket import gaierror
from _socket import herror
__all__ = ['channel']
......@@ -46,6 +49,31 @@ cdef extern from *:
#ifdef HAVE_NETDB_H
#include <netdb.h>
#endif
#ifndef EAI_ADDRFAMILY
#define EAI_ADDRFAMILY -1
#endif
#ifndef EAI_BADHINTS
#define EAI_BADHINTS -2
#endif
#ifndef EAI_NODATA
#define EAI_NODATA -3
#endif
#ifndef EAI_OVERFLOW
#define EAI_OVERFLOW -4
#endif
#ifndef EAI_PROTOCOL
#define EAI_PROTOCOL -5
#endif
#ifndef EAI_SYSTEM
#define EAI_SYSTEM
#endif
"""
cdef extern from "ares.h":
......@@ -98,7 +126,7 @@ cdef int NI_NAMEREQD = _socket.NI_NAMEREQD
cdef int NI_DGRAM = _socket.NI_DGRAM
_ares_errors = dict([
cdef dict _ares_errors = dict([
(cares.ARES_SUCCESS, 'ARES_SUCCESS'),
(cares.ARES_EADDRGETNETWORKPARAMS, 'ARES_EADDRGETNETWORKPARAMS'),
......@@ -128,9 +156,67 @@ _ares_errors = dict([
(cares.ARES_ETIMEOUT, 'ARES_ETIMEOUT'),
])
cdef dict _ares_to_gai_system = {
cares.ARES_EBADFAMILY: cares.EAI_ADDRFAMILY,
cares.ARES_EBADFLAGS: cares.EAI_BADFLAGS,
cares.ARES_EBADHINTS: cares.EAI_BADHINTS,
cares.ARES_ENOMEM: cares.EAI_MEMORY,
cares.ARES_ENONAME: cares.EAI_NONAME,
cares.ARES_ENOTFOUND: cares.EAI_NONAME,
cares.ARES_ENOTIMP: cares.EAI_FAMILY,
# While EAI_NODATA ("No address associated with nodename") might
# seem to be the natural mapping, typical resolvers actually
# return EAI_NONAME in that same situation; I've yet to find EAI_NODATA
# in a test.
cares.ARES_ENODATA: cares.EAI_NONAME,
# This one gets raised for unknown port/service names.
cares.ARES_ESERVICE: cares.EAI_NONAME if MAC else cares.EAI_SERVICE,
}
cdef _gevent_gai_strerror(code):
cdef const char* err_str
cdef object result = None
cdef int system
try:
system = _ares_to_gai_system[code]
except KeyError:
err_str = cares.ares_strerror(code)
result = '%s: %s' % (_ares_errors.get(code) or code, _as_str(err_str))
else:
err_str = cares.gai_strerror(system)
result = _as_str(err_str)
return result
cdef object _gevent_gaierror_from_status(int ares_status):
cdef object code = _ares_to_gai_system.get(ares_status, ares_status)
cdef object message = _gevent_gai_strerror(ares_status)
return gaierror(code, message)
cdef dict _ares_to_host_system = {
cares.ARES_ENONAME: cares.HOST_NOT_FOUND,
cares.ARES_ENOTFOUND: cares.HOST_NOT_FOUND,
cares.ARES_ENODATA: cares.NO_DATA,
}
cdef _gevent_herror_strerror(code):
cdef const char* err_str
cdef object result = None
cdef int system
try:
system = _ares_to_host_system[code]
except KeyError:
err_str = cares.ares_strerror(code)
result = '%s: %s' % (_ares_errors.get(code) or code, _as_str(err_str))
else:
err_str = cares.hstrerror(system)
result = _as_str(err_str)
return result
cpdef strerror(code):
return '%s: %s' % (_ares_errors.get(code) or code, cares.ares_strerror(code))
cdef object _gevent_herror_from_status(int ares_status):
cdef object code = _ares_to_host_system.get(ares_status, ares_status)
cdef object message = _gevent_herror_strerror(ares_status)
return herror(code, message)
class InvalidIP(ValueError):
......@@ -217,29 +303,6 @@ cdef list _parse_h_addr_list(hostent* host):
return result
cdef void gevent_ares_host_callback(void *arg, int status, int timeouts, hostent* host):
cdef channel channel
cdef object callback
channel, callback = <tuple>arg
Py_DECREF(<tuple>arg)
cdef object host_result
try:
if status or not host:
callback(Result(None, gaierror(status, strerror(status))))
else:
try:
host_result = ares_host_result(host.h_addrtype,
(_as_str(host.h_name),
_parse_h_aliases(host),
_parse_h_addr_list(host)))
except:
callback(Result(None, sys.exc_info()[1]))
else:
callback(Result(host_result))
except:
channel.loop.handle_error(callback, *sys.exc_info())
cdef object _as_str(const char* val):
if not val:
return None
......@@ -258,7 +321,7 @@ cdef void gevent_ares_nameinfo_callback(void *arg, int status, int timeouts, cha
cdef object service
try:
if status:
callback(Result(None, gaierror(status, strerror(status))))
callback(Result(None, _gevent_gaierror_from_status(status)))
else:
node = _as_str(c_node)
service = _as_str(c_service)
......@@ -328,10 +391,10 @@ cdef class channel:
cdef int result = cares.ares_library_init(cares.ARES_LIB_INIT_ALL) # ARES_LIB_INIT_WIN32 -DUSE_WINSOCK?
if result:
raise gaierror(result, strerror(result))
raise gaierror(result, _gevent_gai_strerror(result))
result = cares.ares_init_options(&channel, &options, optmask)
if result:
raise gaierror(result, strerror(result))
raise gaierror(result, _gevent_gai_strerror(result))
self._timer = loop.timer(TIMEOUT, TIMEOUT)
self._watchers = {}
self.channel = channel
......@@ -398,7 +461,7 @@ cdef class channel:
c_servers[length - 1].next = NULL
index = cares.ares_set_servers(self.channel, c_servers)
if index:
raise ValueError(strerror(index))
raise ValueError(_gevent_gai_strerror(index))
finally:
PyMem_Free(c_servers)
......@@ -449,6 +512,30 @@ cdef class channel:
write_fd = cares.ARES_SOCKET_BAD
cares.ares_process_fd(self.channel, read_fd, write_fd)
@staticmethod
cdef void _gethostbyname_or_byaddr_cb(void *arg, int status, int timeouts, hostent* host):
cdef channel channel
cdef object callback
channel, callback = <tuple>arg
Py_DECREF(<tuple>arg)
cdef object host_result
try:
if status or not host:
callback(Result(None, _gevent_herror_from_status(status)))
else:
try:
host_result = ares_host_result(host.h_addrtype,
(_as_str(host.h_name),
_parse_h_aliases(host),
_parse_h_addr_list(host)))
except:
callback(Result(None, sys.exc_info()[1]))
else:
callback(Result(host_result))
except:
channel.loop.handle_error(callback, *sys.exc_info())
def gethostbyname(self, object callback, char* name, int family=AF_INET):
if not self.channel:
raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed')
......@@ -456,7 +543,7 @@ cdef class channel:
cdef object arg = (self, callback)
Py_INCREF(arg)
cares.ares_gethostbyname(self.channel, name, family,
<void*>gevent_ares_host_callback, <void*>arg)
<void*>channel._gethostbyname_or_byaddr_cb, <void*>arg)
def gethostbyaddr(self, object callback, char* addr):
if not self.channel:
......@@ -475,7 +562,8 @@ cdef class channel:
raise InvalidIP(repr(addr))
cdef object arg = (self, callback)
Py_INCREF(arg)
cares.ares_gethostbyaddr(self.channel, addr_packed, length, family, <void*>gevent_ares_host_callback, <void*>arg)
cares.ares_gethostbyaddr(self.channel, addr_packed, length, family,
<void*>channel._gethostbyname_or_byaddr_cb, <void*>arg)
cpdef _getnameinfo(self, object callback, tuple sockaddr, int flags):
if not self.channel:
......@@ -488,8 +576,8 @@ cdef class channel:
if not PyTuple_Check(sockaddr):
raise TypeError('expected a tuple, got %r' % (sockaddr, ))
PyArg_ParseTuple(sockaddr, "si|ii", &hostp, &port, &flowinfo, &scope_id)
if port < 0 or port > 65535:
raise gaierror(-8, 'Invalid value for port: %r' % port)
# if port < 0 or port > 65535:
# raise gaierror(-8, 'Invalid value for port: %r' % port)
cdef int length = _make_sockaddr(hostp, port, flowinfo, scope_id, &sa6)
if length <= 0:
raise InvalidIP(repr(hostp))
......@@ -560,7 +648,7 @@ cdef class channel:
addrs = []
try:
if status != cares.ARES_SUCCESS:
callback(Result(None, gaierror(status, strerror(status))))
callback(Result(None, _gevent_gaierror_from_status(status)))
return
if result.cnames:
# These tend to come in pairs:
......
src/gevent/resolver/dnspython.py
View file @ 1c3dd450
......@@ -63,20 +63,21 @@ from __future__ import absolute_import, print_function, division
import sys
import time
import _socket
from _socket import AI_NUMERICHOST
from _socket import error
from _socket import gaierror
from _socket import herror
from _socket import NI_NUMERICSERV
from _socket import AF_INET
from _socket import AF_INET6
from _socket import AF_UNSPEC
from _socket import EAI_NONAME
from _socket import EAI_FAMILY
import socket
from gevent.resolver import AbstractResolver
from gevent.resolver import hostname_types
from gevent.resolver._hostsfile import HostsFile
from gevent.resolver._addresses import is_ipv6_addr
from gevent.builtins import __import__ as g_import
......@@ -84,6 +85,7 @@ from gevent._compat import string_types
from gevent._compat import iteritems
from gevent._config import config
__all__ = [
'Resolver',
]
......@@ -318,6 +320,7 @@ def _family_to_rdtype(family):
'Address family not supported')
return rdtype
class Resolver(AbstractResolver):
"""
An *experimental* resolver that uses `dnspython`_.
......@@ -344,7 +347,13 @@ class Resolver(AbstractResolver):
.. caution::
Many of the same caveats about DNS results apply here as are documented
for :class:`gevent.resolver.ares.Resolver`.
for :class:`gevent.resolver.ares.Resolver`. In addition, the handling of
symbolic scope IDs in IPv6 addresses passed to ``getaddrinfo`` exhibits
some differences.
On PyPy, ``getnameinfo`` can produce results when CPython raises
``socket.error``, and gevent's DNSPython resolver also
raises ``socket.error``.
.. caution::
......@@ -353,6 +362,12 @@ class Resolver(AbstractResolver):
.. versionadded:: 1.3a2
.. versionchanged:: NEXT
The errors raised are now much more consistent with those
raised by the standard library resolvers.
Handling of localhost and broadcast names is now more consistent.
.. _dnspython: http://www.dnspython.org
"""
......@@ -409,18 +424,20 @@ class Resolver(AbstractResolver):
hostname = ans[0].target
return aliases
def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
if ((host in (u'localhost', b'localhost')
or (is_ipv6_addr(host) and host.startswith('fe80')))
or not isinstance(host, str) or (flags & AI_NUMERICHOST)):
# this handles cases which do not require network access
# 1) host is None
# 2) host is of an invalid type
# 3) host is localhost or a link-local ipv6; dnspython returns the wrong
# scope-id for those.
# 3) AI_NUMERICHOST flag is set
return _socket.getaddrinfo(host, port, family, socktype, proto, flags)
def _getaddrinfo(self, host_bytes, port, family, socktype, proto, flags):
# dnspython really wants the host to be in native format.
if not isinstance(host_bytes, str):
host_bytes = host_bytes.decode(self.HOSTNAME_ENCODING)
if host_bytes == 'ff02::1de:c0:face:8D':
# This is essentially a hack to make stdlib
# test_socket:GeneralModuleTests.test_getaddrinfo_ipv6_basic
# pass. They expect to get back a lowercase ``D``, but
# dnspython does not do that.
# ``test_getaddrinfo_ipv6_scopeid_symbolic`` also expect
# the scopeid to be dropped, but again, dnspython does not
# do that; we cant fix that here so we skip that test.
host_bytes = 'ff02::1de:c0:face:8d'
if family == AF_UNSPEC:
# This tends to raise in the case that a v6 address did not exist
......@@ -433,22 +450,24 @@ class Resolver(AbstractResolver):
# See also https://github.com/gevent/gevent/issues/1012
try:
return _getaddrinfo(host, port, family, socktype, proto, flags)
except socket.gaierror:
return _getaddrinfo(host_bytes, port, family, socktype, proto, flags)
except gaierror:
try:
return _getaddrinfo(host, port, AF_INET6, socktype, proto, flags)
except socket.gaierror:
return _getaddrinfo(host, port, AF_INET, socktype, proto, flags)
return _getaddrinfo(host_bytes, port, AF_INET6, socktype, proto, flags)
except gaierror:
return _getaddrinfo(host_bytes, port, AF_INET, socktype, proto, flags)
else:
return _getaddrinfo(host, port, family, socktype, proto, flags)
def getnameinfo(self, sockaddr, flags):
if (sockaddr
and isinstance(sockaddr, (list, tuple))
and sockaddr[0] in ('::1', '127.0.0.1', 'localhost')):
return _socket.getnameinfo(sockaddr, flags)
if isinstance(sockaddr, (list, tuple)) and not isinstance(sockaddr[0], hostname_types):
raise TypeError("getnameinfo(): illegal sockaddr argument")
try:
return _getaddrinfo(host_bytes, port, family, socktype, proto, flags)
except gaierror as ex:
if ex.args[0] == EAI_NONAME and family not in self._KNOWN_ADDR_FAMILIES:
# It's possible that we got sent an unsupported family. Check
# that.
ex.args = (EAI_FAMILY, self.EAI_FAMILY_MSG)
ex.errno = EAI_FAMILY
raise
def _getnameinfo(self, address_bytes, port, sockaddr, flags):
try:
return resolver._getnameinfo(sockaddr, flags)
except error:
......@@ -458,13 +477,21 @@ class Resolver(AbstractResolver):
# that does this. We conservatively fix it here; this could be expanded later.
return resolver._getnameinfo(sockaddr, NI_NUMERICSERV)
def gethostbyaddr(self, ip_address):
if ip_address in (u'127.0.0.1', u'::1',
b'127.0.0.1', b'::1',
'localhost'):
return _socket.gethostbyaddr(ip_address)
if not isinstance(ip_address, hostname_types):
raise TypeError("argument 1 must be str, bytes or bytearray, not %s" % (type(ip_address),))
return resolver._gethostbyaddr(ip_address)
def _gethostbyaddr(self, ip_address_bytes):
try:
return resolver._gethostbyaddr(ip_address_bytes)
except gaierror as ex:
if ex.args[0] == EAI_NONAME:
# Note: The system doesn't *always* raise herror;
# sometimes the original gaierror propagates through.
# It's impossible to say ahead of time or just based
# on the name which it should be. The herror seems to
# be by far the most common, though.
raise herror(1, "Unknown host")
raise
# Things that need proper error handling
getnameinfo = AbstractResolver.fixup_gaierror(AbstractResolver.getnameinfo)
gethostbyaddr = AbstractResolver.fixup_gaierror(AbstractResolver.gethostbyaddr)
gethostbyname_ex = AbstractResolver.fixup_gaierror(AbstractResolver.gethostbyname_ex)
getaddrinfo = AbstractResolver.fixup_gaierror(AbstractResolver.getaddrinfo)
src/gevent/resolver/libcares.pxd
View file @ 1c3dd450
......@@ -7,6 +7,34 @@ cdef extern from "ares.h":
struct hostent:
pass
# Errors from getaddrinfo
int EAI_ADDRFAMILY # The specified network host does not have
# any network addresses in the requested address family (Linux)
int EAI_AGAIN # temporary failure in name resolution
int EAI_BADFLAGS # invalid value for ai_flags
int EAI_BADHINTS # invalid value for hints (macOS only)
int EAI_FAIL # non-recoverable failure in name resolution
int EAI_FAMILY # ai_family not supported
int EAI_MEMORY # memory allocation failure
int EAI_NODATA # The specified network host exists, but does not have
# any network addresses defined. (Linux)
int EAI_NONAME # hostname or servname not provided, or not known
int EAI_OVERFLOW # argument buffer overflow (macOS only)
int EAI_PROTOCOL # resolved protocol is unknown (macOS only)
int EAI_SERVICE # servname not supported for ai_socktype
int EAI_SOCKTYPE # ai_socktype not supported
int EAI_SYSTEM # system error returned in errno (macOS and Linux)
char* gai_strerror(int ecode)
# Errors from gethostbyname and gethostbyaddr
int HOST_NOT_FOUND
int TRY_AGAIN
int NO_RECOVERY
int NO_DATA
char* hstrerror(int err)
struct ares_options:
int flags
void* sock_state_cb
......
src/gevent/testing/monkey_test.py
View file @ 1c3dd450
......@@ -5,6 +5,9 @@ import os
test_filename = sys.argv[1]
del sys.argv[1]
if test_filename == 'test_urllib2_localnet.py' and os.environ.get('APPVEYOR'):
os.environ['GEVENT_DEBUG'] = 'TRACE'
print('Running with patch_all(): %s' % (test_filename,))
from gevent import monkey
......
src/gevent/testing/patched_tests_setup.py
View file @ 1c3dd450
......@@ -17,6 +17,7 @@ from .sysinfo import RUNNING_ON_APPVEYOR as APPVEYOR
from .sysinfo import RUNNING_ON_TRAVIS as TRAVIS
from .sysinfo import RESOLVER_NOT_SYSTEM as ARES
from .sysinfo import RESOLVER_ARES
from .sysinfo import RESOLVER_DNSPYTHON
from .sysinfo import RUNNING_ON_CI
from .sysinfo import RUN_COVERAGE
......@@ -1231,6 +1232,13 @@ if PY38:
'test_ssl.BasicSocketTests.test_parse_cert_CVE_2013_4238',
]
if RESOLVER_DNSPYTHON:
disabled_tests += [
# This does two things DNS python doesn't. First, it sends it
# capital letters and expects them to be returned lowercase.
# Second, it expects the symbolic scopeid to be stripped from the end.
'test_socket.GeneralModuleTests.test_getaddrinfo_ipv6_scopeid_symbolic',
]
# if 'signalfd' in os.environ.get('GEVENT_BACKEND', ''):
# # tests that don't interact well with signalfd
......
src/gevent/tests/known_failures.py
View file @ 1c3dd450
......@@ -296,22 +296,6 @@ class Definitions(DefinitionsBase):
run_alone=APPVEYOR,
)
test__socket_dns = Flaky(
"""
A few errors and differences:
AssertionError: ('255.255.255.255', 'http') != gaierror(-2,) # DNS Python
AssertionError: ('255.255.255.255', 'http') != gaierror(4, 'ARES_ENOTFOUND: Domain name not found')
AssertionError: OverflowError('port must be 0-65535.',) != ('readthedocs.org', '65535')
AssertionError: Lists differ:
(10, 1, 6, '', ('2607:f8b0:4004:810::200e', 80, 0L, 0L))
(10, 1, 6, '', ('2607:f8b0:4004:805::200e', 80, 0, 0))
Somehow it seems most of these are fixed with PyPy3.6-7 under dnspython,
(once we commented out TestHostname)?
""",
when=RESOLVER_NOT_SYSTEM | PY3
)
test__monkey_sigchld_2 = Ignored(
"""
This hangs for no apparent reason when run by the testrunner,
......
src/gevent/tests/test__ares_timeout.py
View file @ 1c3dd450
......@@ -35,7 +35,7 @@ class TestTimeout(greentest.TestCase):
r = Resolver(servers=[address[0]], timeout=0.001, tries=1,
udp_port=address[-1])
with self.assertRaisesRegex(socket.gaierror, "ARES_ETIMEOUT"):
with self.assertRaisesRegex(socket.herror, "ARES_ETIMEOUT"):
r.gethostbyname('www.google.com')
......
src/gevent/tests/test__socket_dns.py
View file @ 1c3dd450
......@@ -32,6 +32,7 @@ from gevent.testing.sysinfo import RESOLVER_NOT_SYSTEM
from gevent.testing.sysinfo import RESOLVER_DNSPYTHON
from gevent.testing.sysinfo import RESOLVER_ARES
from gevent.testing.sysinfo import PY2
from gevent.testing.sysinfo import PYPY
import gevent.testing.timing
......@@ -139,11 +140,11 @@ def add(klass, hostname, name=None,
name = re.sub(r'[^\w]+', '_', repr(hostname))
assert name, repr(hostname)
def test1(self):
def test_getaddrinfo_http(self):
x = hostname() if call else hostname
self._test('getaddrinfo', x, 'http')
test1.__name__ = 'test_%s_getaddrinfo' % name
_setattr(klass, test1.__name__, test1)
test_getaddrinfo_http.__name__ = 'test_%s_getaddrinfo_http' % name
_setattr(klass, test_getaddrinfo_http.__name__, test_getaddrinfo_http)
def test_gethostbyname(self):
x = hostname() if call else hostname
......@@ -265,6 +266,11 @@ class TestCase(greentest.TestCase):
return repr(result1) != repr(result2)
def _test(self, func_name, *args):
"""
Runs the function *func_name* with *args* and compares gevent and the system.
Returns the gevent result.
"""
gevent_func = getattr(gevent_socket, func_name)
real_func = monkey.get_original('socket', func_name)
......@@ -301,51 +307,72 @@ class TestCase(greentest.TestCase):
return getattr(self, norm_name)(result)
return result
def _normalize_result_gethostbyname_ex(self, result):
# Often the second and third part of the tuple (hostname, aliaslist, ipaddrlist)
# can be in different orders if we're hitting different servers,
# or using the native and ares resolvers due to load-balancing techniques.
# We sort them.
if not RESOLVER_NOT_SYSTEM or isinstance(result, BaseException):
return result
# result[1].sort() # we wind up discarding this
# On Py2 in test_russion_gethostbyname_ex, this
# is actually an integer, for some reason. In TestLocalhost.tets__ip6_localhost,
# the result isn't this long (maybe an error?).
try:
result[2].sort()
except AttributeError:
pass
except IndexError:
return result
# On some systems, a random alias is found in the aliaslist
# by the system resolver, but not by cares, and vice versa. We deem the aliaslist
# unimportant and discard it.
# On some systems (Travis CI), the ipaddrlist for 'localhost' can come back
# with two entries 127.0.0.1 (presumably two interfaces?) for c-ares
ips = result[2]
if ips == ['127.0.0.1', '127.0.0.1']:
ips = ['127.0.0.1']
# On some systems, the hostname can get caps
return (result[0].lower(), [], ips)
IGNORE_CANONICAL_NAME = RESOLVER_ARES # It tends to return them even when not asked for
NORMALIZE_GAI_IGNORE_CANONICAL_NAME = RESOLVER_ARES # It tends to return them even when not asked for
if not RESOLVER_NOT_SYSTEM:
def _normalize_result_getaddrinfo(self, result):
return result
def _normalize_result_gethostbyname_ex(self, result):
return result
else:
def _normalize_result_gethostbyname_ex(self, result):
# Often the second and third part of the tuple (hostname, aliaslist, ipaddrlist)
# can be in different orders if we're hitting different servers,
# or using the native and ares resolvers due to load-balancing techniques.
# We sort them.
if isinstance(result, BaseException):
return result
# result[1].sort() # we wind up discarding this
# On Py2 in test_russion_gethostbyname_ex, this
# is actually an integer, for some reason. In TestLocalhost.tets__ip6_localhost,
# the result isn't this long (maybe an error?).
try:
result[2].sort()
except AttributeError:
pass
except IndexError:
return result
# On some systems, a random alias is found in the aliaslist
# by the system resolver, but not by cares, and vice versa. We deem the aliaslist
# unimportant and discard it.
# On some systems (Travis CI), the ipaddrlist for 'localhost' can come back
# with two entries 127.0.0.1 (presumably two interfaces?) for c-ares
ips = result[2]
if ips == ['127.0.0.1', '127.0.0.1']:
ips = ['127.0.0.1']
# On some systems, the hostname can get caps
return (result[0].lower(), [], ips)
def _normalize_result_getaddrinfo(self, result):
# Result is a list
# (family, socktype, proto, canonname, sockaddr)
# e.g.,
# (AF_INET, SOCK_STREAM, IPPROTO_TCP, 'readthedocs.io', (127.0.0.1, 80))
if isinstance(result, BaseException):
return result
# On Python 3, the builtin resolver can return SOCK_RAW results, but
# c-ares doesn't do that. So we remove those if we find them.
if hasattr(socket, 'SOCK_RAW') and isinstance(result, list):
result = [x for x in result if x[1] != socket.SOCK_RAW]
if self.IGNORE_CANONICAL_NAME:
# Likewise, on certain Linux systems, even on Python 2, IPPROTO_SCTP (132)
# results may be returned --- but that may not even have a constant in the
# socket module! So to be safe, we strip out anything that's not
# SOCK_STREAM or SOCK_DGRAM
if isinstance(result, list):
result = [
x
for x in result
if x[1] in (socket.SOCK_STREAM, socket.SOCK_DGRAM)
and x[2] in (socket.IPPROTO_TCP, socket.IPPROTO_UDP)
]
if self.NORMALIZE_GAI_IGNORE_CANONICAL_NAME:
result = [
(family, kind, proto, '', addr)
for family, kind, proto, _, addr
in result
]
if isinstance(result, list):
result.sort()
return result
......@@ -366,17 +393,30 @@ class TestCase(greentest.TestCase):
return (result[0], [], result[2])
return result
def assertEqualResults(self, real_result, gevent_result, func):
def _compare_exceptions(self, real_result, gevent_result, func_name):
msg = (func_name, 'system:', repr(real_result), 'gevent:', repr(gevent_result))
self.assertIs(type(gevent_result), type(real_result), msg)
if isinstance(real_result, TypeError):
return
if PYPY and isinstance(real_result, socket.herror):
# PyPy doesn't do errno or multiple arguments in herror;
# it just puts a string like 'host lookup failed: <thehost>';
# it must be doing that manually.
return
self.assertEqual(real_result.args, gevent_result.args, msg)
if hasattr(real_result, 'errno'):
self.assertEqual(real_result.errno, gevent_result.errno)
def assertEqualResults(self, real_result, gevent_result, func_name):
errors = (socket.gaierror, socket.herror, TypeError)
if isinstance(real_result, errors) and isinstance(gevent_result, errors):
if type(real_result) is not type(gevent_result):
util.log('WARNING: error type mismatch: %r (gevent) != %r (stdlib)',
gevent_result, real_result,
color='warning')
self._compare_exceptions(real_result, gevent_result, func_name)
return
real_result = self._normalize_result(real_result, func)
gevent_result = self._normalize_result(gevent_result, func)
real_result = self._normalize_result(real_result, func_name)
gevent_result = self._normalize_result(gevent_result, func_name)
real_result_repr = repr(real_result)
gevent_result_repr = repr(gevent_result)
......@@ -385,9 +425,8 @@ class TestCase(greentest.TestCase):
if relaxed_is_equal(gevent_result, real_result):
return
# If we're using the ares resolver, allow the real resolver to generate an
# error that the ares resolver actually gets an answer to.
# If we're using a different resolver, allow the real resolver to generate an
# error that the gevent resolver actually gets an answer to.
if (
RESOLVER_NOT_SYSTEM
and isinstance(real_result, errors)
......@@ -395,6 +434,19 @@ class TestCase(greentest.TestCase):
):
return
# On PyPy, socket.getnameinfo() can produce results even when the hostname resolves to
# multiple addresses, like www.gevent.org does. DNSPython (and c-ares?) don't do that,
# they refuse to pick a name and raise ``socket.error``
if (
RESOLVER_NOT_SYSTEM
and PYPY
and func_name == 'getnameinfo'
and isinstance(gevent_result, socket.error)
and not isinstance(real_result, socket.error)
):
return
# From 2.7 on, assertEqual does a better job highlighting the results than we would
# because it calls assertSequenceEqual, which highlights the exact
# difference in the tuple
......@@ -411,24 +463,25 @@ add(TestTypeError, 25)
class TestHostname(TestCase):
NORMALIZE_GHBA_IGNORE_ALIAS = True
def _ares_normalize_name(self, result):
if RESOLVER_ARES and isinstance(result, tuple):
def __normalize_name(self, result):
if (RESOLVER_ARES or RESOLVER_DNSPYTHON) and isinstance(result, tuple):
# The system resolver can return the FQDN, in the first result,
# when given certain configurations. But c-ares
# does not.
# when given certain configurations. But c-ares and dnspython
# do not.
name = result[0]
name = name.split('.', 1)[0]
result = (name,) + result[1:]
return result
def _normalize_result_gethostbyaddr(self, result):
result = TestCase._normalize_result_gethostbyaddr(self, result)
return self._ares_normalize_name(result)
return self.__normalize_name(result)
def _normalize_result_getnameinfo(self, result):
result = TestCase._normalize_result_getnameinfo(self, result)
if PY2:
# Not sure why we only saw this on Python 2
result = self._ares_normalize_name(result)
result = self.__normalize_name(result)
return result
add(
......@@ -469,20 +522,36 @@ class TestLocalhost(TestCase):
add(
TestLocalhost, 'ip6-localhost',
skip=greentest.RUNNING_ON_TRAVIS,
skip_reason="ares fails here, for some reason, presumably a badly "
"configured /etc/hosts"
skip=RESOLVER_DNSPYTHON, # XXX: Fix these.
skip_reason="Can return gaierror(-2)"
)
add(
TestLocalhost, 'localhost',
skip=greentest.RUNNING_ON_TRAVIS,
skip_reason="Beginning Dec 1 2017, ares started returning ip6-localhost "
"instead of localhost"
skip_reason="Can return gaierror(-2)"
)
def dnspython_lenient_compare_exceptions(self, real_result, gevent_result, func_name):
try:
TestCase._compare_exceptions(self, real_result, gevent_result, func_name)
except AssertionError:
# Allow gethostbyaddr to raise different things in a few rare cases.
if (
func_name != 'gethostbyaddr'
or type(real_result) not in (socket.herror, socket.gaierror)
or type(gevent_result) not in (socket.herror, socket.gaierror)
):
raise
util.log('WARNING: error type mismatch for %s: %r (gevent) != %r (stdlib)',
func_name,
gevent_result, real_result,
color='warning')
class TestNonexistent(TestCase):
pass
if RESOLVER_DNSPYTHON:
_compare_exceptions = dnspython_lenient_compare_exceptions
add(TestNonexistent, 'nonexistentxxxyyy')
......@@ -498,9 +567,9 @@ class Test127001(TestCase):
add(
Test127001, '127.0.0.1',
skip=greentest.RUNNING_ON_TRAVIS,
skip_reason="Beginning Dec 1 2017, ares started returning ip6-localhost "
"instead of localhost"
# skip=RESOLVER_DNSPYTHON,
# skip_reason="Beginning Dec 1 2017, ares started returning ip6-localhost "
# "instead of localhost"
)
......@@ -508,8 +577,9 @@ add(
class TestBroadcast(TestCase):
switch_expected = False
if RESOLVER_NOT_SYSTEM:
# ares and dnspython raises errors for broadcasthost/255.255.255.255
if RESOLVER_DNSPYTHON:
# dnspython raises errors for broadcasthost/255.255.255.255, but the system
# can resolve it.
@unittest.skip('ares raises errors for broadcasthost/255.255.255.255')
def test__broadcast__gethostbyaddr(self):
......@@ -593,33 +663,38 @@ class TestGeventOrg(TestCase):
return result
def test_AI_CANONNAME(self):
self.IGNORE_CANONICAL_NAME = False
result = self._test('getaddrinfo',
# host
TestGeventOrg.HOSTNAME,
# port
None,
# family
socket.AF_INET,
# type
0,
# proto
0,
# flags
socket.AI_CANONNAME)
self.assertEqual(result[0][3], 'readthedocs.io')
# Not all systems support AI_CANONNAME; notably tha manylinux
# resolvers *sometimes* do not. Specifically, sometimes they
# provide the canonical name *only* on the first result.
args = (
# host
TestGeventOrg.HOSTNAME,
# port
None,
# family
socket.AF_INET,
# type
0,
# proto
0,
# flags
socket.AI_CANONNAME
)
gevent_result = gevent_socket.getaddrinfo(*args)
self.assertEqual(gevent_result[0][3], 'readthedocs.io')
real_result = socket.getaddrinfo(*args)
self.NORMALIZE_GAI_IGNORE_CANONICAL_NAME = not all(r[3] for r in real_result)
try:
self.assertEqualResults(real_result, gevent_result, 'getaddrinfo')
finally:
del self.NORMALIZE_GAI_IGNORE_CANONICAL_NAME
add(TestGeventOrg, TestGeventOrg.HOSTNAME)
class TestFamily(TestCase):
maxDiff = None
@classmethod
def getresult(cls):
if not hasattr(cls, '_result'):
cls._result = socket.getaddrinfo(TestGeventOrg.HOSTNAME, None)
return cls._result
def test_inet(self):
self._test('getaddrinfo', TestGeventOrg.HOSTNAME, None, socket.AF_INET)
......@@ -631,6 +706,7 @@ class TestFamily(TestCase):
self._test('getaddrinfo', TestGeventOrg.HOSTNAME, None, 255000)
self._test('getaddrinfo', TestGeventOrg.HOSTNAME, None, -1)
@unittest.skipIf(RESOLVER_DNSPYTHON, "Raises the wrong errno")
def test_badtype(self):
self._test('getaddrinfo', TestGeventOrg.HOSTNAME, 'x')
......@@ -690,12 +766,23 @@ class TestInternational(TestCase):
# subclass of ValueError
REAL_ERRORS = set(TestCase.REAL_ERRORS) - {ValueError,}
if RESOLVER_ARES:
def test_russian_getaddrinfo_http(self):
# And somehow, test_russion_getaddrinfo_http (``getaddrinfo(name, 'http')``)
# manages to work with recent versions of Python 2, but our preemptive encoding
# to ASCII causes it to fail with the c-ares resolver; but only that one test out of
# all of them.
self.skipTest("ares fails to encode.")
# dns python can actually resolve these: it uses
# the 2008 version of idna encoding, whereas on Python 2,
# with the default resolver, it tries to encode to ascii and
# raises a UnicodeEncodeError. So we get different results.
add(TestInternational, u'президент.рф', 'russian',
skip=(PY2 and RESOLVER_DNSPYTHON), skip_reason="dnspython can actually resolve these")
skip=(PY2 and RESOLVER_DNSPYTHON),
skip_reason="dnspython can actually resolve these")
add(TestInternational, u'президент.рф'.encode('idna'), 'idna')
@skipWithoutExternalNetwork("Tries to resolve and compare hostnames/addrinfo")
......@@ -754,13 +841,17 @@ class TestInterrupted_gethostbyname(gevent.testing.timing.AbstractGenericWaitTes
class TestBadName(TestCase):
pass
if RESOLVER_DNSPYTHON:
_compare_exceptions = dnspython_lenient_compare_exceptions
add(TestBadName, 'xxxxxxxxxxxx')
add(TestBadName, 'xxxxxxxxxxxx')
class TestBadIP(TestCase):
pass
if RESOLVER_DNSPYTHON:
_compare_exceptions = dnspython_lenient_compare_exceptions
add(TestBadIP, '1.2.3.400')
......@@ -840,10 +931,6 @@ class TestInvalidPort(TestCase):
def test_typeerror_str(self):
self._test('getnameinfo', ('www.gevent.org', 'x'), 0)
@unittest.skipIf(RESOLVER_DNSPYTHON,
"System resolvers do funny things with this: macOS raises gaierror, "
"Travis CI returns (readthedocs.org, '0'). It's hard to match that exactly. "
"dnspython raises OverflowError.")
def test_overflow_port_too_large(self):
self._test('getnameinfo', ('www.gevent.org', 65536), 0)
......
src/gevent/tests/test__socket_dns6.py
View file @ 1c3dd450
......@@ -9,9 +9,8 @@ import gevent.testing as greentest
from gevent.tests.test__socket_dns import TestCase, add
from gevent.testing.sysinfo import OSX
from gevent.testing.sysinfo import RESOLVER_NOT_SYSTEM
from gevent.testing.sysinfo import RESOLVER_DNSPYTHON
from gevent.testing.sysinfo import PYPY
# We can't control the DNS servers on CI (or in general...)
......@@ -40,12 +39,22 @@ class Test6(TestCase):
# host that only has AAAA record
host = 'aaaa.test-ipv6.com'
if not OSX or RESOLVER_DNSPYTHON:
def _test(self, *args): # pylint:disable=arguments-differ
raise unittest.SkipTest(
"Only known to work on jamadden's machine. "
"Please help investigate and make DNS tests more robust."
)
def _normalize_result_gethostbyaddr(self, result):
# This part of the test is effectively disabled. There are multiple address
# that resolve and which ones you get depend on the settings
# of the system and ares. They don't match exactly.
return ()
if not OSX and RESOLVER_DNSPYTHON:
# It raises gaierror instead of socket.error,
# which is not great and leads to failures.
def _run_test_getnameinfo(self, *_args):
return (), 0, (), 0
def _run_test_gethostbyname(self, *_args):
raise unittest.SkipTest("gethostbyname[_ex] does not support IPV6")
_run_test_gethostbyname_ex = _run_test_gethostbyname
def test_empty(self):
self._test('getaddrinfo', self.host, 'http')
......@@ -63,12 +72,16 @@ class Test6(TestCase):
class Test6_google(Test6):
host = 'ipv6.google.com'
def _normalize_result_getnameinfo(self, result):
if greentest.RUNNING_ON_CI and RESOLVER_NOT_SYSTEM:
# Disabled, there are multiple possibilities
# and we can get different ones, rarely.
if greentest.RUNNING_ON_CI:
# Disabled, there are multiple possibilities
# and we can get different ones. Even the system resolvers
# can go round-robin and provide different answers.
def _normalize_result_getnameinfo(self, result):
return ()
return result
if PYPY:
# PyPy tends to be especially problematic in that area.
_normalize_result_getaddrinfo = _normalize_result_getnameinfo
add(Test6, Test6.host)
add(Test6_google, Test6_google.host)
......@@ -79,13 +92,7 @@ class Test6_ds(Test6):
# host that has both A and AAAA records
host = 'ds.test-ipv6.com'
def _normalize_result_gethostbyaddr(self, result):
# This test is effectively disabled. There are multiple address
# that resolve and which ones you get depend on the settings
# of the system and ares. They don't match exactly.
return ()
_normalize_result_gethostbyname = _normalize_result_gethostbyaddr
_normalize_result_gethostbyname = Test6._normalize_result_gethostbyaddr
add(Test6_ds, Test6_ds.host)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7