Commit 29fe22c8 authored by Jason Madden's avatar Jason Madden

Add support for /etc/hosts to dnspython resolver.

Based on code from eventlet, but heavily refactored and modified:

- Split host parsing into its own class, re-used from the test case
- Hosts handle PTR lookups for ipv4
- Some bug fixes around what should be strings where.
- Allow strings for the rdtype paramater
- Watch last modified time instead of reloading the file at fixed
  intervals.

More of our DNS tests pass now (notable TestEtcHosts is now enabled
for this resolver), and we tend to generate more conformant results
than the ares resolver (by observation, not tested).
parent 6fffdb4c
......@@ -45,6 +45,9 @@ from gevent.hub import InvalidSwitchError
__implements__ = ['Queue', 'PriorityQueue', 'LifoQueue']
__extensions__ = ['JoinableQueue', 'Channel']
__imports__ = ['Empty', 'Full']
if hasattr(__queue__, 'SimpleQueue'):
__imports__.append('SimpleQueue') # New in 3.7
SimpleQueue = __queue__.SimpleQueue
__all__ = __implements__ + __extensions__ + __imports__
......
......@@ -63,8 +63,12 @@ def _lookup_port(port, socktype):
socktypes.append(socktype)
return port, socktypes
hostname_types = tuple(set(string_types + (bytearray, bytes)))
def _resolve_special(hostname, family):
if not isinstance(hostname, hostname_types):
raise TypeError("argument 1 must be str, bytes or bytearray, not %s" % (type(hostname),))
if hostname == '':
result = getaddrinfo(None, 0, family, SOCK_DGRAM, 0, AI_PASSIVE)
if len(result) != 1:
......@@ -80,7 +84,7 @@ class AbstractResolver(object):
return self.gethostbyname_ex(hostname, family)[-1][0]
def gethostbyname_ex(self, hostname, family=AF_INET):
aliases = []
aliases = self._getaliases(hostname, family)
addresses = []
tuples = self.getaddrinfo(hostname, 0, family,
SOCK_STREAM,
......@@ -93,3 +97,7 @@ class AbstractResolver(object):
def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
raise NotImplementedError()
def _getaliases(self, hostname, family):
# pylint:disable=unused-argument
return []
This diff is collapsed.
This diff is collapsed.
......@@ -50,7 +50,7 @@ def format_call(function, args):
if args.endswith(',)'):
args = args[:-2] + ')'
try:
module = function.__module__.replace('gevent.socket', 'gevent').replace('_socket', 'stdlib')
module = function.__module__.replace('gevent._socketcommon', 'gevent')
name = function.__name__
return '%s:%s%s' % (module, name, args)
except AttributeError:
......@@ -166,13 +166,16 @@ def relaxed_is_equal(a, b):
return all(relaxed_is_equal(x, y) for (x, y) in zip(a, b))
def add(klass, hostname, name=None):
def add(klass, hostname, name=None,
skip=None, skip_reason=None):
call = callable(hostname)
def _setattr(k, n, v):
def _setattr(k, n, func):
if skip:
func = greentest.skipIf(skip, skip_reason,)(func)
if not hasattr(k, n):
setattr(k, n, v)
setattr(k, n, func)
if name is None:
if call:
......@@ -283,7 +286,8 @@ class TestCase(greentest.TestCase):
ips = result[2]
if ips == ['127.0.0.1', '127.0.0.1']:
ips = ['127.0.0.1']
return (result[0], [], ips)
# On some systems, the hostname can get caps
return (result[0].lower(), [], ips)
def _normalize_result_getaddrinfo(self, result):
if not RESOLVER_NOT_SYSTEM:
......@@ -345,8 +349,6 @@ add(TestTypeError, None)
add(TestTypeError, 25)
@unittest.skipIf(RESOLVER_DNSPYTHON,
"This commonly needs /etc/hosts to function, and dnspython doesn't do that.")
class TestHostname(TestCase):
pass
......@@ -423,33 +425,45 @@ class TestBroadcast(TestCase):
add(TestBroadcast, '<broadcast>')
@unittest.skipIf(RESOLVER_DNSPYTHON, "/etc/hosts completely ignored under dnspython")
from gevent.resolver.dnspython import HostsFile
class SanitizedHostsFile(HostsFile):
def iter_all_host_addr_pairs(self):
for name, addr in super(SanitizedHostsFile, self).iter_all_host_addr_pairs():
if (RESOLVER_NOT_SYSTEM
and (name.endswith('local') # ignore bonjour, ares can't find them
# ignore common aliases that ares can't find
or addr == '255.255.255.255'
or name == 'broadcasthost'
# We get extra results from some impls, like OS X
# it returns DGRAM results
or name == 'localhost')):
continue
if name.endswith('local'):
# These can only be found if bonjour is running,
# and are very slow to do so with the system resolver on OS X
continue
yield name, addr
class TestEtcHosts(TestCase):
pass
try:
with open('/etc/hosts') as f:
etc_hosts = f.read()
except IOError:
etc_hosts = ''
for ip, host in re.findall(r'^\s*(\d+\.\d+\.\d+\.\d+)\s+([^\s]+)', etc_hosts, re.M)[:10]:
if (RESOLVER_NOT_SYSTEM
and (host.endswith('local') # ignore bonjour, ares can't find them
# ignore common aliases that ares can't find
or ip == '255.255.255.255'
or host == 'broadcasthost'
# We get extra results from some impls, like OS X
# it returns DGRAM results
or host == 'localhost')):
continue
if host.endswith('local'):
# These can only be found if bonjour is running,
# and are very slow to do so with the system resolver on OS X
continue
add(TestEtcHosts, host)
add(TestEtcHosts, ip)
del host, ip
MAX_HOSTS = os.getenv('GEVENTTEST_MAX_ETC_HOSTS', 10)
@classmethod
def populate_tests(cls):
hf = SanitizedHostsFile(os.path.join(os.path.dirname(__file__),
'hosts_file.txt'))
all_etc_hosts = sorted(hf.iter_all_host_addr_pairs())
if len(all_etc_hosts) > cls.MAX_HOSTS and not DEBUG:
all_etc_hosts = all_etc_hosts[:cls.MAX_HOSTS]
for host, ip in all_etc_hosts:
add(cls, host)
add(cls, ip)
TestEtcHosts.populate_tests()
class TestGeventOrg(TestCase):
......@@ -544,12 +558,12 @@ class Test_getaddrinfo(TestCase):
class TestInternational(TestCase):
pass
if not (PY2 and RESOLVER_DNSPYTHON):
# dns python can actually resolve these: it uses
# the 2008 version of idna encoding, whereas on Python 2,
# with the default resolver, it tries to encode to ascii and
# raises a UnicodeEncodeError. So we get different results.
add(TestInternational, u'президент.рф', 'russian')
# dns python can actually resolve these: it uses
# the 2008 version of idna encoding, whereas on Python 2,
# with the default resolver, it tries to encode to ascii and
# raises a UnicodeEncodeError. So we get different results.
add(TestInternational, u'президент.рф', 'russian',
skip=(PY2 and RESOLVER_DNSPYTHON), skip_reason="dnspython can actually resolve these")
add(TestInternational, u'президент.рф'.encode('idna'), 'idna')
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment