Merge pull request #1094 from gevent/dnspython-hosts

Add support for /etc/hosts to dnspython resolver.

appveyor.yml

@@ -131,7 +131,7 @@ install:
   # pip will build them from source using the MSVC compiler matching the
   # target Python version and architecture
   # Note that psutil won't build under PyPy on Windows.
-  - "%CMD_IN_ENV% pip install -U setuptools wheel cython greenlet cffi"
+  - "%CMD_IN_ENV% pip install -U setuptools wheel cython greenlet cffi dnspython idna"
 
   - ps:
       if ("${env:PYTHON_ID}" -ne "pypy") {

src/gevent/queue.py

@@ -45,6 +45,9 @@ from gevent.hub import InvalidSwitchError
 __implements__ = ['Queue', 'PriorityQueue', 'LifoQueue']
 __extensions__ = ['JoinableQueue', 'Channel']
 __imports__ = ['Empty', 'Full']
+if hasattr(__queue__, 'SimpleQueue'):
+    __imports__.append('SimpleQueue') # New in 3.7
+    SimpleQueue = __queue__.SimpleQueue
 __all__ = __implements__ + __extensions__ + __imports__
 
 

src/gevent/resolver/__init__.py

@@ -63,8 +63,12 @@ def _lookup_port(port, socktype):
         socktypes.append(socktype)
     return port, socktypes
 
+hostname_types = tuple(set(string_types + (bytearray, bytes)))
 
 def _resolve_special(hostname, family):
+    if not isinstance(hostname, hostname_types):
+        raise TypeError("argument 1 must be str, bytes or bytearray, not %s" % (type(hostname),))
+
     if hostname == '':
         result = getaddrinfo(None, 0, family, SOCK_DGRAM, 0, AI_PASSIVE)
         if len(result) != 1:
@@ -80,7 +84,7 @@ class AbstractResolver(object):
         return self.gethostbyname_ex(hostname, family)[-1][0]
 
     def gethostbyname_ex(self, hostname, family=AF_INET):
-        aliases = []
+        aliases = self._getaliases(hostname, family)
         addresses = []
         tuples = self.getaddrinfo(hostname, 0, family,
                                   SOCK_STREAM,
@@ -93,3 +97,7 @@ class AbstractResolver(object):
 
     def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
         raise NotImplementedError()
+
+    def _getaliases(self, hostname, family):
+        # pylint:disable=unused-argument
+        return []

src/gevent/resolver/dnspython.py

 # Copyright (c) 2018  gevent contributors. See LICENSE for details.
 
+# Portions of this code taken from the gogreen project:
+#   http://github.com/slideinc/gogreen
+#
+# Copyright (c) 2005-2010 Slide, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#     * Neither the name of the author nor the names of other
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Portions of this code taken from the eventlet project:
+# https://github.com/eventlet/eventlet/blob/master/eventlet/support/greendns.py
+
+# Unless otherwise noted, the files in Eventlet are under the following MIT license:
+
+# Copyright (c) 2005-2006, Bob Ippolito
+# Copyright (c) 2007-2010, Linden Research, Inc.
+# Copyright (c) 2008-2010, Eventlet Contributors (see AUTHORS)
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+from __future__ import absolute_import, print_function, division
+
+import time
+import re
+import os
+import sys
 
 import _socket
 from _socket import AI_NUMERICHOST
 from _socket import error
 from _socket import NI_NUMERICSERV
+from _socket import AF_INET
+from _socket import AF_INET6
+from _socket import AF_UNSPEC
 
 import socket
 
 from . import AbstractResolver
+from . import hostname_types
 
+from gevent._compat import string_types
+from gevent._compat import iteritems
 from gevent._patcher import import_patched
 
 __all__ = [
@@ -46,11 +116,14 @@ dns.rdata.__import__ = _dns_import_patched
 
 resolver = dns.resolver
 
+_exc_clear = getattr(sys, 'exc_clear', lambda: None)
 
 # This is a copy of resolver._getaddrinfo with the crucial change that it
 # doesn't have a bare except:, because that breaks Timeout and KeyboardInterrupt
+# A secondary change is that calls to sys.exc_clear() have been inserted to avoid
+# failing tests in test__refcount.py (timeouts).
 # See https://github.com/rthalley/dnspython/pull/300
-def _getaddrinfo(host=None, service=None, family=socket.AF_UNSPEC, socktype=0,
+def _getaddrinfo(host=None, service=None, family=AF_UNSPEC, socktype=0,
                  proto=0, flags=0):
     # pylint:disable=too-many-locals,broad-except,too-many-statements
     # pylint:disable=too-many-branches
@@ -82,12 +155,14 @@ def _getaddrinfo(host=None, service=None, family=socket.AF_UNSPEC, socktype=0,
             v6addrs.append(host)
             canonical_name = host
     except Exception:
+        _exc_clear()
         try:
             # Is it a V4 address literal?
             addr = dns.ipv4.inet_aton(host)
             v4addrs.append(host)
             canonical_name = host
         except Exception:
+            _exc_clear()
             if flags & socket.AI_NUMERICHOST == 0:
                 try:
                     if family == socket.AF_INET6 or family == socket.AF_UNSPEC:
@@ -109,8 +184,10 @@ def _getaddrinfo(host=None, service=None, family=socket.AF_UNSPEC, socktype=0,
                             for rdata in v4.rrset:
                                 v4addrs.append(rdata.address)
                 except dns.resolver.NXDOMAIN:
+                    _exc_clear()
                     raise socket.gaierror(socket.EAI_NONAME)
                 except Exception:
+                    _exc_clear()
                     raise socket.gaierror(socket.EAI_SYSTEM)
     port = None
     try:
@@ -120,11 +197,13 @@ def _getaddrinfo(host=None, service=None, family=socket.AF_UNSPEC, socktype=0,
         else:
             port = int(service)
     except Exception:
+        _exc_clear()
         if flags & socket.AI_NUMERICSERV == 0:
             try:
                 port = socket.getservbyname(service)
             except Exception:
-                pass
+                _exc_clear()
+
     if port is None:
         raise socket.gaierror(socket.EAI_NONAME)
     tuples = []
@@ -155,6 +234,281 @@ def _getaddrinfo(host=None, service=None, family=socket.AF_UNSPEC, socktype=0,
 
 resolver._getaddrinfo = _getaddrinfo
 
+HOSTS_TTL = 300.0
+
+def _is_addr(host, parse=dns.ipv4.inet_aton):
+    if not host:
+        return False
+    assert isinstance(host, hostname_types), repr(host)
+    try:
+        parse(host)
+    except dns.exception.SyntaxError:
+        return False
+    else:
+        return True
+
+# Return True if host is a valid IPv4 address
+_is_ipv4_addr = _is_addr
+
+
+def _is_ipv6_addr(host):
+    # Return True if host is a valid IPv6 address
+    host = host.split('%', 1)[0] if host else host
+    return _is_addr(host, dns.ipv6.inet_aton)
+
+class HostsFile(object):
+    """
+    A class to read the contents of a hosts file (/etc/hosts).
+    """
+
+    LINES_RE = re.compile(r"""
+        \s*  # Leading space
+        ([^\r\n#]+?)  # The actual match, non-greedy so as not to include trailing space
+        \s*  # Trailing space
+        (?:[#][^\r\n]+)?  # Comments
+        (?:$|[\r\n]+)  # EOF or newline
+    """, re.VERBOSE)
+
+    def __init__(self, fname=None):
+        self.v4 = {} # name -> ipv4
+        self.v6 = {} # name -> ipv6
+        self.aliases = {} # name -> canonical_name
+        self.reverse = {} # ip addr -> some name
+        if fname is None:
+            if os.name == 'posix':
+                fname = '/etc/hosts'
+            elif os.name == 'nt': # pragma: no cover
+                fname = os.path.expandvars(
+                    r'%SystemRoot%\system32\drivers\etc\hosts')
+        self.fname = fname
+        assert self.fname
+        self._last_load = 0
+
+
+    def _readlines(self):
+        # Read the contents of the hosts file.
+        #
+        # Return list of lines, comment lines and empty lines are
+        # excluded. Note that this performs disk I/O so can be
+        # blocking.
+        with open(self.fname, 'rb') as fp:
+            fdata = fp.read()
+
+
+        # XXX: Using default decoding. Is that correct?
+        udata = fdata.decode(errors='ignore') if not isinstance(fdata, str) else fdata
+
+        return self.LINES_RE.findall(udata)
+
+    def load(self): # pylint:disable=too-many-locals
+        # Load hosts file
+
+        # This will (re)load the data from the hosts
+        # file if it has changed.
+
+        try:
+            load_time = os.stat(self.fname).st_mtime
+            needs_load = load_time > self._last_load
+        except (IOError, OSError):
+            from gevent import get_hub
+            get_hub().handle_error(self, *sys.exc_info())
+            needs_load = False
+
+        if not needs_load:
+            return
+
+        v4 = {}
+        v6 = {}
+        aliases = {}
+        reverse = {}
+
+        for line in self._readlines():
+            parts = line.split()
+            if len(parts) < 2:
+                continue
+            ip = parts.pop(0)
+            if _is_ipv4_addr(ip):
+                ipmap = v4
+            elif _is_ipv6_addr(ip):
+                if ip.startswith('fe80'):
+                    # Do not use link-local addresses, OSX stores these here
+                    continue
+                ipmap = v6
+            else:
+                continue
+            cname = parts.pop(0).lower()
+            ipmap[cname] = ip
+            for alias in parts:
+                alias = alias.lower()
+                ipmap[alias] = ip
+                aliases[alias] = cname
+
+            # XXX: This is wrong for ipv6
+            if ipmap is v4:
+                ptr = '.'.join(reversed(ip.split('.'))) + '.in-addr.arpa'
+            else:
+                ptr = ip + '.ip6.arpa.'
+            if ptr not in reverse:
+                reverse[ptr] = cname
+
+        self._last_load = load_time
+        self.v4 = v4
+        self.v6 = v6
+        self.aliases = aliases
+        self.reverse = reverse
+
+    def iter_all_host_addr_pairs(self):
+        self.load()
+        for name, addr in iteritems(self.v4):
+            yield name, addr
+        for name, addr in iteritems(self.v6):
+            yield name, addr
+
+class _HostsAnswer(dns.resolver.Answer):
+    # Answer class for HostsResolver object
+
+    def __init__(self, qname, rdtype, rdclass, rrset, raise_on_no_answer=True):
+        self.response = None
+        self.qname = qname
+        self.rdtype = rdtype
+        self.rdclass = rdclass
+        self.canonical_name = qname
+        if not rrset and raise_on_no_answer:
+            raise dns.resolver.NoAnswer()
+        self.rrset = rrset
+        self.expiration = (time.time() +
+                           rrset.ttl if hasattr(rrset, 'ttl') else 0)
+
+
+class _HostsResolver(object):
+    """
+    Class to parse the hosts file
+    """
+
+    def __init__(self, fname=None, interval=HOSTS_TTL):
+        self.hosts_file = HostsFile(fname)
+        self.interval = interval
+        self._last_load = 0
+
+    def query(self, qname, rdtype=dns.rdatatype.A, rdclass=dns.rdataclass.IN,
+              tcp=False, source=None, raise_on_no_answer=True): # pylint:disable=unused-argument
+        # Query the hosts file
+        #
+        # The known rdtypes are dns.rdatatype.A, dns.rdatatype.AAAA and
+        # dns.rdatatype.CNAME.
+        # The ``rdclass`` parameter must be dns.rdataclass.IN while the
+        # ``tcp`` and ``source`` parameters are ignored.
+        # Return a HostAnswer instance or raise a dns.resolver.NoAnswer
+        # exception.
+
+        now = time.time()
+        hosts_file = self.hosts_file
+        if self._last_load + self.interval < now:
+            self._last_load = now
+            hosts_file.load()
+
+        rdclass = dns.rdataclass.IN # Always
+        if isinstance(qname, string_types):
+            name = qname
+            qname = dns.name.from_text(qname)
+        else:
+            name = str(qname)
+
+        name = name.lower()
+        rrset = dns.rrset.RRset(qname, rdclass, rdtype)
+        rrset.ttl = self._last_load + self.interval - now
+
+        if rdtype == dns.rdatatype.A:
+            mapping = hosts_file.v4
+            kind = dns.rdtypes.IN.A.A
+        elif rdtype == dns.rdatatype.AAAA:
+            mapping = hosts_file.v6
+            kind = dns.rdtypes.IN.AAAA.AAAA
+        elif rdtype == dns.rdatatype.CNAME:
+            mapping = hosts_file.aliases
+            kind = lambda c, t, addr: dns.rdtypes.ANY.CNAME.CNAME(c, t, dns.name.from_text(addr))
+        elif rdtype == dns.rdatatype.PTR:
+            mapping = hosts_file.reverse
+            kind = lambda c, t, addr: dns.rdtypes.ANY.PTR.PTR(c, t, dns.name.from_text(addr))
+
+
+        addr = mapping.get(name)
+        if not addr and qname.is_absolute():
+            addr = mapping.get(name[:-1])
+        if addr:
+            rrset.add(kind(rdclass, rdtype, addr))
+        return _HostsAnswer(qname, rdtype, rdclass, rrset, raise_on_no_answer)
+
+    def getaliases(self, hostname):
+        # Return a list of all the aliases of a given cname
+
+        # Due to the way store aliases this is a bit inefficient, this
+        # clearly was an afterthought.  But this is only used by
+        # gethostbyname_ex so it's probably fine.
+        aliases = self.hosts_file.aliases
+        result = []
+        if hostname in aliases:
+            cannon = aliases[hostname]
+        else:
+            cannon = hostname
+        result.append(cannon)
+        for alias, cname in iteritems(aliases):
+            if cannon == cname:
+                result.append(alias)
+        result.remove(hostname)
+        return result
+
+class _DualResolver(object):
+
+    def __init__(self):
+        self.hosts_resolver = _HostsResolver()
+        self.network_resolver = resolver.get_default_resolver()
+        self.network_resolver.cache = resolver.LRUCache()
+
+    def query(self, qname, rdtype=dns.rdatatype.A, rdclass=dns.rdataclass.IN,
+              tcp=False, source=None, raise_on_no_answer=True,
+              _hosts_rdtypes=(dns.rdatatype.A, dns.rdatatype.AAAA, dns.rdatatype.PTR)):
+        # Query the resolver, using /etc/hosts
+
+        # Behavior:
+        # 1. if hosts is enabled and contains answer, return it now
+        # 2. query nameservers for qname
+        if qname is None:
+            qname = '0.0.0.0'
+
+        if not isinstance(qname, string_types):
+            if isinstance(qname, bytes):
+                qname = qname.decode("idna")
+
+        if isinstance(qname, string_types):
+            qname = dns.name.from_text(qname, None)
+
+        if isinstance(rdtype, string_types):
+            rdtype = dns.rdatatype.from_text(rdtype)
+
+        if rdclass == dns.rdataclass.IN and rdtype in _hosts_rdtypes:
+            try:
+                answer = self.hosts_resolver.query(qname, rdtype, raise_on_no_answer=False)
+            except Exception: # pylint: disable=broad-except
+                from gevent import get_hub
+                get_hub().handle_error(self, *sys.exc_info())
+            else:
+                if answer.rrset:
+                    return answer
+
+        return self.network_resolver.query(qname, rdtype, rdclass,
+                                           tcp, source, raise_on_no_answer=raise_on_no_answer)
+
+def _family_to_rdtype(family):
+    if family == socket.AF_INET:
+        rdtype = dns.rdatatype.A
+    elif family == socket.AF_INET6:
+        rdtype = dns.rdatatype.AAAA
+    else:
+        raise socket.gaierror(socket.EAI_FAMILY,
+                              'Address family not supported')
+    return rdtype
+
 class Resolver(AbstractResolver):
     """
     An *experimental* resolver that uses `dnspython`_.
@@ -180,11 +534,6 @@ class Resolver(AbstractResolver):
 
     .. caution::
 
-        This completely ignores the contents of ``/etc/hosts``, but it is
-        configured by ``/etc/resolv.conf`` (on Unix) or the registry (on
-        Windows). There are some measures in place to be able to resolve
-        ``localhost`` related names and addresses through the system resolver.
-
         Many of the same caveats about DNS results apply here as are documented
         for :class:`gevent.resolver.ares.Resolver`.
 
@@ -199,10 +548,12 @@ class Resolver(AbstractResolver):
     """
 
     def __init__(self, hub=None): # pylint: disable=unused-argument
+        self._resolver = _DualResolver()
         if resolver._resolver is None:
-            resolver._resolver = resolver.get_default_resolver()
-            # Add a default cache
-            resolver._resolver.cache = resolver.LRUCache()
+            resolver._resolver = self._resolver
+        # Different hubs in different threads could be sharing the same
+        # resolver.
+        assert isinstance(resolver._resolver, _DualResolver)
 
     @property
     def resolver(self):
@@ -217,20 +568,57 @@ class Resolver(AbstractResolver):
         * ``nameservers`` controls which nameservers to talk to
         * ``lifetime`` configures a timeout for each individual query.
         """
-        return resolver._resolver
+        return self._resolver.network_resolver
 
     def close(self):
         pass
 
+    def _getaliases(self, hostname, family):
+        if not isinstance(hostname, str):
+            if isinstance(hostname, bytes):
+                hostname = hostname.decode("idna")
+        aliases = self._resolver.hosts_resolver.getaliases(hostname)
+        net_resolver = self._resolver.network_resolver
+        rdtype = _family_to_rdtype(family)
+        while True:
+            try:
+                ans = net_resolver.query(hostname, dns.rdatatype.CNAME, rdtype)
+            except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.resolver.NoNameservers):
+                break
+            else:
+                aliases.extend(str(rr.target) for rr in ans.rrset)
+                hostname = ans[0].target
+        return aliases
+
     def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
-        if ((host == u'localhost' or host == b'localhost')
+        if ((host in (u'localhost', b'localhost')
+             or (_is_ipv6_addr(host) and host.startswith('fe80')))
                 or not isinstance(host, str) or (flags & AI_NUMERICHOST)):
             # this handles cases which do not require network access
             # 1) host is None
             # 2) host is of an invalid type
+            # 3) host is localhost or a link-local ipv6; dnspython returns the wrong
+            #    scope-id for those.
             # 3) AI_NUMERICHOST flag is set
+
             return _socket.getaddrinfo(host, port, family, socktype, proto, flags)
 
+        if family == AF_UNSPEC:
+            # This tends to raise in the case that a v6 address did not exist
+            # but a v4 does. So we break it into two parts.
+
+            # Note that if there is no ipv6 in the hosts file, but there *is*
+            # an ipv4, and there *is* an ipv6 in the nameservers, we will return
+            # both (from the first call). The system resolver on OS X only returns
+            # the results from the hosts file. doubleclick.com is one example.
+            try:
+                return _getaddrinfo(host, port, family, socktype, proto, flags)
+            except socket.gaierror:
+                try:
+                    return _getaddrinfo(host, port, AF_INET6, socktype, proto, flags)
+                except socket.gaierror:
+                    return _getaddrinfo(host, port, AF_INET, socktype, proto, flags)
+        else:
             return _getaddrinfo(host, port, family, socktype, proto, flags)
 
     def getnameinfo(self, sockaddr, flags):
@@ -238,6 +626,8 @@ class Resolver(AbstractResolver):
                 and isinstance(sockaddr, (list, tuple))
                 and sockaddr[0] in ('::1', '127.0.0.1', 'localhost')):
             return _socket.getnameinfo(sockaddr, flags)
+        if isinstance(sockaddr, (list, tuple)) and not isinstance(sockaddr[0], hostname_types):
+            raise TypeError("getnameinfo(): illegal sockaddr argument")
         try:
             return resolver._getnameinfo(sockaddr, flags)
         except error:
@@ -252,4 +642,8 @@ class Resolver(AbstractResolver):
                           b'127.0.0.1', b'::1',
                           'localhost'):
             return _socket.gethostbyaddr(ip_address)
+
+        if not isinstance(ip_address, hostname_types):
+            raise TypeError("argument 1 must be str, bytes or bytearray, not %s" % (type(ip_address),))
+
         return resolver._gethostbyaddr(ip_address)

src/greentest/test__socket_dns.py

@@ -50,7 +50,7 @@ def format_call(function, args):
     if args.endswith(',)'):
         args = args[:-2] + ')'
     try:
-        module = function.__module__.replace('gevent.socket', 'gevent').replace('_socket', 'stdlib')
+        module = function.__module__.replace('gevent._socketcommon', 'gevent')
         name = function.__name__
         return '%s:%s%s' % (module, name, args)
     except AttributeError:
@@ -166,13 +166,16 @@ def relaxed_is_equal(a, b):
     return all(relaxed_is_equal(x, y) for (x, y) in zip(a, b))
 
 
-def add(klass, hostname, name=None):
+def add(klass, hostname, name=None,
+        skip=None, skip_reason=None):
 
     call = callable(hostname)
 
-    def _setattr(k, n, v):
+    def _setattr(k, n, func):
+        if skip:
+            func = greentest.skipIf(skip, skip_reason,)(func)
         if not hasattr(k, n):
-            setattr(k, n, v)
+            setattr(k, n, func)
 
     if name is None:
         if call:
@@ -283,7 +286,8 @@ class TestCase(greentest.TestCase):
         ips = result[2]
         if ips == ['127.0.0.1', '127.0.0.1']:
             ips = ['127.0.0.1']
-        return (result[0], [], ips)
+        # On some systems, the hostname can get caps
+        return (result[0].lower(), [], ips)
 
     def _normalize_result_getaddrinfo(self, result):
         if not RESOLVER_NOT_SYSTEM:
@@ -345,8 +349,6 @@ add(TestTypeError, None)
 add(TestTypeError, 25)
 
 
-@unittest.skipIf(RESOLVER_DNSPYTHON,
-                 "This commonly needs /etc/hosts to function, and dnspython doesn't do that.")
 class TestHostname(TestCase):
     pass
 
@@ -423,33 +425,45 @@ class TestBroadcast(TestCase):
 add(TestBroadcast, '<broadcast>')
 
 
-@unittest.skipIf(RESOLVER_DNSPYTHON, "/etc/hosts completely ignored under dnspython")
-class TestEtcHosts(TestCase):
-    pass
-
-try:
-    with open('/etc/hosts') as f:
-        etc_hosts = f.read()
-except IOError:
-    etc_hosts = ''
-
-for ip, host in re.findall(r'^\s*(\d+\.\d+\.\d+\.\d+)\s+([^\s]+)', etc_hosts, re.M)[:10]:
+from gevent.resolver.dnspython import HostsFile
+class SanitizedHostsFile(HostsFile):
+    def iter_all_host_addr_pairs(self):
+        for name, addr in super(SanitizedHostsFile, self).iter_all_host_addr_pairs():
             if (RESOLVER_NOT_SYSTEM
-        and (host.endswith('local') # ignore bonjour, ares can't find them
+                    and (name.endswith('local') # ignore bonjour, ares can't find them
                          # ignore common aliases that ares can't find
-             or ip == '255.255.255.255'
-             or host == 'broadcasthost'
+                         or addr == '255.255.255.255'
+                         or name == 'broadcasthost'
                          # We get extra results from some impls, like OS X
                          # it returns DGRAM results
-             or host == 'localhost')):
+                         or name == 'localhost')):
                 continue
-    if host.endswith('local'):
+            if name.endswith('local'):
                 # These can only be found if bonjour is running,
                 # and are very slow to do so with the system resolver on OS X
                 continue
-    add(TestEtcHosts, host)
-    add(TestEtcHosts, ip)
-    del host, ip
+            yield name, addr
+
+class TestEtcHosts(TestCase):
+
+    MAX_HOSTS = os.getenv('GEVENTTEST_MAX_ETC_HOSTS', 10)
+
+    @classmethod
+    def populate_tests(cls):
+        hf = SanitizedHostsFile(os.path.join(os.path.dirname(__file__),
+                                             'hosts_file.txt'))
+        all_etc_hosts = sorted(hf.iter_all_host_addr_pairs())
+        if len(all_etc_hosts) > cls.MAX_HOSTS and not DEBUG:
+            all_etc_hosts = all_etc_hosts[:cls.MAX_HOSTS]
+
+        for host, ip in all_etc_hosts:
+            add(cls, host)
+            add(cls, ip)
+
+
+
+TestEtcHosts.populate_tests()
+
 
 
 class TestGeventOrg(TestCase):
@@ -544,12 +558,12 @@ class Test_getaddrinfo(TestCase):
 class TestInternational(TestCase):
     pass
 
-if not (PY2 and RESOLVER_DNSPYTHON):
-    # dns python can actually resolve these: it uses
-    # the 2008 version of idna encoding, whereas on Python 2,
-    # with the default resolver, it tries to encode to ascii and
-    # raises a UnicodeEncodeError. So we get different results.
-    add(TestInternational, u'президент.рф', 'russian')
+# dns python can actually resolve these: it uses
+# the 2008 version of idna encoding, whereas on Python 2,
+# with the default resolver, it tries to encode to ascii and
+# raises a UnicodeEncodeError. So we get different results.
+add(TestInternational, u'президент.рф', 'russian',
+    skip=(PY2 and RESOLVER_DNSPYTHON), skip_reason="dnspython can actually resolve these")
 add(TestInternational, u'президент.рф'.encode('idna'), 'idna')