Add support for /etc/hosts to dnspython resolver.

Based on code from eventlet, but heavily refactored and modified:

- Split host parsing into its own class, re-used from the test case
- Hosts handle PTR lookups for ipv4
- Some bug fixes around what should be strings where.
- Allow strings for the rdtype paramater
- Watch last modified time instead of reloading the file at fixed
  intervals.

More of our DNS tests pass now (notable TestEtcHosts is now enabled
for this resolver), and we tend to generate more conformant results
than the ares resolver (by observation, not tested).

src/gevent/queue.py

@@ -45,6 +45,9 @@ from gevent.hub import InvalidSwitchError
 __implements__ = ['Queue', 'PriorityQueue', 'LifoQueue']
 __extensions__ = ['JoinableQueue', 'Channel']
 __imports__ = ['Empty', 'Full']
+if hasattr(__queue__, 'SimpleQueue'):
+    __imports__.append('SimpleQueue') # New in 3.7
+    SimpleQueue = __queue__.SimpleQueue
 __all__ = __implements__ + __extensions__ + __imports__
 
 

src/gevent/resolver/__init__.py

@@ -63,8 +63,12 @@ def _lookup_port(port, socktype):
         socktypes.append(socktype)
     return port, socktypes
 
+hostname_types = tuple(set(string_types + (bytearray, bytes)))
 
 def _resolve_special(hostname, family):
+    if not isinstance(hostname, hostname_types):
+        raise TypeError("argument 1 must be str, bytes or bytearray, not %s" % (type(hostname),))
+
     if hostname == '':
         result = getaddrinfo(None, 0, family, SOCK_DGRAM, 0, AI_PASSIVE)
         if len(result) != 1:
@@ -80,7 +84,7 @@ class AbstractResolver(object):
         return self.gethostbyname_ex(hostname, family)[-1][0]
 
     def gethostbyname_ex(self, hostname, family=AF_INET):
-        aliases = []
+        aliases = self._getaliases(hostname, family)
         addresses = []
         tuples = self.getaddrinfo(hostname, 0, family,
                                   SOCK_STREAM,
@@ -93,3 +97,7 @@ class AbstractResolver(object):
 
     def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
         raise NotImplementedError()
+
+    def _getaliases(self, hostname, family):
+        # pylint:disable=unused-argument
+        return []

src/gevent/resolver/dnspython.py

 # Copyright (c) 2018  gevent contributors. See LICENSE for details.
 
+# Portions of this code taken from the gogreen project:
+#   http://github.com/slideinc/gogreen
+#
+# Copyright (c) 2005-2010 Slide, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#     * Neither the name of the author nor the names of other
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Portions of this code taken from the eventlet project:
+# https://github.com/eventlet/eventlet/blob/master/eventlet/support/greendns.py
+
+# Unless otherwise noted, the files in Eventlet are under the following MIT license:
+
+# Copyright (c) 2005-2006, Bob Ippolito
+# Copyright (c) 2007-2010, Linden Research, Inc.
+# Copyright (c) 2008-2010, Eventlet Contributors (see AUTHORS)
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+from __future__ import absolute_import, print_function, division
+
+import time
+import re
+import os
+import sys
 
 import _socket
 from _socket import AI_NUMERICHOST
 from _socket import error
 from _socket import NI_NUMERICSERV
+from _socket import AF_INET
+from _socket import AF_INET6
+from _socket import AF_UNSPEC
 
 import socket
 
 from . import AbstractResolver
+from . import hostname_types
 
+from gevent._compat import string_types
+from gevent._compat import iteritems
 from gevent._patcher import import_patched
 
 __all__ = [
@@ -46,11 +116,14 @@ dns.rdata.__import__ = _dns_import_patched
 
 resolver = dns.resolver
 
+_exc_clear = getattr(sys, 'exc_clear', lambda: None)
 
 # This is a copy of resolver._getaddrinfo with the crucial change that it
 # doesn't have a bare except:, because that breaks Timeout and KeyboardInterrupt
+# A secondary change is that calls to sys.exc_clear() have been inserted to avoid
+# failing tests in test__refcount.py (timeouts).
 # See https://github.com/rthalley/dnspython/pull/300
-def _getaddrinfo(host=None, service=None, family=socket.AF_UNSPEC, socktype=0,
+def _getaddrinfo(host=None, service=None, family=AF_UNSPEC, socktype=0,
                  proto=0, flags=0):
     # pylint:disable=too-many-locals,broad-except,too-many-statements
     # pylint:disable=too-many-branches
@@ -82,12 +155,14 @@ def _getaddrinfo(host=None, service=None, family=socket.AF_UNSPEC, socktype=0,
             v6addrs.append(host)
             canonical_name = host
     except Exception:
+        _exc_clear()
         try:
             # Is it a V4 address literal?
             addr = dns.ipv4.inet_aton(host)
             v4addrs.append(host)
             canonical_name = host
         except Exception:
+            _exc_clear()
             if flags & socket.AI_NUMERICHOST == 0:
                 try:
                     if family == socket.AF_INET6 or family == socket.AF_UNSPEC:
@@ -109,8 +184,10 @@ def _getaddrinfo(host=None, service=None, family=socket.AF_UNSPEC, socktype=0,
                             for rdata in v4.rrset:
                                 v4addrs.append(rdata.address)
                 except dns.resolver.NXDOMAIN:
+                    _exc_clear()
                     raise socket.gaierror(socket.EAI_NONAME)
                 except Exception:
+                    _exc_clear()
                     raise socket.gaierror(socket.EAI_SYSTEM)
     port = None
     try:
@@ -120,11 +197,13 @@ def _getaddrinfo(host=None, service=None, family=socket.AF_UNSPEC, socktype=0,
         else:
             port = int(service)
     except Exception:
+        _exc_clear()
         if flags & socket.AI_NUMERICSERV == 0:
             try:
                 port = socket.getservbyname(service)
             except Exception:
-                pass
+                _exc_clear()
+
     if port is None:
         raise socket.gaierror(socket.EAI_NONAME)
     tuples = []
@@ -155,6 +234,281 @@ def _getaddrinfo(host=None, service=None, family=socket.AF_UNSPEC, socktype=0,
 
 resolver._getaddrinfo = _getaddrinfo
 
+HOSTS_TTL = 300.0
+
+def _is_addr(host, parse=dns.ipv4.inet_aton):
+    if not host:
+        return False
+    assert isinstance(host, hostname_types), repr(host)
+    try:
+        parse(host)
+    except dns.exception.SyntaxError:
+        return False
+    else:
+        return True
+
+# Return True if host is a valid IPv4 address
+_is_ipv4_addr = _is_addr
+
+
+def _is_ipv6_addr(host):
+    # Return True if host is a valid IPv6 address
+    host = host.split('%', 1)[0] if host else host
+    return _is_addr(host, dns.ipv6.inet_aton)
+
+class HostsFile(object):
+    """
+    A class to read the contents of a hosts file (/etc/hosts).
+    """
+
+    LINES_RE = re.compile(r"""
+        \s*  # Leading space
+        ([^\r\n#]+?)  # The actual match, non-greedy so as not to include trailing space
+        \s*  # Trailing space
+        (?:[#][^\r\n]+)?  # Comments
+        (?:$|[\r\n]+)  # EOF or newline
+    """, re.VERBOSE)
+
+    def __init__(self, fname=None):
+        self.v4 = {} # name -> ipv4
+        self.v6 = {} # name -> ipv6
+        self.aliases = {} # name -> canonical_name
+        self.reverse = {} # ip addr -> some name
+        if fname is None:
+            if os.name == 'posix':
+                fname = '/etc/hosts'
+            elif os.name == 'nt': # pragma: no cover
+                fname = os.path.expandvars(
+                    r'%SystemRoot%\system32\drivers\etc\hosts')
+        self.fname = fname
+        assert self.fname
+        self._last_load = 0
+
+
+    def _readlines(self):
+        # Read the contents of the hosts file.
+        #
+        # Return list of lines, comment lines and empty lines are
+        # excluded. Note that this performs disk I/O so can be
+        # blocking.
+        with open(self.fname, 'rb') as fp:
+            fdata = fp.read()
+
+
+        # XXX: Using default decoding. Is that correct?
+        udata = fdata.decode(errors='ignore') if not isinstance(fdata, str) else fdata
+
+        return self.LINES_RE.findall(udata)
+
+    def load(self): # pylint:disable=too-many-locals
+        # Load hosts file
+
+        # This will (re)load the data from the hosts
+        # file if it has changed.
+
+        try:
+            load_time = os.stat(self.fname).st_mtime
+            needs_load = load_time > self._last_load
+        except (IOError, OSError):
+            from gevent import get_hub
+            get_hub().handle_error(self, *sys.exc_info())
+            needs_load = False
+
+        if not needs_load:
+            return
+
+        v4 = {}
+        v6 = {}
+        aliases = {}
+        reverse = {}
+
+        for line in self._readlines():
+            parts = line.split()
+            if len(parts) < 2:
+                continue
+            ip = parts.pop(0)
+            if _is_ipv4_addr(ip):
+                ipmap = v4
+            elif _is_ipv6_addr(ip):
+                if ip.startswith('fe80'):
+                    # Do not use link-local addresses, OSX stores these here
+                    continue
+                ipmap = v6
+            else:
+                continue
+            cname = parts.pop(0).lower()
+            ipmap[cname] = ip
+            for alias in parts:
+                alias = alias.lower()
+                ipmap[alias] = ip
+                aliases[alias] = cname
+
+            # XXX: This is wrong for ipv6
+            if ipmap is v4:
+                ptr = '.'.join(reversed(ip.split('.'))) + '.in-addr.arpa'
+            else:
+                ptr = ip + '.ip6.arpa.'
+            if ptr not in reverse:
+                reverse[ptr] = cname
+
+        self._last_load = load_time
+        self.v4 = v4
+        self.v6 = v6
+        self.aliases = aliases
+        self.reverse = reverse
+
+    def iter_all_host_addr_pairs(self):
+        self.load()
+        for name, addr in iteritems(self.v4):
+            yield name, addr
+        for name, addr in iteritems(self.v6):
+            yield name, addr
+
+class _HostsAnswer(dns.resolver.Answer):
+    # Answer class for HostsResolver object
+
+    def __init__(self, qname, rdtype, rdclass, rrset, raise_on_no_answer=True):
+        self.response = None
+        self.qname = qname
+        self.rdtype = rdtype
+        self.rdclass = rdclass
+        self.canonical_name = qname
+        if not rrset and raise_on_no_answer:
+            raise dns.resolver.NoAnswer()
+        self.rrset = rrset
+        self.expiration = (time.time() +
+                           rrset.ttl if hasattr(rrset, 'ttl') else 0)
+
+
+class _HostsResolver(object):
+    """
+    Class to parse the hosts file
+    """
+
+    def __init__(self, fname=None, interval=HOSTS_TTL):
+        self.hosts_file = HostsFile(fname)
+        self.interval = interval
+        self._last_load = 0
+
+    def query(self, qname, rdtype=dns.rdatatype.A, rdclass=dns.rdataclass.IN,
+              tcp=False, source=None, raise_on_no_answer=True): # pylint:disable=unused-argument
+        # Query the hosts file
+        #
+        # The known rdtypes are dns.rdatatype.A, dns.rdatatype.AAAA and
+        # dns.rdatatype.CNAME.
+        # The ``rdclass`` parameter must be dns.rdataclass.IN while the
+        # ``tcp`` and ``source`` parameters are ignored.
+        # Return a HostAnswer instance or raise a dns.resolver.NoAnswer
+        # exception.
+
+        now = time.time()
+        hosts_file = self.hosts_file
+        if self._last_load + self.interval < now:
+            self._last_load = now
+            hosts_file.load()
+
+        rdclass = dns.rdataclass.IN # Always
+        if isinstance(qname, string_types):
+            name = qname
+            qname = dns.name.from_text(qname)
+        else:
+            name = str(qname)
+
+        name = name.lower()
+        rrset = dns.rrset.RRset(qname, rdclass, rdtype)
+        rrset.ttl = self._last_load + self.interval - now
+
+        if rdtype == dns.rdatatype.A:
+            mapping = hosts_file.v4
+            kind = dns.rdtypes.IN.A.A
+        elif rdtype == dns.rdatatype.AAAA:
+            mapping = hosts_file.v6
+            kind = dns.rdtypes.IN.AAAA.AAAA
+        elif rdtype == dns.rdatatype.CNAME:
+            mapping = hosts_file.aliases
+            kind = lambda c, t, addr: dns.rdtypes.ANY.CNAME.CNAME(c, t, dns.name.from_text(addr))
+        elif rdtype == dns.rdatatype.PTR:
+            mapping = hosts_file.reverse
+            kind = lambda c, t, addr: dns.rdtypes.ANY.PTR.PTR(c, t, dns.name.from_text(addr))
+
+
+        addr = mapping.get(name)
+        if not addr and qname.is_absolute():
+            addr = mapping.get(name[:-1])
+        if addr:
+            rrset.add(kind(rdclass, rdtype, addr))
+        return _HostsAnswer(qname, rdtype, rdclass, rrset, raise_on_no_answer)
+
+    def getaliases(self, hostname):
+        # Return a list of all the aliases of a given cname
+
+        # Due to the way store aliases this is a bit inefficient, this
+        # clearly was an afterthought.  But this is only used by
+        # gethostbyname_ex so it's probably fine.
+        aliases = self.hosts_file.aliases
+        result = []
+        if hostname in aliases:
+            cannon = aliases[hostname]
+        else:
+            cannon = hostname
+        result.append(cannon)
+        for alias, cname in iteritems(aliases):
+            if cannon == cname:
+                result.append(alias)
+        result.remove(hostname)
+        return result
+
+class _DualResolver(object):
+
+    def __init__(self):
+        self.hosts_resolver = _HostsResolver()
+        self.network_resolver = resolver.get_default_resolver()
+        self.network_resolver.cache = resolver.LRUCache()
+
+    def query(self, qname, rdtype=dns.rdatatype.A, rdclass=dns.rdataclass.IN,
+              tcp=False, source=None, raise_on_no_answer=True,
+              _hosts_rdtypes=(dns.rdatatype.A, dns.rdatatype.AAAA, dns.rdatatype.PTR)):
+        # Query the resolver, using /etc/hosts
+
+        # Behavior:
+        # 1. if hosts is enabled and contains answer, return it now
+        # 2. query nameservers for qname
+        if qname is None:
+            qname = '0.0.0.0'
+
+        if not isinstance(qname, string_types):
+            if isinstance(qname, bytes):
+                qname = qname.decode("idna")
+
+        if isinstance(qname, string_types):
+            qname = dns.name.from_text(qname, None)
+
+        if isinstance(rdtype, string_types):
+            rdtype = dns.rdatatype.from_text(rdtype)
+
+        if rdclass == dns.rdataclass.IN and rdtype in _hosts_rdtypes:
+            try:
+                answer = self.hosts_resolver.query(qname, rdtype, raise_on_no_answer=False)
+            except Exception: # pylint: disable=broad-except
+                from gevent import get_hub
+                get_hub().handle_error(self, *sys.exc_info())
+            else:
+                if answer.rrset:
+                    return answer
+
+        return self.network_resolver.query(qname, rdtype, rdclass,
+                                           tcp, source, raise_on_no_answer=raise_on_no_answer)
+
+def _family_to_rdtype(family):
+    if family == socket.AF_INET:
+        rdtype = dns.rdatatype.A
+    elif family == socket.AF_INET6:
+        rdtype = dns.rdatatype.AAAA
+    else:
+        raise socket.gaierror(socket.EAI_FAMILY,
+                              'Address family not supported')
+    return rdtype
+
 class Resolver(AbstractResolver):
     """
     An *experimental* resolver that uses `dnspython`_.
@@ -180,11 +534,6 @@ class Resolver(AbstractResolver):
 
     .. caution::
 
-        This completely ignores the contents of ``/etc/hosts``, but it is
-        configured by ``/etc/resolv.conf`` (on Unix) or the registry (on
-        Windows). There are some measures in place to be able to resolve
-        ``localhost`` related names and addresses through the system resolver.
-
         Many of the same caveats about DNS results apply here as are documented
         for :class:`gevent.resolver.ares.Resolver`.
 
@@ -199,10 +548,12 @@ class Resolver(AbstractResolver):
     """
 
     def __init__(self, hub=None): # pylint: disable=unused-argument
+        self._resolver = _DualResolver()
         if resolver._resolver is None:
-            resolver._resolver = resolver.get_default_resolver()
-            # Add a default cache
-            resolver._resolver.cache = resolver.LRUCache()
+            resolver._resolver = self._resolver
+        # Different hubs in different threads could be sharing the same
+        # resolver.
+        assert isinstance(resolver._resolver, _DualResolver)
 
     @property
     def resolver(self):
@@ -217,27 +568,66 @@ class Resolver(AbstractResolver):
         * ``nameservers`` controls which nameservers to talk to
         * ``lifetime`` configures a timeout for each individual query.
         """
-        return resolver._resolver
+        return self._resolver.network_resolver
 
     def close(self):
         pass
 
+    def _getaliases(self, hostname, family):
+        if not isinstance(hostname, str):
+            if isinstance(hostname, bytes):
+                hostname = hostname.decode("idna")
+        aliases = self._resolver.hosts_resolver.getaliases(hostname)
+        net_resolver = self._resolver.network_resolver
+        rdtype = _family_to_rdtype(family)
+        while True:
+            try:
+                ans = net_resolver.query(hostname, dns.rdatatype.CNAME, rdtype)
+            except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.resolver.NoNameservers):
+                break
+            else:
+                aliases.extend(str(rr.target) for rr in ans.rrset)
+                hostname = ans[0].target
+        return aliases
+
     def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
-        if ((host == u'localhost' or host == b'localhost')
+        if ((host in (u'localhost', b'localhost')
+             or (_is_ipv6_addr(host) and host.startswith('fe80')))
                 or not isinstance(host, str) or (flags & AI_NUMERICHOST)):
             # this handles cases which do not require network access
             # 1) host is None
             # 2) host is of an invalid type
+            # 3) host is localhost or a link-local ipv6; dnspython returns the wrong
+            #    scope-id for those.
             # 3) AI_NUMERICHOST flag is set
+
             return _socket.getaddrinfo(host, port, family, socktype, proto, flags)
 
-        return _getaddrinfo(host, port, family, socktype, proto, flags)
+        if family == AF_UNSPEC:
+            # This tends to raise in the case that a v6 address did not exist
+            # but a v4 does. So we break it into two parts.
+
+            # Note that if there is no ipv6 in the hosts file, but there *is*
+            # an ipv4, and there *is* an ipv6 in the nameservers, we will return
+            # both (from the first call). The system resolver on OS X only returns
+            # the results from the hosts file. doubleclick.com is one example.
+            try:
+                return _getaddrinfo(host, port, family, socktype, proto, flags)
+            except socket.gaierror:
+                try:
+                    return _getaddrinfo(host, port, AF_INET6, socktype, proto, flags)
+                except socket.gaierror:
+                    return _getaddrinfo(host, port, AF_INET, socktype, proto, flags)
+        else:
+            return _getaddrinfo(host, port, family, socktype, proto, flags)
 
     def getnameinfo(self, sockaddr, flags):
         if (sockaddr
                 and isinstance(sockaddr, (list, tuple))
                 and sockaddr[0] in ('::1', '127.0.0.1', 'localhost')):
             return _socket.getnameinfo(sockaddr, flags)
+        if isinstance(sockaddr, (list, tuple)) and not isinstance(sockaddr[0], hostname_types):
+            raise TypeError("getnameinfo(): illegal sockaddr argument")
         try:
             return resolver._getnameinfo(sockaddr, flags)
         except error:
@@ -252,4 +642,8 @@ class Resolver(AbstractResolver):
                           b'127.0.0.1', b'::1',
                           'localhost'):
             return _socket.gethostbyaddr(ip_address)
+
+        if not isinstance(ip_address, hostname_types):
+            raise TypeError("argument 1 must be str, bytes or bytearray, not %s" % (type(ip_address),))
+
         return resolver._gethostbyaddr(ip_address)

src/greentest/test__socket_dns.py

@@ -50,7 +50,7 @@ def format_call(function, args):
     if args.endswith(',)'):
         args = args[:-2] + ')'
     try:
-        module = function.__module__.replace('gevent.socket', 'gevent').replace('_socket', 'stdlib')
+        module = function.__module__.replace('gevent._socketcommon', 'gevent')
         name = function.__name__
         return '%s:%s%s' % (module, name, args)
     except AttributeError:
@@ -166,13 +166,16 @@ def relaxed_is_equal(a, b):
     return all(relaxed_is_equal(x, y) for (x, y) in zip(a, b))
 
 
-def add(klass, hostname, name=None):
+def add(klass, hostname, name=None,
+        skip=None, skip_reason=None):
 
     call = callable(hostname)
 
-    def _setattr(k, n, v):
+    def _setattr(k, n, func):
+        if skip:
+            func = greentest.skipIf(skip, skip_reason,)(func)
         if not hasattr(k, n):
-            setattr(k, n, v)
+            setattr(k, n, func)
 
     if name is None:
         if call:
@@ -283,7 +286,8 @@ class TestCase(greentest.TestCase):
         ips = result[2]
         if ips == ['127.0.0.1', '127.0.0.1']:
             ips = ['127.0.0.1']
-        return (result[0], [], ips)
+        # On some systems, the hostname can get caps
+        return (result[0].lower(), [], ips)
 
     def _normalize_result_getaddrinfo(self, result):
         if not RESOLVER_NOT_SYSTEM:
@@ -345,8 +349,6 @@ add(TestTypeError, None)
 add(TestTypeError, 25)
 
 
-@unittest.skipIf(RESOLVER_DNSPYTHON,
-                 "This commonly needs /etc/hosts to function, and dnspython doesn't do that.")
 class TestHostname(TestCase):
     pass
 
@@ -423,33 +425,45 @@ class TestBroadcast(TestCase):
 add(TestBroadcast, '<broadcast>')
 
 
-@unittest.skipIf(RESOLVER_DNSPYTHON, "/etc/hosts completely ignored under dnspython")
+from gevent.resolver.dnspython import HostsFile
+class SanitizedHostsFile(HostsFile):
+    def iter_all_host_addr_pairs(self):
+        for name, addr in super(SanitizedHostsFile, self).iter_all_host_addr_pairs():
+            if (RESOLVER_NOT_SYSTEM
+                    and (name.endswith('local') # ignore bonjour, ares can't find them
+                         # ignore common aliases that ares can't find
+                         or addr == '255.255.255.255'
+                         or name == 'broadcasthost'
+                         # We get extra results from some impls, like OS X
+                         # it returns DGRAM results
+                         or name == 'localhost')):
+                continue
+            if name.endswith('local'):
+                # These can only be found if bonjour is running,
+                # and are very slow to do so with the system resolver on OS X
+                continue
+            yield name, addr
+
 class TestEtcHosts(TestCase):
-    pass
 
-try:
-    with open('/etc/hosts') as f:
-        etc_hosts = f.read()
-except IOError:
-    etc_hosts = ''
-
-for ip, host in re.findall(r'^\s*(\d+\.\d+\.\d+\.\d+)\s+([^\s]+)', etc_hosts, re.M)[:10]:
-    if (RESOLVER_NOT_SYSTEM
-        and (host.endswith('local') # ignore bonjour, ares can't find them
-             # ignore common aliases that ares can't find
-             or ip == '255.255.255.255'
-             or host == 'broadcasthost'
-             # We get extra results from some impls, like OS X
-             # it returns DGRAM results
-             or host == 'localhost')):
-        continue
-    if host.endswith('local'):
-        # These can only be found if bonjour is running,
-        # and are very slow to do so with the system resolver on OS X
-        continue
-    add(TestEtcHosts, host)
-    add(TestEtcHosts, ip)
-    del host, ip
+    MAX_HOSTS = os.getenv('GEVENTTEST_MAX_ETC_HOSTS', 10)
+
+    @classmethod
+    def populate_tests(cls):
+        hf = SanitizedHostsFile(os.path.join(os.path.dirname(__file__),
+                                             'hosts_file.txt'))
+        all_etc_hosts = sorted(hf.iter_all_host_addr_pairs())
+        if len(all_etc_hosts) > cls.MAX_HOSTS and not DEBUG:
+            all_etc_hosts = all_etc_hosts[:cls.MAX_HOSTS]
+
+        for host, ip in all_etc_hosts:
+            add(cls, host)
+            add(cls, ip)
+
+
+
+TestEtcHosts.populate_tests()
+
 
 
 class TestGeventOrg(TestCase):
@@ -544,12 +558,12 @@ class Test_getaddrinfo(TestCase):
 class TestInternational(TestCase):
     pass
 
-if not (PY2 and RESOLVER_DNSPYTHON):
-    # dns python can actually resolve these: it uses
-    # the 2008 version of idna encoding, whereas on Python 2,
-    # with the default resolver, it tries to encode to ascii and
-    # raises a UnicodeEncodeError. So we get different results.
-    add(TestInternational, u'президент.рф', 'russian')
+# dns python can actually resolve these: it uses
+# the 2008 version of idna encoding, whereas on Python 2,
+# with the default resolver, it tries to encode to ascii and
+# raises a UnicodeEncodeError. So we get different results.
+add(TestInternational, u'президент.рф', 'russian',
+    skip=(PY2 and RESOLVER_DNSPYTHON), skip_reason="dnspython can actually resolve these")
 add(TestInternational, u'президент.рф'.encode('idna'), 'idna')