Merge pull request #1094 from gevent/dnspython-hosts

Add support for /etc/hosts to dnspython resolver.

appveyor.yml

@@ -131,7 +131,7 @@ install:
   # pip will build them from source using the MSVC compiler matching the
   # target Python version and architecture
   # Note that psutil won't build under PyPy on Windows.
-  - "%CMD_IN_ENV% pip install -U setuptools wheel cython greenlet cffi"
+  - "%CMD_IN_ENV% pip install -U setuptools wheel cython greenlet cffi dnspython idna"
 
   - ps:
       if ("${env:PYTHON_ID}" -ne "pypy") {

src/gevent/queue.py

@@ -45,6 +45,9 @@ from gevent.hub import InvalidSwitchError
 __implements__ = ['Queue', 'PriorityQueue', 'LifoQueue']
 __extensions__ = ['JoinableQueue', 'Channel']
 __imports__ = ['Empty', 'Full']
+if hasattr(__queue__, 'SimpleQueue'):
+    __imports__.append('SimpleQueue') # New in 3.7
+    SimpleQueue = __queue__.SimpleQueue
 __all__ = __implements__ + __extensions__ + __imports__
 
 

src/gevent/resolver/__init__.py

@@ -63,8 +63,12 @@ def _lookup_port(port, socktype):
         socktypes.append(socktype)
     return port, socktypes
 
+hostname_types = tuple(set(string_types + (bytearray, bytes)))
 
 def _resolve_special(hostname, family):
+    if not isinstance(hostname, hostname_types):
+        raise TypeError("argument 1 must be str, bytes or bytearray, not %s" % (type(hostname),))
+
     if hostname == '':
         result = getaddrinfo(None, 0, family, SOCK_DGRAM, 0, AI_PASSIVE)
         if len(result) != 1:
@@ -80,7 +84,7 @@ class AbstractResolver(object):
         return self.gethostbyname_ex(hostname, family)[-1][0]
 
     def gethostbyname_ex(self, hostname, family=AF_INET):
-        aliases = []
+        aliases = self._getaliases(hostname, family)
         addresses = []
         tuples = self.getaddrinfo(hostname, 0, family,
                                   SOCK_STREAM,
@@ -93,3 +97,7 @@ class AbstractResolver(object):
 
     def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
         raise NotImplementedError()
+
+    def _getaliases(self, hostname, family):
+        # pylint:disable=unused-argument
+        return []

src/greentest/known_failures.py

@@ -157,12 +157,12 @@ if PYPY:
             'test__socket_dns.py',
         ]
 
-        if TRAVIS:
-            FAILING_TESTS += [
-                # This fails to get the correct results, sometimes. I can't reproduce locally
-                'FLAKY test__example_udp_server.py',
-                'FLAKY test__example_udp_client.py',
-            ]
+    if TRAVIS:
+        FAILING_TESTS += [
+            # This fails to get the correct results, sometimes. I can't reproduce locally
+            'FLAKY test__example_udp_server.py',
+            'FLAKY test__example_udp_client.py',
+        ]
 
     if PY3 and TRAVIS:
         FAILING_TESTS += [

src/greentest/test__socket_dns.py

@@ -50,7 +50,7 @@ def format_call(function, args):
     if args.endswith(',)'):
         args = args[:-2] + ')'
     try:
-        module = function.__module__.replace('gevent.socket', 'gevent').replace('_socket', 'stdlib')
+        module = function.__module__.replace('gevent._socketcommon', 'gevent')
         name = function.__name__
         return '%s:%s%s' % (module, name, args)
     except AttributeError:
@@ -166,13 +166,16 @@ def relaxed_is_equal(a, b):
     return all(relaxed_is_equal(x, y) for (x, y) in zip(a, b))
 
 
-def add(klass, hostname, name=None):
+def add(klass, hostname, name=None,
+        skip=None, skip_reason=None):
 
     call = callable(hostname)
 
-    def _setattr(k, n, v):
+    def _setattr(k, n, func):
+        if skip:
+            func = greentest.skipIf(skip, skip_reason,)(func)
         if not hasattr(k, n):
-            setattr(k, n, v)
+            setattr(k, n, func)
 
     if name is None:
         if call:
@@ -283,7 +286,8 @@ class TestCase(greentest.TestCase):
         ips = result[2]
         if ips == ['127.0.0.1', '127.0.0.1']:
             ips = ['127.0.0.1']
-        return (result[0], [], ips)
+        # On some systems, the hostname can get caps
+        return (result[0].lower(), [], ips)
 
     def _normalize_result_getaddrinfo(self, result):
         if not RESOLVER_NOT_SYSTEM:
@@ -345,8 +349,6 @@ add(TestTypeError, None)
 add(TestTypeError, 25)
 
 
-@unittest.skipIf(RESOLVER_DNSPYTHON,
-                 "This commonly needs /etc/hosts to function, and dnspython doesn't do that.")
 class TestHostname(TestCase):
     pass
 
@@ -423,33 +425,45 @@ class TestBroadcast(TestCase):
 add(TestBroadcast, '<broadcast>')
 
 
-@unittest.skipIf(RESOLVER_DNSPYTHON, "/etc/hosts completely ignored under dnspython")
+from gevent.resolver.dnspython import HostsFile
+class SanitizedHostsFile(HostsFile):
+    def iter_all_host_addr_pairs(self):
+        for name, addr in super(SanitizedHostsFile, self).iter_all_host_addr_pairs():
+            if (RESOLVER_NOT_SYSTEM
+                    and (name.endswith('local') # ignore bonjour, ares can't find them
+                         # ignore common aliases that ares can't find
+                         or addr == '255.255.255.255'
+                         or name == 'broadcasthost'
+                         # We get extra results from some impls, like OS X
+                         # it returns DGRAM results
+                         or name == 'localhost')):
+                continue
+            if name.endswith('local'):
+                # These can only be found if bonjour is running,
+                # and are very slow to do so with the system resolver on OS X
+                continue
+            yield name, addr
+
 class TestEtcHosts(TestCase):
-    pass
 
-try:
-    with open('/etc/hosts') as f:
-        etc_hosts = f.read()
-except IOError:
-    etc_hosts = ''
-
-for ip, host in re.findall(r'^\s*(\d+\.\d+\.\d+\.\d+)\s+([^\s]+)', etc_hosts, re.M)[:10]:
-    if (RESOLVER_NOT_SYSTEM
-        and (host.endswith('local') # ignore bonjour, ares can't find them
-             # ignore common aliases that ares can't find
-             or ip == '255.255.255.255'
-             or host == 'broadcasthost'
-             # We get extra results from some impls, like OS X
-             # it returns DGRAM results
-             or host == 'localhost')):
-        continue
-    if host.endswith('local'):
-        # These can only be found if bonjour is running,
-        # and are very slow to do so with the system resolver on OS X
-        continue
-    add(TestEtcHosts, host)
-    add(TestEtcHosts, ip)
-    del host, ip
+    MAX_HOSTS = os.getenv('GEVENTTEST_MAX_ETC_HOSTS', 10)
+
+    @classmethod
+    def populate_tests(cls):
+        hf = SanitizedHostsFile(os.path.join(os.path.dirname(__file__),
+                                             'hosts_file.txt'))
+        all_etc_hosts = sorted(hf.iter_all_host_addr_pairs())
+        if len(all_etc_hosts) > cls.MAX_HOSTS and not DEBUG:
+            all_etc_hosts = all_etc_hosts[:cls.MAX_HOSTS]
+
+        for host, ip in all_etc_hosts:
+            add(cls, host)
+            add(cls, ip)
+
+
+
+TestEtcHosts.populate_tests()
+
 
 
 class TestGeventOrg(TestCase):
@@ -544,12 +558,12 @@ class Test_getaddrinfo(TestCase):
 class TestInternational(TestCase):
     pass
 
-if not (PY2 and RESOLVER_DNSPYTHON):
-    # dns python can actually resolve these: it uses
-    # the 2008 version of idna encoding, whereas on Python 2,
-    # with the default resolver, it tries to encode to ascii and
-    # raises a UnicodeEncodeError. So we get different results.
-    add(TestInternational, u'президент.рф', 'russian')
+# dns python can actually resolve these: it uses
+# the 2008 version of idna encoding, whereas on Python 2,
+# with the default resolver, it tries to encode to ascii and
+# raises a UnicodeEncodeError. So we get different results.
+add(TestInternational, u'президент.рф', 'russian',
+    skip=(PY2 and RESOLVER_DNSPYTHON), skip_reason="dnspython can actually resolve these")
 add(TestInternational, u'президент.рф'.encode('idna'), 'idna')