asyncio: Skip getaddrinfo if host is already resolved.

getaddrinfo takes an exclusive lock on some platforms, causing clients to queue
up waiting for the lock if many names are being resolved concurrently. Users
may want to handle name resolution in their own code, for the sake of caching,
using an alternate resolver, or to measure DNS duration separately from
connection duration. Skip getaddrinfo if the "host" passed into
create_connection is already resolved.

See https://github.com/python/asyncio/pull/302 for details.

Patch by A. Jesse Jiryu Davis.

Lib/asyncio/base_events.py

@@ -16,8 +16,10 @@ to modify the meaning of the API call itself.
 
 import collections
 import concurrent.futures
+import functools
 import heapq
 import inspect
+import ipaddress
 import itertools
 import logging
 import os
@@ -70,49 +72,83 @@ def _format_pipe(fd):
         return repr(fd)
 
 
-def _check_resolved_address(sock, address):
-    # Ensure that the address is already resolved to avoid the trap of hanging
-    # the entire event loop when the address requires doing a DNS lookup.
-    #
-    # getaddrinfo() is slow (around 10 us per call): this function should only
-    # be called in debug mode
-    family = sock.family
-
-    if family == socket.AF_INET:
-        host, port = address
-    elif family == socket.AF_INET6:
-        host, port = address[:2]
+# Linux's sock.type is a bitmask that can include extra info about socket.
+_SOCKET_TYPE_MASK = 0
+if hasattr(socket, 'SOCK_NONBLOCK'):
+    _SOCKET_TYPE_MASK |= socket.SOCK_NONBLOCK
+if hasattr(socket, 'SOCK_CLOEXEC'):
+    _SOCKET_TYPE_MASK |= socket.SOCK_CLOEXEC
+
+
+@functools.lru_cache(maxsize=1024)
+def _ipaddr_info(host, port, family, type, proto):
+    # Try to skip getaddrinfo if "host" is already an IP. Since getaddrinfo
+    # blocks on an exclusive lock on some platforms, users might handle name
+    # resolution in their own code and pass in resolved IPs.
+    if proto not in {0, socket.IPPROTO_TCP, socket.IPPROTO_UDP} or host is None:
+        return None
+
+    type &= ~_SOCKET_TYPE_MASK
+    if type == socket.SOCK_STREAM:
+        proto = socket.IPPROTO_TCP
+    elif type == socket.SOCK_DGRAM:
+        proto = socket.IPPROTO_UDP
     else:
-        return
+        return None
 
-    # On Windows, socket.inet_pton() is only available since Python 3.4
     if hasattr(socket, 'inet_pton'):
-        # getaddrinfo() is slow and has known issue: prefer inet_pton()
-        # if available
+        if family == socket.AF_UNSPEC:
+            afs = [socket.AF_INET, socket.AF_INET6]
+        else:
+            afs = [family]
+
+        for af in afs:
+            # Linux's inet_pton doesn't accept an IPv6 zone index after host,
+            # like '::1%lo0', so strip it. If we happen to make an invalid
+            # address look valid, we fail later in sock.connect or sock.bind.
             try:
-            socket.inet_pton(family, host)
-        except OSError as exc:
-            raise ValueError("address must be resolved (IP address), "
-                             "got host %r: %s"
-                             % (host, exc))
+                if af == socket.AF_INET6:
+                    socket.inet_pton(af, host.partition('%')[0])
                 else:
-        # Use getaddrinfo(flags=AI_NUMERICHOST) to ensure that the address is
-        # already resolved.
-        type_mask = 0
-        if hasattr(socket, 'SOCK_NONBLOCK'):
-            type_mask |= socket.SOCK_NONBLOCK
-        if hasattr(socket, 'SOCK_CLOEXEC'):
-            type_mask |= socket.SOCK_CLOEXEC
+                    socket.inet_pton(af, host)
+                return af, type, proto, '', (host, port)
+            except OSError:
+                pass
+
+        # "host" is not an IP address.
+        return None
+
+    # No inet_pton. (On Windows it's only available since Python 3.4.)
+    # Even though getaddrinfo with AI_NUMERICHOST would be non-blocking, it
+    # still requires a lock on some platforms, and waiting for that lock could
+    # block the event loop. Use ipaddress instead, it's just text parsing.
+    try:
+        addr = ipaddress.IPv4Address(host)
+    except ValueError:
         try:
-            socket.getaddrinfo(host, port,
-                               family=family,
-                               type=(sock.type & ~type_mask),
-                               proto=sock.proto,
-                               flags=socket.AI_NUMERICHOST)
-        except socket.gaierror as err:
-            raise ValueError("address must be resolved (IP address), "
-                             "got host %r: %s"
-                             % (host, err))
+            addr = ipaddress.IPv6Address(host.partition('%')[0])
+        except ValueError:
+            return None
+
+    af = socket.AF_INET if addr.version == 4 else socket.AF_INET6
+    if family not in (socket.AF_UNSPEC, af):
+        # "host" is wrong IP version for "family".
+        return None
+
+    return af, type, proto, '', (host, port)
+
+
+def _check_resolved_address(sock, address):
+    # Ensure that the address is already resolved to avoid the trap of hanging
+    # the entire event loop when the address requires doing a DNS lookup.
+
+    if hasattr(socket, 'AF_UNIX') and sock.family == socket.AF_UNIX:
+        return
+
+    host, port = address[:2]
+    if _ipaddr_info(host, port, sock.family, sock.type, sock.proto) is None:
+        raise ValueError("address must be resolved (IP address),"
+                         " got host %r" % host)
 
 
 def _run_until_complete_cb(fut):
@@ -535,7 +571,12 @@ class BaseEventLoop(events.AbstractEventLoop):
 
     def getaddrinfo(self, host, port, *,
                     family=0, type=0, proto=0, flags=0):
-        if self._debug:
+        info = _ipaddr_info(host, port, family, type, proto)
+        if info is not None:
+            fut = futures.Future(loop=self)
+            fut.set_result([info])
+            return fut
+        elif self._debug:
             return self.run_in_executor(None, self._getaddrinfo_debug,
                                         host, port, family, type, proto, flags)
         else:

Lib/asyncio/proactor_events.py

@@ -441,7 +441,6 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
 
     def sock_connect(self, sock, address):
         try:
-            if self._debug:
             base_events._check_resolved_address(sock, address)
         except ValueError as err:
             fut = futures.Future(loop=self)

Lib/asyncio/selector_events.py

@@ -397,7 +397,6 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop):
             raise ValueError("the socket must be non-blocking")
         fut = futures.Future(loop=self)
         try:
-            if self._debug:
             base_events._check_resolved_address(sock, address)
         except ValueError as err:
             fut.set_exception(err)

Lib/asyncio/test_utils.py

@@ -446,9 +446,14 @@ def disable_logger():
     finally:
         logger.setLevel(old_level)
 
-def mock_nonblocking_socket():
+
+def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM,
+                            family=socket.AF_INET):
     """Create a mock of a non-blocking socket."""
-    sock = mock.Mock(socket.socket)
+    sock = mock.MagicMock(socket.socket)
+    sock.proto = proto
+    sock.type = type
+    sock.family = family
     sock.gettimeout.return_value = 0.0
     return sock
 

Lib/test/test_asyncio/test_events.py

@@ -1573,10 +1573,6 @@ class EventLoopTestsMixin:
              'selector': self.loop._selector.__class__.__name__})
 
     def test_sock_connect_address(self):
-        # In debug mode, sock_connect() must ensure that the address is already
-        # resolved (call _check_resolved_address())
-        self.loop.set_debug(True)
-
         addresses = [(socket.AF_INET, ('www.python.org', 80))]
         if support.IPV6_ENABLED:
             addresses.extend((

Lib/test/test_asyncio/test_proactor_events.py

@@ -436,7 +436,7 @@ class ProactorSocketTransportTests(test_utils.TestCase):
 class BaseProactorEventLoopTests(test_utils.TestCase):
 
     def setUp(self):
-        self.sock = mock.Mock(socket.socket)
+        self.sock = test_utils.mock_nonblocking_socket()
         self.proactor = mock.Mock()
 
         self.ssock, self.csock = mock.Mock(), mock.Mock()
@@ -491,8 +491,8 @@ class BaseProactorEventLoopTests(test_utils.TestCase):
         self.proactor.send.assert_called_with(self.sock, b'data')
 
     def test_sock_connect(self):
-        self.loop.sock_connect(self.sock, 123)
-        self.proactor.connect.assert_called_with(self.sock, 123)
+        self.loop.sock_connect(self.sock, ('1.2.3.4', 123))
+        self.proactor.connect.assert_called_with(self.sock, ('1.2.3.4', 123))
 
     def test_sock_accept(self):
         self.loop.sock_accept(self.sock)