Enable the dnspython resolver to work in non-monkey-patched processes

Do this by importing dns specially with sys.modules containing the
gevent modules. This is the technique eventlet uses (but we don't
leave the patched modules sitting around in sys.modules after we're
done; eventlet may have good reason to do that, but this is not a
public API here and is specialized just for dns).

Makefile

@@ -67,7 +67,7 @@ alltest: basictest
 	cd src/greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 ${PYTHON} testrunner.py --config known_failures.py --ignore tests_that_dont_use_resolver.txt --quiet
 	${PYTHON} scripts/travis.py fold_end ares
 	${PYTHON} scripts/travis.py fold_start dnspython "Running dnspython tests"
-	cd src/greentest && GEVENT_RESOLVER=dnspython ${PYTHON} testrunner.py --config known_failures.py --ignore tests_that_dont_use_resolver.txt,tests_that_dont_monkeypatch.txt --quiet
+	cd src/greentest && GEVENT_RESOLVER=dnspython ${PYTHON} testrunner.py --config known_failures.py --ignore tests_that_dont_use_resolver.txt --quiet
 	${PYTHON} scripts/travis.py fold_end dnspython
 # In the past, we included all test files that had a reference to 'subprocess'' somewhere in their
 # text. The monkey-patched stdlib tests were specifically included here.

src/gevent/_patcher.py

+# Copyright 2018 gevent. See LICENSE for details.
+
+# Portions of the following are inspired by code from eventlet. I
+# believe they are distinct enough that no eventlet copyright would
+# apply (they are not a copy or substantial portion of the eventlot
+# code).
+
+# Added in gevent 1.3a2. Not public in that release.
+
+from __future__ import absolute_import, print_function
+
+import imp
+import importlib
+import sys
+
+from gevent._compat import PY3
+from gevent._compat import iteritems
+
+from gevent.builtins import __import__ as _import
+
+
+MAPPING = {
+    'gevent.local': '_threading_local',
+    'gevent.socket': 'socket',
+    'gevent.select': 'select',
+    'gevent.ssl': 'ssl',
+    'gevent.thread': '_thread' if PY3 else 'thread',
+    'gevent.subprocess': 'subprocess',
+    'gevent.os': 'os',
+    'gevent.threading': 'threading',
+    'gevent.builtins': 'builtins' if PY3 else '__builtin__',
+    'gevent.signal': 'signal',
+    'gevent.time': 'time',
+    'gevent.queue': 'queue' if PY3 else 'Queue',
+}
+
+_PATCH_PREFIX = '__g_patched_module_'
+
+class _SysModulesPatcher(object):
+
+    def __init__(self, importing):
+        self._saved = {}
+        self.importing = importing
+        self.green_modules = {
+            stdlib_name: importlib.import_module(gevent_name)
+            for gevent_name, stdlib_name
+            in iteritems(MAPPING)
+        }
+        self.orig_imported = frozenset(sys.modules)
+
+    def _save(self):
+        for modname in self.green_modules:
+            self._saved[modname] = sys.modules.get(modname, None)
+
+        self._saved[self.importing] = sys.modules.get(self.importing, None)
+        # Anything we've already patched regains its original name during this
+        # process
+        for mod_name, mod in iteritems(sys.modules):
+            if mod_name.startswith(_PATCH_PREFIX):
+                orig_mod_name = mod_name[len(_PATCH_PREFIX):]
+                self._saved[mod_name] = sys.modules.get(orig_mod_name, None)
+                self.green_modules[orig_mod_name] = mod
+
+    def _replace(self):
+        # Cover the target modules so that when you import the module it
+        # sees only the patched versions
+        for name, mod in iteritems(self.green_modules):
+            sys.modules[name] = mod
+
+    def _restore(self):
+        for modname, mod in iteritems(self._saved):
+            if mod is not None:
+                sys.modules[modname] = mod
+            else:
+                try:
+                    del sys.modules[modname]
+                except KeyError:
+                    pass
+        # Anything from the same package tree we imported this time
+        # needs to be saved so we can restore it later, and so it doesn't
+        # leak into the namespace.
+        pkg_prefix = self.importing.split('.', 1)[0]
+        for modname, mod in list(iteritems(sys.modules)):
+            if (modname not in self.orig_imported
+                    and modname != self.importing
+                    and not modname.startswith(_PATCH_PREFIX)
+                    and modname.startswith(pkg_prefix)):
+                sys.modules[_PATCH_PREFIX + modname] = mod
+                del sys.modules[modname]
+
+    def __exit__(self, t, v, tb):
+        try:
+            self._restore()
+        finally:
+            imp.release_lock()
+
+    def __enter__(self):
+        imp.acquire_lock()
+        self._save()
+        self._replace()
+
+
+def import_patched(module_name):
+    """
+    Import *module_name* with gevent monkey-patches active,
+    and return the greened module.
+
+    Any sub-modules that were imported by the package are also
+    saved.
+
+    """
+    patched_name = _PATCH_PREFIX + module_name
+    if patched_name in sys.modules:
+        return sys.modules[patched_name]
+
+
+    # Save the current module state, and restore on exit,
+    # capturing desirable changes in the modules package.
+    with _SysModulesPatcher(module_name):
+        sys.modules.pop(module_name, None)
+
+        module = _import(module_name, {}, {}, module_name.split('.')[:-1])
+        sys.modules[patched_name] = module
+
+    return module

src/gevent/builtins.py

@@ -14,15 +14,15 @@ from gevent.lock import RLock
 # So we test for the old, deprecated version first
 
 try: # Py2
-    import __builtin__ as builtins
+    import __builtin__ as __gbuiltins__
     _allowed_module_name_types = (basestring,) # pylint:disable=undefined-variable
     __target__ = '__builtin__'
 except ImportError:
-    import builtins # pylint: disable=import-error
+    import builtins as __gbuiltins__ # pylint: disable=import-error
     _allowed_module_name_types = (str,)
     __target__ = 'builtins'
 
-_import = builtins.__import__
+_import = __gbuiltins__.__import__
 
 # We need to protect imports both across threads and across greenlets.
 # And the order matters. Note that under 3.4, the global import lock
@@ -120,6 +120,13 @@ def _lock_imports():
 
 if sys.version_info[:2] >= (3, 3):
     __implements__ = []
+    __import__ = _import
 else:
     __implements__ = ['__import__']
 __all__ = __implements__
+
+
+from gevent._util import copy_globals
+
+__imports__ = copy_globals(__gbuiltins__, globals(),
+                           names_to_ignore=__implements__)

src/gevent/queue.py

@@ -42,7 +42,10 @@ from gevent.hub import get_hub, Waiter, getcurrent
 from gevent.hub import InvalidSwitchError
 
 
-__all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue', 'JoinableQueue', 'Channel']
+__implements__ = ['Queue', 'PriorityQueue', 'LifoQueue']
+__extensions__ = ['JoinableQueue', 'Channel']
+__imports__ = ['Empty', 'Full']
+__all__ = __implements__ + __extensions__ + __imports__
 
 
 def _safe_remove(deq, item):

src/gevent/resolver/dnspython.py

 # Copyright (c) 2018  gevent contributors. See LICENSE for details.
 
+
 import _socket
 from _socket import AI_NUMERICHOST
 from _socket import error
@@ -9,14 +10,41 @@ import socket
 
 from . import AbstractResolver
 
-from dns import resolver
-import dns
-
+from gevent._patcher import import_patched
 
 __all__ = [
     'Resolver',
 ]
 
+# Import the DNS packages to use the gevent modules,
+# even if the system is not monkey-patched.
+dns = import_patched('dns')
+
+for pkg in ('dns',
+            'dns.rdtypes',
+            'dns.rdtypes.IN',
+            'dns.rdtypes.ANY'):
+    mod = import_patched(pkg)
+    for name in mod.__all__:
+        setattr(mod, name, import_patched(pkg + '.' + name))
+
+
+def _dns_import_patched(name):
+    assert name.startswith('dns')
+    import_patched(name)
+    return dns
+
+# This module tries to dynamically import classes
+# using __import__, and it's important that they match
+# the ones we just created, otherwise exceptions won't be caught
+# as expected. It uses a one-arg __import__ statement and then
+# tries to walk down the sub-modules using getattr, so we can't
+# directly use import_patched as-is.
+dns.rdata.__import__ = _dns_import_patched
+
+resolver = dns.resolver
+
+
 # This is a copy of resolver._getaddrinfo with the crucial change that it
 # doesn't have a bare except:, because that breaks Timeout and KeyboardInterrupt
 # See https://github.com/rthalley/dnspython/pull/300
@@ -123,6 +151,8 @@ def _getaddrinfo(host=None, service=None, family=socket.AF_UNSPEC, socktype=0,
     return tuples
 
 
+resolver._getaddrinfo = _getaddrinfo
+
 class Resolver(AbstractResolver):
     """
     An *experimental* resolver that uses `dnspython`_.
@@ -136,17 +166,15 @@ class Resolver(AbstractResolver):
     resolver can resolve Unicode host names that the system resolver
     cannot.
 
-    This uses thread locks and sockets, so it only functions if the
-    system has been monkey-patched. Otherwise it will raise a
-    ``ValueError``.
+    .. note::
 
-    This uses dnspython's default resolver object. This object has
-    several useful attributes that can be used to adjust the behaviour
-    of the DNS system; in particular, the ``cache`` attribute could be
-    set to an instance of :class:`dns.resolver.Cache` or
-    :class:`dns.resolver.LRUCache` (by default a ``LRUCache`` is
-    used), and ``nameservers`` controls which nameservers to talk to,
-    and ``lifetime`` configures a timeout for each individual query.
+        This **does not** use dnspython's default resolver object, or share any
+        classes with ``import dns``. A separate copy of the objects is imported to
+        be able to function in a non monkey-patched process. The documentation for the resolver
+        object still applies.
+
+        The resolver that we use is available as the :attr:`resolver` attribute
+        of this object (typically ``gevent.get_hub().resolver.resolver``).
 
     .. caution::
 
@@ -169,15 +197,25 @@ class Resolver(AbstractResolver):
     """
 
     def __init__(self, hub=None): # pylint: disable=unused-argument
-        from gevent import monkey
-        if not all(monkey.is_module_patched(m) for m in ['threading', 'socket', 'select']):
-            raise ValueError("Can only be used when monkey-patched")
         if resolver._resolver is None:
             resolver._resolver = resolver.get_default_resolver()
             # Add a default cache
             resolver._resolver.cache = resolver.LRUCache()
-        if resolver._getaddrinfo is not _getaddrinfo:
-            resolver._getaddrinfo = _getaddrinfo
+
+    @property
+    def resolver(self):
+        """
+        The dnspython resolver object we use.
+
+        This object has several useful attributes that can be used to
+        adjust the behaviour of the DNS system:
+
+        * ``cache`` is a :class:`dns.resolver.LRUCache`. Its maximum size
+           can be configured by calling :meth:`resolver.cache.set_max_size`
+        *  ``nameservers`` controls which nameservers to talk to
+        * ``lifetime`` configures a timeout for each individual query.
+        """
+        return resolver._resolver
 
     def close(self):
         pass

src/greentest/test__all__.py

@@ -7,21 +7,7 @@ import types
 from greentest.modules import walk_modules
 from greentest.sysinfo import PLATFORM_SPECIFIC_SUFFIXES
 
-
-MAPPING = {
-    'gevent.local': '_threading_local',
-    'gevent.socket': 'socket',
-    'gevent.select': 'select',
-    'gevent.ssl': 'ssl',
-    'gevent.thread': '_thread' if six.PY3 else 'thread',
-    'gevent.subprocess': 'subprocess',
-    'gevent.os': 'os',
-    'gevent.threading': 'threading',
-    'gevent.builtins': 'builtins' if six.PY3 else '__builtin__',
-    'gevent.signal': 'signal',
-    'gevent.time': 'time',
-}
-
+from gevent._patcher import MAPPING
 
 class ANY(object):
     def __contains__(self, item):
@@ -53,6 +39,7 @@ NO_ALL = [
     'gevent._fileobjectposix',
     'gevent._tblib',
     'gevent._corecffi',
+    'gevent._patcher',
 ]
 
 # A list of modules that may contain things that aren't actually, technically,
@@ -90,7 +77,8 @@ class Test(unittest.TestCase):
     def check_implements_presence_justified(self):
         "Check that __implements__ is present only if the module is modeled after a module from stdlib (like gevent.socket)."
         if self.__implements__ is not None and self.stdlib_module is None:
-            raise AssertionError('%r has __implements__ but no stdlib counterpart' % self.modname)
+            raise AssertionError('%r has __implements__ but no stdlib counterpart (%s)'
+                                 % (self.modname, self.stdlib_name))
 
     def set_stdlib_all(self):
         self.assertIsNotNone(self.stdlib_module)

src/greentest/test__socket_dns.py

@@ -4,10 +4,6 @@
 import gevent
 from gevent import monkey
 
-if ['gevent.resolver.dnspython.Resolver'] == gevent.get_hub().resolver_class:
-    # dnspython requires monkey-patching
-    monkey.patch_all()
-
 import os
 import re
 import greentest