Merge pull request #1369 from gevent/py2-future-shadow

 Patch the 'thread' module provided by 'future' if it's already imported

CHANGES.rst

@@ -30,6 +30,14 @@
   <https://bugs.python.org/issue32270>`_ applied to all versions
   gevent runs on.
 
+- Python 2: If the backport of the ``_thread_`` module from
+  ``futures`` has already been imported at monkey-patch time, also
+  patch this module to be consistent. The ``pkg_resources`` package
+  imports this, and ``pkg_resources`` is often imported early on
+  Python 2 for namespace packages, so if ``futures`` is installed this
+  will likely be the case.
+
+
 1.4.0 (2019-01-04)
 ==================
 

src/gevent/_compat.py

@@ -5,12 +5,19 @@ internal gevent python 2/python 3 bridges. Not for external use.
 
 from __future__ import print_function, absolute_import, division
 
+## Important: This module should generally not have any other gevent
+## imports (the exception is _util_py2)
+
 import sys
 import os
 
 
 PY2 = sys.version_info[0] == 2
 PY3 = sys.version_info[0] >= 3
+PY35 = sys.version_info[:2] >= (3, 5)
+PY36 = sys.version_info[:2] >= (3, 6)
+PY37 = sys.version_info[:2] >= (3, 7)
+PY38 = sys.version_info[:2] >= (3, 8)
 PYPY = hasattr(sys, 'pypy_version_info')
 WIN = sys.platform.startswith("win")
 LINUX = sys.platform.startswith('linux')

src/gevent/_socket3.py

@@ -607,25 +607,23 @@ class socket(object):
         """
         return self._sendfile_use_send(file, offset, count)
 
-    # get/set_inheritable new in 3.4
-    if hasattr(os, 'get_inheritable') or hasattr(os, 'get_handle_inheritable'):
-        # pylint:disable=no-member
-        if os.name == 'nt':
-            def get_inheritable(self):
-                return os.get_handle_inheritable(self.fileno())
-
-            def set_inheritable(self, inheritable):
-                os.set_handle_inheritable(self.fileno(), inheritable)
-        else:
-            def get_inheritable(self):
-                return os.get_inheritable(self.fileno())
 
-            def set_inheritable(self, inheritable):
-                os.set_inheritable(self.fileno(), inheritable)
-        _added = "\n\n.. versionadded:: 1.1rc4 Added in Python 3.4"
-        get_inheritable.__doc__ = "Get the inheritable flag of the socket" + _added
-        set_inheritable.__doc__ = "Set the inheritable flag of the socket" + _added
-        del _added
+    if os.name == 'nt':
+        def get_inheritable(self):
+            return os.get_handle_inheritable(self.fileno())
+
+        def set_inheritable(self, inheritable):
+            os.set_handle_inheritable(self.fileno(), inheritable)
+    else:
+        def get_inheritable(self):
+            return os.get_inheritable(self.fileno())
+
+        def set_inheritable(self, inheritable):
+            os.set_inheritable(self.fileno(), inheritable)
+
+    get_inheritable.__doc__ = "Get the inheritable flag of the socket"
+    set_inheritable.__doc__ = "Set the inheritable flag of the socket"
+
 
 
 SocketType = socket
@@ -652,6 +650,7 @@ if hasattr(_socket.socket, "share"):
 
     __implements__.append('fromshare')
 
+
 if hasattr(_socket, "socketpair"):
 
     def socketpair(family=None, type=SOCK_STREAM, proto=0):
@@ -679,9 +678,8 @@ if hasattr(_socket, "socketpair"):
 else: # pragma: no cover
     # Origin: https://gist.github.com/4325783, by Geert Jansen.  Public domain.
 
-    # gevent: taken from 3.6 release. Expected to be used only on Win. Added to Win/3.5
-    # gevent: for < 3.5, pass the default value of 128 to lsock.listen()
-    # (3.5+ uses this as a default and the original code passed no value)
+    # gevent: taken from 3.6 release, confirmed unchanged in 3.7 and
+    # 3.8a1. Expected to be used only on Win. Added to Win/3.5
 
     _LOCALHOST = '127.0.0.1'
     _LOCALHOST_V6 = '::1'
@@ -704,7 +702,7 @@ else: # pragma: no cover
         lsock = socket(family, type, proto)
         try:
             lsock.bind((host, 0))
-            lsock.listen(128)
+            lsock.listen()
             # On IPv6, ignore flow_info and scope_id
             addr, port = lsock.getsockname()[:2]
             csock = socket(family, type, proto)
@@ -723,14 +721,6 @@ else: # pragma: no cover
             lsock.close()
         return (ssock, csock)
 
-    if sys.version_info[:2] < (3, 5):
-        # Not provided natively
-        if 'socketpair' in __implements__:
-            # Multiple imports can cause this to be missing if _socketcommon
-            # was successfully imported, leading to subsequent imports to cause
-            # ValueError
-            __implements__.remove('socketpair')
-
 
 if hasattr(__socket__, 'close'): # Python 3.7b1+
     close = __socket__.close # pylint:disable=no-member

src/gevent/backdoor.py

@@ -17,6 +17,7 @@ from gevent.greenlet import Greenlet
 from gevent.hub import getcurrent
 from gevent.server import StreamServer
 from gevent.pool import Pool
+from gevent._compat import PY36
 
 __all__ = ['BackdoorServer']
 
@@ -145,7 +146,7 @@ class BackdoorServer(StreamServer):
         getcurrent().switch_in()
         try:
             console = InteractiveConsole(self._create_interactive_locals())
-            if sys.version_info[:3] >= (3, 6, 0):
+            if PY36:
                 # Beginning in 3.6, the console likes to print "now exiting <class>"
                 # but probably our socket is already closed, so this just causes problems.
                 console.interact(banner=self.banner, exitmsg='') # pylint:disable=unexpected-keyword-arg

src/gevent/builtins.py

@@ -2,10 +2,10 @@
 """gevent friendly implementations of builtin functions."""
 from __future__ import absolute_import
 
-import sys
 import weakref
 
 from gevent.lock import RLock
+from gevent._compat import PY3
 from gevent._compat import imp_acquire_lock
 from gevent._compat import imp_release_lock
 
@@ -121,7 +121,7 @@ def _lock_imports():
     global __lock_imports
     __lock_imports = True
 
-if sys.version_info[:2] >= (3, 3):
+if PY3:
     __implements__ = []
     __import__ = _import
 else:

src/gevent/monkey.py

 # Copyright (c) 2009-2012 Denis Bilenko. See LICENSE for details.
-# pylint: disable=redefined-outer-name
+# pylint: disable=redefined-outer-name,too-many-lines
 """
 Make the standard library cooperative.
 
@@ -146,12 +146,15 @@ __all__ = [
 if sys.version_info[0] >= 3:
     string_types = (str,)
     PY3 = True
+    PY2 = False
 else:
     import __builtin__ # pylint:disable=import-error
     string_types = (__builtin__.basestring,)
     PY3 = False
+    PY2 = True
 
 WIN = sys.platform.startswith("win")
+PY36 = sys.version_info[:2] >= (3, 6)
 
 class MonkeyPatchWarning(RuntimeWarning):
     """
@@ -283,7 +286,9 @@ def __call_module_hook(gevent_module, name, module, items, _warnings):
 
 def patch_module(target_module, source_module, items=None,
                  _warnings=None,
-                 _notify_did_subscribers=True):
+                 _notify_will_subscribers=True,
+                 _notify_did_subscribers=True,
+                 _call_hooks=True):
     """
     patch_module(target_module, source_module, items=None)
 
@@ -318,18 +323,21 @@ def patch_module(target_module, source_module, items=None,
             raise AttributeError('%r does not have __implements__' % source_module)
 
     try:
-        __call_module_hook(source_module, 'will', target_module, items, _warnings)
-        _notify_patch(
-            events.GeventWillPatchModuleEvent(target_module.__name__, source_module,
-                                              target_module, items),
-            _warnings)
+        if _call_hooks:
+            __call_module_hook(source_module, 'will', target_module, items, _warnings)
+        if _notify_will_subscribers:
+            _notify_patch(
+                events.GeventWillPatchModuleEvent(target_module.__name__, source_module,
+                                                  target_module, items),
+                _warnings)
     except events.DoNotPatch:
         return False
 
     for attr in items:
         patch_item(target_module, attr, getattr(source_module, attr))
 
-    __call_module_hook(source_module, 'did', target_module, items, _warnings)
+    if _call_hooks:
+        __call_module_hook(source_module, 'did', target_module, items, _warnings)
 
     if _notify_did_subscribers:
         # We allow turning off the broadcast of the 'did' event for the benefit
@@ -342,7 +350,12 @@ def patch_module(target_module, source_module, items=None,
 
     return True
 
-def _patch_module(name, items=None, _warnings=None, _notify_did_subscribers=True):
+def _patch_module(name,
+                  items=None,
+                  _warnings=None,
+                  _notify_will_subscribers=True,
+                  _notify_did_subscribers=True,
+                  _call_hooks=True):
 
     gevent_module = getattr(__import__('gevent.' + name), name)
     module_name = getattr(gevent_module, '__target__', name)
@@ -350,7 +363,29 @@ def _patch_module(name, items=None, _warnings=None, _notify_did_subscribers=True
 
     patch_module(target_module, gevent_module, items=items,
                  _warnings=_warnings,
-                 _notify_did_subscribers=_notify_did_subscribers)
+                 _notify_will_subscribers=_notify_will_subscribers,
+                 _notify_did_subscribers=_notify_did_subscribers,
+                 _call_hooks=_call_hooks)
+
+    # On Python 2, the `futures` package will install
+    # a bunch of modules with the same name as those from Python 3,
+    # such as `_thread`; primarily these just do `from thread import *`,
+    # meaning we have alternate references. If that's already been imported,
+    # we need to attempt to patch that too.
+
+    # Be sure to keep the original states matching also.
+
+    alternate_names = getattr(gevent_module, '__alternate_targets__', ())
+    for alternate_name in alternate_names:
+        alternate_module = sys.modules.get(alternate_name)
+        if alternate_module is not None and alternate_module is not target_module:
+            saved.pop(alternate_name, None)
+            patch_module(alternate_module, gevent_module, items=items,
+                         _warnings=_warnings,
+                         _notify_will_subscribers=False,
+                         _notify_did_subscribers=False,
+                         _call_hooks=False)
+            saved[alternate_name] = saved[module_name]
 
     return gevent_module, target_module
 
@@ -583,11 +618,14 @@ def patch_thread(threading=True, _threading_local=True, Event=True, logging=True
         orig_current_thread = None
 
     gevent_thread_mod, thread_mod = _patch_module('thread',
-                                                  _warnings=_warnings, _notify_did_subscribers=False)
+                                                  _warnings=_warnings,
+                                                  _notify_did_subscribers=False)
+
 
     if threading:
         gevent_threading_mod, _ = _patch_module('threading',
-                                                _warnings=_warnings, _notify_did_subscribers=False)
+                                                _warnings=_warnings,
+                                                _notify_did_subscribers=False)
 
         if Event:
             from gevent.event import Event
@@ -648,7 +686,7 @@ def patch_thread(threading=True, _threading_local=True, Event=True, logging=True
                 continue
             thread.join = make_join_func(thread, None)
 
-    if sys.version_info[0] >= 3:
+    if PY3:
 
         # Issue 18808 changes the nature of Thread.join() to use
         # locks. This means that a greenlet spawned in the main thread
@@ -757,7 +795,7 @@ def patch_ssl(_warnings=None, _first_time=True):
     """
     may_need_warning = (
         _first_time
-        and sys.version_info[:2] >= (3, 6)
+        and PY36
         and 'ssl' in sys.modules
         and hasattr(sys.modules['ssl'], 'SSLContext'))
     # Previously, we didn't warn on Python 2 if pkg_resources has been imported
@@ -892,7 +930,7 @@ def patch_builtins():
     .. _greenlet safe: https://github.com/gevent/gevent/issues/108
 
     """
-    if sys.version_info[:2] < (3, 3):
+    if PY2:
         _patch_module('builtins')
 
 @_ignores_DoNotPatch

src/gevent/subprocess.py

@@ -44,6 +44,9 @@ from gevent.hub import sleep
 from gevent.hub import getcurrent
 from gevent._compat import integer_types, string_types, xrange
 from gevent._compat import PY3
+from gevent._compat import PY35
+from gevent._compat import PY36
+from gevent._compat import PY37
 from gevent._compat import reraise
 from gevent._compat import fspath
 from gevent._compat import fsencode
@@ -118,7 +121,7 @@ __extra__ = [
     'CompletedProcess',
 ]
 
-if sys.version_info[:2] >= (3, 3):
+if PY3:
     __imports__ += [
         'DEVNULL',
         'getstatusoutput',
@@ -130,7 +133,7 @@ else:
     __extra__.append("TimeoutExpired")
 
 
-if sys.version_info[:2] >= (3, 5):
+if PY35:
     __extra__.remove('run')
     __extra__.remove('CompletedProcess')
     __implements__.append('run')
@@ -144,12 +147,12 @@ if sys.version_info[:2] >= (3, 5):
     except:
         MAXFD = 256
 
-if sys.version_info[:2] >= (3, 6):
+if PY36:
     # This was added to __all__ for windows in 3.6
     __extra__.remove('STARTUPINFO')
     __imports__.append('STARTUPINFO')
 
-if sys.version_info[:2] >= (3, 7):
+if PY37:
     __imports__.extend([
         'ABOVE_NORMAL_PRIORITY_CLASS', 'BELOW_NORMAL_PRIORITY_CLASS',
         'HIGH_PRIORITY_CLASS', 'IDLE_PRIORITY_CLASS',
@@ -479,7 +482,7 @@ class Popen(object):
             if preexec_fn is not None:
                 raise ValueError("preexec_fn is not supported on Windows "
                                  "platforms")
-            if sys.version_info[:2] >= (3, 7):
+            if PY37:
                 if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
                     close_fds = True
             else:

src/gevent/tests/test__monkey_futures_thread.py

+# -*- coding: utf-8 -*-
+"""
+Tests that on Python 2, if the futures backport of 'thread' is already
+imported before we monkey-patch, it gets patched too.
+"""
+
+import unittest
+
+try:
+    import thread
+    import _thread
+    HAS_BOTH = True
+except ImportError:
+    HAS_BOTH = False
+
+class TestMonkey(unittest.TestCase):
+
+    @unittest.skipUnless(HAS_BOTH, "Python 2, needs future backport installed")
+    def test_patches_both(self):
+        thread_lt = thread.LockType
+        _thread_lt = _thread.LockType
+        self.assertIs(thread_lt, _thread_lt)
+
+        from gevent.thread import LockType as gLockType
+
+        self.assertIsNot(thread_lt, gLockType)
+
+        import gevent.monkey
+        gevent.monkey.patch_all()
+
+        thread_lt2 = thread.LockType
+        _thread_lt2 = _thread.LockType
+
+        self.assertIs(thread_lt2, gLockType)
+        self.assertIs(_thread_lt2, gLockType)
+
+        self.assertIs(thread_lt2, _thread_lt2)
+        self.assertIsNot(thread_lt2, thread_lt)
+
+        # Retrieving the original on the old name still works
+        orig_locktype = gevent.monkey.get_original('thread', 'LockType')
+        self.assertIs(orig_locktype, thread_lt)
+
+        # And the new name
+        orig__locktype = gevent.monkey.get_original('_thread', 'LockType')
+        self.assertIs(orig__locktype, thread_lt)
+
+if __name__ == '__main__':
+    unittest.main()

src/gevent/tests/test__socket_dns.py

@@ -15,6 +15,7 @@ import gevent.socket as gevent_socket
 from gevent.testing.util import log
 from gevent.testing import six
 from gevent.testing.six import xrange
+from gevent.testing import flaky
 
 
 resolver = gevent.get_hub().resolver
@@ -224,6 +225,18 @@ class TestCase(greentest.TestCase):
     switch_expected = None
     verbose_dns = False
 
+    def setUp(self):
+        super(TestCase, self).setUp()
+        if not self.verbose_dns:
+            # Silence the default reporting of errors from the ThreadPool,
+            # we handle those here.
+            gevent.get_hub().exception_stream = None
+
+    def tearDown(self):
+        if not self.verbose_dns:
+            del gevent.get_hub().exception_stream
+        super(TestCase, self).tearDown()
+
     def should_log_results(self, result1, result2):
         if not self.verbose_dns:
             return False
@@ -697,7 +710,14 @@ class Test_getnameinfo_fail(TestCase):
 
 class TestInvalidPort(TestCase):
 
+    @flaky.reraises_flaky_race_condition()
     def test1(self):
+        # An Appveyor beginning 2019-03-21, the system resolver
+        # sometimes returns ('23.100.69.251', '65535') instead of
+        # raising an error. That IP address belongs to
+        # readthedocs[.io?] which is where www.gevent.org is a CNAME
+        # to...but it doesn't actually *reverse* to readthedocs.io.
+        # Can't reproduce locally, not sure what's happening
         self._test('getnameinfo', ('www.gevent.org', -1), 0)
 
     def test2(self):

src/gevent/tests/test__threading_monkey_in_thread.py

 # We can monkey-patch in a thread, but things don't work as expected.
 from __future__ import print_function
-import sys
+
 import threading
 from gevent import monkey
 import gevent.testing as greentest

src/gevent/thread.py

@@ -11,19 +11,28 @@ Implementation of the standard :mod:`thread` module that spawns greenlets.
 from __future__ import absolute_import
 import sys
 
-__implements__ = ['allocate_lock',
-                  'get_ident',
-                  'exit',
-                  'LockType',
-                  'stack_size',
-                  'start_new_thread',
-                  '_local']
+__implements__ = [
+    'allocate_lock',
+    'get_ident',
+    'exit',
+    'LockType',
+    'stack_size',
+    'start_new_thread',
+    '_local',
+]
 
 __imports__ = ['error']
-if sys.version_info[0] <= 2:
+if sys.version_info[0] == 2:
     import thread as __thread__ # pylint:disable=import-error
+    PY2 = True
+    PY3 = False
+    # Name the `future` backport that might already have been imported;
+    # Importing `pkg_resources` imports this, for example.
+    __alternate_targets__ = ('_thread',)
 else:
     import _thread as __thread__ # pylint:disable=import-error
+    PY2 = False
+    PY3 = True
     __target__ = '_thread'
     __imports__ += [
         'TIMEOUT_MAX',
@@ -33,13 +42,9 @@ else:
         'start_new'
     ]
 
-if hasattr(__thread__, 'RLock'):
-    assert sys.version_info[0] >= 3 or hasattr(sys, 'pypy_version_info')
-    # Added in Python 3.4, backported to PyPy 2.7-7.0
-    __imports__.append("RLock")
 
 error = __thread__.error
-from gevent._compat import PY3
+
 from gevent._compat import PYPY
 from gevent._util import copy_globals
 from gevent.hub import getcurrent, GreenletExit
@@ -47,6 +52,12 @@ from gevent.greenlet import Greenlet
 from gevent.lock import BoundedSemaphore
 from gevent.local import local as _local
 
+if hasattr(__thread__, 'RLock'):
+    assert PY3 or PYPY
+    # Added in Python 3.4, backported to PyPy 2.7-7.0
+    __imports__.append("RLock")
+
+
 
 def get_ident(gr=None):
     if gr is None:
@@ -117,5 +128,6 @@ __imports__ = copy_globals(__thread__, globals(),
 __all__ = __implements__ + __imports__
 __all__.remove('_local')
 
+
 # XXX interrupt_main
 # XXX _count()

src/gevent/threading.py

@@ -39,9 +39,14 @@ __implements__ = [
 import threading as __threading__
 _DummyThread_ = __threading__._DummyThread
 from gevent.local import local
-from gevent.thread import start_new_thread as _start_new_thread, allocate_lock as _allocate_lock, get_ident as _get_ident
+from gevent.thread import start_new_thread as _start_new_thread
+from gevent.thread import allocate_lock as _allocate_lock
+from gevent.thread import get_ident as _get_ident
 from gevent.hub import sleep as _sleep, getcurrent
 
+from gevent._compat import PY3
+from gevent._compat import PYPY
+
 # Exports, prevent unused import warnings
 local = local
 start_new_thread = _start_new_thread
@@ -152,8 +157,7 @@ else:
 
         return main_threads[0]
 
-import sys
-if sys.version_info[0] >= 3:
+if PY3:
     # XXX: Issue 18808 breaks us on Python 3.4+.
     # Thread objects now expect a callback from the interpreter itself
     # (threadmodule.c:release_sentinel). Because this never happens
@@ -202,7 +206,7 @@ if sys.version_info[0] >= 3:
     # The main thread is patched up with more care
     # in _gevent_will_monkey_patch
 
-if sys.version_info[0] >= 3:
+if PY3:
     __implements__.remove('_get_ident')
     __implements__.append('get_ident')
     get_ident = _get_ident
@@ -217,7 +221,7 @@ if hasattr(__threading__, '_CRLock'):
     # Fortunately they left the Python fallback in place.
 
     # This was also backported to PyPy 2.7-7.0
-    assert sys.version_info[0] >= 3 or hasattr(sys, 'pypy_version_info'), "Unsupported Python version"
+    assert PY3 or PYPY, "Unsupported Python version"
     _CRLock = None
     __implements__.append('_CRLock')