Merge pull request #1253 from gevent/issue1248

Monkey-patch queue.SimpleQueue to be queue._PySimpleQueue.

CHANGES.rst

@@ -7,10 +7,16 @@
 1.3.5 (unreleased)
 ==================
 
-- Nothing changed yet.
-
 - Update Python versions tested on Travis, including PyPy to 6.0. See :issue:`1195`.
 
+- :mod:`gevent.queue` imports ``_PySimpleQueue`` instead of
+  ``SimpleQueue`` so that it doesn't block the event loop.
+  :func:`gevent.monkey.patch_all` makes this same substitution in
+  :mod:`queue`. This fixes issues with
+  :class:`concurrent.futures.ThreadPoolExecutor` as well. Reported in
+  :issue:`1248` by wwqgtxx and :issue:`1251` by pyld.
+
+
 1.3.4 (2018-06-20)
 ==================
 

src/gevent/monkey.py

@@ -123,6 +123,7 @@ __all__ = [
     'patch_builtins',
     'patch_dns',
     'patch_os',
+    'patch_queue',
     'patch_select',
     'patch_signal',
     'patch_socket',
@@ -245,6 +246,7 @@ def get_original(mod_name, item_name):
         return _get_original(mod_name, [item_name])[0]
     return _get_original(mod_name, item_name)
 
+
 _NONE = object()
 
 
@@ -434,6 +436,20 @@ def patch_os():
     _patch_module('os')
 
 
+@_ignores_DoNotPatch
+def patch_queue():
+    """
+    On Python 3.7 and above, replace :class:`queue.SimpleQueue` (implemented
+    in C) with its Python counterpart.
+
+    .. versionadded:: 1.3.5
+    """
+
+    import gevent.queue
+    if 'SimpleQueue' in gevent.queue.__all__:
+        _patch_module('queue', items=['SimpleQueue'])
+
+
 @_ignores_DoNotPatch
 def patch_time():
     """
@@ -908,6 +924,7 @@ def patch_all(socket=True, dns=True, time=True, select=True, thread=True, os=Tru
               httplib=False, # Deprecated, to be removed.
               subprocess=True, sys=False, aggressive=True, Event=True,
               builtins=True, signal=True,
+              queue=True,
               **kwargs):
     """
     Do all of the default monkey patching (calls every other applicable
@@ -932,6 +949,8 @@ def patch_all(socket=True, dns=True, time=True, select=True, thread=True, os=Tru
        Add ``**kwargs`` for the benefit of event subscribers. CAUTION: gevent may add
        and interpret additional arguments in the future, so it is suggested to use prefixes
        for kwarg values to be interpreted by plugins, for example, `patch_all(mylib_futures=True)`.
+    .. versionchanged:: 1.3.5
+       Add *queue*, defaulting to True, for Python 3.7.
     """
     # pylint:disable=too-many-locals,too-many-branches
 
@@ -973,6 +992,8 @@ def patch_all(socket=True, dns=True, time=True, select=True, thread=True, os=Tru
         patch_builtins()
     if signal:
         patch_signal()
+    if queue:
+        patch_queue()
 
     _notify_patch(events.GeventDidPatchBuiltinModulesEvent(modules_to_patch, kwargs), _warnings)
     _notify_patch(events.GeventDidPatchAllEvent(modules_to_patch, kwargs), _warnings)

src/gevent/queue.py

@@ -46,14 +46,16 @@ from gevent._hub_local import get_hub_noargs as get_hub
 from greenlet import getcurrent
 from gevent.exceptions import InvalidSwitchError
 
-
+__all__ = []
 __implements__ = ['Queue', 'PriorityQueue', 'LifoQueue']
 __extensions__ = ['JoinableQueue', 'Channel']
 __imports__ = ['Empty', 'Full']
 if hasattr(__queue__, 'SimpleQueue'):
-    __imports__.append('SimpleQueue') # New in 3.7
-    SimpleQueue = __queue__.SimpleQueue # pylint:disable=no-member
-__all__ = __implements__ + __extensions__ + __imports__
+    __all__.append('SimpleQueue') # New in 3.7
+    # SimpleQueue is implemented in C and directly allocates locks
+    # unaffected by monkey patching. We need the Python version.
+    SimpleQueue = __queue__._PySimpleQueue # pylint:disable=no-member
+__all__ += (__implements__ + __extensions__ + __imports__)
 
 
 # pylint 2.0.dev2 things collections.dequeue.popleft() doesn't return

src/greentest/test__monkey.py

@@ -134,6 +134,18 @@ class TestMonkey(SubscriberCleanupMixin, unittest.TestCase):
             self.assertFalse(isinstance(e, events.GeventDidPatchModuleEvent)
                              and e.module_name == 'ssl')
 
+    def test_patch_queue(self):
+        try:
+            import queue
+        except ImportError:
+            # Python 2 called this Queue. Note that having
+            # python-future installed gives us a queue module on
+            # Python 2 as well.
+            queue = None
+        if not hasattr(queue, 'SimpleQueue'):
+            raise unittest.SkipTest("Needs SimpleQueue")
+        # pylint:disable=no-member
+        self.assertIs(queue.SimpleQueue, queue._PySimpleQueue)
 
 if __name__ == '__main__':
     unittest.main()

src/greentest/test__queue.py

+import unittest
+
 import greentest
 from greentest import TestCase, main
 import gevent
@@ -444,6 +446,18 @@ class TestPutInterruptChannel(TestPutInterrupt):
         return self.kind()
 
 
+if hasattr(queue, 'SimpleQueue'):
+
+    class TestGetInterruptSimpleQueue(TestGetInterrupt):
+        kind = queue.SimpleQueue
+
+        def test_raises_timeout_Timeout(self):
+            raise unittest.SkipTest("Not supported")
+
+        test_raises_timeout_Timeout_exc_customized = test_raises_timeout_Timeout
+        test_outer_timeout_is_not_lost = test_raises_timeout_Timeout
+
+
 del AbstractGenericGetTestCase