Add support for the new onerror attribute in CFFI 1.2.0/PyPy 2.7.0. This fixes...

Add support for the new onerror attribute in CFFI 1.2.0/PyPy 2.7.0. This fixes our signal handling issues.

changelog.rst

@@ -28,6 +28,9 @@ Unreleased
 - ``gevent.socket.socket.sendall`` supports arbitrary objects that
   implement the buffer protocol (such as ctypes structurs), just like
   native sockets. Reported in :issue:`466` by tzickel.
+- Added support for the ``onerror`` attribute present in CFFI 1.2.0
+  for better signal handling under PyPy. Thanks to Armin Rigo and Omer
+  Katz. (See https://bitbucket.org/cffi/cffi/issue/152/handling-errors-from-signal-handlers-in)
 
 1.1a1 (Jun 29, 2015)
 ====================
@@ -35,7 +38,8 @@ Unreleased
 - Add support for Python 3.3 and 3.4. Many people have contributed to
   this effort, including but not limited to Fantix King, hashstat,
   Elizabeth Myers, jander, Luke Woydziak, and others. See :issue:`38`.
-- Add support for PyPy. See :issue:`248`.
+- Add support for PyPy. See :issue:`248`. Note that for best results,
+  you'll need a very recent PyPy build including CFFI 1.2.0.
 - Drop support for Python 2.5. Python 2.5 users can continue to use
   gevent 1.0.x.
 - Fix ``gevent.greenlet.joinall`` to not ignore ``count`` when

gevent/corecffi.py

@@ -26,6 +26,19 @@ def st_nlink_type():
         return "unsigned long"
     return "long long"
 
+import cffi
+if cffi.__version_info__ >= (1, 2, 0):
+    # See https://bitbucket.org/cffi/cffi/issue/152/handling-errors-from-signal-handlers-in.
+    # With this version, bundled with PyPy 2.7.0 and above, we can more reliably
+    # handle signals and other exceptions. With this support, we could simplify
+    # _python_callback and _python_handle_error in addition to the simplifications for
+    # signals and KeyboardInterrupt. However, because we need to support PyPy 2.5.0+,
+    # we keep as much as practical shared.
+    _cffi_supports_on_error = True
+else:
+    # In older versions, including the 2.5.0 currently on Travis CI, we
+    # have to use a kludge
+    _cffi_supports_on_error = False
 
 from cffi import FFI
 ffi = FFI()
@@ -499,6 +512,13 @@ def time():
 _default_loop_destroyed = False
 
 
+def _loop_callback(*args, **kwargs):
+    if _cffi_supports_on_error:
+        return ffi.callback(*args, **kwargs)
+    kwargs.pop('onerror')
+    return ffi.callback(*args, **kwargs)
+
+
 class loop(object):
 
     error_handler = None
@@ -510,13 +530,17 @@ class loop(object):
         # self._check is a watcher that runs in each iteration of the
         # mainloop, just after the blocking call
         self._check = ffi.new("struct ev_check *")
-        self._check_callback_ffi = ffi.callback("void(*)(struct ev_loop *, void*, int)", self._check_callback)
+        self._check_callback_ffi = _loop_callback("void(*)(struct ev_loop *, void*, int)",
+                                                  self._check_callback,
+                                                  onerror=self._check_callback_handle_error)
         libev.ev_check_init(self._check, self._check_callback_ffi)
 
         # self._prepare is a watcher that runs in each iteration of the mainloop,
         # just before the blocking call
         self._prepare = ffi.new("struct ev_prepare *")
-        self._prepare_callback_ffi = ffi.callback("void(*)(struct ev_loop *, void*, int)", self._run_callbacks)
+        self._prepare_callback_ffi = _loop_callback("void(*)(struct ev_loop *, void*, int)",
+                                                    self._run_callbacks,
+                                                    onerror=self._check_callback_handle_error)
         libev.ev_prepare_init(self._prepare, self._prepare_callback_ffi)
 
         self._timer0 = ffi.new("struct ev_timer *")
@@ -546,21 +570,34 @@ class loop(object):
         libev.ev_check_start(self._ptr, self._check)
         self.unref()
 
-        if default:
-            signalmodule.signal(2, self.int_handler)
-        self.ate_keyboard_interrupt = False
-        self.keyboard_interrupt_allowed = True
         self._keepaliveset = set()
 
-    def _check_callback(self, *args):
-        if self.ate_keyboard_interrupt:
-            self.handle_error(self, KeyboardInterrupt, KeyboardInterrupt(), None)
+        if not _cffi_supports_on_error:
+            if default:
+                signalmodule.signal(2, self.int_handler)
             self.ate_keyboard_interrupt = False
+            self.keyboard_interrupt_allowed = True
+
+    def _check_callback_handle_error(self, t, v, tb):
+        # None as the context argument causes the exception to be raised
+        # in the main greenlet.
+        self.handle_error(None, t, v, tb)
+
+    if _cffi_supports_on_error:
+        def _check_callback(self, *args):
+            # If we have the onerror callback, this is a no-op; all the real
+            # work to rethrow the exception is done by the onerror callback
+            pass
+    else:
+        def _check_callback(self, *args):
+            if self.ate_keyboard_interrupt:
+                self.handle_error(self, KeyboardInterrupt, KeyboardInterrupt(), None)
+                self.ate_keyboard_interrupt = False
 
-    def int_handler(self, *args):
-        if self.keyboard_interrupt_allowed:
-            raise KeyboardInterrupt
-        self.ate_keyboard_interrupt = True
+        def int_handler(self, *args):
+            if self.keyboard_interrupt_allowed:
+                raise KeyboardInterrupt
+            self.ate_keyboard_interrupt = True
 
     def _run_callbacks(self, evloop, _, revents):
         count = 1000
@@ -608,7 +645,7 @@ class loop(object):
                 _default_loop_destroyed = True
             libev.ev_loop_destroy(self._ptr)
             self._ptr = ffi.NULL
-        # XXX restore default_int_signal handler
+        # XXX restore default_int_signal handler if we set it (_cffi_supports_on_error is False)
 
     @property
     def ptr(self):

greentest/2.7pypy/test_ssl.py

@@ -181,7 +181,11 @@ class BasicSocketTests(unittest.TestCase):
         self.assertGreaterEqual(fix, 0)
         self.assertLess(fix, 256)
         self.assertGreaterEqual(patch, 0)
-        self.assertLessEqual(patch, 26)
+        if sys.pypy_version_info[:3] < (2, 7, 0):
+            # gevent: compat with 2.7.0+; actually, this might depend on the local build environment
+            self.assertLessEqual(patch, 26)
+        else:
+            self.assertLessEqual(patch, 29)
         self.assertGreaterEqual(status, 0)
         self.assertLessEqual(status, 15)
         # Version string as returned by OpenSSL, the format might change

greentest/2.7pypy/test_urllib2.py

@@ -279,6 +279,7 @@ class MockHTTPClass:
         self.data = None
         self.raise_on_endheaders = False
         self._tunnel_headers = {}
+        self.sock = None # gevent: compat with 2.7.0+
 
     def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
         self.host = host

greentest/patched_tests_setup.py

@@ -187,12 +187,16 @@ if hasattr(sys, 'pypy_version_info'):
         # in PyPy's forking. Only runs on linux and is specific to the PyPy
         # implementation of subprocess (possibly explains the extra parameter to
         # _execut_child)
-
-        'test_signal.InterProcessSignalTests.test_main',
-        # Fails to get the signal to the correct handler due to
-        # https://bitbucket.org/cffi/cffi/issue/152/handling-errors-from-signal-handlers-in
     ]
 
+    import cffi
+    if cffi.__version_info__ < (1, 2, 0):
+        disabled_tests += [
+            'test_signal.InterProcessSignalTests.test_main',
+            # Fails to get the signal to the correct handler due to
+            # https://bitbucket.org/cffi/cffi/issue/152/handling-errors-from-signal-handlers-in
+        ]
+
 # if 'signalfd' in os.environ.get('GEVENT_BACKEND', ''):
 #     # tests that don't interact well with signalfd
 #     disabled_tests.extend([

known_failures.py

@@ -77,13 +77,17 @@ if PYPY:
         # non-traceability? (Is it even repeatable? Possibly not; a lot of the test time is spent in,
         # e.g., test__socket_dns.py doing network stuff.)
         'test__threading_vs_settrace.py',
+    ]
 
+    import cffi
+    if cffi.__version_info__ < (1, 2, 0):
+        FAILING_TESTS += [
 
-        # check_sendall_interrupted and testInterruptedTimeout fail due to
-        # https://bitbucket.org/cffi/cffi/issue/152/handling-errors-from-signal-handlers-in
-        # See also patched_tests_setup and 'test_signal.InterProcessSignalTests.test_main'
-        'test_socket.py',
-    ]
+            # check_sendall_interrupted and testInterruptedTimeout fail due to
+            # https://bitbucket.org/cffi/cffi/issue/152/handling-errors-from-signal-handlers-in
+            # See also patched_tests_setup and 'test_signal.InterProcessSignalTests.test_main'
+            'test_socket.py',
+        ]
 
 
 if PY3: