Make callbacks.c stop relying on implementation details of Cython and Python....

Make callbacks.c stop relying on implementation details of Cython and Python. Instead use documented ways to interact.

CHANGES.rst

@@ -81,7 +81,6 @@ Build Changes
   one pointer less memory per object, and one pointer less memory per
   greenlet. See :pr:`1024`.
 
-
 - The Cython ares 'channel' class is no longer declared to be publicly
   accessible from a named C structure. Doing so caused a conflict with
   the c-ares header files.
@@ -289,6 +288,11 @@ libev
     attributes are not portable and not implemented by libuv or the
     CFFI backend. See :issue:`1076`.
 
+  - Certain private helper functions (``gevent_handle_error``, part of
+    ``gevent_call``) are now implemented in Cython instead of C. This
+    reduces our reliance on internal undocumented implementation
+    details of Cython and Python that could change.
+
 
 1.2.2 (2017-06-05)
 ==================

_setuplibev.py

@@ -61,7 +61,10 @@ def configure_libev(bext, ext):
         os.chdir(cwd)
 
 CORE = Extension(name='gevent.libev.corecext',
-                 sources=['src/gevent/libev/corecext.pyx'],
+                 sources=[
+                     'src/gevent/libev/corecext.pyx',
+                     'src/gevent/libev/callbacks.c',
+                 ],
                  include_dirs=['src/gevent/libev'] + [dep_abspath('libev')] if LIBEV_EMBED else [],
                  libraries=list(LIBRARIES),
                  define_macros=list(DEFINE_MACROS),
@@ -89,13 +92,3 @@ else:
     CORE.libraries.append('ev')
 
 CORE = cythonize1(CORE)
-
-# XXX The include of callbacks.c must go at the end of the
-# file because it references things cython generates.
-# How can we do this automatically, or relax that restriction?
-with open(CORE.sources[0]) as f:
-    core_data = f.read()
-
-if '#include "callbacks.c"' not in core_data:
-    with open(CORE.sources[0], 'a') as f:
-        f.write('\n#include "callbacks.c"\n')

src/gevent/libev/callbacks.c

 /* Copyright (c) 2011-2012 Denis Bilenko. See LICENSE for details. */
+#include <stddef.h>
+#include "Python.h"
+#include "ev.h"
+#include "corecext.h"
+#include "callbacks.h"
 #ifdef Py_PYTHON_H
 
-/* the name changes depending on our file layout and --module-name option */
-#define _GEVENTLOOP struct __pyx_vtabstruct_6gevent_5libev_8corecext_loop
-
-
-static void gevent_handle_error(struct PyGeventLoopObject* loop, PyObject* context) {
-    PyThreadState *tstate;
-    PyObject *type, *value, *traceback, *result;
-    tstate = PyThreadState_GET();
-    type = tstate->curexc_type;
-    if (!type)
-        return;
-    value = tstate->curexc_value;
-    traceback = tstate->curexc_traceback;
-    if (!value) value = Py_None;
-    if (!traceback) traceback = Py_None;
-
-    Py_INCREF(type);
-    Py_INCREF(value);
-    Py_INCREF(traceback);
-
-    PyErr_Clear();
+#if PY_MAJOR_VERSION >= 3
+  #define PyInt_FromLong               PyLong_FromLong
+#endif
 
-    result = ((_GEVENTLOOP *)loop->__pyx_vtab)->handle_error(loop, context, type, value, traceback, 0);
 
-    if (result) {
-        Py_DECREF(result);
-    }
-    else {
-        PyErr_Print();
-        PyErr_Clear();
-    }
-
-    Py_DECREF(type);
-    Py_DECREF(value);
-    Py_DECREF(traceback);
-}
+#ifndef CYTHON_INLINE
+  #if defined(__clang__)
+    #define CYTHON_INLINE __inline__ __attribute__ ((__unused__))
+  #elif defined(__GNUC__)
+    #define CYTHON_INLINE __inline__
+  #elif defined(_MSC_VER)
+    #define CYTHON_INLINE __inline
+  #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+    #define CYTHON_INLINE inline
+  #else
+    #define CYTHON_INLINE
+  #endif
+#endif
 
 
 static CYTHON_INLINE void gevent_check_signals(struct PyGeventLoopObject* loop) {
@@ -69,7 +56,7 @@ static void gevent_stop(PyObject* watcher, struct PyGeventLoopObject* loop) {
     error = 1;
     method = PyObject_GetAttrString(watcher, "stop");
     if (method) {
-        result = PyObject_Call(method, __pyx_empty_tuple, NULL);
+        result = PyObject_Call(method, _empty_tuple, NULL);
         if (result) {
             Py_DECREF(result);
             error = 0;
@@ -94,7 +81,7 @@ static void gevent_callback(struct PyGeventLoopObject* loop, PyObject* callback,
     Py_INCREF(watcher);
     gevent_check_signals(loop);
     if (args == Py_None) {
-        args = __pyx_empty_tuple;
+        args = _empty_tuple;
     }
     length = PyTuple_Size(args);
     if (length < 0) {
@@ -143,7 +130,7 @@ end:
 }
 
 
-static void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallbackObject* cb) {
+void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallbackObject* cb) {
     /* no need for GIL here because it is only called from run_callbacks which already has GIL */
     PyObject *result, *callback, *args;
     if (!loop || !cb)
@@ -187,7 +174,7 @@ static void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallback
 
 #undef DEFINE_CALLBACK
 #define DEFINE_CALLBACK(WATCHER_LC, WATCHER_TYPE) \
-    static void gevent_callback_##WATCHER_LC(struct ev_loop *_loop, void *c_watcher, int revents) {                  \
+    void gevent_callback_##WATCHER_LC(struct ev_loop *_loop, void *c_watcher, int revents) {                  \
         struct PyGeventWatcherObject* watcher = (struct PyGeventWatcherObject*)GET_OBJECT(PyGevent##WATCHER_TYPE##Object, c_watcher, _watcher);    \
         gevent_callback(watcher->loop, watcher->_callback, watcher->args, (PyObject*)watcher, c_watcher, revents); \
     }
@@ -196,7 +183,7 @@ static void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallback
 DEFINE_CALLBACKS
 
 
-static void gevent_run_callbacks(struct ev_loop *_loop, void *watcher, int revents) {
+void gevent_run_callbacks(struct ev_loop *_loop, void *watcher, int revents) {
     struct PyGeventLoopObject* loop;
     PyObject *result;
     GIL_DECLARE;
@@ -204,7 +191,7 @@ static void gevent_run_callbacks(struct ev_loop *_loop, void *watcher, int reven
     loop = GET_OBJECT(PyGeventLoopObject, watcher, _prepare);
     Py_INCREF(loop);
     gevent_check_signals(loop);
-    result = ((_GEVENTLOOP *)loop->__pyx_vtab)->_run_callbacks(loop);
+    result = gevent_loop_run_callbacks(loop);
     if (result) {
         Py_DECREF(result);
     }
@@ -218,7 +205,7 @@ static void gevent_run_callbacks(struct ev_loop *_loop, void *watcher, int reven
 
 /* This is only used on Win32 */
 
-static void gevent_periodic_signal_check(struct ev_loop *_loop, void *watcher, int revents) {
+void gevent_periodic_signal_check(struct ev_loop *_loop, void *watcher, int revents) {
     GIL_DECLARE;
     GIL_ENSURE;
     gevent_check_signals(GET_OBJECT(PyGeventLoopObject, watcher, _periodic_signal_checker));

src/gevent/libev/callbacks.h

+struct ev_loop;
+struct PyGeventLoopObject;
+struct PyGeventCallbackObject;
+
 #define DEFINE_CALLBACK(WATCHER_LC, WATCHER_TYPE) \
-    static void gevent_callback_##WATCHER_LC(struct ev_loop *, void *, int);
+    void gevent_callback_##WATCHER_LC(struct ev_loop *, void *, int);
 
 
 #define DEFINE_CALLBACKS0              \
@@ -21,14 +25,14 @@
 DEFINE_CALLBACKS
 
 
-static void gevent_run_callbacks(struct ev_loop *, void *, int);
-struct PyGeventLoopObject;
-static void gevent_handle_error(struct PyGeventLoopObject* loop, PyObject* context);
-struct PyGeventCallbackObject;
-static void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallbackObject* cb);
+void gevent_run_callbacks(struct ev_loop *, void *, int);
+
+
+
+void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallbackObject* cb);
 
 static void gevent_noop(struct ev_loop *_loop, void *watcher, int revents) {
 }
 
 /* Only used on Win32 */
-static void gevent_periodic_signal_check(struct ev_loop *, void *, int);
+void gevent_periodic_signal_check(struct ev_loop *, void *, int);

src/gevent/libev/corecext.pyx

@@ -1172,3 +1172,51 @@ EV_USE_INOTIFY = libev.EV_USE_INOTIFY
 EV_USE_SIGNALFD = libev.EV_USE_SIGNALFD
 EV_USE_EVENTFD = libev.EV_USE_EVENTFD
 EV_USE_4HEAP = libev.EV_USE_4HEAP
+
+# Things used in callbacks.c
+
+from cpython cimport PyErr_Fetch
+from cpython cimport PyObject
+
+cdef public void gevent_handle_error(loop loop, object context):
+    cdef PyObject* typep
+    cdef PyObject* valuep
+    cdef PyObject* tracebackp
+
+    cdef object type
+    cdef object value = None
+    cdef object traceback = None
+    cdef object result
+
+    # If it was set, this will clear it, and we will own
+    # the references.
+    PyErr_Fetch(&typep, &valuep, &tracebackp)
+    # TODO: Should we call PyErr_Normalize? There's code in
+    # Hub.handle_error that works around what looks like an
+    # unnormalized exception.
+
+    if not typep:
+        return
+    # This assignment will do a Py_INCREF
+    # on the value. We already own the reference
+    # returned from PyErr_Fetch,
+    # so we must decref immediately
+    type = <object>typep
+    Py_DECREF(type)
+
+    if valuep:
+        value = <object>valuep
+        Py_DECREF(value)
+    if tracebackp:
+        traceback = <object>tracebackp
+        Py_DECREF(traceback)
+
+    # If this method fails by raising an exception,
+    # cython will print it for us because we don't return a
+    # Python object and we don't declare an `except` clause.
+    loop.handle_error(context, type, value, traceback)
+
+cdef public tuple _empty_tuple = ()
+
+cdef public object gevent_loop_run_callbacks(loop loop):
+    return loop._run_callbacks()