Stop using #ifdef LIBEV_EMBED in corecext.ppyx

This eliminates the need to call cythonpp.py on it. It also leads to a
more consistent API.

We could take the same approach to bring back sig_fd if necessary, but
that's still not available to cython.

CHANGES.rst

@@ -249,11 +249,22 @@ libev
 
 - The C extension has been updated to use more modern Cython idioms
   and generate less code for simplicity, faster compilation and better
-  cache usage. See :issue:`1076`.
-
-- The ``sigfd`` property that was only conditionally available on
-  certain platforms and when libev was embedded, and only in the C
-  implementation, not the CFFI implementation, has been removed.
+  cache usage.
+
+  - Watcher objects may be slightly larger. On a 64-bit platform, a
+    typical watcher may be 16 bytes (2 pointers) larger. This is
+    offset by slight performance gains.
+
+  - Cython is no longer preprocessed. Certain attributes that were
+    previously only defined in certain compilation modes (notably
+    LIBEV_EMBED) are now always defined, but will raise ``AttributeError``
+    or have a negative value when not available. In general these
+    attributes are not portable and not implemented by libuv or the
+    CFFI backend. See :issue:`1076`.
+
+  - The ``sigfd`` property that was only conditionally available on
+    certain platforms and when libev was embedded, and only in the C
+    implementation, not the CFFI implementation, has been removed.
 
 1.2.2 (2017-06-05)
 ==================

Makefile

@@ -14,8 +14,8 @@ export LC_ALL=C.UTF-8
 
 all: src/gevent/libev/gevent.corecext.c src/gevent/gevent.ares.c src/gevent/gevent._semaphore.c src/gevent/gevent._local.c
 
-src/gevent/libev/gevent.corecext.c: src/gevent/libev/corecext.ppyx src/gevent/libev/libev.pxd util/cythonpp.py
-	$(PYTHON) util/cythonpp.py -o gevent.corecext.c --module-name gevent.libev.corecext.pyx src/gevent/libev/corecext.ppyx
+src/gevent/libev/gevent.corecext.c: src/gevent/libev/corecext.ppyx src/gevent/libev/libev.pxd src/gevent/libev/libev.h
+	$(CYTHON) -o gevent.corecext.c src/gevent/libev/corecext.ppyx
 	echo '#include "callbacks.c"' >> gevent.corecext.c
 	mv gevent.corecext.* src/gevent/libev/
 

appveyor/make.cmd

 IF "%PYTHON_EXE%" == "python" (
-    %PYEXE% util\cythonpp.py -o gevent.corecext.c --module-name gevent.libev.corecext.pyx src\gevent\libev\corecext.ppyx
+    cython -o gevent.corecext.c src\gevent\libev\corecext.ppyx
 	type src\gevent\libev\callbacks.c >> gevent.corecext.c
     move gevent.corecext.* src\gevent\libev
 )

src/gevent/libev/corecext.ppyx

@@ -518,13 +518,9 @@ cdef public class loop [object PyGeventLoopObject, type PyGeventLoop_Type]:
         if self.default:
             msg += ' default'
         msg += ' pending=%s' % self.pendingcnt
-#ifdef LIBEV_EMBED
         msg += self._format_details()
-#endif
         return msg
 
-#ifdef LIBEV_EMBED
-
     def _format_details(self):
         cdef str msg = ''
         cdef object fileno = self.fileno()
@@ -546,31 +542,28 @@ cdef public class loop [object PyGeventLoopObject, type PyGeventLoop_Type]:
     def fileno(self):
         cdef int fd
         if self._ptr:
-            fd = self._ptr.backend_fd
+            fd = libev.gevent_ev_loop_backend_fd(self._ptr)
             if fd >= 0:
                 return fd
 
     @property
     def activecnt(self):
         _check_loop(self)
-        return self._ptr.activecnt
+        return libev.gevent_ev_loop_activecnt(self._ptr)
 
     @property
     def sig_pending(self):
         _check_loop(self)
-        return self._ptr.sig_pending
+        return libev.gevent_ev_loop_sig_pending(self._ptr)
 
     @property
     def origflags(self):
-        _check_loop(self)
-        return _flags_to_list(self._ptr.origflags)
+        return _flags_to_list(self.origflags_int)
 
     @property
     def origflags_int(self):
         _check_loop(self)
-        return self._ptr.origflags
-
-#endif
+        return libev.gevent_ev_loop_origflags(self._ptr)
 
 
 cdef public class callback [object PyGeventCallbackObject, type PyGeventCallback_Type]:
@@ -667,15 +660,29 @@ cdef start_and_stop make_ss(void* start, void* stop):
     cdef start_and_stop result = start_and_stop(<start_stop_func>start, <start_stop_func>stop)
     return result
 
+cdef bint _watcher_start(watcher self, object callback, tuple args) except -1:
+    # This method should be called by subclasses of watcher, if they
+    # override the python-level `start` function: they've already paid
+    # for argument unpacking, and `start` cannot be cpdef since it
+    # uses varargs.
+
+    # We keep this as a function, not a cdef method of watcher.
+    # If it's a cdef method, it could potentially be overridden
+    # by a subclass, which means that the watcher gains a pointer to a
+    # function table (vtable), making each object 8 bytes larger.
+
+    _check_loop(self.loop)
+    if callback is None or not callable(callback):
+        raise TypeError("Expected callable, not %r" % (callback, ))
+    self._callback = callback
+    self.args = args
+    _libev_unref(self)
+    _python_incref(self)
+    self.__ss.start(self.loop._ptr, self.__watcher)
+    return 1
 
 cdef public class watcher [object PyGeventWatcherObject, type PyGeventWatcher_Type]:
     """Abstract base class for all the watchers"""
-    # Our subclasses will declare the ev_X struct
-    # as an inline member. This is good for locality, but
-    # probably bad for alignment, as it will get tacked on
-    # immediately after our data.
-    cdef readonly unsigned int _flags
-
     ## pointer members
     cdef public loop loop
     cdef object _callback
@@ -708,6 +715,17 @@ cdef public class watcher [object PyGeventWatcherObject, type PyGeventWatcher_Ty
     # are then 144 bytes.
     cdef start_and_stop* __ss
 
+    ## Int members
+
+    # Our subclasses will declare the ev_X struct
+    # as an inline member. This is good for locality, but
+    # probably bad for alignment, as it will get tacked on
+    # immediately after our data.
+
+    # But all ev_watchers start with some ints, so maybe we can help that
+    # out by putting our ints here.
+    cdef readonly unsigned int _flags
+
     def __init__(self, loop loop, ref=True, priority=None):
         if not self.__watcher or not self.__ss.start or not self.__ss.stop:
             raise ValueError("Cannot construct a bare watcher")
@@ -771,23 +789,9 @@ cdef public class watcher [object PyGeventWatcherObject, type PyGeventWatcher_Ty
         return True if libev.ev_is_pending(self.__watcher) else False
 
     def start(self, object callback, *args):
-        self._watcher_start(callback, args)
+        _watcher_start(self, callback, args)
 
-    cdef bint _watcher_start(self, object callback, tuple args) except -1:
-        # This method should be called by subclasses if they override the python-level
-        # `start` function: they've already paid for argument unpacking, and
-        # `start` cannot be cpdef since it uses varargs
-        _check_loop(self.loop)
-        if callback is None or not callable(callback):
-            raise TypeError("Expected callable, not %r" % (callback, ))
-        self._callback = callback
-        self.args = args
-        _libev_unref(self)
-        _python_incref(self)
-        self.__ss.start(self.loop._ptr, self.__watcher)
-        return 1
-
-    cpdef stop(self):
+    def stop(self):
         _check_loop(self.loop)
         _libev_ref(self)
         # The callback cannot possibly fire while we are executing,
@@ -845,7 +849,7 @@ cdef public class io(watcher) [object PyGeventIOObject, type PyGeventIO_Type]:
     def start(self, object callback, *args, pass_events=False):
         if pass_events:
             args = (GEVENT_CORE_EVENTS, ) + args
-        self._watcher_start(callback, args)
+        _watcher_start(self, callback, args)
 
     def __init__(self, loop loop, libev.vfd_socket_t fd, int events, ref=True, priority=None):
         watcher.__init__(self, loop, ref, priority)
@@ -906,7 +910,7 @@ cdef public class timer(watcher) [object PyGeventTimerObject, type PyGeventTimer
             update = self.update_loop_time_on_start
         if update:
             libev.ev_now_update(self.loop._ptr)
-        self._watcher_start(callback, args)
+        _watcher_start(self, callback, args)
 
     def __cinit__(self, loop loop, double after=0.0, double repeat=0.0, ref=True, priority=None):
         if repeat < 0.0:
@@ -1147,8 +1151,8 @@ cpdef set_syserr_cb(callback):
         raise TypeError('Expected callable or None, got %r' % (callback, ))
 
 
-#ifdef LIBEV_EMBED
-LIBEV_EMBED = True
+
+LIBEV_EMBED = bool(libev.LIBEV_EMBED)
 EV_USE_FLOOR = libev.EV_USE_FLOOR
 EV_USE_CLOCK_SYSCALL = libev.EV_USE_CLOCK_SYSCALL
 EV_USE_REALTIME = libev.EV_USE_REALTIME
@@ -1158,6 +1162,3 @@ EV_USE_INOTIFY = libev.EV_USE_INOTIFY
 EV_USE_SIGNALFD = libev.EV_USE_SIGNALFD
 EV_USE_EVENTFD = libev.EV_USE_EVENTFD
 EV_USE_4HEAP = libev.EV_USE_4HEAP
-#else
-LIBEV_EMBED = False
-#endif

src/gevent/libev/libev.h

 #if defined(LIBEV_EMBED)
 #include "ev.c"
+#undef LIBEV_EMBED
+#define LIBEV_EMBED 1
+#define gevent_ev_loop_origflags(loop) ((loop)->origflags)
+#define gevent_ev_loop_sig_pending(loop) ((loop))->sig_pending
+#define gevent_ev_loop_backend_fd(loop) ((loop))->backend_fd
+#define gevent_ev_loop_activecnt(loop) ((loop))->activecnt
 #else /* !LIBEV_EMBED */
 #include "ev.h"
 
+#define gevent_ev_loop_origflags(loop) -1
+#define gevent_ev_loop_sig_pending(loop) -1
+#define gevent_ev_loop_backend_fd(loop) -1
+#define gevent_ev_loop_activecnt(loop) -1
+
+#define LIBEV_EMBED 0
+#define EV_USE_FLOOR -1
+#define EV_USE_CLOCK_SYSCALL -1
+#define EV_USE_REALTIME -1
+#define EV_USE_MONOTONIC -1
+#define EV_USE_NANOSLEEP -1
+#define EV_USE_INOTIFY -1
+#define EV_USE_SIGNALFD -1
+#define EV_USE_EVENTFD -1
+#define EV_USE_4HEAP -1
+
+
 #ifndef _WIN32
 #include <signal.h>
 #endif /* !_WIN32 */

src/gevent/libev/libev.pxd

@@ -16,6 +16,7 @@ cdef extern from "libev_vfd.h":
     void vfd_free(int)
 
 cdef extern from "libev.h" nogil:
+    int LIBEV_EMBED
     int EV_MINPRI
     int EV_MAXPRI
 
@@ -216,6 +217,14 @@ cdef extern from "libev.h" nogil:
     void ev_break(ev_loop*, int)
     unsigned int ev_pending_count(ev_loop*)
 
+    # gevent extra functions. These are defined in libev.h.
     ev_loop* gevent_ev_default_loop(unsigned int flags)
     void gevent_install_sigchld_handler()
     void gevent_reset_sigchld_handler()
+
+    # These compensate for lack of access to ev_loop struct definition
+    # when LIBEV_EMBED is false.
+    unsigned int gevent_ev_loop_origflags(ev_loop*);
+    int gevent_ev_loop_sig_pending(ev_loop*);
+    int gevent_ev_loop_backend_fd(ev_loop*);
+    int gevent_ev_loop_activecnt(ev_loop*);

util/cythonpp.py

 #!/usr/bin/env python
 # Copyright (C) 2011-2012 Denis Bilenko (http://denisbilenko.com)
 # Copyright (C) 2015-2016 gevent contributors
+
+######################################
+######################################
+######################################
+### WARNING WARNING WARNING WARNING
+##
+## This script is unmaintained and no
+## longer in use in this project due to
+## bugs.
+## See https://github.com/gevent/gevent/issues/1076
+##
+### WARNING WARNING WARNING WARNING
+######################################
+######################################
+######################################
+
+
 from __future__ import print_function
 import sys
 import os