Restore libev/corecffi refactoring and incorporate fork for libuv

Makefile

@@ -167,7 +167,7 @@ develop:
 	${PIP} install -U -r dev-requirements.txt
 
 lint-py27: $(PY27)
-	PYTHON=python2.7.13 PATH=$(BUILD_RUNTIMES)/versions/python2.7.13/bin:$(PATH) make develop travis_test_linters
+	PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop travis_test_linters
 
 test-py27: $(PY27)
 	PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop fulltoxtest

src/gevent/_ffi/loop.py

@@ -91,7 +91,7 @@ class _Callbacks(object):
                 # Legacy behaviour from corecext: convert None into ()
                 # See test__core_watcher.py
                 args = _NOARGS
-            if len(args) > 0 and args[0] == GEVENT_CORE_EVENTS:
+            if args and args[0] == GEVENT_CORE_EVENTS:
                 args = (revents, ) + args[1:]
             the_watcher.callback(*args)
         except: # pylint:disable=bare-except
@@ -148,7 +148,7 @@ def assign_standard_callbacks(ffi, lib):
 
 if sys.version_info[0] >= 3:
     basestring = (bytes, str)
-    integer_types = int,
+    integer_types = (int,)
 else:
     import __builtin__ # pylint:disable=import-error
     basestring = __builtin__.basestring,

src/gevent/_ffi/watcher.py

@@ -136,10 +136,10 @@ class AbstractWatcherType(type):
             meth.__name__ = watcher_name
             return meth
 
-        for name in 'start', 'stop', 'init':
-            watcher_name = '_watcher' + '_' + name
+        for meth_name in 'start', 'stop', 'init':
+            watcher_name = '_watcher' + '_' + meth_name
             if watcher_name not in cls_dict:
-                LazyOnClass.lazy(cls_dict, _make_meth(name, watcher_name))
+                LazyOnClass.lazy(cls_dict, _make_meth(meth_name, watcher_name))
 
     def new_handle(cls, obj):
         return cls._FFI.new_handle(obj)
@@ -174,11 +174,11 @@ class watcher(object):
         self._watcher_ffi_set_init_ref(ref)
 
     @classmethod
-    def _watcher_ffi_close(cls, ffi_handle):
+    def _watcher_ffi_close(cls, ffi_watcher):
         pass
 
     def _watcher_create(self, ref): # pylint:disable=unused-argument
-        self._handle = type(self).new_handle(self) # This is a GC cycle
+        self._handle = type(self).new_handle(self) # This is a GC cycle pylint:disable=no-member
         self._watcher = self._watcher_new()
         # This call takes care of calling _watcher_ffi_close when
         # self goes away, making sure self._watcher stays alive
@@ -188,7 +188,7 @@ class watcher(object):
         self._watcher.data = self._handle
 
     def _watcher_new(self):
-        return type(self).new(self._watcher_struct_pointer_type)
+        return type(self).new(self._watcher_struct_pointer_type) # pylint:disable=no-member
 
     def _watcher_ffi_set_init_ref(self, ref):
         pass

src/gevent/_fileobjectposix.py

@@ -256,7 +256,7 @@ class FileObjectPosix(FileObjectBase):
                         @functools.wraps(m)
                         def wrapped(*args, **kwargs):
                             result = m(*args, **kwargs)
-                            assert isinstance(result, unicode)
+                            assert isinstance(result, unicode) # pylint:disable=undefined-variable
                             return result.encode('latin-1')
                         return wrapped
                     return m

src/gevent/libev/watcher.py

 # pylint: disable=too-many-lines, protected-access, redefined-outer-name, not-callable
+# pylint: disable=no-member
 from __future__ import absolute_import, print_function
 import sys
 

src/gevent/libuv/_corecffi_cdef.c

@@ -10,14 +10,36 @@
 #define UV_VERSION_PATCH ...
 
 typedef enum {
-    UV_RUN_DEFAULT = 0,
-    UV_RUN_ONCE,
-    UV_RUN_NOWAIT
+	UV_RUN_DEFAULT = 0,
+	UV_RUN_ONCE,
+	UV_RUN_NOWAIT
 } uv_run_mode;
 
+typedef enum {
+  UV_UNKNOWN_HANDLE = 0,
+  UV_ASYNC,
+  UV_CHECK,
+  UV_FS_EVENT,
+  UV_FS_POLL,
+  UV_HANDLE,
+  UV_IDLE,
+  UV_NAMED_PIPE,
+  UV_POLL,
+  UV_PREPARE,
+  UV_PROCESS,
+  UV_STREAM,
+  UV_TCP,
+  UV_TIMER,
+  UV_TTY,
+  UV_UDP,
+  UV_SIGNAL,
+  UV_FILE,
+  UV_HANDLE_TYPE_MAX
+} uv_handle_type;
+
 enum uv_poll_event {
-    UV_READABLE = 1,
-    UV_WRITABLE = 2,
+	UV_READABLE = 1,
+	UV_WRITABLE = 2,
 	/* new in 1.9 */
 	UV_DISCONNECT = 4
 };
@@ -28,28 +50,28 @@ enum uv_fs_event {
 };
 
 enum uv_fs_event_flags {
-    /*
-    * By default, if the fs event watcher is given a directory name, we will
-    * watch for all events in that directory. This flags overrides this behavior
-    * and makes fs_event report only changes to the directory entry itself. This
-    * flag does not affect individual files watched.
-    * This flag is currently not implemented yet on any backend.
-    */
-    UV_FS_EVENT_WATCH_ENTRY = 1,
-    /*
-    * By default uv_fs_event will try to use a kernel interface such as inotify
-    * or kqueue to detect events. This may not work on remote filesystems such
-    * as NFS mounts. This flag makes fs_event fall back to calling stat() on a
-    * regular interval.
-    * This flag is currently not implemented yet on any backend.
-    */
-    UV_FS_EVENT_STAT = 2,
-    /*
-    * By default, event watcher, when watching directory, is not registering
-    * (is ignoring) changes in it's subdirectories.
-    * This flag will override this behaviour on platforms that support it.
-    */
-    UV_FS_EVENT_RECURSIVE = 4
+	/*
+	* By default, if the fs event watcher is given a directory name, we will
+	* watch for all events in that directory. This flags overrides this behavior
+	* and makes fs_event report only changes to the directory entry itself. This
+	* flag does not affect individual files watched.
+	* This flag is currently not implemented yet on any backend.
+	*/
+	UV_FS_EVENT_WATCH_ENTRY = 1,
+	/*
+	* By default uv_fs_event will try to use a kernel interface such as inotify
+	* or kqueue to detect events. This may not work on remote filesystems such
+	* as NFS mounts. This flag makes fs_event fall back to calling stat() on a
+	* regular interval.
+	* This flag is currently not implemented yet on any backend.
+	*/
+	UV_FS_EVENT_STAT = 2,
+	/*
+	* By default, event watcher, when watching directory, is not registering
+	* (is ignoring) changes in it's subdirectories.
+	* This flag will override this behaviour on platforms that support it.
+	*/
+	UV_FS_EVENT_RECURSIVE = 4
 };
 
 const char* uv_strerror(int);
@@ -63,48 +85,67 @@ struct uv_loop_s {
 };
 struct uv_handle_s {
 	struct uv_loop_s* loop;
+	uv_handle_type type;
 	void *data;
 	GEVENT_STRUCT_DONE _;
 };
 struct uv_idle_s {
+	struct uv_loop_s* loop;
+	uv_handle_type type;
 	void *data;
 	GEVENT_STRUCT_DONE _;
 };
 struct uv_prepare_s {
+	struct uv_loop_s* loop;
+	uv_handle_type type;
 	void *data;
 	GEVENT_STRUCT_DONE _;
 };
 struct uv_timer_s {
+	struct uv_loop_s* loop;
+	uv_handle_type type;
 	void *data;
 	GEVENT_STRUCT_DONE _;
 };
 struct uv_signal_s {
+	struct uv_loop_s* loop;
+	uv_handle_type type;
 	void *data;
 	GEVENT_STRUCT_DONE _;
 };
 struct uv_poll_s {
+	struct uv_loop_s* loop;
+	uv_handle_type type;
 	void *data;
 	GEVENT_STRUCT_DONE _;
 };
 
 struct uv_check_s {
+	struct uv_loop_s* loop;
+	uv_handle_type type;
 	void *data;
 	GEVENT_STRUCT_DONE _;
 };
 
 struct uv_async_s {
+	struct uv_loop_s* loop;
+	uv_handle_type type;
 	void *data;
 	void (*async_cb)(void*);
 	GEVENT_STRUCT_DONE _;
 };
 
 struct uv_fs_event_s {
-	void* data;
+	struct uv_loop_s* loop;
+	uv_handle_type type;
+	void *data;
 	GEVENT_STRUCT_DONE _;
 };
 
 struct uv_fs_poll_s {
-	void* data;
+	struct uv_loop_s* loop;
+	uv_handle_type type;
+	void *data;
 	GEVENT_STRUCT_DONE _;
 };
 
@@ -142,27 +183,27 @@ typedef void (*uv_signal_cb)(void *handle, int signum);
 typedef void (*uv_fs_event_cb)(void* handle, const char* filename, int events, int status);
 
 typedef struct {
-    long tv_sec;
-    long tv_nsec;
+	long tv_sec;
+	long tv_nsec;
 } uv_timespec_t;
 
 typedef struct {
-    uint64_t st_dev;
-    uint64_t st_mode;
-    uint64_t st_nlink;
-    uint64_t st_uid;
-    uint64_t st_gid;
-    uint64_t st_rdev;
-    uint64_t st_ino;
-    uint64_t st_size;
-    uint64_t st_blksize;
-    uint64_t st_blocks;
-    uint64_t st_flags;
-    uint64_t st_gen;
-    uv_timespec_t st_atim;
-    uv_timespec_t st_mtim;
-    uv_timespec_t st_ctim;
-    uv_timespec_t st_birthtim;
+	uint64_t st_dev;
+	uint64_t st_mode;
+	uint64_t st_nlink;
+	uint64_t st_uid;
+	uint64_t st_gid;
+	uint64_t st_rdev;
+	uint64_t st_ino;
+	uint64_t st_size;
+	uint64_t st_blksize;
+	uint64_t st_blocks;
+	uint64_t st_flags;
+	uint64_t st_gen;
+	uv_timespec_t st_atim;
+	uv_timespec_t st_mtim;
+	uv_timespec_t st_ctim;
+	uv_timespec_t st_birthtim;
 } uv_stat_t;
 
 typedef void (*uv_fs_poll_cb)(void* handle, int status, const uv_stat_t* prev, const uv_stat_t* curr);
@@ -171,6 +212,7 @@ typedef void (*uv_fs_poll_cb)(void* handle, int status, const uv_stat_t* prev, c
 uv_loop_t *uv_default_loop();
 uv_loop_t* uv_loop_new(); // not documented; neither is uv_loop_delete
 int uv_loop_init(uv_loop_t* loop);
+int uv_loop_fork(uv_loop_t* loop);
 int uv_loop_alive(const uv_loop_t *loop);
 int uv_loop_close(uv_loop_t* loop);
 uint64_t uv_backend_timeout(uv_loop_t* loop);

src/gevent/libuv/loop.py

@@ -15,8 +15,8 @@ from gevent._ffi.loop import AbstractLoop
 from gevent.libuv import _corecffi # pylint:disable=no-name-in-module,import-error
 from gevent._ffi.loop import assign_standard_callbacks
 
-ffi = _corecffi.ffi
-libuv = _corecffi.lib
+ffi = _corecffi.ffi # pylint:disable=no-member
+libuv = _corecffi.lib # pylint:disable=no-member
 
 __all__ = [
 ]
@@ -26,7 +26,7 @@ _callbacks = assign_standard_callbacks(ffi, libuv)
 from gevent._ffi.loop import EVENTS
 GEVENT_CORE_EVENTS = EVENTS # export
 
-from gevent.libuv import watcher as _watchers
+from gevent.libuv import watcher as _watchers # pylint:disable=no-name-in-module
 
 _events_to_str = _watchers._events_to_str # export
 
@@ -226,7 +226,7 @@ class loop(AbstractLoop):
         # re-__init__ this whole class? Does it matter?
         # OR maybe we need to uv_walk() and close all the handles?
 
-        # XXX: libuv <= 1.9 simply CANNOT handle a fork unless you immediately
+        # XXX: libuv < 1.12 simply CANNOT handle a fork unless you immediately
         # exec() in the child. There are multiple calls to abort() that
         # will kill the child process:
         # - The OS X poll implementation (kqueue) aborts on an error return
@@ -242,8 +242,8 @@ class loop(AbstractLoop):
         # had already been closed
         # (https://github.com/joyent/libuv/issues/1405)
 
-        #raise NotImplementedError()
-        pass
+        # In 1.12, the uv_loop_fork function was added (by gevent!)
+        libuv.uv_loop_fork(self._ptr)
 
 
     def run(self, nowait=False, once=False):
@@ -301,6 +301,18 @@ class loop(AbstractLoop):
                               self._sigchld_callback_ffi,
                               signal.SIGCHLD)
 
+    def reset_sigchld(self):
+        if not self.default or not self._sigchld_watcher:
+            return
+
+        libuv.uv_signal_stop(self._sigchld_watcher)
+        # Must go through this to manage the memory lifetime
+        # correctly. Alternately, we could just stop it and restart
+        # it in install_sigchld?
+        _watchers.watcher._watcher_ffi_close(self._sigchld_watcher)
+        del self._sigchld_watcher
+        del self._sigchld_callback_ffi
+
     def __sigchld_callback(self, _handler, _signum):
         while True:
             try:

src/gevent/libuv/watcher.py

@@ -25,6 +25,14 @@ def _dbg(*args, **kwargs):
 
 #_dbg = print
 
+def _pid_dbg(*args, **kwargs):
+    import os
+    import sys
+    kwargs['file'] = sys.stderr
+    print(os.getpid(), *args, **kwargs)
+
+# _dbg = _pid_dbg
+
 _events = [(libuv.UV_READABLE, "READ"),
            (libuv.UV_WRITABLE, "WRITE")]
 
@@ -37,7 +45,7 @@ class UVFuncallError(ValueError):
 class libuv_error_wrapper(object):
     # Makes sure that everything stored as a function
     # on the wrapper instances (classes, actually,
-    # because this is used my the metaclass)
+    # because this is used by the metaclass)
     # checks its return value and raises an error.
     # This expects that everything we call has an int
     # or void return value and follows the conventions
@@ -50,7 +58,7 @@ class libuv_error_wrapper(object):
 
         @functools.wraps(libuv_func)
         def wrap(*args, **kwargs):
-            if len(args) > 0 and isinstance(args[0], watcher):
+            if args and isinstance(args[0], watcher):
                 args = args[1:]
             res = libuv_func(*args, **kwargs)
             if res is not None and res < 0:
@@ -116,9 +124,13 @@ class watcher(_base.watcher):
         # Instead, this is arranged as a callback to GC when the
         # watcher class dies. Obviously it's important to keep the ffi
         # watcher alive.
-
-        if not libuv.uv_is_closing(ffi_watcher):
-            #print("Closing handle", self._watcher)
+        _dbg("Request to close handle", ffi_watcher, ffi_watcher.type)
+        if ffi_watcher.type and not libuv.uv_is_closing(ffi_watcher):
+            # If the type isn't set, we were never properly initialized,
+            # and trying to close it results in libuv terminating the process.
+            # Sigh. Same thing if it's already in the process of being
+            # closed.
+            _dbg("Closing handle", ffi_watcher, ffi_watcher.type)
             _closing_handles.add(ffi_watcher)
             libuv.uv_close(ffi_watcher, _uv_close_callback)
 
@@ -165,6 +177,8 @@ class watcher(_base.watcher):
 
     def _get_ref(self):
         # Convert 1/0 to True/False
+        if self._watcher is None:
+            return None
         return True if libuv.uv_has_ref(self._watcher) else False
 
     def _set_ref(self, value):

src/greentest/known_failures.py

@@ -158,9 +158,6 @@ if PYPY:
 if LIBUV:
     if sys.platform.startswith("darwin"):
         FAILING_TESTS += [
-            # libuv doesn't support fork without an immediate exec
-            # on all platforms. It does appear to work with linux/epall
-            'test__core_fork.py',
         ]
 
 if PY3:

src/greentest/test__core_fork.py

@@ -48,8 +48,4 @@ if __name__ == '__main__':
     # fork watchers weren't firing in multi-threading processes.
     # This test is designed to prove that they are.
     # However, it fails on Windows: The fork watcher never runs!
-    if hasattr(gevent.core, 'libuv') and sys.platform.startswith("darwin"):
-        # XXX: Formalize this check better.
-        print("ERROR: forking crashes the child process if it doesn't exec")
-        sys.exit(1)
     test()

src/greentest/test__monkey_sigchld_3.py

@@ -46,7 +46,6 @@ if hasattr(signal, 'SIGCHLD'):
         popen.stdout.read()
         popen.wait() # This hangs if it doesn't.
 
-
         sys.exit(0)
 else:
     print("No SIGCHLD, not testing")