Fix #1126 by moving the run queue to the heap from the stack.

Cherry pick 1ef16f38491000f4fdafdc01dc809d30fc785867

CHANGES.rst

@@ -126,6 +126,11 @@
 - Add `gevent.util.GreenletTree` to visualize the greenlet tree. This
   is used by `gevent.util.format_run_info`.
 
+- gevent now **requires** the patched version of libuv it is
+  distributed with. Building gevent with a non-embedded libuv, while
+  not previously supported, is not possible now. See
+  :issue:`1226`.
+
 1.3a1 (2018-01-27)
 ==================
 

deps/README.rst

 - Modify the c-ares Makefile.in[c] to empty out the MANPAGES variables
   so that we don't have to ship those in the sdist.
+
+  XXX: We need a patch for that.
+
+- Apply the gevent-libuv.patch to updates of libuv.

deps/gevent-libuv.patch

+diff --git a/deps/libuv/src/unix/loop-watcher.c b/deps/libuv/src/unix/loop-watcher.c
+index 340bb0df..ff92c8e3 100644
+--- a/deps/libuv/src/unix/loop-watcher.c
++++ b/deps/libuv/src/unix/loop-watcher.c
+@@ -22,6 +22,17 @@
+ #include "uv.h"
+ #include "internal.h"
+ 
++/*
++ * gevent: Fix for https://github.com/gevent/gevent/issues/1126
++ *
++ * Using a stack-based queue variable in uv__run_* badly breaks
++ * for certain stack manipulations when greenlets switch.
++ * Windows keeps the stack in the loop. In ordor to minimize changes,
++ * we move the stack to the heap by changing just this file. We can't
++ * use global static variables because of multiple threads.
++ */
++#include <stdlib.h>
++
+ #define UV_LOOP_WATCHER_DEFINE(name, type)                                    \
+   int uv_##name##_init(uv_loop_t* loop, uv_##name##_t* handle) {              \
+     uv__handle_init(loop, (uv_handle_t*)handle, UV_##type);                   \
+@@ -47,16 +58,17 @@
+                                                                               \
+   void uv__run_##name(uv_loop_t* loop) {                                      \
+     uv_##name##_t* h;                                                         \
+-    QUEUE queue;                                                              \
++    QUEUE* queue = malloc(sizeof(QUEUE));                                     \
+     QUEUE* q;                                                                 \
+-    QUEUE_MOVE(&loop->name##_handles, &queue);                                \
+-    while (!QUEUE_EMPTY(&queue)) {                                            \
++    QUEUE_MOVE(&loop->name##_handles, queue);                                 \
++    while (!QUEUE_EMPTY(queue)) {                                             \
+       q = QUEUE_HEAD(&queue);                                                 \
+       h = QUEUE_DATA(q, uv_##name##_t, queue);                                \
+       QUEUE_REMOVE(q);                                                        \
+       QUEUE_INSERT_TAIL(&loop->name##_handles, q);                            \
+       h->name##_cb(h);                                                        \
+     }                                                                         \
++    free(queue);                                                              \
+   }                                                                           \
+                                                                               \
+   void uv__##name##_close(uv_##name##_t* handle) {                            \

deps/libuv/src/unix/loop-watcher.c

@@ -22,6 +22,17 @@
 #include "uv.h"
 #include "internal.h"
 
+/*
+ * gevent: Fix for https://github.com/gevent/gevent/issues/1126
+ *
+ * Using a stack-based queue variable in uv__run_* badly breaks
+ * for certain stack manipulations when greenlets switch.
+ * Windows keeps the stack in the loop. In ordor to minimize changes,
+ * we move the stack to the heap by changing just this file. We can't
+ * use global static variables because of multiple threads.
+ */
+#include <stdlib.h>
+
 #define UV_LOOP_WATCHER_DEFINE(name, type)                                    \
   int uv_##name##_init(uv_loop_t* loop, uv_##name##_t* handle) {              \
     uv__handle_init(loop, (uv_handle_t*)handle, UV_##type);                   \
@@ -47,16 +58,17 @@
                                                                               \
   void uv__run_##name(uv_loop_t* loop) {                                      \
     uv_##name##_t* h;                                                         \
-    QUEUE queue;                                                              \
+    QUEUE* queue = malloc(sizeof(QUEUE));                                     \
     QUEUE* q;                                                                 \
-    QUEUE_MOVE(&loop->name##_handles, &queue);                                \
-    while (!QUEUE_EMPTY(&queue)) {                                            \
+    QUEUE_MOVE(&loop->name##_handles, queue);                                 \
+    while (!QUEUE_EMPTY(queue)) {                                             \
       q = QUEUE_HEAD(&queue);                                                 \
       h = QUEUE_DATA(q, uv_##name##_t, queue);                                \
       QUEUE_REMOVE(q);                                                        \
       QUEUE_INSERT_TAIL(&loop->name##_handles, q);                            \
       h->name##_cb(h);                                                        \
     }                                                                         \
+    free(queue);                                                              \
   }                                                                           \
                                                                               \
   void uv__##name##_close(uv_##name##_t* handle) {                            \

src/gevent/libuv/watcher.py

@@ -13,12 +13,18 @@ libuv = _corecffi.lib
 from gevent._ffi import watcher as _base
 from gevent._ffi import _dbg
 
-_closing_handles = set()
+_closing_watchers = set()
 
-
-@ffi.def_extern()
-def _uv_close_callback(handle):
-    _closing_handles.remove(handle)
+# In debug mode, it would be nice to be able to clear the memory of
+# the watcher (its size determined by
+# libuv.uv_handle_size(ffi_watcher.type)) using memset so that if we
+# are using it after it's supposedly been closed and deleted, we'd
+# catch it sooner. BUT doing so breaks test__threadpool. We get errors
+# about `pthread_mutex_lock[3]: Invalid argument` (and sometimes we
+# crash) suggesting either that we're writing on memory that doesn't
+# belong to us, somehow, or that we haven't actually lost all
+# references...
+_uv_close_callback = ffi.def_extern(name='_uv_close_callback')(_closing_watchers.remove)
 
 
 _events = [(libuv.UV_READABLE, "READ"),
@@ -125,7 +131,7 @@ class watcher(_base.watcher):
             # Sigh. Same thing if it's already in the process of being
             # closed.
             _dbg("Closing handle", ffi_watcher)
-            _closing_handles.add(ffi_watcher)
+            _closing_watchers.add(ffi_watcher)
             libuv.uv_close(ffi_watcher, libuv._uv_close_callback)
 
         ffi_handle_watcher.data = ffi.NULL

src/greentest/3.7/test_subprocess.py

@@ -372,7 +372,7 @@ class ProcessTestCase(BaseTestCase):
         # is relative.
         python_dir, python_base = self._split_python_path()
         rel_python = os.path.join(os.curdir, python_base)
-        with support.temp_cwd('test_cwd_with_relative_arg', quiet=True) as wrong_dir: # gevent: use distinct name, avoid Travis CI failure)
+        with support.temp_cwd('test_cwd_with_relative_arg' + str(os.getpid()), quiet=True) as wrong_dir: # gevent: use distinct name, avoid Travis CI failure)
             # Before calling with the correct cwd, confirm that the call fails
             # without cwd and with the wrong cwd.
             self.assertRaises(FileNotFoundError, subprocess.Popen,
@@ -389,7 +389,7 @@ class ProcessTestCase(BaseTestCase):
         python_dir, python_base = self._split_python_path()
         rel_python = os.path.join(os.curdir, python_base)
         doesntexist = "somethingyoudonthave"
-        with support.temp_cwd('test_cwd_with_relative_executable', quiet=True) as wrong_dir: # gevent: use distinct name, avoid Travis CI failure)
+        with support.temp_cwd('test_cwd_with_relative_executable' + str(os.getpid()), quiet=True) as wrong_dir: # gevent: use distinct name, avoid Travis CI failure)
             # Before calling with the correct cwd, confirm that the call fails
             # without cwd and with the wrong cwd.
             self.assertRaises(FileNotFoundError, subprocess.Popen,
@@ -407,7 +407,7 @@ class ProcessTestCase(BaseTestCase):
         python_dir, python_base = self._split_python_path()
         abs_python = os.path.join(python_dir, python_base)
         rel_python = os.path.join(os.curdir, python_base)
-        with support.temp_dir() as wrong_dir:
+        with support.temp_cwd('test_cwd_with_absolute_arg' + str(os.getpid()), quiet=True) as wrong_dir: # gevent: use distinct name, avoid Travis CI failure)
             # Before calling with an absolute path, confirm that using a
             # relative path fails.
             self.assertRaises(FileNotFoundError, subprocess.Popen,