Basic support for CPython 3.8.0b4.

Still needs the specific networking test classes added, but all the basics pass for me. Lets see about CI.

.travis.yml

@@ -42,6 +42,7 @@ env:
       - TRAVIS_PYTHON_VERSION=3.5
       - TRAVIS_PYTHON_VERSION=3.6
       - TRAVIS_PYTHON_VERSION=3.7
+      - TRAVIS_PYTHON_VERSION=3.8
       - TRAVIS_PYTHON_VERSION=pypy2.7
       - TRAVIS_PYTHON_VERSION=pypy3.6
       - TRAVIS_PYTHON_VERSION=2.7 GEVENTSETUP_EMBED=0
@@ -144,6 +145,8 @@ jobs:
       script: ccache -s
       before_script: true
       after_success: true
+    - <<: *build-gevent
+      env: TRAVIS_PYTHON_VERSION=3.8
     - <<: *build-gevent
       env: TRAVIS_PYTHON_VERSION=3.5
     - <<: *build-gevent
@@ -278,6 +281,14 @@ jobs:
       env: TRAVIS_PYTHON_VERSION=3.7
       name: pure37
 
+    # 3.8
+    - <<: *test-libuv-jobs
+      env: TRAVIS_PYTHON_VERSION=3.8
+      name: libuv36
+    - <<: *test-libev-jobs
+      env: TRAVIS_PYTHON_VERSION=3.8
+      name: libev-cffi36
+
 
     # 2.7, no-embed. Run the tests that exercise the libraries we
     # linked to.

CHANGES.rst

@@ -12,6 +12,10 @@
 - Add an ``--module`` option to ``gevent.monkey`` allowing to run a Python
   module rather than a script. See :pr:`1440`.
 
+- Add support for CPython 3.8.0b4.
+
+- Improve the way joining the main thread works on Python 3.
+
 1.5a1 (2019-05-02)
 ==================
 

examples/webproxy.py

@@ -15,7 +15,12 @@ from gevent import monkey; monkey.patch_all()
 import sys
 import re
 import traceback
-from cgi import escape
+
+try:
+    from cgi import escape
+except ImportError:
+    # Python 3.8 removed this API
+    from html import escape
 
 try:
     import urllib2

pyproject.toml

@@ -12,8 +12,9 @@ requires = [
      # 0.28 is faster, and (important!) lets us specify the target module
      # name to be created so that we can have both foo.py and _foo.so
      # at the same time. 0.29 fixes some issues with Python 3.7,
-     # and adds the 3str mode for transition to Python 3.
-     "Cython >= 0.29.7",
+     # and adds the 3str mode for transition to Python 3. 0.29.12+ is
+     # required for Python 3.8
+     "Cython >= 0.29.13",
      # See version requirements in setup.py
      "cffi >= 1.12.3 ; platform_python_implementation == 'CPython'",
      # Python 3.7 requires at least 0.4.14, which is ABI incompatible with earlier

scripts/install.sh

@@ -99,6 +99,9 @@ for var in "$@"; do
         3.7)
             install 3.7.2 python3.7 3.7.d
             ;;
+        3.8)
+            install 3.8.0b4 python3.8 3.8.d
+            ;;
         pypy2.7)
             install pypy2.7-7.1.0 pypy2.7 pypy2.7.d
             ;;

src/gevent/__semaphore.pxd

@@ -13,7 +13,7 @@ cdef class Semaphore(AbstractLinkable):
     # threadpool uses it
     cpdef _start_notify(self)
     cpdef int wait(self, object timeout=*) except -1000
-    cpdef bint acquire(self, int blocking=*, object timeout=*) except -1000
+    cpdef bint acquire(self, bint blocking=*, object timeout=*) except -1000
     cpdef __enter__(self)
     cpdef __exit__(self, object t, object v, object tb)
 

src/gevent/_compat.py

@@ -21,7 +21,7 @@ PY38 = sys.version_info[:2] >= (3, 8)
 PYPY = hasattr(sys, 'pypy_version_info')
 WIN = sys.platform.startswith("win")
 LINUX = sys.platform.startswith('linux')
-OSX = sys.platform == 'darwin'
+OSX = MAC = sys.platform == 'darwin'
 
 
 PURE_PYTHON = PYPY or os.getenv('PURE_PYTHON')

src/gevent/_socket2.py

@@ -6,6 +6,7 @@ from __future__ import absolute_import
 
 # Our import magic sadly makes this warning useless
 # pylint: disable=undefined-variable
+import sys
 
 from gevent import _socketcommon
 from gevent._util import copy_globals

src/gevent/_socketcommon.py

@@ -69,13 +69,19 @@ __py3_imports__ = [
 __imports__.extend(__py3_imports__)
 
 import time
-import sys
+
 from gevent._hub_local import get_hub_noargs as get_hub
 from gevent._compat import string_types, integer_types, PY3
+from gevent._compat import PY38
+from gevent._compat import WIN as is_windows
+from gevent._compat import OSX as is_macos
 from gevent._util import copy_globals
 
-is_windows = sys.platform == 'win32'
-is_macos = sys.platform == 'darwin'
+if PY38:
+    __imports__.extend([
+        'create_server',
+        'has_dualstack_ipv6',
+    ])
 
 # pylint:disable=no-name-in-module,unused-import
 if is_windows:

src/gevent/monkey.py

@@ -668,6 +668,22 @@ def patch_thread(threading=True, _threading_local=True, Event=True, logging=True
                 raise RuntimeError("Cannot join current thread")
             if thread_greenlet is not None and thread_greenlet.dead:
                 return
+            # You may ask: Why not call thread_greenlet.join()?
+            # Well, in the one case we actually have a greenlet, it's the
+            # low-level greenlet.greenlet object for the main thread, which
+            # doesn't have a join method.
+            #
+            # You may ask: Why not become the main greenlet's *parent*
+            # so you can get notified when it finishes? Because you can't
+            # create a greenlet cycle (the current greenlet is a descendent
+            # of the parent), and nor can you set a greenlet's parent to None,
+            # so there can only ever be one greenlet with a parent of None: the main
+            # greenlet, the one we need to watch.
+            #
+            # You may ask: why not swizzle out the problematic lock on the main thread
+            # into a gevent friendly lock? Well, the interpreter actually depends on that
+            # for the main thread in threading._shutdown; see below.
+
             if not thread.is_alive():
                 return
 
@@ -700,8 +716,34 @@ def patch_thread(threading=True, _threading_local=True, Event=True, logging=True
         if orig_current_thread == threading_mod.main_thread():
             main_thread = threading_mod.main_thread()
             _greenlet = main_thread._greenlet = greenlet.getcurrent()
-
-            main_thread.join = make_join_func(main_thread, _greenlet)
+            main_thread.__real_tstate_lock = main_thread._tstate_lock
+
+            # The interpreter will call threading._shutdown
+            # when the main thread exits and is about to
+            # go away. It is called *in* the main thread. This
+            # is a perfect place to notify other greenlets that
+            # the main thread is done. We do this by overriding the
+            # lock of the main thread during operation, and only restoring
+            # it to the native blocking version at shutdown time
+            # (the interpreter also has a reference to this lock in a
+            # C data structure).
+            main_thread._tstate_lock = threading_mod.Lock()
+            main_thread._tstate_lock.acquire()
+            orig_shutdown = threading_mod._shutdown
+            def _shutdown():
+                # Release anyone trying to join() me,
+                # and let us switch to them.
+                if main_thread._tstate_lock:
+                    main_thread._tstate_lock.release()
+                    from gevent import sleep
+                    sleep()
+                # The only truly blocking native shutdown lock to
+                # acquire should be our own (hopefully)
+                main_thread._tstate_lock = main_thread.__real_tstate_lock
+                main_thread.__real_tstate_lock = None
+                orig_shutdown()
+
+            threading_mod._shutdown = _shutdown
 
             # Patch up the ident of the main thread to match. This
             # matters if threading was imported before monkey-patching

src/gevent/os.py

@@ -385,6 +385,12 @@ if hasattr(os, 'fork'):
             # we're not watching it
             return _waitpid(pid, options)
 
+        def _watch_child(pid, callback=None, loop=None, ref=False):
+            loop = loop or get_hub().loop
+            watcher = loop.child(pid, ref=ref)
+            _watched_children[pid] = watcher
+            watcher.start(_on_child, watcher, callback)
+
         def fork_and_watch(callback=None, loop=None, ref=False, fork=fork_gevent):
             """
             Fork a child process and start a child watcher for it in the parent process.
@@ -413,10 +419,7 @@ if hasattr(os, 'fork'):
             pid = fork()
             if pid:
                 # parent
-                loop = loop or get_hub().loop
-                watcher = loop.child(pid, ref=ref)
-                _watched_children[pid] = watcher
-                watcher.start(_on_child, watcher, callback)
+                _watch_child(pid, callback, loop, ref)
             return pid
 
         __extensions__.append('fork_and_watch')
@@ -474,6 +477,23 @@ if hasattr(os, 'fork'):
                     # take any args to match fork_and_watch
                     return forkpty_and_watch(*args, **kwargs)
             __implements__.append("waitpid")
+
+            if hasattr(os, 'posix_spawn'):
+                _raw_posix_spawn = os.posix_spawn
+                _raw_posix_spawnp = os.posix_spawnp
+
+                def posix_spawn(*args, **kwargs):
+                    pid = _raw_posix_spawn(*args, **kwargs)
+                    _watch_child(pid)
+                    return pid
+
+                def posix_spawnp(*args, **kwargs):
+                    pid = _raw_posix_spawnp(*args, **kwargs)
+                    _watch_child(pid)
+                    return pid
+
+                __implements__.append("posix_spawn")
+                __implements__.append("posix_spawnp")
         else:
             def fork():
                 """
@@ -503,6 +523,7 @@ if hasattr(os, 'fork'):
 else:
     __implements__.remove('fork')
 
+
 __imports__ = copy_globals(os, globals(),
                            names_to_ignore=__implements__ + __extensions__,
                            dunder_names_to_keep=())

src/gevent/subprocess.py

@@ -47,6 +47,7 @@ from gevent._compat import PY3
 from gevent._compat import PY35
 from gevent._compat import PY36
 from gevent._compat import PY37
+from gevent._compat import PY38
 from gevent._compat import reraise
 from gevent._compat import fspath
 from gevent._compat import fsencode
@@ -69,6 +70,25 @@ if PY3 and not sys.platform.startswith('win32'):
     __implements__.append("_posixsubprocess")
     _posixsubprocess = None
 
+if PY38:
+    # Using os.posix_spawn() to start subprocesses
+    # bypasses our child watchers on certain operating systems,
+    # and with certain library versions. Possibly the right
+    # fix is to monkey-patch os.posix_spawn like we do os.fork?
+    # These have no effect, they're just here to match the stdlib.
+    # TODO: When available, given a monkey patch on them, I think
+    # we ought to be able to use them if the stdlib has identified them
+    # as suitable.
+    __implements__.extend([
+        '_use_posix_spawn',
+        '_USE_POSIX_SPAWN'
+    ])
+
+    def _use_posix_spawn():
+        return False
+
+    _USE_POSIX_SPAWN = False
+
 # Some symbols we define that we expect to export;
 # useful for static analysis
 PIPE = "PIPE should be imported"
@@ -1720,3 +1740,15 @@ def run(*popenargs, **kwargs):
             raise _with_stdout_stderr(CalledProcessError(retcode, process.args, stdout), stderr)
 
     return CompletedProcess(process.args, retcode, stdout, stderr)
+
+def _gevent_did_monkey_patch(*_args):
+    # Beginning on 3.8 on Mac, the 'spawn' method became the default
+    # start method. That doesn't fire fork watchers and we can't
+    # easily patch to make it do so: multiprocessing uses the private
+    # c accelerated _subprocess module to implement this. Instead we revert
+    # back to using fork.
+    from gevent._compat import MAC
+    if MAC:
+        import multiprocessing
+        if hasattr(multiprocessing, 'set_start_method'):
+            multiprocessing.set_start_method('fork', force=True)

src/gevent/testing/testrunner.py

@@ -480,17 +480,26 @@ def _setup_environ(debug=False):
     if 'GEVENT_DEBUG' not in os.environ and debug:
         os.environ['GEVENT_DEBUG'] = 'debug'
 
-    if 'PYTHONTRACEMALLOC' not in os.environ:
+    if 'PYTHONTRACEMALLOC' not in os.environ and debug:
+        # This slows the tests down quite a bit. Reserve
+        # for debugging.
         os.environ['PYTHONTRACEMALLOC'] = '10'
 
     if 'PYTHONDEVMODE' not in os.environ:
-        # Python 3.7
+        # Python 3.7 and above.
         os.environ['PYTHONDEVMODE'] = '1'
 
-    if 'PYTHONMALLOC' not in os.environ:
-        # Python 3.6
+    if 'PYTHONMALLOC' not in os.environ and debug:
+        # Python 3.6 and above.
+        # This slows the tests down some, but
+        # can detect memory corruption. Unfortunately
+        # it can also be flaky, especially in pre-release
+        # versions of Python (e.g., lots of crashes on Python 3.8b4).
         os.environ['PYTHONMALLOC'] = 'debug'
 
+    if sys.version_info.releaselevel != 'final' and not debug:
+        os.environ['PYTHONMALLOC'] = 'default'
+        os.environ['PYTHONDEVMODE'] = ''
 
 
 def main():

src/gevent/tests/test__core_fork.py

 from __future__ import print_function
 import gevent.monkey
 gevent.monkey.patch_all()
-import gevent
-import os
 
+import os
 import multiprocessing
 
+import gevent
+from gevent._compat import MAC
+
 hub = gevent.get_hub()
 pid = os.getpid()
 newpid = None
@@ -46,6 +48,7 @@ def test():
 if __name__ == '__main__':
     # Must call for Windows to fork properly; the fork can't be in the top-level
     multiprocessing.freeze_support()
+
     # fork watchers weren't firing in multi-threading processes.
     # This test is designed to prove that they are.
     # However, it fails on Windows: The fork watcher never runs!

src/gevent/tests/test__threading_2.py

@@ -33,6 +33,7 @@ try:
     from test.support import verbose
 except ImportError:
     from test.test_support import verbose
+
 import random
 import re
 import sys
@@ -46,7 +47,7 @@ import unittest
 import weakref
 
 from gevent.tests import lock_tests
-
+verbose = False
 # A trivial mutable counter.
 
 def skipDueToHang(cls):
@@ -132,7 +133,7 @@ class ThreadTests(unittest.TestCase):
             print('waiting for all tasks to complete')
         for t in threads:
             t.join(NUMTASKS)
-            self.assertFalse(t.is_alive())
+            self.assertFalse(t.is_alive(), t.__dict__)
             if hasattr(t, 'ident'):
                 self.assertNotEqual(t.ident, 0)
                 self.assertFalse(t.ident is None)
@@ -351,28 +352,33 @@ class ThreadTests(unittest.TestCase):
         # Issue 1722344
         # Raising SystemExit skipped threading._shutdown
         import subprocess
-        p = subprocess.Popen([sys.executable, "-W", "ignore", "-c", """if 1:
+        script = """if 1:
 %s
                 import threading
                 from time import sleep
 
                 def child():
-                    sleep(1)
+                    sleep(0.3)
                     # As a non-daemon thread we SHOULD wake up and nothing
                     # should be torn down yet
-                    print("Woke up, sleep function is: %%r" %% sleep)
+                    print("Woke up, sleep function is: %%s.%%s" %% (sleep.__module__, sleep.__name__))
 
                 threading.Thread(target=child).start()
                 raise SystemExit
-            """ % setup_4],
+        """ % setup_4
+        p = subprocess.Popen([sys.executable, "-W", "ignore", "-c", script],
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE)
         stdout, stderr = p.communicate()
         stdout = stdout.strip()
         stdout = stdout.decode('utf-8')
         stderr = stderr.decode('utf-8')
-        assert re.match('^Woke up, sleep function is: <.*?sleep.*?>$', stdout), repr(stdout)
-        stderr = re.sub(r"^\[\d+ refs\]", "", stderr, re.MULTILINE).strip()
+
+
+        self.assertEqual(
+            'Woke up, sleep function is: gevent.hub.sleep',
+            stdout)
+
         # On Python 2, importing pkg_resources tends to result in some 'ImportWarning'
         # being printed to stderr about packages missing __init__.py; the -W ignore is...
         # ignored.
@@ -410,7 +416,7 @@ class ThreadTests(unittest.TestCase):
                     self.should_raise = should_raise
                     self.thread = threading.Thread(target=self._run,
                                                    args=(self,),
-                                                   kwargs={'yet_another': self})
+                                                   kwargs={'_yet_another': self})
                     self.thread.start()
 
                 def _run(self, _other_ref, _yet_another):
@@ -463,7 +469,7 @@ class ThreadJoinOnShutdown(unittest.TestCase):
             t = threading.Thread(target=joiningfunc,
                                  args=(threading.current_thread(),))
             t.start()
-            time.sleep(0.1)
+            time.sleep(0.2)
             print('end of main')
             """
         self._run_and_join(script)

src/gevent/thread.py

@@ -41,6 +41,12 @@ else:
         'interrupt_main',
         'start_new'
     ]
+    if sys.version_info[:2] >= (3, 8):
+        # We can't actually produce a value that "may be used
+        # to identify this particular thread system-wide", right?
+        # Even if we could, I imagine people will want to pass this to
+        # non-Python (native) APIs, so we shouldn't mess with it.
+        __imports__.append('get_native_id')
 
 
 error = __thread__.error

src/gevent/threading.py

@@ -160,41 +160,25 @@ else:
 if PY3:
     # XXX: Issue 18808 breaks us on Python 3.4+.
     # Thread objects now expect a callback from the interpreter itself
-    # (threadmodule.c:release_sentinel). Because this never happens
+    # (threadmodule.c:release_sentinel) when the C-level PyThreadState
+    # object is being deallocated. Because this never happens
     # when a greenlet exits, join() and friends will block forever.
-    # The solution below involves capturing the greenlet when it is
-    # started and deferring the known broken methods to it.
+    # Fortunately this is easy to fix: just ensure that the allocation of the
+    # lock, _set_sentinel, creates a *gevent* lock, and release it when
+    # we're done. The main _shutdown code is in Python and deals with
+    # this gracefully.
 
     class Thread(__threading__.Thread):
-        _greenlet = None
-
-        def is_alive(self):
-            return bool(self._greenlet)
-
-        isAlive = is_alive
 
         def _set_tstate_lock(self):
-            self._greenlet = getcurrent()
-
-        def run(self):
-            try:
-                super(Thread, self).run()
-            finally:
-                # avoid ref cycles, but keep in __dict__ so we can
-                # distinguish the started/never-started case
-                self._greenlet = None
-                self._stop() # mark as finished
-
-        def join(self, timeout=None):
-            if '_greenlet' not in self.__dict__:
-                raise RuntimeError("Cannot join an inactive thread")
-            if self._greenlet is None:
-                return
-            self._greenlet.join(timeout=timeout)
-
-        def _wait_for_tstate_lock(self, *args, **kwargs):
-            # pylint:disable=arguments-differ
-            raise NotImplementedError()
+            super(Thread, self)._set_tstate_lock()
+            greenlet = getcurrent()
+            greenlet.rawlink(self.__greenlet_finished)
+
+        def __greenlet_finished(self, _):
+            if self._tstate_lock:
+                self._tstate_lock.release()
+                self._stop()
 
     __implements__.append('Thread')
 
@@ -203,6 +187,8 @@ if PY3:
 
     __implements__.append('Timer')
 
+    _set_sentinel = allocate_lock
+    __implements__.append('_set_sentinel')
     # The main thread is patched up with more care
     # in _gevent_will_monkey_patch