Commit 47fac0a9 authored by Kirill Smelkov's avatar Kirill Smelkov

libgolang/{gevent,thread}: Preserve Python-level exception in runtime calls

Gevent runtime uses python-level calls internally which might interfere
with current python state. For example if current python exception is
set, and e.g. go or makesema runtime call is made, the following
happens:

    golang/golang_test.py::test_pyx_runtime_vs_pyexc RuntimeError: abc

    The above exception was the direct cause of the following exception:

    SystemError: <class 'gevent.__semaphore.Semaphore'> returned a result with an error set
    Exception ignored in: 'golang.runtime._runtime_gevent._sema_alloc'
    SystemError: <class 'gevent.__semaphore.Semaphore'> returned a result with an error set
    terminate called after throwing an instance of 'golang::PanicError'
      what():  makesema: alloc failed
    Fatal Python error: Aborted

-> Fix all functions in the runtimes that work at Python level to
   save/restore Python-level exception on entry/exit.

This is mostly gevent runtime, but also a couple of non-posix fallbacks in
thread runtime.

The bug was there from day 1 of runtimes - from ce8152a2 (pyx api:
Provide sleep), f971a2a8 (pyx api: Provide go) and 69db91bf (libgolang:
Add internal semaphores).
parent 689dc862
...@@ -28,6 +28,8 @@ from __future__ import print_function, absolute_import ...@@ -28,6 +28,8 @@ from __future__ import print_function, absolute_import
from golang cimport go, chan, _chan, makechan, pychan, nil, select, \ from golang cimport go, chan, _chan, makechan, pychan, nil, select, \
default, structZ, panic, pypanic, topyexc, cbool default, structZ, panic, pypanic, topyexc, cbool
from golang cimport time
from cpython cimport PyObject, PyErr_SetString, PyErr_Clear, PyErr_Occurred
cdef extern from "golang/libgolang.h" namespace "golang" nogil: cdef extern from "golang/libgolang.h" namespace "golang" nogil:
int _tchanrecvqlen(_chan *ch) int _tchanrecvqlen(_chan *ch)
...@@ -151,6 +153,51 @@ def test_go_nogil(): ...@@ -151,6 +153,51 @@ def test_go_nogil():
_test_go_nogil() _test_go_nogil()
# verify that runtime calls preserve current Python exception
# ( for example gevent runtime uses python-level calls internally which might
# interfere with current py state )
def test_runtime_vs_pyexc():
cdef PyObject *pyexc
assert PyErr_Occurred() == NULL # no exception initially
# set "current" exception
PyErr_SetString(RuntimeError, "abc")
pyexc = PyErr_Occurred()
assert pyexc != NULL
assert pyexc == PyErr_Occurred()
# makechan (also tests sema alloc)
cdef chan[int] ch = makechan[int](1)
assert PyErr_Occurred() == pyexc
# chan send/recv (also test sema acquire/release)
ch.send(3)
assert PyErr_Occurred() == pyexc
assert ch.recv() == 3
assert PyErr_Occurred() == pyexc
# chan free (also tests sema free)
ch = nil
# go
go(_noop)
assert PyErr_Occurred() == pyexc
# sleep
time.sleep(0.001)
assert PyErr_Occurred() == pyexc
# now
time.now()
assert PyErr_Occurred() == pyexc
# clear current exception, or else test driver will see calling us as failure
PyErr_Clear()
assert PyErr_Occurred() == NULL
cdef void _noop() nogil:
pass
# runtime/libgolang_test_c.c # runtime/libgolang_test_c.c
cdef extern from * nogil: cdef extern from * nogil:
""" """
......
...@@ -44,6 +44,7 @@ from cython cimport final ...@@ -44,6 +44,7 @@ from cython cimport final
from golang.runtime._libgolang cimport _libgolang_runtime_ops, _libgolang_sema, \ from golang.runtime._libgolang cimport _libgolang_runtime_ops, _libgolang_sema, \
STACK_DEAD_WHILE_PARKED, panic STACK_DEAD_WHILE_PARKED, panic
from golang.runtime cimport _runtime_thread from golang.runtime cimport _runtime_thread
from golang.runtime._runtime_pymisc cimport PyExc, pyexc_fetch, pyexc_restore
# _goviapy & _togo serve go # _goviapy & _togo serve go
...@@ -58,43 +59,37 @@ cdef class _togo: ...@@ -58,43 +59,37 @@ cdef class _togo:
# internal functions that work under gil # internal functions that work under gil
cdef nogil: cdef:
# XXX better panic with pyexc object and detect that at recover side? # XXX better panic with pyexc object and detect that at recover side?
bint _go(void (*f)(void *), void *arg): bint _go(void (*f)(void *) nogil, void *arg):
with gil:
_ = _togo(); _.f = f; _.arg = arg _ = _togo(); _.f = f; _.arg = arg
g = Greenlet(_goviapy, _) g = Greenlet(_goviapy, _)
g.start() g.start()
return True return True
_libgolang_sema* _sema_alloc(): _libgolang_sema* _sema_alloc():
with gil:
pygsema = Semaphore() pygsema = Semaphore()
Py_INCREF(pygsema) Py_INCREF(pygsema)
return <_libgolang_sema*>pygsema return <_libgolang_sema*>pygsema
bint _sema_free(_libgolang_sema *gsema): bint _sema_free(_libgolang_sema *gsema):
with gil:
pygsema = <PYGSema>gsema pygsema = <PYGSema>gsema
Py_DECREF(pygsema) Py_DECREF(pygsema)
return True return True
bint _sema_acquire(_libgolang_sema *gsema): bint _sema_acquire(_libgolang_sema *gsema):
with gil:
pygsema = <PYGSema>gsema pygsema = <PYGSema>gsema
pygsema.acquire() pygsema.acquire()
return True return True
bint _sema_release(_libgolang_sema *gsema): bint _sema_release(_libgolang_sema *gsema):
with gil:
pygsema = <PYGSema>gsema pygsema = <PYGSema>gsema
pygsema.release() pygsema.release()
return True return True
bint _nanosleep(uint64_t dt): bint _nanosleep(uint64_t dt):
cdef double dt_s = dt * 1E-9 cdef double dt_s = dt * 1E-9
with gil:
pygsleep(dt_s) pygsleep(dt_s)
return True return True
...@@ -103,31 +98,55 @@ cdef nogil: ...@@ -103,31 +98,55 @@ cdef nogil:
cdef nogil: cdef nogil:
void go(void (*f)(void *), void *arg): void go(void (*f)(void *), void *arg):
cdef PyExc exc
with gil:
pyexc_fetch(&exc)
ok = _go(f, arg) ok = _go(f, arg)
pyexc_restore(exc)
if not ok: if not ok:
panic("pyxgo: gevent: go: failed") panic("pyxgo: gevent: go: failed")
_libgolang_sema* sema_alloc(): _libgolang_sema* sema_alloc():
cdef PyExc exc
with gil:
pyexc_fetch(&exc)
sema = _sema_alloc() sema = _sema_alloc()
pyexc_restore(exc)
return sema # libgolang checks for NULL return return sema # libgolang checks for NULL return
void sema_free(_libgolang_sema *gsema): void sema_free(_libgolang_sema *gsema):
cdef PyExc exc
with gil:
pyexc_fetch(&exc)
ok = _sema_free(gsema) ok = _sema_free(gsema)
pyexc_restore(exc)
if not ok: if not ok:
panic("pyxgo: gevent: sema: free: failed") panic("pyxgo: gevent: sema: free: failed")
void sema_acquire(_libgolang_sema *gsema): void sema_acquire(_libgolang_sema *gsema):
cdef PyExc exc
with gil:
pyexc_fetch(&exc)
ok = _sema_acquire(gsema) ok = _sema_acquire(gsema)
pyexc_restore(exc)
if not ok: if not ok:
panic("pyxgo: gevent: sema: acquire: failed") panic("pyxgo: gevent: sema: acquire: failed")
void sema_release(_libgolang_sema *gsema): void sema_release(_libgolang_sema *gsema):
cdef PyExc exc
with gil:
pyexc_fetch(&exc)
ok = _sema_release(gsema) ok = _sema_release(gsema)
pyexc_restore(exc)
if not ok: if not ok:
panic("pyxgo: gevent: sema: release: failed") panic("pyxgo: gevent: sema: release: failed")
void nanosleep(uint64_t dt): void nanosleep(uint64_t dt):
cdef PyExc exc
with gil:
pyexc_fetch(&exc)
ok = _nanosleep(dt) ok = _nanosleep(dt)
pyexc_restore(exc)
if not ok: if not ok:
panic("pyxgo: gevent: sleep: failed") panic("pyxgo: gevent: sleep: failed")
......
# cython: language_level=2
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""_runtime_pymisc.pxd provides miscellaneous Python-related utilities for _runtime_*.pyx"""
# PyExc and pyexc_fetch/pyexc_restore provide non-noisy way to save/restore
# current python exception.
cdef extern from "Python.h":
"""
typedef struct PyExc {
PyObject *pyexc_type;
PyObject *pyexc_value;
PyObject *pyexc_traceback;
} PyExc;
static inline void pyexc_fetch(PyExc *e) {
PyErr_Fetch(&e->pyexc_type, &e->pyexc_value, &e->pyexc_traceback);
}
static inline void pyexc_restore(PyExc e) {
PyErr_Restore(e.pyexc_type, e.pyexc_value, e.pyexc_traceback);
}
"""
struct PyExc:
pass
void pyexc_fetch(PyExc*)
void pyexc_restore(PyExc)
...@@ -97,16 +97,15 @@ IF POSIX: ...@@ -97,16 +97,15 @@ IF POSIX:
ELSE: ELSE:
# !posix via-gil timing fallback # !posix via-gil timing fallback
import time as pytimemod import time as pytimemod
from golang.runtime._runtime_pymisc cimport PyExc, pyexc_fetch, pyexc_restore
cdef nogil: cdef:
bint _nanosleep(double dt_s): bint _nanosleep(double dt_s):
with gil:
pytimemod.sleep(dt_s) pytimemod.sleep(dt_s)
return True return True
(double, bint) _nanotime(): (double, bint) _nanotime():
cdef double t_s cdef double t_s
with gil:
t_s = pytimemod.time() t_s = pytimemod.time()
return t_s, True return t_s, True
...@@ -154,7 +153,11 @@ cdef nogil: ...@@ -154,7 +153,11 @@ cdef nogil:
ELSE: ELSE:
void nanosleep(uint64_t dt): void nanosleep(uint64_t dt):
cdef double dt_s = dt * 1E-9 # no overflow possible cdef double dt_s = dt * 1E-9 # no overflow possible
cdef PyExc exc
with gil:
pyexc_fetch(&exc)
ok = _nanosleep(dt_s) ok = _nanosleep(dt_s)
pyexc_restore(exc)
if not ok: if not ok:
panic("pyxgo: thread: nanosleep: pytime.sleep failed") panic("pyxgo: thread: nanosleep: pytime.sleep failed")
...@@ -171,7 +174,12 @@ cdef nogil: ...@@ -171,7 +174,12 @@ cdef nogil:
return ts.tv_sec*i1E9 + ts.tv_nsec return ts.tv_sec*i1E9 + ts.tv_nsec
ELSE: ELSE:
uint64_t nanotime(): uint64_t nanotime():
cdef double t_s
cdef PyExc exc
with gil:
pyexc_fetch(&exc)
t_s, ok = _nanotime() t_s, ok = _nanotime()
pyexc_restore(exc)
if not ok: if not ok:
panic("pyxgo: thread: nanotime: pytime.time failed") panic("pyxgo: thread: nanotime: pytime.time failed")
t_ns = t_s * 1E9 t_ns = t_s * 1E9
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment