Commit ae3087c6 authored by Mark Shannon's avatar Mark Shannon Committed by Antoine Pitrou

Move exc state to generator. Fixes bpo-25612 (#1773)

Move exception state information from frame objects to coroutine (generator/thread) object where it belongs.
parent 91dc64ba
......@@ -30,14 +30,6 @@ typedef struct _frame {
char f_trace_lines; /* Emit per-line trace events? */
char f_trace_opcodes; /* Emit per-opcode trace events? */
/* In a generator, we need to be able to swap between the exception
state inside the generator and the exception state of the calling
frame (which shouldn't be impacted when the generator "yields"
from an except handler).
These three fields exist exactly for that, and are unused for
non-generator frames. See the save_exc_state and swap_exc_state
functions in ceval.c for details of their use. */
PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
/* Borrowed reference to a generator, or NULL */
PyObject *f_gen;
......
......@@ -25,7 +25,8 @@ struct _frame; /* Avoid including frameobject.h */
/* Name of the generator. */ \
PyObject *prefix##_name; \
/* Qualified name of the generator. */ \
PyObject *prefix##_qualname;
PyObject *prefix##_qualname; \
_PyErr_StackItem prefix##_exc_state;
typedef struct {
/* The gi_ prefix is intended to remind of generator-iterator. */
......
......@@ -78,6 +78,7 @@ PyAPI_FUNC(void) PyErr_SetNone(PyObject *);
PyAPI_FUNC(void) PyErr_SetObject(PyObject *, PyObject *);
#ifndef Py_LIMITED_API
PyAPI_FUNC(void) _PyErr_SetKeyError(PyObject *);
_PyErr_StackItem *_PyErr_GetTopmostException(PyThreadState *tstate);
#endif
PyAPI_FUNC(void) PyErr_SetString(
PyObject *exception,
......
......@@ -123,6 +123,21 @@ typedef int (*Py_tracefunc)(PyObject *, struct _frame *, int, PyObject *);
#ifdef Py_LIMITED_API
typedef struct _ts PyThreadState;
#else
typedef struct _err_stackitem {
/* This struct represents an entry on the exception stack, which is a
* per-coroutine state. (Coroutine in the computer science sense,
* including the thread and generators).
* This ensures that the exception state is not impacted by "yields"
* from an except handler.
*/
PyObject *exc_type, *exc_value, *exc_traceback;
struct _err_stackitem *previous_item;
} _PyErr_StackItem;
typedef struct _ts {
/* See Python/ceval.c for comments explaining most fields */
......@@ -147,13 +162,19 @@ typedef struct _ts {
PyObject *c_profileobj;
PyObject *c_traceobj;
/* The exception currently being raised */
PyObject *curexc_type;
PyObject *curexc_value;
PyObject *curexc_traceback;
PyObject *exc_type;
PyObject *exc_value;
PyObject *exc_traceback;
/* The exception currently being handled, if no coroutines/generators
* are present. Always last element on the stack referred to be exc_info.
*/
_PyErr_StackItem exc_state;
/* Pointer to the top of the stack of the exceptions currently
* being handled */
_PyErr_StackItem *exc_info;
PyObject *dict; /* Stores per-thread state */
......
......@@ -1097,6 +1097,62 @@ class ExceptionTests(unittest.TestCase):
self.assertIn("test message", report)
self.assertTrue(report.endswith("\n"))
def test_yield_in_nested_try_excepts(self):
#Issue #25612
class MainError(Exception):
pass
class SubError(Exception):
pass
def main():
try:
raise MainError()
except MainError:
try:
yield
except SubError:
pass
raise
coro = main()
coro.send(None)
with self.assertRaises(MainError):
coro.throw(SubError())
def test_generator_doesnt_retain_old_exc2(self):
#Issue 28884#msg282532
def g():
try:
raise ValueError
except ValueError:
yield 1
self.assertEqual(sys.exc_info(), (None, None, None))
yield 2
gen = g()
try:
raise IndexError
except IndexError:
self.assertEqual(next(gen), 1)
self.assertEqual(next(gen), 2)
def test_raise_in_generator(self):
#Issue 25612#msg304117
def g():
yield 1
raise
yield 2
with self.assertRaises(ZeroDivisionError):
i = g()
try:
1/0
except:
next(i)
next(i)
class ImportErrorTests(unittest.TestCase):
......
......@@ -971,7 +971,7 @@ class SizeofTest(unittest.TestCase):
nfrees = len(x.f_code.co_freevars)
extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
ncells + nfrees - 1
check(x, vsize('8P2c4P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
check(x, vsize('5P2c4P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
# function
def func(): pass
check(func, size('12P'))
......@@ -988,7 +988,7 @@ class SizeofTest(unittest.TestCase):
check(bar, size('PP'))
# generator
def get_gen(): yield 1
check(get_gen(), size('Pb2PPP'))
check(get_gen(), size('Pb2PPP4P'))
# iterator
check(iter('abc'), size('lP'))
# callable-iterator
......
Move the current exception state from the frame object to the co-routine.
This simplifies the interpreter and fixes a couple of obscure bugs caused by
having swap exception state when entering or exiting a generator.
......@@ -379,8 +379,7 @@ static PyGetSetDef frame_getsetlist[] = {
* ob_type, ob_size, f_code, f_valuestack;
* f_locals, f_trace,
f_exc_type, f_exc_value, f_exc_traceback are NULL;
* f_locals, f_trace are NULL;
* f_localsplus does not require re-allocation and
the local variables in f_localsplus are NULL.
......@@ -438,9 +437,6 @@ frame_dealloc(PyFrameObject *f)
Py_DECREF(f->f_globals);
Py_CLEAR(f->f_locals);
Py_CLEAR(f->f_trace);
Py_CLEAR(f->f_exc_type);
Py_CLEAR(f->f_exc_value);
Py_CLEAR(f->f_exc_traceback);
co = f->f_code;
if (co->co_zombieframe == NULL)
......@@ -469,9 +465,6 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg)
Py_VISIT(f->f_globals);
Py_VISIT(f->f_locals);
Py_VISIT(f->f_trace);
Py_VISIT(f->f_exc_type);
Py_VISIT(f->f_exc_value);
Py_VISIT(f->f_exc_traceback);
/* locals */
slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars);
......@@ -502,9 +495,6 @@ frame_tp_clear(PyFrameObject *f)
f->f_stacktop = NULL;
f->f_executing = 0;
Py_CLEAR(f->f_exc_type);
Py_CLEAR(f->f_exc_value);
Py_CLEAR(f->f_exc_traceback);
Py_CLEAR(f->f_trace);
/* locals */
......@@ -698,7 +688,6 @@ _PyFrame_New_NoTrack(PyThreadState *tstate, PyCodeObject *code,
f->f_localsplus[i] = NULL;
f->f_locals = NULL;
f->f_trace = NULL;
f->f_exc_type = f->f_exc_value = f->f_exc_traceback = NULL;
}
f->f_stacktop = f->f_valuestack;
f->f_builtins = builtins;
......
......@@ -16,6 +16,15 @@ static char *NON_INIT_CORO_MSG = "can't send non-None value to a "
static char *ASYNC_GEN_IGNORED_EXIT_MSG =
"async generator ignored GeneratorExit";
static inline int
exc_state_traverse(_PyErr_StackItem *exc_state, visitproc visit, void *arg)
{
Py_VISIT(exc_state->exc_type);
Py_VISIT(exc_state->exc_value);
Py_VISIT(exc_state->exc_traceback);
return 0;
}
static int
gen_traverse(PyGenObject *gen, visitproc visit, void *arg)
{
......@@ -23,7 +32,7 @@ gen_traverse(PyGenObject *gen, visitproc visit, void *arg)
Py_VISIT(gen->gi_code);
Py_VISIT(gen->gi_name);
Py_VISIT(gen->gi_qualname);
return 0;
return exc_state_traverse(&gen->gi_exc_state, visit, arg);
}
void
......@@ -87,6 +96,21 @@ _PyGen_Finalize(PyObject *self)
PyErr_Restore(error_type, error_value, error_traceback);
}
static inline void
exc_state_clear(_PyErr_StackItem *exc_state)
{
PyObject *t, *v, *tb;
t = exc_state->exc_type;
v = exc_state->exc_value;
tb = exc_state->exc_traceback;
exc_state->exc_type = NULL;
exc_state->exc_value = NULL;
exc_state->exc_traceback = NULL;
Py_XDECREF(t);
Py_XDECREF(v);
Py_XDECREF(tb);
}
static void
gen_dealloc(PyGenObject *gen)
{
......@@ -116,6 +140,7 @@ gen_dealloc(PyGenObject *gen)
Py_CLEAR(gen->gi_code);
Py_CLEAR(gen->gi_name);
Py_CLEAR(gen->gi_qualname);
exc_state_clear(&gen->gi_exc_state);
PyObject_GC_Del(gen);
}
......@@ -187,7 +212,11 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing)
f->f_back = tstate->frame;
gen->gi_running = 1;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
result = PyEval_EvalFrameEx(f, exc);
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
gen->gi_running = 0;
/* Don't keep the reference to f_back any longer than necessary. It
......@@ -281,16 +310,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing)
if (!result || f->f_stacktop == NULL) {
/* generator can't be rerun, so release the frame */
/* first clean reference cycle through stored exception traceback */
PyObject *t, *v, *tb;
t = f->f_exc_type;
v = f->f_exc_value;
tb = f->f_exc_traceback;
f->f_exc_type = NULL;
f->f_exc_value = NULL;
f->f_exc_traceback = NULL;
Py_XDECREF(t);
Py_XDECREF(v);
Py_XDECREF(tb);
exc_state_clear(&gen->gi_exc_state);
gen->gi_frame->f_gen = NULL;
gen->gi_frame = NULL;
Py_DECREF(f);
......@@ -810,6 +830,10 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f,
gen->gi_code = (PyObject *)(f->f_code);
gen->gi_running = 0;
gen->gi_weakreflist = NULL;
gen->gi_exc_state.exc_type = NULL;
gen->gi_exc_state.exc_value = NULL;
gen->gi_exc_state.exc_traceback = NULL;
gen->gi_exc_state.previous_item = NULL;
if (name != NULL)
gen->gi_name = name;
else
......
......@@ -511,9 +511,6 @@ enum why_code {
WHY_SILENCED = 0x0080 /* Exception silenced by 'with' */
};
static void save_exc_state(PyThreadState *, PyFrameObject *);
static void swap_exc_state(PyThreadState *, PyFrameObject *);
static void restore_and_clear_exc_state(PyThreadState *, PyFrameObject *);
static int do_raise(PyObject *, PyObject *);
static int unpack_iterable(PyObject *, int, int, PyObject **);
......@@ -813,17 +810,19 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
#define UNWIND_EXCEPT_HANDLER(b) \
do { \
PyObject *type, *value, *traceback; \
_PyErr_StackItem *exc_info; \
assert(STACK_LEVEL() >= (b)->b_level + 3); \
while (STACK_LEVEL() > (b)->b_level + 3) { \
value = POP(); \
Py_XDECREF(value); \
} \
type = tstate->exc_type; \
value = tstate->exc_value; \
traceback = tstate->exc_traceback; \
tstate->exc_type = POP(); \
tstate->exc_value = POP(); \
tstate->exc_traceback = POP(); \
exc_info = tstate->exc_info; \
type = exc_info->exc_type; \
value = exc_info->exc_value; \
traceback = exc_info->exc_traceback; \
exc_info->exc_type = POP(); \
exc_info->exc_value = POP(); \
exc_info->exc_traceback = POP(); \
Py_XDECREF(type); \
Py_XDECREF(value); \
Py_XDECREF(traceback); \
......@@ -910,16 +909,6 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
f->f_stacktop = NULL; /* remains NULL unless yield suspends frame */
f->f_executing = 1;
if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
if (!throwflag && f->f_exc_type != NULL && f->f_exc_type != Py_None) {
/* We were in an except handler when we left,
restore the exception state which was put aside
(see YIELD_VALUE). */
swap_exc_state(tstate, f);
}
else
save_exc_state(tstate, f);
}
#ifdef LLTRACE
lltrace = _PyDict_GetItemId(f->f_globals, &PyId___ltrace__) != NULL;
......@@ -3447,12 +3436,13 @@ fast_block_end:
|| b->b_type == SETUP_FINALLY)) {
PyObject *exc, *val, *tb;
int handler = b->b_handler;
_PyErr_StackItem *exc_info = tstate->exc_info;
/* Beware, this invalidates all b->b_* fields */
PyFrame_BlockSetup(f, EXCEPT_HANDLER, -1, STACK_LEVEL());
PUSH(tstate->exc_traceback);
PUSH(tstate->exc_value);
if (tstate->exc_type != NULL) {
PUSH(tstate->exc_type);
PUSH(exc_info->exc_traceback);
PUSH(exc_info->exc_value);
if (exc_info->exc_type != NULL) {
PUSH(exc_info->exc_type);
}
else {
Py_INCREF(Py_None);
......@@ -3470,10 +3460,10 @@ fast_block_end:
else
PyException_SetTraceback(val, Py_None);
Py_INCREF(exc);
tstate->exc_type = exc;
exc_info->exc_type = exc;
Py_INCREF(val);
tstate->exc_value = val;
tstate->exc_traceback = tb;
exc_info->exc_value = val;
exc_info->exc_traceback = tb;
if (tb == NULL)
tb = Py_None;
Py_INCREF(tb);
......@@ -3516,28 +3506,6 @@ fast_block_end:
assert((retval != NULL) ^ (PyErr_Occurred() != NULL));
fast_yield:
if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
/* The purpose of this block is to put aside the generator's exception
state and restore that of the calling frame. If the current
exception state is from the caller, we clear the exception values
on the generator frame, so they are not swapped back in latter. The
origin of the current exception state is determined by checking for
except handler blocks, which we must be in iff a new exception
state came into existence in this frame. (An uncaught exception
would have why == WHY_EXCEPTION, and we wouldn't be here). */
int i;
for (i = 0; i < f->f_iblock; i++) {
if (f->f_blockstack[i].b_type == EXCEPT_HANDLER) {
break;
}
}
if (i == f->f_iblock)
/* We did not create this exception. */
restore_and_clear_exc_state(tstate, f);
else
swap_exc_state(tstate, f);
}
if (tstate->use_tracing) {
if (tstate->c_tracefunc) {
......@@ -4057,60 +4025,6 @@ special_lookup(PyObject *o, _Py_Identifier *id)
}
/* These 3 functions deal with the exception state of generators. */
static void
save_exc_state(PyThreadState *tstate, PyFrameObject *f)
{
PyObject *type, *value, *traceback;
Py_XINCREF(tstate->exc_type);
Py_XINCREF(tstate->exc_value);
Py_XINCREF(tstate->exc_traceback);
type = f->f_exc_type;
value = f->f_exc_value;
traceback = f->f_exc_traceback;
f->f_exc_type = tstate->exc_type;
f->f_exc_value = tstate->exc_value;
f->f_exc_traceback = tstate->exc_traceback;
Py_XDECREF(type);
Py_XDECREF(value);
Py_XDECREF(traceback);
}
static void
swap_exc_state(PyThreadState *tstate, PyFrameObject *f)
{
PyObject *tmp;
tmp = tstate->exc_type;
tstate->exc_type = f->f_exc_type;
f->f_exc_type = tmp;
tmp = tstate->exc_value;
tstate->exc_value = f->f_exc_value;
f->f_exc_value = tmp;
tmp = tstate->exc_traceback;
tstate->exc_traceback = f->f_exc_traceback;
f->f_exc_traceback = tmp;
}
static void
restore_and_clear_exc_state(PyThreadState *tstate, PyFrameObject *f)
{
PyObject *type, *value, *tb;
type = tstate->exc_type;
value = tstate->exc_value;
tb = tstate->exc_traceback;
tstate->exc_type = f->f_exc_type;
tstate->exc_value = f->f_exc_value;
tstate->exc_traceback = f->f_exc_traceback;
f->f_exc_type = NULL;
f->f_exc_value = NULL;
f->f_exc_traceback = NULL;
Py_XDECREF(type);
Py_XDECREF(value);
Py_XDECREF(tb);
}
/* Logic for the raise statement (too complicated for inlining).
This *consumes* a reference count to each of its arguments. */
static int
......@@ -4121,10 +4035,11 @@ do_raise(PyObject *exc, PyObject *cause)
if (exc == NULL) {
/* Reraise */
PyThreadState *tstate = PyThreadState_GET();
_PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate);
PyObject *tb;
type = tstate->exc_type;
value = tstate->exc_value;
tb = tstate->exc_traceback;
type = exc_info->exc_type;
value = exc_info->exc_value;
tb = exc_info->exc_traceback;
if (type == Py_None || type == NULL) {
PyErr_SetString(PyExc_RuntimeError,
"No active exception to reraise");
......
......@@ -53,6 +53,18 @@ PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback)
Py_XDECREF(oldtraceback);
}
_PyErr_StackItem *
_PyErr_GetTopmostException(PyThreadState *tstate)
{
_PyErr_StackItem *exc_info = tstate->exc_info;
while ((exc_info->exc_type == NULL || exc_info->exc_type == Py_None) &&
exc_info->previous_item != NULL)
{
exc_info = exc_info->previous_item;
}
return exc_info;
}
static PyObject*
_PyErr_CreateException(PyObject *exception, PyObject *value)
{
......@@ -83,7 +95,7 @@ PyErr_SetObject(PyObject *exception, PyObject *value)
}
Py_XINCREF(value);
exc_value = tstate->exc_value;
exc_value = _PyErr_GetTopmostException(tstate)->exc_value;
if (exc_value != NULL && exc_value != Py_None) {
/* Implicit exception chaining */
Py_INCREF(exc_value);
......@@ -335,9 +347,11 @@ PyErr_GetExcInfo(PyObject **p_type, PyObject **p_value, PyObject **p_traceback)
{
PyThreadState *tstate = PyThreadState_GET();
*p_type = tstate->exc_type;
*p_value = tstate->exc_value;
*p_traceback = tstate->exc_traceback;
_PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate);
*p_type = exc_info->exc_type;
*p_value = exc_info->exc_value;
*p_traceback = exc_info->exc_traceback;
Py_XINCREF(*p_type);
Py_XINCREF(*p_value);
......@@ -350,13 +364,13 @@ PyErr_SetExcInfo(PyObject *p_type, PyObject *p_value, PyObject *p_traceback)
PyObject *oldtype, *oldvalue, *oldtraceback;
PyThreadState *tstate = PyThreadState_GET();
oldtype = tstate->exc_type;
oldvalue = tstate->exc_value;
oldtraceback = tstate->exc_traceback;
oldtype = tstate->exc_info->exc_type;
oldvalue = tstate->exc_info->exc_value;
oldtraceback = tstate->exc_info->exc_traceback;
tstate->exc_type = p_type;
tstate->exc_value = p_value;
tstate->exc_traceback = p_traceback;
tstate->exc_info->exc_type = p_type;
tstate->exc_info->exc_value = p_value;
tstate->exc_info->exc_traceback = p_traceback;
Py_XDECREF(oldtype);
Py_XDECREF(oldvalue);
......
......@@ -257,9 +257,11 @@ new_threadstate(PyInterpreterState *interp, int init)
tstate->curexc_value = NULL;
tstate->curexc_traceback = NULL;
tstate->exc_type = NULL;
tstate->exc_value = NULL;
tstate->exc_traceback = NULL;
tstate->exc_state.exc_type = NULL;
tstate->exc_state.exc_value = NULL;
tstate->exc_state.exc_traceback = NULL;
tstate->exc_state.previous_item = NULL;
tstate->exc_info = &tstate->exc_state;
tstate->c_profilefunc = NULL;
tstate->c_tracefunc = NULL;
......@@ -444,9 +446,16 @@ PyThreadState_Clear(PyThreadState *tstate)
Py_CLEAR(tstate->curexc_value);
Py_CLEAR(tstate->curexc_traceback);
Py_CLEAR(tstate->exc_type);
Py_CLEAR(tstate->exc_value);
Py_CLEAR(tstate->exc_traceback);
Py_CLEAR(tstate->exc_state.exc_type);
Py_CLEAR(tstate->exc_state.exc_value);
Py_CLEAR(tstate->exc_state.exc_traceback);
/* The stack of exception states should contain just this thread. */
assert(tstate->exc_info->previous_item == NULL);
if (Py_VerboseFlag && tstate->exc_info != &tstate->exc_state) {
fprintf(stderr,
"PyThreadState_Clear: warning: thread still has a generator\n");
}
tstate->c_profilefunc = NULL;
tstate->c_tracefunc = NULL;
......
......@@ -311,14 +311,13 @@ PyDoc_STRVAR(excepthook_doc,
static PyObject *
sys_exc_info(PyObject *self, PyObject *noargs)
{
PyThreadState *tstate;
tstate = PyThreadState_GET();
_PyErr_StackItem *err_info = _PyErr_GetTopmostException(PyThreadState_GET());
return Py_BuildValue(
"(OOO)",
tstate->exc_type != NULL ? tstate->exc_type : Py_None,
tstate->exc_value != NULL ? tstate->exc_value : Py_None,
tstate->exc_traceback != NULL ?
tstate->exc_traceback : Py_None);
err_info->exc_type != NULL ? err_info->exc_type : Py_None,
err_info->exc_value != NULL ? err_info->exc_value : Py_None,
err_info->exc_traceback != NULL ?
err_info->exc_traceback : Py_None);
}
PyDoc_STRVAR(exc_info_doc,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment