Issue #23353: Fix the exception handling of generators in PyEval_EvalFrameEx().

At entry, save or swap the exception state even if PyEval_EvalFrameEx() is
called with throwflag=0. At exit, the exception state is now always restored or
swapped, not only if why is WHY_YIELD or WHY_RETURN. Patch co-written with
Antoine Pitrou.

Lib/test/test_generators.py

@@ -50,6 +50,115 @@ class FinalizationTest(unittest.TestCase):
         self.assertEqual(gc.garbage, old_garbage)
 
 
+class ExceptionTest(unittest.TestCase):
+    # Tests for the issue #23353: check that the currently handled exception
+    # is correctly saved/restored in PyEval_EvalFrameEx().
+
+    def test_except_throw(self):
+        def store_raise_exc_generator():
+            try:
+                self.assertEqual(sys.exc_info()[0], None)
+                yield
+            except Exception as exc:
+                # exception raised by gen.throw(exc)
+                self.assertEqual(sys.exc_info()[0], ValueError)
+                self.assertIsNone(exc.__context__)
+                yield
+
+                # ensure that the exception is not lost
+                self.assertEqual(sys.exc_info()[0], ValueError)
+                yield
+
+                # we should be able to raise back the ValueError
+                raise
+
+        make = store_raise_exc_generator()
+        next(make)
+
+        try:
+            raise ValueError()
+        except Exception as exc:
+            try:
+                make.throw(exc)
+            except Exception:
+                pass
+
+        next(make)
+        with self.assertRaises(ValueError) as cm:
+            next(make)
+        self.assertIsNone(cm.exception.__context__)
+
+        self.assertEqual(sys.exc_info(), (None, None, None))
+
+    def test_except_next(self):
+        def gen():
+            self.assertEqual(sys.exc_info()[0], ValueError)
+            yield "done"
+
+        g = gen()
+        try:
+            raise ValueError
+        except Exception:
+            self.assertEqual(next(g), "done")
+        self.assertEqual(sys.exc_info(), (None, None, None))
+
+    def test_except_gen_except(self):
+        def gen():
+            try:
+                self.assertEqual(sys.exc_info()[0], None)
+                yield
+                # we are called from "except ValueError:", TypeError must
+                # inherit ValueError in its context
+                raise TypeError()
+            except TypeError as exc:
+                self.assertEqual(sys.exc_info()[0], TypeError)
+                self.assertEqual(type(exc.__context__), ValueError)
+            # here we are still called from the "except ValueError:"
+            self.assertEqual(sys.exc_info()[0], ValueError)
+            yield
+            self.assertIsNone(sys.exc_info()[0])
+            yield "done"
+
+        g = gen()
+        next(g)
+        try:
+            raise ValueError
+        except Exception:
+            next(g)
+
+        self.assertEqual(next(g), "done")
+        self.assertEqual(sys.exc_info(), (None, None, None))
+
+    def test_except_throw_exception_context(self):
+        def gen():
+            try:
+                try:
+                    self.assertEqual(sys.exc_info()[0], None)
+                    yield
+                except ValueError:
+                    # we are called from "except ValueError:"
+                    self.assertEqual(sys.exc_info()[0], ValueError)
+                    raise TypeError()
+            except Exception as exc:
+                self.assertEqual(sys.exc_info()[0], TypeError)
+                self.assertEqual(type(exc.__context__), ValueError)
+            # we are still called from "except ValueError:"
+            self.assertEqual(sys.exc_info()[0], ValueError)
+            yield
+            self.assertIsNone(sys.exc_info()[0])
+            yield "done"
+
+        g = gen()
+        next(g)
+        try:
+            raise ValueError
+        except Exception as exc:
+            g.throw(exc)
+
+        self.assertEqual(next(g), "done")
+        self.assertEqual(sys.exc_info(), (None, None, None))
+
+
 tutorial_tests = """
 Let's try a simple generator:
 

Misc/NEWS

@@ -50,6 +50,12 @@ Core and Builtins
 Library
 -------
 
+- Issue #23353: Fix the exception handling of generators in
+  PyEval_EvalFrameEx(). At entry, save or swap the exception state even if
+  PyEval_EvalFrameEx() is called with throwflag=0. At exit, the exception state
+  is now always restored or swapped, not only if why is WHY_YIELD or
+  WHY_RETURN. Patch co-written with Antoine Pitrou.
+
 - Issue #18518: timeit now rejects statements which can't be compiled outside
   a function or a loop (e.g. "return" or "break").
 

Python/ceval.c

@@ -1189,8 +1189,8 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
     f->f_stacktop = NULL;       /* remains NULL unless yield suspends frame */
     f->f_executing = 1;
 
-    if (co->co_flags & CO_GENERATOR && !throwflag) {
-        if (f->f_exc_type != NULL && f->f_exc_type != Py_None) {
+    if (co->co_flags & CO_GENERATOR) {
+        if (!throwflag && f->f_exc_type != NULL && f->f_exc_type != Py_None) {
             /* We were in an except handler when we left,
                restore the exception state which was put aside
                (see YIELD_VALUE). */
@@ -3172,7 +3172,8 @@ fast_block_end:
             || (retval == NULL && PyErr_Occurred()));
 
 fast_yield:
-    if (co->co_flags & CO_GENERATOR && (why == WHY_YIELD || why == WHY_RETURN)) {
+    if (co->co_flags & CO_GENERATOR) {
+
         /* The purpose of this block is to put aside the generator's exception
            state and restore that of the calling frame. If the current
            exception state is from the caller, we clear the exception values