#3021: Antoine Pitrou's Lexical exception handlers

Doc/library/dis.rst

@@ -397,6 +397,14 @@ Miscellaneous opcodes.
    denoting nested loops, try statements, and such.
 
 
+.. opcode:: POP_EXCEPT ()
+
+   Removes one block from the block stack. The popped block must be an exception
+   handler block, as implicitly created when entering an except handler.
+   In addition to popping extraneous values from the frame stack, the
+   last three popped values are used to restore the exception state.
+
+
 .. opcode:: END_FINALLY ()
 
    Terminates a :keyword:`finally` clause.  The interpreter recalls whether the
@@ -412,24 +420,22 @@ Miscellaneous opcodes.
 
 .. opcode:: WITH_CLEANUP ()
 
-   Cleans up the stack when a :keyword:`with` statement block exits.  On top of
-   the stack are 1--3 values indicating how/why the finally clause was entered:
-
-   * TOP = ``None``
-   * (TOP, SECOND) = (``WHY_{RETURN,CONTINUE}``), retval
-   * TOP = ``WHY_*``; no retval below it
-   * (TOP, SECOND, THIRD) = exc_info()
+   Cleans up the stack when a :keyword:`with` statement block exits.  TOS is
+   the context manager's :meth:`__exit__` bound method. Below TOS are 1--3
+   values indicating how/why the finally clause was entered:
 
-   Under them is EXIT, the context manager's :meth:`__exit__` bound method.
+   * SECOND = ``None``
+   * (SECOND, THIRD) = (``WHY_{RETURN,CONTINUE}``), retval
+   * SECOND = ``WHY_*``; no retval below it
+   * (SECOND, THIRD, FOURTH) = exc_info()
 
-   In the last case, ``EXIT(TOP, SECOND, THIRD)`` is called, otherwise
-   ``EXIT(None, None, None)``.
+   In the last case, ``TOS(SECOND, THIRD, FOURTH)`` is called, otherwise
+   ``TOS(None, None, None)``.  In addition, TOS is removed from the stack.
 
-   EXIT is removed from the stack, leaving the values above it in the same
-   order. In addition, if the stack represents an exception, *and* the function
-   call returns a 'true' value, this information is "zapped", to prevent
-   ``END_FINALLY`` from re-raising the exception.  (But non-local gotos should
-   still be resumed.)
+   If the stack represents an exception, *and* the function call returns
+   a 'true' value, this information is "zapped" and replaced with a single
+   ``WHY_SILENCED`` to prevent ``END_FINALLY`` from re-raising the exception.
+   (But non-local gotos will still be resumed.)
 
    .. XXX explain the WHY stuff!
 

Doc/library/inspect.rst

@@ -94,17 +94,6 @@ attributes:
 |           | f_code          | code object being         |
 |           |                 | executed in this frame    |
 +-----------+-----------------+---------------------------+
-|           | f_exc_traceback | traceback if raised in    |
-|           |                 | this frame, or ``None``   |
-+-----------+-----------------+---------------------------+
-|           | f_exc_type      | exception type if raised  |
-|           |                 | in this frame, or         |
-|           |                 | ``None``                  |
-+-----------+-----------------+---------------------------+
-|           | f_exc_value     | exception value if raised |
-|           |                 | in this frame, or         |
-|           |                 | ``None``                  |
-+-----------+-----------------+---------------------------+
 |           | f_globals       | global namespace seen by  |
 |           |                 | this frame                |
 +-----------+-----------------+---------------------------+

Doc/library/sys.rst

@@ -136,8 +136,8 @@ always available.
    frame is not handling an exception, the information is taken from the calling
    stack frame, or its caller, and so on until a stack frame is found that is
    handling an exception.  Here, "handling an exception" is defined as "executing
-   or having executed an except clause."  For any stack frame, only information
-   about the most recently handled exception is accessible.
+   an except clause."  For any stack frame, only information about the exception
+   being currently handled is accessible.
 
    .. index:: object: traceback
 

Doc/reference/datamodel.rst

@@ -875,19 +875,14 @@ Internal types
 
       .. index::
          single: f_trace (frame attribute)
-         single: f_exc_type (frame attribute)
-         single: f_exc_value (frame attribute)
-         single: f_exc_traceback (frame attribute)
          single: f_lineno (frame attribute)
 
       Special writable attributes: :attr:`f_trace`, if not ``None``, is a function
       called at the start of each source code line (this is used by the debugger);
-      :attr:`f_exc_type`, :attr:`f_exc_value`, :attr:`f_exc_traceback` represent the
-      last exception raised in the parent frame provided another exception was ever
-      raised in the current frame (in all other cases they are None); :attr:`f_lineno`
-      is the current line number of the frame --- writing to this from within a trace
-      function jumps to the given line (only for the bottom-most frame).  A debugger
-      can implement a Jump command (aka Set Next Statement) by writing to f_lineno.
+      :attr:`f_lineno` is the current line number of the frame --- writing to this
+      from within a trace function jumps to the given line (only for the bottom-most
+      frame).  A debugger can implement a Jump command (aka Set Next Statement)
+      by writing to f_lineno.
 
    Traceback objects
       .. index::

Include/frameobject.h

@@ -27,13 +27,13 @@ typedef struct _frame {
     PyObject **f_stacktop;
     PyObject *f_trace;		/* Trace function */
 
-    /* If an exception is raised in this frame, the next three are used to
-     * record the exception info (if any) originally in the thread state.  See
-     * comments before set_exc_info() -- it's not obvious.
-     * Invariant:  if _type is NULL, then so are _value and _traceback.
-     * Desired invariant:  all three are NULL, or all three are non-NULL.  That
-     * one isn't currently true, but "should be".
-     */
+	/* In a generator, we need to be able to swap between the exception
+	   state inside the generator and the exception state of the calling
+	   frame (which shouldn't be impacted when the generator "yields"
+	   from an except handler).
+	   These three fields exist exactly for that, and are unused for
+	   non-generator frames. See the SAVE_EXC_STATE and SWAP_EXC_STATE
+	   macros in ceval.c for details of their use. */
     PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
 
     PyThreadState *f_tstate;

Include/opcode.h

@@ -70,6 +70,7 @@ extern "C" {
 #define YIELD_VALUE	86
 #define POP_BLOCK	87
 #define END_FINALLY	88
+#define POP_EXCEPT	89
 
 #define HAVE_ARGUMENT	90	/* Opcodes from here have an argument: */
 
@@ -133,6 +134,13 @@ extern "C" {
 #define EXTENDED_ARG  143
 
 
+/* EXCEPT_HANDLER is a special, implicit block type which is created when
+   entering an except handler. It is not an opcode but we define it here
+   as we want it to be available to both frameobject.c and ceval.c, while
+   remaining private.*/
+#define EXCEPT_HANDLER 257
+
+
 enum cmp_op {PyCmp_LT=Py_LT, PyCmp_LE=Py_LE, PyCmp_EQ=Py_EQ, PyCmp_NE=Py_NE, PyCmp_GT=Py_GT, PyCmp_GE=Py_GE,
 	     PyCmp_IN, PyCmp_NOT_IN, PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD};
 

Lib/doctest.py

@@ -1242,10 +1242,9 @@ class DocTestRunner:
 
             # The example raised an exception:  check if it was expected.
             else:
-                exc_info = sys.exc_info()
-                exc_msg = traceback.format_exception_only(*exc_info[:2])[-1]
+                exc_msg = traceback.format_exception_only(*exception[:2])[-1]
                 if not quiet:
-                    got += _exception_traceback(exc_info)
+                    got += _exception_traceback(exception)
 
                 # If `example.exc_msg` is None, then we weren't expecting
                 # an exception.
@@ -1275,7 +1274,7 @@ class DocTestRunner:
             elif outcome is BOOM:
                 if not quiet:
                     self.report_unexpected_exception(out, test, example,
-                                                     exc_info)
+                                                     exception)
                 failures += 1
             else:
                 assert False, ("unknown outcome", outcome)

Lib/inspect.py

@@ -197,9 +197,6 @@ def isframe(object):
         f_back          next outer frame object (this frame's caller)
         f_builtins      built-in namespace seen by this frame
         f_code          code object being executed in this frame
-        f_exc_traceback traceback if raised in this frame, or None
-        f_exc_type      exception type if raised in this frame, or None
-        f_exc_value     exception value if raised in this frame, or None
         f_globals       global namespace seen by this frame
         f_lasti         index of last attempted instruction in bytecode
         f_lineno        current line number in Python source code

Lib/opcode.py

@@ -105,6 +105,7 @@ def_op('MAKE_BYTES', 85)
 def_op('YIELD_VALUE', 86)
 def_op('POP_BLOCK', 87)
 def_op('END_FINALLY', 88)
+def_op('POP_EXCEPT', 89)
 
 HAVE_ARGUMENT = 90              # Opcodes from here have an argument:
 

Lib/test/test_exceptions.py

@@ -427,6 +427,7 @@ class ExceptionTests(unittest.TestCase):
             local_ref = obj
             raise MyException(obj)
 
+        # Qualified "except" with "as"
         obj = MyObj()
         wr = weakref.ref(obj)
         try:
@@ -437,6 +438,113 @@ class ExceptionTests(unittest.TestCase):
         obj = wr()
         self.failUnless(obj is None, "%s" % obj)
 
+        # Qualified "except" without "as"
+        obj = MyObj()
+        wr = weakref.ref(obj)
+        try:
+            inner_raising_func()
+        except MyException:
+            pass
+        obj = None
+        obj = wr()
+        self.failUnless(obj is None, "%s" % obj)
+
+        # Bare "except"
+        obj = MyObj()
+        wr = weakref.ref(obj)
+        try:
+            inner_raising_func()
+        except:
+            pass
+        obj = None
+        obj = wr()
+        self.failUnless(obj is None, "%s" % obj)
+
+        # "except" with premature block leave
+        obj = MyObj()
+        wr = weakref.ref(obj)
+        for i in [0]:
+            try:
+                inner_raising_func()
+            except:
+                break
+        obj = None
+        obj = wr()
+        self.failUnless(obj is None, "%s" % obj)
+
+        # "except" block raising another exception
+        obj = MyObj()
+        wr = weakref.ref(obj)
+        try:
+            try:
+                inner_raising_func()
+            except:
+                raise KeyError
+        except KeyError:
+            obj = None
+            obj = wr()
+            self.failUnless(obj is None, "%s" % obj)
+
+        # Some complicated construct
+        obj = MyObj()
+        wr = weakref.ref(obj)
+        try:
+            inner_raising_func()
+        except MyException:
+            try:
+                try:
+                    raise
+                finally:
+                    raise
+            except MyException:
+                pass
+        obj = None
+        obj = wr()
+        self.failUnless(obj is None, "%s" % obj)
+
+        # Inside an exception-silencing "with" block
+        class Context:
+            def __enter__(self):
+                return self
+            def __exit__ (self, exc_type, exc_value, exc_tb):
+                return True
+        obj = MyObj()
+        wr = weakref.ref(obj)
+        with Context():
+            inner_raising_func()
+        obj = None
+        obj = wr()
+        self.failUnless(obj is None, "%s" % obj)
+
+    def test_generator_leaking(self):
+        # Test that generator exception state doesn't leak into the calling
+        # frame
+        def yield_raise():
+            try:
+                raise KeyError("caught")
+            except KeyError:
+                yield sys.exc_info()[0]
+                yield sys.exc_info()[0]
+            yield sys.exc_info()[0]
+        g = yield_raise()
+        self.assertEquals(next(g), KeyError)
+        self.assertEquals(sys.exc_info()[0], None)
+        self.assertEquals(next(g), KeyError)
+        self.assertEquals(sys.exc_info()[0], None)
+        self.assertEquals(next(g), None)
+
+        # Same test, but inside an exception handler
+        try:
+            raise TypeError("foo")
+        except TypeError:
+            g = yield_raise()
+            self.assertEquals(next(g), KeyError)
+            self.assertEquals(sys.exc_info()[0], TypeError)
+            self.assertEquals(next(g), KeyError)
+            self.assertEquals(sys.exc_info()[0], TypeError)
+            self.assertEquals(next(g), TypeError)
+            del g
+            self.assertEquals(sys.exc_info()[0], TypeError)
 
 def test_main():
     run_unittest(ExceptionTests)

Lib/test/test_raise.py

@@ -16,6 +16,13 @@ def get_tb():
         return sys.exc_info()[2]
 
 
+class Context:
+    def __enter__(self):
+        return self
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        return True
+
+
 class TestRaise(unittest.TestCase):
     def test_invalid_reraise(self):
         try:
@@ -37,6 +44,71 @@ class TestRaise(unittest.TestCase):
         else:
             self.fail("No exception raised")
 
+    def test_except_reraise(self):
+        def reraise():
+            try:
+                raise TypeError("foo")
+            except:
+                try:
+                    raise KeyError("caught")
+                except KeyError:
+                    pass
+                raise
+        self.assertRaises(TypeError, reraise)
+
+    def test_finally_reraise(self):
+        def reraise():
+            try:
+                raise TypeError("foo")
+            except:
+                try:
+                    raise KeyError("caught")
+                finally:
+                    raise
+        self.assertRaises(KeyError, reraise)
+
+    def test_nested_reraise(self):
+        def nested_reraise():
+            raise
+        def reraise():
+            try:
+                raise TypeError("foo")
+            except:
+                nested_reraise()
+        self.assertRaises(TypeError, reraise)
+
+    def test_with_reraise1(self):
+        def reraise():
+            try:
+                raise TypeError("foo")
+            except:
+                with Context():
+                    pass
+                raise
+        self.assertRaises(TypeError, reraise)
+
+    def test_with_reraise2(self):
+        def reraise():
+            try:
+                raise TypeError("foo")
+            except:
+                with Context():
+                    raise KeyError("caught")
+                raise
+        self.assertRaises(TypeError, reraise)
+
+    def test_yield_reraise(self):
+        def reraise():
+            try:
+                raise TypeError("foo")
+            except:
+                yield 1
+                raise
+        g = reraise()
+        next(g)
+        self.assertRaises(TypeError, lambda: next(g))
+        self.assertRaises(StopIteration, lambda: next(g))
+
     def test_erroneous_exception(self):
         class MyException(Exception):
             def __init__(self):
@@ -158,6 +230,5 @@ class TestRemovedFunctionality(unittest.TestCase):
 def test_main():
     support.run_unittest(__name__)
 
-
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS

@@ -49,6 +49,10 @@ Core and Builtins
   Exception (KeyboardInterrupt, and SystemExit) propagate instead of
   ignoring them.
 
+- #3021 Exception reraising sematics have been significantly improved.  However,
+  f_exc_type, f_exc_value, and f_exc_traceback cannot be accessed from Python
+  code anymore.
+
 Extension Modules
 -----------------
 

Misc/cheatsheet

@@ -1262,9 +1262,6 @@ Special informative state attributes for some types:
         f_lineno (int, R/O): current line number
         f_lasti (int, R/O): precise instruction (index into bytecode)
         f_trace (function/None, R/W): debug hook called at start of each source line
-        f_exc_type (Type/None, R/W): Most recent exception type
-        f_exc_value (any, R/W): Most recent exception value
-        f_exc_traceback (traceback/None, R/W): Most recent exception traceback
     Tracebacks:
         tb_next (frame/None, R/O): next level in stack trace (toward the frame where
                                   the exception occurred)

Objects/frameobject.c

@@ -20,9 +20,6 @@ static PyMemberDef frame_memberlist[] = {
 	{"f_builtins",	T_OBJECT,	OFF(f_builtins),READONLY},
 	{"f_globals",	T_OBJECT,	OFF(f_globals),	READONLY},
 	{"f_lasti",	T_INT,		OFF(f_lasti),	READONLY},
-	{"f_exc_type",	T_OBJECT,	OFF(f_exc_type)},
-	{"f_exc_value",	T_OBJECT,	OFF(f_exc_value)},
-	{"f_exc_traceback", T_OBJECT,	OFF(f_exc_traceback)},
 	{NULL}	/* Sentinel */
 };
 

Python/compile.c

@@ -760,6 +760,8 @@ opcode_stack_effect(int opcode, int oparg)
 
 		case POP_BLOCK:
 			return 0;
+		case POP_EXCEPT:
+			return 0;  /* -3 except if bad bytecode */
 		case END_FINALLY:
 			return -1; /* or -2 or -3 if exception occurred */
 
@@ -818,7 +820,8 @@ opcode_stack_effect(int opcode, int oparg)
 			return 0;
 		case SETUP_EXCEPT:
 		case SETUP_FINALLY:
-			return 3; /* actually pushed by an exception */
+			return 6; /* can push 3 values for the new exception
+				+ 3 others for the previous exception state */
 
 		case LOAD_FAST:
 			return 1;
@@ -2031,6 +2034,7 @@ compiler_try_except(struct compiler *c, stmt_ty s)
             /* second # body */
 		VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body);
 	        ADDOP(c, POP_BLOCK);
+	        ADDOP(c, POP_EXCEPT);
 	        compiler_pop_fblock(c, FINALLY_TRY, cleanup_body);
 
             /* finally: */
@@ -2050,9 +2054,20 @@ compiler_try_except(struct compiler *c, stmt_ty s)
 	        compiler_pop_fblock(c, FINALLY_END, cleanup_end);
 		}
 		else {
+            basicblock *cleanup_body;
+
+            cleanup_body = compiler_new_block(c);
+            if(!cleanup_body)
+                return 0;
+
+			ADDOP(c, POP_TOP);
             ADDOP(c, POP_TOP);
-            ADDOP(c, POP_TOP);
+	        compiler_use_next_block(c, cleanup_body);
+	        if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body))
+		        return 0;
 		    VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body);
+	        ADDOP(c, POP_EXCEPT);
+	        compiler_pop_fblock(c, FINALLY_TRY, cleanup_body);
 		}
 		ADDOP_JREL(c, JUMP_FORWARD, end);
 		compiler_use_next_block(c, except);
@@ -3109,7 +3124,7 @@ compiler_with(struct compiler *c, stmt_ty s)
 {
     static identifier enter_attr, exit_attr;
     basicblock *block, *finally;
-    identifier tmpvalue = NULL;
+    identifier tmpvalue = NULL, tmpexit = NULL;
 
     assert(s->kind == With_kind);
 
@@ -3144,6 +3159,10 @@ compiler_with(struct compiler *c, stmt_ty s)
 	    return 0;
 	PyArena_AddPyObject(c->c_arena, tmpvalue);
     }
+	tmpexit = compiler_new_tmpname(c);
+	if (tmpexit == NULL)
+	    return 0;
+	PyArena_AddPyObject(c->c_arena, tmpexit);
 
     /* Evaluate EXPR */
     VISIT(c, expr, s->v.With.context_expr);
@@ -3151,7 +3170,8 @@ compiler_with(struct compiler *c, stmt_ty s)
     /* Squirrel away context.__exit__ by stuffing it under context */
     ADDOP(c, DUP_TOP);
     ADDOP_O(c, LOAD_ATTR, exit_attr, names);
-    ADDOP(c, ROT_TWO);
+	if (!compiler_nameop(c, tmpexit, Store))
+	    return 0;
 
     /* Call context.__enter__() */
     ADDOP_O(c, LOAD_ATTR, enter_attr, names);
@@ -3198,6 +3218,9 @@ compiler_with(struct compiler *c, stmt_ty s)
     /* Finally block starts; context.__exit__ is on the stack under
        the exception or return information. Just issue our magic
        opcode. */
+	if (!compiler_nameop(c, tmpexit, Load) ||
+		!compiler_nameop(c, tmpexit, Del))
+		return 0;
     ADDOP(c, WITH_CLEANUP);
 
     /* Finally block ends. */

Python/import.c

@@ -86,8 +86,9 @@ extern time_t PyOS_GetLastModificationTime(char *, FILE *);
 		      3100 (merge from 2.6a0, see 62151)
 		      3102 (__file__ points to source file)
        Python 3.0a4: 3110 (WITH_CLEANUP optimization).
+       Python 3.0a5: 3130 (lexical exception stacking, including POP_EXCEPT)
 */
-#define MAGIC (3110 | ((long)'\r'<<16) | ((long)'\n'<<24))
+#define MAGIC (3130 | ((long)'\r'<<16) | ((long)'\n'<<24))
 
 /* Magic word as global; note that _PyImport_Init() can change the
    value of this global to accommodate for alterations of how the