bpo-30579: Allow TracebackType creation and tb_next mutation from Python (GH-4793)

Third party projects may wish to hide their own internal machinery in
order to present more comprehensible tracebacks to end users
(e.g. Jinja2 and Trio both do this).

Previously such projects have had to rely on ctypes to do so:

  https://github.com/pallets/jinja/blob/fe3dadacdf4cf411d0a5b6bbd4d5234697a28af2/jinja2/debug.py#L345
  https://github.com/python-trio/trio/blob/1e86b1aee8c0c759f6f239ae53a05d0d3963c629/trio/_core/_multierror.py#L296

This provides a Python level API for creating and modifying real
Traceback objects, allowing tracebacks to be edited at runtime.

Patch by Nathaniel Smith.

Lib/test/test_raise.py

@@ -228,6 +228,72 @@ class TestTraceback(unittest.TestCase):
             self.fail("No exception raised")
 
 
+class TestTracebackType(unittest.TestCase):
+
+    def raiser(self):
+        raise ValueError
+
+    def test_attrs(self):
+        try:
+            self.raiser()
+        except Exception as exc:
+            tb = exc.__traceback__
+
+        self.assertIsInstance(tb.tb_next, types.TracebackType)
+        self.assertIs(tb.tb_frame, sys._getframe())
+        self.assertIsInstance(tb.tb_lasti, int)
+        self.assertIsInstance(tb.tb_lineno, int)
+
+        self.assertIs(tb.tb_next.tb_next, None)
+
+        # Invalid assignments
+        with self.assertRaises(TypeError):
+            del tb.tb_next
+
+        with self.assertRaises(TypeError):
+            tb.tb_next = "asdf"
+
+        # Loops
+        with self.assertRaises(ValueError):
+            tb.tb_next = tb
+
+        with self.assertRaises(ValueError):
+            tb.tb_next.tb_next = tb
+
+        # Valid assignments
+        tb.tb_next = None
+        self.assertIs(tb.tb_next, None)
+
+        new_tb = get_tb()
+        tb.tb_next = new_tb
+        self.assertIs(tb.tb_next, new_tb)
+
+    def test_constructor(self):
+        other_tb = get_tb()
+        frame = sys._getframe()
+
+        tb = types.TracebackType(other_tb, frame, 1, 2)
+        self.assertEqual(tb.tb_next, other_tb)
+        self.assertEqual(tb.tb_frame, frame)
+        self.assertEqual(tb.tb_lasti, 1)
+        self.assertEqual(tb.tb_lineno, 2)
+
+        tb = types.TracebackType(None, frame, 1, 2)
+        self.assertEqual(tb.tb_next, None)
+
+        with self.assertRaises(TypeError):
+            types.TracebackType("no", frame, 1, 2)
+
+        with self.assertRaises(TypeError):
+            types.TracebackType(other_tb, "no", 1, 2)
+
+        with self.assertRaises(TypeError):
+            types.TracebackType(other_tb, frame, "no", 2)
+
+        with self.assertRaises(TypeError):
+            types.TracebackType(other_tb, frame, 1, "nuh-uh")
+
+
 class TestContext(unittest.TestCase):
     def test_instance_context_instance_raise(self):
         context = IndexError()

Misc/NEWS.d/next/Core and Builtins/2017-12-11-01-52-42.bpo-30579.X6cEzf.rst

+Implement TracebackType.__new__ to allow Python-level creation of
+traceback objects, and make TracebackType.tb_next mutable.

Python/clinic/traceback.c.h

+/*[clinic input]
+preserve
+[clinic start generated code]*/
+
+PyDoc_STRVAR(tb_new__doc__,
+"TracebackType(tb_next, tb_frame, tb_lasti, tb_lineno)\n"
+"--\n"
+"\n"
+"Create a new traceback object.");
+
+static PyObject *
+tb_new_impl(PyTypeObject *type, PyObject *tb_next, PyFrameObject *tb_frame,
+            int tb_lasti, int tb_lineno);
+
+static PyObject *
+tb_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
+{
+    PyObject *return_value = NULL;
+    static const char * const _keywords[] = {"tb_next", "tb_frame", "tb_lasti", "tb_lineno", NULL};
+    static _PyArg_Parser _parser = {"OO!ii:TracebackType", _keywords, 0};
+    PyObject *tb_next;
+    PyFrameObject *tb_frame;
+    int tb_lasti;
+    int tb_lineno;
+
+    if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser,
+        &tb_next, &PyFrame_Type, &tb_frame, &tb_lasti, &tb_lineno)) {
+        goto exit;
+    }
+    return_value = tb_new_impl(type, tb_next, tb_frame, tb_lasti, tb_lineno);
+
+exit:
+    return return_value;
+}
+/*[clinic end generated code: output=0133130d7d19556f input=a9049054013a1b77]*/

Python/traceback.c

@@ -27,6 +27,65 @@ _Py_IDENTIFIER(close);
 _Py_IDENTIFIER(open);
 _Py_IDENTIFIER(path);
 
+/*[clinic input]
+class TracebackType "PyTracebackObject *" "&PyTraceback_Type"
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=928fa06c10151120]*/
+
+#include "clinic/traceback.c.h"
+
+static PyObject *
+tb_create_raw(PyTracebackObject *next, PyFrameObject *frame, int lasti,
+              int lineno)
+{
+    PyTracebackObject *tb;
+    if ((next != NULL && !PyTraceBack_Check(next)) ||
+                    frame == NULL || !PyFrame_Check(frame)) {
+        PyErr_BadInternalCall();
+        return NULL;
+    }
+    tb = PyObject_GC_New(PyTracebackObject, &PyTraceBack_Type);
+    if (tb != NULL) {
+        Py_XINCREF(next);
+        tb->tb_next = next;
+        Py_XINCREF(frame);
+        tb->tb_frame = frame;
+        tb->tb_lasti = lasti;
+        tb->tb_lineno = lineno;
+        PyObject_GC_Track(tb);
+    }
+    return (PyObject *)tb;
+}
+
+/*[clinic input]
+@classmethod
+TracebackType.__new__ as tb_new
+
+  tb_next: object
+  tb_frame: object(type='PyFrameObject *', subclass_of='&PyFrame_Type')
+  tb_lasti: int
+  tb_lineno: int
+
+Create a new traceback object.
+[clinic start generated code]*/
+
+static PyObject *
+tb_new_impl(PyTypeObject *type, PyObject *tb_next, PyFrameObject *tb_frame,
+            int tb_lasti, int tb_lineno)
+/*[clinic end generated code: output=fa077debd72d861a input=01cbe8ec8783fca7]*/
+{
+    if (tb_next == Py_None) {
+        tb_next = NULL;
+    } else if (!PyTraceBack_Check(tb_next)) {
+        return PyErr_Format(PyExc_TypeError,
+                            "expected traceback object or None, got '%s'",
+                            Py_TYPE(tb_next)->tp_name);
+    }
+
+    return tb_create_raw((PyTracebackObject *)tb_next, tb_frame, tb_lasti,
+                         tb_lineno);
+}
+
 static PyObject *
 tb_dir(PyTracebackObject *self)
 {
@@ -34,19 +93,72 @@ tb_dir(PyTracebackObject *self)
                                    "tb_lasti", "tb_lineno");
 }
 
+static PyObject *
+tb_next_get(PyTracebackObject *self, void *Py_UNUSED(_))
+{
+    PyObject* ret = (PyObject*)self->tb_next;
+    if (!ret) {
+        ret = Py_None;
+    }
+    Py_INCREF(ret);
+    return ret;
+}
+
+static int
+tb_next_set(PyTracebackObject *self, PyObject *new_next, void *Py_UNUSED(_))
+{
+    if (!new_next) {
+        PyErr_Format(PyExc_TypeError, "can't delete tb_next attribute");
+        return -1;
+    }
+
+    /* We accept None or a traceback object, and map None -> NULL (inverse of
+       tb_next_get) */
+    if (new_next == Py_None) {
+        new_next = NULL;
+    } else if (!PyTraceBack_Check(new_next)) {
+        PyErr_Format(PyExc_TypeError,
+                     "expected traceback object, got '%s'",
+                     Py_TYPE(new_next)->tp_name);
+        return -1;
+    }
+
+    /* Check for loops */
+    PyTracebackObject *cursor = (PyTracebackObject *)new_next;
+    while (cursor) {
+        if (cursor == self) {
+            PyErr_Format(PyExc_ValueError, "traceback loop detected");
+            return -1;
+        }
+        cursor = cursor->tb_next;
+    }
+
+    PyObject *old_next = (PyObject*)self->tb_next;
+    Py_XINCREF(new_next);
+    self->tb_next = (PyTracebackObject *)new_next;
+    Py_XDECREF(old_next);
+
+    return 0;
+}
+
+
 static PyMethodDef tb_methods[] = {
    {"__dir__", (PyCFunction)tb_dir, METH_NOARGS},
    {NULL, NULL, 0, NULL},
 };
 
 static PyMemberDef tb_memberlist[] = {
-    {"tb_next",         T_OBJECT,       OFF(tb_next),   READONLY},
     {"tb_frame",        T_OBJECT,       OFF(tb_frame),  READONLY},
     {"tb_lasti",        T_INT,          OFF(tb_lasti),  READONLY},
     {"tb_lineno",       T_INT,          OFF(tb_lineno), READONLY},
     {NULL}      /* Sentinel */
 };
 
+static PyGetSetDef tb_getsetters[] = {
+    {"tb_next", (getter)tb_next_get, (setter)tb_next_set, NULL, NULL},
+    {NULL}      /* Sentinel */
+};
+
 static void
 tb_dealloc(PyTracebackObject *tb)
 {
@@ -94,7 +206,7 @@ PyTypeObject PyTraceBack_Type = {
     0,                  /* tp_setattro */
     0,                                          /* tp_as_buffer */
     Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */
-    0,                                          /* tp_doc */
+    tb_new__doc__,                              /* tp_doc */
     (traverseproc)tb_traverse,                  /* tp_traverse */
     (inquiry)tb_clear,                          /* tp_clear */
     0,                                          /* tp_richcompare */
@@ -103,39 +215,24 @@ PyTypeObject PyTraceBack_Type = {
     0,                                          /* tp_iternext */
     tb_methods,         /* tp_methods */
     tb_memberlist,      /* tp_members */
-    0,                                          /* tp_getset */
+    tb_getsetters,                              /* tp_getset */
     0,                                          /* tp_base */
     0,                                          /* tp_dict */
+    0,                                          /* tp_descr_get */
+    0,                                          /* tp_descr_set */
+    0,                                          /* tp_dictoffset */
+    0,                                          /* tp_init */
+    0,                                          /* tp_alloc */
+    tb_new,                                     /* tp_new */
 };
 
-static PyTracebackObject *
-newtracebackobject(PyTracebackObject *next, PyFrameObject *frame)
-{
-    PyTracebackObject *tb;
-    if ((next != NULL && !PyTraceBack_Check(next)) ||
-                    frame == NULL || !PyFrame_Check(frame)) {
-        PyErr_BadInternalCall();
-        return NULL;
-    }
-    tb = PyObject_GC_New(PyTracebackObject, &PyTraceBack_Type);
-    if (tb != NULL) {
-        Py_XINCREF(next);
-        tb->tb_next = next;
-        Py_XINCREF(frame);
-        tb->tb_frame = frame;
-        tb->tb_lasti = frame->f_lasti;
-        tb->tb_lineno = PyFrame_GetLineNumber(frame);
-        PyObject_GC_Track(tb);
-    }
-    return tb;
-}
-
 int
 PyTraceBack_Here(PyFrameObject *frame)
 {
     PyObject *exc, *val, *tb, *newtb;
     PyErr_Fetch(&exc, &val, &tb);
-    newtb = (PyObject *)newtracebackobject((PyTracebackObject *)tb, frame);
+    newtb = tb_create_raw((PyTracebackObject *)tb, frame, frame->f_lasti,
+                          PyFrame_GetLineNumber(frame));
     if (newtb == NULL) {
         _PyErr_ChainExceptions(exc, val, tb);
         return -1;