bpo-31949: Fixed several issues in printing tracebacks (PyTraceBack_Print()). (#4289)

* Setting sys.tracebacklimit to 0 or less now suppresses printing tracebacks.
* Setting sys.tracebacklimit to None now causes using the default limit.
* Setting sys.tracebacklimit to an integer larger than LONG_MAX now means using
  the limit LONG_MAX rather than the default limit.
* Fixed integer overflows in the case of more than 2**31 traceback items on
  Windows.
* Fixed output errors handling.

Lib/test/test_sys.py

@@ -808,6 +808,39 @@ class SysModuleTest(unittest.TestCase):
         self.assertIsInstance(level, int)
         self.assertGreater(level, 0)
 
+    def test_sys_tracebacklimit(self):
+        code = """if 1:
+            import sys
+            def f1():
+                1 / 0
+            def f2():
+                f1()
+            sys.tracebacklimit = %r
+            f2()
+        """
+        def check(tracebacklimit, expected):
+            p = subprocess.Popen([sys.executable, '-c', code % tracebacklimit],
+                                 stderr=subprocess.PIPE)
+            out = p.communicate()[1]
+            self.assertEqual(out.splitlines(), expected)
+
+        traceback = [
+            b'Traceback (most recent call last):',
+            b'  File "<string>", line 8, in <module>',
+            b'  File "<string>", line 6, in f2',
+            b'  File "<string>", line 4, in f1',
+            b'ZeroDivisionError: division by zero'
+        ]
+        check(10, traceback)
+        check(3, traceback)
+        check(2, traceback[:1] + traceback[2:])
+        check(1, traceback[:1] + traceback[3:])
+        check(0, [traceback[-1]])
+        check(-1, [traceback[-1]])
+        check(1<<1000, traceback)
+        check(-1<<1000, [traceback[-1]])
+        check(None, traceback)
+
 
 @test.support.cpython_only
 class SizeofTest(unittest.TestCase):

Misc/NEWS.d/next/Core and Builtins/2017-11-05-16-11-07.bpo-31949.2yNC_z.rst

+Fixed several issues in printing tracebacks (PyTraceBack_Print()).
+
+* Setting sys.tracebacklimit to 0 or less now suppresses printing tracebacks.
+* Setting sys.tracebacklimit to None now causes using the default limit.
+* Setting sys.tracebacklimit to an integer larger than LONG_MAX now means using
+  the limit LONG_MAX rather than the default limit.
+* Fixed integer overflows in the case of more than 2**31 traceback items on
+  Windows.
+* Fixed output errors handling.

Python/traceback.c

@@ -414,57 +414,68 @@ tb_displayline(PyObject *f, PyObject *filename, int lineno, PyObject *name)
     return err;
 }
 
+static int
+tb_print_line_repeated(PyObject *f, long cnt)
+{
+    int err;
+    PyObject *line = PyUnicode_FromFormat(
+            "  [Previous line repeated %ld more times]\n", cnt-3);
+    if (line == NULL) {
+        return -1;
+    }
+    err = PyFile_WriteObject(line, f, Py_PRINT_RAW);
+    Py_DECREF(line);
+    return err;
+}
+
 static int
 tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit)
 {
     int err = 0;
-    long depth = 0;
+    Py_ssize_t depth = 0;
     PyObject *last_file = NULL;
     int last_line = -1;
     PyObject *last_name = NULL;
     long cnt = 0;
-    PyObject *line;
     PyTracebackObject *tb1 = tb;
     while (tb1 != NULL) {
         depth++;
         tb1 = tb1->tb_next;
     }
+    while (tb != NULL && depth > limit) {
+        depth--;
+        tb = tb->tb_next;
+    }
     while (tb != NULL && err == 0) {
-        if (depth <= limit) {
         if (last_file != NULL &&
             tb->tb_frame->f_code->co_filename == last_file &&
             last_line != -1 && tb->tb_lineno == last_line &&
-                last_name != NULL &&
-                tb->tb_frame->f_code->co_name == last_name) {
+            last_name != NULL && tb->tb_frame->f_code->co_name == last_name)
+        {
             cnt++;
-                } else {
+        }
+        else {
             if (cnt > 3) {
-                        line = PyUnicode_FromFormat(
-                        "  [Previous line repeated %d more times]\n", cnt-3);
-                        err = PyFile_WriteObject(line, f, Py_PRINT_RAW);
-                        Py_DECREF(line);
+                err = tb_print_line_repeated(f, cnt);
             }
             last_file = tb->tb_frame->f_code->co_filename;
             last_line = tb->tb_lineno;
             last_name = tb->tb_frame->f_code->co_name;
             cnt = 0;
         }
-            if (cnt < 3)
+        if (err == 0 && cnt < 3) {
             err = tb_displayline(f,
                                  tb->tb_frame->f_code->co_filename,
                                  tb->tb_lineno,
                                  tb->tb_frame->f_code->co_name);
+            if (err == 0) {
+                err = PyErr_CheckSignals();
+            }
         }
-        depth--;
         tb = tb->tb_next;
-        if (err == 0)
-            err = PyErr_CheckSignals();
     }
-    if (cnt > 3) {
-        line = PyUnicode_FromFormat(
-        "  [Previous line repeated %d more times]\n", cnt-3);
-        err = PyFile_WriteObject(line, f, Py_PRINT_RAW);
-        Py_DECREF(line);
+    if (err == 0 && cnt > 3) {
+        err = tb_print_line_repeated(f, cnt);
     }
     return err;
 }
@@ -485,26 +496,15 @@ PyTraceBack_Print(PyObject *v, PyObject *f)
         return -1;
     }
     limitv = PySys_GetObject("tracebacklimit");
-    if (limitv) {
-        PyObject *exc_type, *exc_value, *exc_tb;
-
-        PyErr_Fetch(&exc_type, &exc_value, &exc_tb);
-        limit = PyLong_AsLong(limitv);
-        if (limit == -1 && PyErr_Occurred()) {
-            if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
-                limit = PyTraceBack_LIMIT;
-            }
-            else {
-                Py_XDECREF(exc_type);
-                Py_XDECREF(exc_value);
-                Py_XDECREF(exc_tb);
-                return 0;
-            }
+    if (limitv && PyLong_Check(limitv)) {
+        int overflow;
+        limit = PyLong_AsLongAndOverflow(limitv, &overflow);
+        if (overflow > 0) {
+            limit = LONG_MAX;
         }
         else if (limit <= 0) {
-            limit = PyTraceBack_LIMIT;
+            return 0;
         }
-        PyErr_Restore(exc_type, exc_value, exc_tb);
     }
     err = PyFile_WriteString("Traceback (most recent call last):\n", f);
     if (!err)