fix overflow detection of strop.expandtabs

Lib/test/test_strop.py

@@ -4,6 +4,7 @@ warnings.filterwarnings("ignore", "strop functions are obsolete;",
                         r'test.test_strop|unittest')
 import strop
 import unittest
+import sys
 from test import test_support
 
 
@@ -115,6 +116,11 @@ class StropFunctionTestCase(unittest.TestCase):
         strop.uppercase
         strop.whitespace
 
+    @unittest.skipUnless(sys.maxsize == 2147483647, "only for 32-bit")
+    def test_expandtabs_overflow(self):
+        s = '\t\n' * 0x10000 + 'A' * 0x1000000
+        self.assertRaises(OverflowError, strop.expandtabs, s, 0x10001)
+
     @test_support.precisionbigmemtest(size=test_support._2G - 1, memuse=5)
     def test_stropjoin_huge_list(self, size):
         a = "A" * size

Misc/NEWS

@@ -40,6 +40,9 @@ Core and Builtins
 Library
 -------
 
+- Fix possible overflow bug in strop.expandtabs. You shouldn't be using this
+  module!
+
 - Issue #20145: `assertRaisesRegex` now raises a TypeError if the second
   argument is not a string or compiled regex.
 

Modules/stropmodule.c

@@ -593,7 +593,7 @@ strop_expandtabs(PyObject *self, PyObject *args)
     char* e;
     char* p;
     char* q;
-    Py_ssize_t i, j, old_j;
+    Py_ssize_t i, j;
     PyObject* out;
     char* string;
     Py_ssize_t stringlen;
@@ -610,30 +610,29 @@ strop_expandtabs(PyObject *self, PyObject *args)
     }
 
     /* First pass: determine size of output string */
-    i = j = old_j = 0; /* j: current column; i: total of previous lines */
+    i = j = 0; /* j: current column; i: total of previous lines */
     e = string + stringlen;
     for (p = string; p < e; p++) {
         if (*p == '\t') {
-            j += tabsize - (j%tabsize);
-            if (old_j > j) {
-                PyErr_SetString(PyExc_OverflowError,
-                                "new string is too long");
-                return NULL;
-            }
-            old_j = j;
+            Py_ssize_t incr = tabsize - (j%tabsize);
+            if (j > PY_SSIZE_T_MAX - incr)
+                goto overflow;
+            j += incr;
         } else {
+            if (j > PY_SSIZE_T_MAX - 1)
+                goto overflow;
             j++;
             if (*p == '\n') {
+                if (i > PY_SSIZE_T_MAX - j)
+                    goto overflow;
                 i += j;
                 j = 0;
             }
         }
     }
 
-    if ((i + j) < 0) {
-        PyErr_SetString(PyExc_OverflowError, "new string is too long");
-        return NULL;
-    }
+    if (i > PY_SSIZE_T_MAX - j)
+        goto overflow;
 
     /* Second pass: create output string and fill it */
     out = PyString_FromStringAndSize(NULL, i+j);
@@ -658,6 +657,9 @@ strop_expandtabs(PyObject *self, PyObject *args)
     }
 
     return out;
+  overflow:
+    PyErr_SetString(PyExc_OverflowError, "result is too long");
+    return NULL;
 }