fix integer overflow in unicode case operations (closes #22643)

Lib/test/test_unicode.py

@@ -661,6 +661,11 @@ class UnicodeTest(string_tests.CommonTest,
         self.assertEqual('x'.center(4, '\U0010FFFF'),
                          '\U0010FFFFx\U0010FFFF\U0010FFFF')
 
+    @unittest.skipUnless(sys.maxsize == 2**31 - 1, "requires 32-bit system")
+    def test_case_operation_overflow(self):
+        # Issue #22643
+        self.assertRaises(OverflowError, ("ü"*(2**32//12 + 1)).upper)
+
     def test_contains(self):
         # Testing Unicode contains method
         self.assertIn('a', 'abdb')

Misc/NEWS

@@ -10,6 +10,9 @@ What's New in Python 3.3.6?
 Core and Builtins
 -----------------
 
+- Issue #22643: Fix integer overflow in Unicode case operations (upper, lower,
+  title, swapcase, casefold).
+
 - Issue #22518: Fixed integer overflow issues in "backslashreplace",
   "xmlcharrefreplace", and "surrogatepass" error handlers.
 

Objects/unicodeobject.c

@@ -9484,6 +9484,11 @@ case_operation(PyObject *self,
     kind = PyUnicode_KIND(self);
     data = PyUnicode_DATA(self);
     length = PyUnicode_GET_LENGTH(self);
+    if (length > PY_SSIZE_T_MAX / 3 ||
+        length > PY_SIZE_MAX / (3 * sizeof(Py_UCS4))) {
+        PyErr_SetString(PyExc_OverflowError, "string is too long");
+        return NULL;
+    }
     tmp = PyMem_MALLOC(sizeof(Py_UCS4) * 3 * length);
     if (tmp == NULL)
         return PyErr_NoMemory();