Issue #22335: Fix crash when trying to enlarge a bytearray to 0x7fffffff bytes...

Issue #22335: Fix crash when trying to enlarge a bytearray to 0x7fffffff bytes on a 32-bit platform.

Lib/test/test_bytes.py

@@ -13,9 +13,11 @@ import functools
 import pickle
 import tempfile
 import unittest
+
 import test.support
 import test.string_tests
 import test.buffer_tests
+from test.support import bigaddrspacetest, MAX_Py_ssize_t
 
 
 if sys.flags.bytes_warning:
@@ -111,6 +113,17 @@ class BaseBytesTest:
         self.assertRaises(ValueError, self.type2test, [sys.maxsize+1])
         self.assertRaises(ValueError, self.type2test, [10**100])
 
+    @bigaddrspacetest
+    def test_constructor_overflow(self):
+        size = MAX_Py_ssize_t
+        self.assertRaises((OverflowError, MemoryError), self.type2test, size)
+        try:
+            # Should either pass or raise an error (e.g. on debug builds with
+            # additional malloc() overhead), but shouldn't crash.
+            bytearray(size - 4)
+        except (OverflowError, MemoryError):
+            pass
+
     def test_compare(self):
         b1 = self.type2test([1, 2, 3])
         b2 = self.type2test([1, 2, 3])

Misc/NEWS

@@ -11,6 +11,9 @@ Release date: TBA
 Core and Builtins
 -----------------
 
+- Issue #22335: Fix crash when trying to enlarge a bytearray to 0x7fffffff
+  bytes on a 32-bit platform.
+
 - Issue #22653: Fix an assertion failure in debug mode when doing a reentrant
   dict insertion in debug mode.
 

Objects/bytearrayobject.c

@@ -175,20 +175,22 @@ PyByteArray_AsString(PyObject *self)
 }
 
 int
-PyByteArray_Resize(PyObject *self, Py_ssize_t size)
+PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size)
 {
     void *sval;
     PyByteArrayObject *obj = ((PyByteArrayObject *)self);
-    Py_ssize_t alloc = obj->ob_alloc;
-    Py_ssize_t logical_offset = obj->ob_start - obj->ob_bytes;
+    /* All computations are done unsigned to avoid integer overflows
+       (see issue #22335). */
+    size_t alloc = (size_t) obj->ob_alloc;
+    size_t logical_offset = (size_t) (obj->ob_start - obj->ob_bytes);
+    size_t size = (size_t) requested_size;
 
     assert(self != NULL);
     assert(PyByteArray_Check(self));
-    assert(size >= 0);
-    assert(logical_offset >= 0);
     assert(logical_offset <= alloc);
+    assert(requested_size >= 0);
 
-    if (size == Py_SIZE(self)) {
+    if (requested_size == Py_SIZE(self)) {
         return 0;
     }
     if (!_canresize(obj)) {
@@ -220,6 +222,10 @@ PyByteArray_Resize(PyObject *self, Py_ssize_t size)
             alloc = size + 1;
         }
     }
+    if (alloc > PY_SSIZE_T_MAX) {
+        PyErr_NoMemory();
+        return -1;
+    }
 
     if (logical_offset > 0) {
         sval = PyObject_Malloc(alloc);
@@ -227,7 +233,8 @@ PyByteArray_Resize(PyObject *self, Py_ssize_t size)
             PyErr_NoMemory();
             return -1;
         }
-        memcpy(sval, PyByteArray_AS_STRING(self), Py_MIN(size, Py_SIZE(self)));
+        memcpy(sval, PyByteArray_AS_STRING(self),
+               Py_MIN(requested_size, Py_SIZE(self)));
         PyObject_Free(obj->ob_bytes);
     }
     else {

Objects/obmalloc.c

@@ -1754,8 +1754,8 @@ _PyMem_DebugMalloc(void *ctx, size_t nbytes)
 
     bumpserialno();
     total = nbytes + 4*SST;
-    if (total < nbytes)
-        /* overflow:  can't represent total as a size_t */
+    if (nbytes > PY_SSIZE_T_MAX - 4*SST)
+        /* overflow:  can't represent total as a Py_ssize_t */
         return NULL;
 
     p = (uchar *)api->alloc.malloc(api->alloc.ctx, total);
@@ -1817,8 +1817,8 @@ _PyMem_DebugRealloc(void *ctx, void *p, size_t nbytes)
     bumpserialno();
     original_nbytes = read_size_t(q - 2*SST);
     total = nbytes + 4*SST;
-    if (total < nbytes)
-        /* overflow:  can't represent total as a size_t */
+    if (nbytes > PY_SSIZE_T_MAX - 4*SST)
+        /* overflow:  can't represent total as a Py_ssize_t */
         return NULL;
 
     /* Resize and add decorations. We may get a new pointer here, in which