Issue #16096: Fix several occurrences of potential signed integer overflow. ...

Issue #16096: Fix several occurrences of potential signed integer overflow.  Thanks Serhiy Storchaka.

Modules/_codecsmodule.c

@@ -177,12 +177,12 @@ escape_encode(PyObject *self,
         return NULL;
 
     size = PyBytes_GET_SIZE(str);
-    newsize = 4*size;
-    if (newsize > PY_SSIZE_T_MAX || newsize / 4 != size) {
+    if (size > PY_SSIZE_T_MAX / 4) {
         PyErr_SetString(PyExc_OverflowError,
             "string is too large to encode");
             return NULL;
     }
+    newsize = 4*size;
     v = PyBytes_FromStringAndSize(NULL, newsize);
 
     if (v == NULL) {

Modules/_datetimemodule.c

@@ -1265,14 +1265,13 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple,
         assert(ptoappend != NULL);
         assert(ntoappend > 0);
         while (usednew + ntoappend > totalnew) {
-            size_t bigger = totalnew << 1;
-            if ((bigger >> 1) != totalnew) { /* overflow */
+            if (totalnew > (PY_SSIZE_T_MAX >> 1)) { /* overflow */
                 PyErr_NoMemory();
                 goto Done;
             }
-            if (_PyBytes_Resize(&newfmt, bigger) < 0)
+            totalnew <<= 1;
+            if (_PyBytes_Resize(&newfmt, totalnew) < 0)
                 goto Done;
-            totalnew = bigger;
             pnew = PyBytes_AsString(newfmt) + usednew;
         }
         memcpy(pnew, ptoappend, ntoappend);

Modules/_randommodule.c

@@ -284,7 +284,8 @@ random_seed(RandomObject *self, PyObject *args)
         n = newn;
         if (keyused >= keymax) {
             unsigned long bigger = keymax << 1;
-            if ((bigger >> 1) != keymax) {
+            if ((bigger >> 1) != keymax ||
+                bigger > PY_SSIZE_T_MAX / sizeof(*key)) {
                 PyErr_NoMemory();
                 goto Done;
             }

Modules/arraymodule.c

@@ -483,11 +483,11 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, struct arraydescr *descr)
         return NULL;
     }
 
-    nbytes = size * descr->itemsize;
     /* Check for overflow */
-    if (nbytes / descr->itemsize != (size_t)size) {
+    if (size > PY_SSIZE_T_MAX / descr->itemsize) {
         return PyErr_NoMemory();
     }
+    nbytes = size * descr->itemsize;
     op = (arrayobject *) type->tp_alloc(type, 0);
     if (op == NULL) {
         return NULL;
@@ -1251,11 +1251,15 @@ array_fromfile(arrayobject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "On:fromfile", &f, &n))
         return NULL;
 
-    nbytes = n * itemsize;
-    if (nbytes < 0 || nbytes/itemsize != n) {
+    if (n < 0) {
+        PyErr_SetString(PyExc_ValueError, "negative count");
+        return NULL;
+    }
+    if (n > PY_SSIZE_T_MAX / itemsize) {
         PyErr_NoMemory();
         return NULL;
     }
+    nbytes = n * itemsize;
 
     b = _PyObject_CallMethodId(f, &PyId_read, "n", nbytes);
     if (b == NULL)

Modules/audioop.c

@@ -1108,8 +1108,7 @@ audioop_ratecv(PyObject *self, PyObject *args)
         PyErr_SetString(AudioopError, "# of channels should be >= 1");
         return NULL;
     }
-    bytes_per_frame = size * nchannels;
-    if (bytes_per_frame / nchannels != size) {
+    if (size > INT_MAX / nchannels) {
         /* This overflow test is rigorously correct because
            both multiplicands are >= 1.  Use the argument names
            from the docs for the error msg. */
@@ -1117,6 +1116,7 @@ audioop_ratecv(PyObject *self, PyObject *args)
                         "width * nchannels too big for a C int");
         return NULL;
     }
+    bytes_per_frame = size * nchannels;
     if (weightA < 1 || weightB < 0) {
         PyErr_SetString(AudioopError,
             "weightA should be >= 1, weightB should be >= 0");

Objects/tupleobject.c

@@ -96,15 +96,11 @@ PyTuple_New(register Py_ssize_t size)
     else
 #endif
     {
-        Py_ssize_t nbytes = size * sizeof(PyObject *);
         /* Check for overflow */
-        if (nbytes / sizeof(PyObject *) != (size_t)size ||
-            (nbytes > PY_SSIZE_T_MAX - sizeof(PyTupleObject) - sizeof(PyObject *)))
-        {
+        if (size > (PY_SSIZE_T_MAX - sizeof(PyTupleObject) -
+                    sizeof(PyObject *)) / sizeof(PyObject *)) {
             return PyErr_NoMemory();
         }
-        /* nbytes += sizeof(PyTupleObject) - sizeof(PyObject *); */
-
         op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
         if (op == NULL)
             return NULL;
@@ -481,9 +477,9 @@ tuplerepeat(PyTupleObject *a, Py_ssize_t n)
         if (Py_SIZE(a) == 0)
             return PyTuple_New(0);
     }
-    size = Py_SIZE(a) * n;
-    if (size/Py_SIZE(a) != n)
+    if (n > PY_SSIZE_T_MAX / Py_SIZE(a))
         return PyErr_NoMemory();
+    size = Py_SIZE(a) * n;
     np = (PyTupleObject *) PyTuple_New(size);
     if (np == NULL)
         return NULL;

Objects/unicodeobject.c

@@ -4492,7 +4492,6 @@ _PyUnicode_EncodeUTF7(PyObject *str,
     void *data;
     Py_ssize_t len;
     PyObject *v;
-    Py_ssize_t allocated;
     int inShift = 0;
     Py_ssize_t i;
     unsigned int base64bits = 0;
@@ -4510,11 +4509,9 @@ _PyUnicode_EncodeUTF7(PyObject *str,
         return PyBytes_FromStringAndSize(NULL, 0);
 
     /* It might be possible to tighten this worst case */
-    allocated = 8 * len;
-    if (allocated / 8 != len)
+    if (len > PY_SSIZE_T_MAX / 8)
         return PyErr_NoMemory();
-
-    v = PyBytes_FromStringAndSize(NULL, allocated);
+    v = PyBytes_FromStringAndSize(NULL, len * 8);
     if (v == NULL)
         return NULL;
 
@@ -5092,7 +5089,7 @@ _PyUnicode_EncodeUTF32(PyObject *str,
     Py_ssize_t len;
     PyObject *v;
     unsigned char *p;
-    Py_ssize_t nsize, bytesize, i;
+    Py_ssize_t nsize, i;
     /* Offsets from p for storing byte pairs in the right order. */
 #ifdef BYTEORDER_IS_LITTLE_ENDIAN
     int iorder[] = {0, 1, 2, 3};
@@ -5120,10 +5117,9 @@ _PyUnicode_EncodeUTF32(PyObject *str,
     len = PyUnicode_GET_LENGTH(str);
 
     nsize = len + (byteorder == 0);
-    bytesize = nsize * 4;
-    if (bytesize / 4 != nsize)
+    if (nsize > PY_SSIZE_T_MAX / 4)
         return PyErr_NoMemory();
-    v = PyBytes_FromStringAndSize(NULL, bytesize);
+    v = PyBytes_FromStringAndSize(NULL, nsize * 4);
     if (v == NULL)
         return NULL;
 
@@ -10159,7 +10155,7 @@ replace(PyObject *self, PyObject *str1,
     }
     else {
         Py_ssize_t n, i, j, ires;
-        Py_ssize_t product, new_size;
+        Py_ssize_t new_size;
         int rkind = skind;
         char *res;
 
@@ -10191,19 +10187,18 @@ replace(PyObject *self, PyObject *str1,
         }
         /* new_size = PyUnicode_GET_LENGTH(self) + n * (PyUnicode_GET_LENGTH(str2) -
            PyUnicode_GET_LENGTH(str1))); */
-        product = n * (len2-len1);
-        if ((product / (len2-len1)) != n) {
+        if (len2 > len1 && len2 - len1 > (PY_SSIZE_T_MAX - slen) / n) {
                 PyErr_SetString(PyExc_OverflowError,
                                 "replace string is too long");
                 goto error;
         }
-        new_size = slen + product;
+        new_size = slen + n * (len2 - len1);
         if (new_size == 0) {
             Py_INCREF(unicode_empty);
             u = unicode_empty;
             goto done;
         }
-        if (new_size < 0 || new_size > (PY_SSIZE_T_MAX >> (rkind-1))) {
+        if (new_size > (PY_SSIZE_T_MAX >> (rkind-1))) {
             PyErr_SetString(PyExc_OverflowError,
                             "replace string is too long");
             goto error;