add some overflow checks before multiplying (closes #23165)

Misc/NEWS

@@ -10,6 +10,9 @@ What's New in Python 3.2.6?
 Core and Builtins
 -----------------
 
+- Issue #23165: Perform overflow checks before allocating memory in the
+  _Py_char2wchar function.
+
 - Issue #19529: Fix a potential crash in converting Unicode objects to wchar_t
   when Py_UNICODE is 4 bytes but wchar_t is 2 bytes, for example on AIX.
 

Python/fileutils.c

@@ -169,8 +169,11 @@ decode_ascii_surrogateescape(const char *arg, size_t *size)
     wchar_t *res;
     unsigned char *in;
     wchar_t *out;
+    size_t argsize = strlen(arg) + 1;
 
-    res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t));
+    if (argsize > PY_SSIZE_T_MAX/sizeof(wchar_t))
+        return NULL;
+    res = PyMem_Malloc(argsize*sizeof(wchar_t));
     if (!res)
         return NULL;
 
@@ -250,10 +253,15 @@ _Py_char2wchar(const char* arg, size_t *size)
     argsize = mbstowcs(NULL, arg, 0);
 #endif
     if (argsize != (size_t)-1) {
-        res = (wchar_t *)PyMem_Malloc((argsize+1)*sizeof(wchar_t));
+        if (argsize == PY_SSIZE_T_MAX)
+            goto oom;
+        argsize += 1;
+        if (argsize > PY_SSIZE_T_MAX/sizeof(wchar_t))
+            goto oom;
+        res = (wchar_t *)PyMem_Malloc(argsize*sizeof(wchar_t));
         if (!res)
             goto oom;
-        count = mbstowcs(res, arg, argsize+1);
+        count = mbstowcs(res, arg, argsize);
         if (count != (size_t)-1) {
             wchar_t *tmp;
             /* Only use the result if it contains no
@@ -276,6 +284,8 @@ _Py_char2wchar(const char* arg, size_t *size)
     /* Overallocate; as multi-byte characters are in the argument, the
        actual output could use less memory. */
     argsize = strlen(arg) + 1;
+    if (argsize > PY_SSIZE_T_MAX/sizeof(wchar_t))
+        goto oom;
     res = (wchar_t*)PyMem_Malloc(argsize*sizeof(wchar_t));
     if (!res)
         goto oom;