Merged revisions 81045 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r81045 | mark.dickinson | 2010-05-10 17:07:42 +0100 (Mon, 10 May 2010) | 3 lines

  Issue #8674: Fix incorrect and UB-inducing overflow checks in audioop
  module.  Thanks Tomas Hoger for the patch.
........

Misc/ACKS

@@ -315,6 +315,7 @@ Joerg-Cyril Hoehle
 Gregor Hoffleit
 Chris Hoffman
 Albert Hofkamp
+Tomas Hoger
 Jonathan Hogg
 Gerrit Holl
 Shane Holloway

Misc/NEWS

@@ -39,6 +39,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #8674: Fixed a number of incorrect or undefined-behaviour-inducing
+  overflow checks in the audioop module.
+
 - Issue #8571: Fix an internal error when compressing or decompressing a
   chunk larger than 1GB with the zlib module's compressor and decompressor
   objects.

Modules/audioop.c

@@ -829,7 +829,7 @@ static PyObject *
 audioop_tostereo(PyObject *self, PyObject *args)
 {
     signed char *cp, *ncp;
-    int len, new_len, size, val1, val2, val = 0;
+    int len, size, val1, val2, val = 0;
     double fac1, fac2, fval, maxval;
     PyObject *rv;
     int i;
@@ -846,14 +846,13 @@ audioop_tostereo(PyObject *self, PyObject *args)
         return 0;
     }
 
-    new_len = len*2;
-    if (new_len < 0) {
+    if (len > INT_MAX/2) {
         PyErr_SetString(PyExc_MemoryError,
                         "not enough memory for output buffer");
         return 0;
     }
 
-    rv = PyString_FromStringAndSize(NULL, new_len);
+    rv = PyString_FromStringAndSize(NULL, len*2);
     if ( rv == 0 )
         return 0;
     ncp = (signed char *)PyString_AsString(rv);
@@ -1016,7 +1015,7 @@ audioop_lin2lin(PyObject *self, PyObject *args)
 {
     signed char *cp;
     unsigned char *ncp;
-    int len, new_len, size, size2, val = 0;
+    int len, size, size2, val = 0;
     PyObject *rv;
     int i, j;
 
@@ -1030,13 +1029,12 @@ audioop_lin2lin(PyObject *self, PyObject *args)
         return 0;
     }
 
-    new_len = (len/size)*size2;
-    if (new_len < 0) {
+    if (len/size > INT_MAX/size2) {
         PyErr_SetString(PyExc_MemoryError,
                         "not enough memory for output buffer");
         return 0;
     }
-    rv = PyString_FromStringAndSize(NULL, new_len);
+    rv = PyString_FromStringAndSize(NULL, (len/size)*size2);
     if ( rv == 0 )
         return 0;
     ncp = (unsigned char *)PyString_AsString(rv);
@@ -1072,7 +1070,6 @@ audioop_ratecv(PyObject *self, PyObject *args)
     int chan, d, *prev_i, *cur_i, cur_o;
     PyObject *state, *samps, *str, *rv = NULL;
     int bytes_per_frame;
-    size_t alloc_size;
 
     weightA = 1;
     weightB = 0;
@@ -1115,14 +1112,13 @@ audioop_ratecv(PyObject *self, PyObject *args)
     inrate /= d;
     outrate /= d;
 
-    alloc_size = sizeof(int) * (unsigned)nchannels;
-    if (alloc_size < nchannels) {
+    if ((size_t)nchannels > PY_SIZE_MAX/sizeof(int)) {
         PyErr_SetString(PyExc_MemoryError,
                         "not enough memory for output buffer");
         return 0;
     }
-    prev_i = (int *) malloc(alloc_size);
-    cur_i = (int *) malloc(alloc_size);
+    prev_i = (int *) malloc(nchannels * sizeof(int));
+    cur_i = (int *) malloc(nchannels * sizeof(int));
     if (prev_i == NULL || cur_i == NULL) {
         (void) PyErr_NoMemory();
         goto exit;
@@ -1296,7 +1292,7 @@ audioop_ulaw2lin(PyObject *self, PyObject *args)
     unsigned char *cp;
     unsigned char cval;
     signed char *ncp;
-    int len, new_len, size, val;
+    int len, size, val;
     PyObject *rv;
     int i;
 
@@ -1309,18 +1305,17 @@ audioop_ulaw2lin(PyObject *self, PyObject *args)
         return 0;
     }
 
-    new_len = len*size;
-    if (new_len < 0) {
+    if (len > INT_MAX/size) {
         PyErr_SetString(PyExc_MemoryError,
                         "not enough memory for output buffer");
         return 0;
     }
-    rv = PyString_FromStringAndSize(NULL, new_len);
+    rv = PyString_FromStringAndSize(NULL, len*size);
     if ( rv == 0 )
         return 0;
     ncp = (signed char *)PyString_AsString(rv);
 
-    for ( i=0; i < new_len; i += size ) {
+    for ( i=0; i < len*size; i += size ) {
         cval = *cp++;
         val = st_ulaw2linear16(cval);
 
@@ -1370,7 +1365,7 @@ audioop_alaw2lin(PyObject *self, PyObject *args)
     unsigned char *cp;
     unsigned char cval;
     signed char *ncp;
-    int len, new_len, size, val;
+    int len, size, val;
     PyObject *rv;
     int i;
 
@@ -1383,18 +1378,17 @@ audioop_alaw2lin(PyObject *self, PyObject *args)
         return 0;
     }
 
-    new_len = len*size;
-    if (new_len < 0) {
+    if (len > INT_MAX/size) {
         PyErr_SetString(PyExc_MemoryError,
                         "not enough memory for output buffer");
         return 0;
     }
-    rv = PyString_FromStringAndSize(NULL, new_len);
+    rv = PyString_FromStringAndSize(NULL, len*size);
     if ( rv == 0 )
         return 0;
     ncp = (signed char *)PyString_AsString(rv);
 
-    for ( i=0; i < new_len; i += size ) {
+    for ( i=0; i < len*size; i += size ) {
         cval = *cp++;
         val = st_alaw2linear16(cval);
 
@@ -1519,7 +1513,7 @@ audioop_adpcm2lin(PyObject *self, PyObject *args)
 {
     signed char *cp;
     signed char *ncp;
-    int len, new_len, size, valpred, step, delta, index, sign, vpdiff;
+    int len, size, valpred, step, delta, index, sign, vpdiff;
     PyObject *rv, *str, *state;
     int i, inputbuffer = 0, bufferstep;
 
@@ -1541,13 +1535,12 @@ audioop_adpcm2lin(PyObject *self, PyObject *args)
     } else if ( !PyArg_ParseTuple(state, "ii", &valpred, &index) )
         return 0;
 
-    new_len = len*size*2;
-    if (new_len < 0) {
+    if (len > (INT_MAX/2)/size) {
         PyErr_SetString(PyExc_MemoryError,
                         "not enough memory for output buffer");
         return 0;
     }
-    str = PyString_FromStringAndSize(NULL, new_len);
+    str = PyString_FromStringAndSize(NULL, len*size*2);
     if ( str == 0 )
         return 0;
     ncp = (signed char *)PyString_AsString(str);
@@ -1555,7 +1548,7 @@ audioop_adpcm2lin(PyObject *self, PyObject *args)
     step = stepsizeTable[index];
     bufferstep = 0;
 
-    for ( i=0; i < new_len; i += size ) {
+    for ( i=0; i < len*size*2; i += size ) {
         /* Step 1 - get the delta value and compute next index */
         if ( bufferstep ) {
             delta = inputbuffer & 0xf;