Merged revisions 81046 from the python2.6 branch:

Issue #8674: Fix incorrect and UB-inducing overflow checks in audioop
module.  Thanks Tomas Hoger for the patch.

Misc/ACKS

@@ -288,6 +288,7 @@ Joerg-Cyril Hoehle
 Gregor Hoffleit
 Chris Hoffman
 Albert Hofkamp
+Tomas Hoger
 Jonathan Hogg
 Gerrit Holl
 Rune Holm

Misc/NEWS

@@ -9,6 +9,12 @@ What's New in Python 2.5.6c1?
 
 *Release date: XX-XXX-2010*
 
+Library
+-------
+
+- Issue #8674: Fixed a number of incorrect or undefined-behaviour-inducing
+  overflow checks in the audioop module (CVE-2010-1634).
+
 
 What's New in Python 2.5.5?
 ===========================

Modules/audioop.c

@@ -824,7 +824,7 @@ static PyObject *
 audioop_tostereo(PyObject *self, PyObject *args)
 {
         signed char *cp, *ncp;
-        int len, new_len, size, val1, val2, val = 0;
+        int len, size, val1, val2, val = 0;
         double fac1, fac2, fval, maxval;
         PyObject *rv;
         int i;
@@ -841,14 +841,13 @@ audioop_tostereo(PyObject *self, PyObject *args)
                 return 0;
         }
     
-        new_len = len*2;
-        if (new_len < 0) {
+        if (len > INT_MAX/2) {
                 PyErr_SetString(PyExc_MemoryError,
                                 "not enough memory for output buffer");
                 return 0;
         }
 
-        rv = PyString_FromStringAndSize(NULL, new_len);
+        rv = PyString_FromStringAndSize(NULL, len*2);
         if ( rv == 0 )
                 return 0;
         ncp = (signed char *)PyString_AsString(rv);
@@ -1011,7 +1010,7 @@ audioop_lin2lin(PyObject *self, PyObject *args)
 {
         signed char *cp;
         unsigned char *ncp;
-        int len, new_len, size, size2, val = 0;
+        int len, size, size2, val = 0;
         PyObject *rv;
         int i, j;
 
@@ -1025,13 +1024,12 @@ audioop_lin2lin(PyObject *self, PyObject *args)
                 return 0;
         }
     
-        new_len = (len/size)*size2;
-        if (new_len < 0) {
+        if (len/size > INT_MAX/size2) {
                 PyErr_SetString(PyExc_MemoryError,
                                 "not enough memory for output buffer");
                 return 0;
         }
-        rv = PyString_FromStringAndSize(NULL, new_len);
+        rv = PyString_FromStringAndSize(NULL, (len/size)*size2);
         if ( rv == 0 )
                 return 0;
         ncp = (unsigned char *)PyString_AsString(rv);
@@ -1067,7 +1065,6 @@ audioop_ratecv(PyObject *self, PyObject *args)
         int chan, d, *prev_i, *cur_i, cur_o;
         PyObject *state, *samps, *str, *rv = NULL;
         int bytes_per_frame;
-        size_t alloc_size;
 
         weightA = 1;
         weightB = 0;
@@ -1110,14 +1107,13 @@ audioop_ratecv(PyObject *self, PyObject *args)
         inrate /= d;
         outrate /= d;
 
-        alloc_size = sizeof(int) * (unsigned)nchannels;
-        if (alloc_size < nchannels) {
+        if ((size_t)nchannels > PY_SIZE_MAX/sizeof(int)) {
                 PyErr_SetString(PyExc_MemoryError,
                                 "not enough memory for output buffer");
                 return 0;
         }
-        prev_i = (int *) malloc(alloc_size);
-        cur_i = (int *) malloc(alloc_size);
+        prev_i = (int *) malloc(nchannels * sizeof(int));
+        cur_i = (int *) malloc(nchannels * sizeof(int));
         if (prev_i == NULL || cur_i == NULL) {
                 (void) PyErr_NoMemory();
                 goto exit;
@@ -1291,7 +1287,7 @@ audioop_ulaw2lin(PyObject *self, PyObject *args)
         unsigned char *cp;
         unsigned char cval;
         signed char *ncp;
-        int len, new_len, size, val;
+        int len, size, val;
         PyObject *rv;
         int i;
 
@@ -1304,18 +1300,17 @@ audioop_ulaw2lin(PyObject *self, PyObject *args)
                 return 0;
         }
     
-        new_len = len*size;
-        if (new_len < 0) {
+        if (len > INT_MAX/size) {
                 PyErr_SetString(PyExc_MemoryError,
                                 "not enough memory for output buffer");
                 return 0;
         }
-        rv = PyString_FromStringAndSize(NULL, new_len);
+        rv = PyString_FromStringAndSize(NULL, len*size);
         if ( rv == 0 )
                 return 0;
         ncp = (signed char *)PyString_AsString(rv);
     
-        for ( i=0; i < new_len; i += size ) {
+        for ( i=0; i < len*size; i += size ) {
                 cval = *cp++;
                 val = st_ulaw2linear16(cval);
         
@@ -1365,7 +1360,7 @@ audioop_alaw2lin(PyObject *self, PyObject *args)
         unsigned char *cp;
         unsigned char cval;
         signed char *ncp;
-        int len, new_len, size, val;
+        int len, size, val;
         PyObject *rv;
         int i;
 
@@ -1378,18 +1373,17 @@ audioop_alaw2lin(PyObject *self, PyObject *args)
                 return 0;
         }
     
-        new_len = len*size;
-        if (new_len < 0) {
+        if (len > INT_MAX/size) {
                 PyErr_SetString(PyExc_MemoryError,
                                 "not enough memory for output buffer");
                 return 0;
         }
-        rv = PyString_FromStringAndSize(NULL, new_len);
+        rv = PyString_FromStringAndSize(NULL, len*size);
         if ( rv == 0 )
                 return 0;
         ncp = (signed char *)PyString_AsString(rv);
     
-        for ( i=0; i < new_len; i += size ) {
+        for ( i=0; i < len*size; i += size ) {
                 cval = *cp++;
                 val = st_alaw2linear16(cval);
         
@@ -1514,7 +1508,7 @@ audioop_adpcm2lin(PyObject *self, PyObject *args)
 {
         signed char *cp;
         signed char *ncp;
-        int len, new_len, size, valpred, step, delta, index, sign, vpdiff;
+        int len, size, valpred, step, delta, index, sign, vpdiff;
         PyObject *rv, *str, *state;
         int i, inputbuffer = 0, bufferstep;
 
@@ -1536,13 +1530,12 @@ audioop_adpcm2lin(PyObject *self, PyObject *args)
         } else if ( !PyArg_ParseTuple(state, "ii", &valpred, &index) )
                 return 0;
     
-        new_len = len*size*2;
-        if (new_len < 0) {
+        if (len > (INT_MAX/2)/size) {
                 PyErr_SetString(PyExc_MemoryError,
                                 "not enough memory for output buffer");
                 return 0;
         }
-        str = PyString_FromStringAndSize(NULL, new_len);
+        str = PyString_FromStringAndSize(NULL, len*size*2);
         if ( str == 0 )
                 return 0;
         ncp = (signed char *)PyString_AsString(str);
@@ -1550,7 +1543,7 @@ audioop_adpcm2lin(PyObject *self, PyObject *args)
         step = stepsizeTable[index];
         bufferstep = 0;
     
-        for ( i=0; i < new_len; i += size ) {
+        for ( i=0; i < len*size*2; i += size ) {
                 /* Step 1 - get the delta value and compute next index */
                 if ( bufferstep ) {
                         delta = inputbuffer & 0xf;