Merged revisions 81908 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/branches/py3k

................
  r81908 | antoine.pitrou | 2010-06-11 23:46:32 +0200 (ven., 11 juin 2010) | 11 lines

  Merged revisions 81907 via svnmerge from
  svn+ssh://pythondev@svn.python.org/python/trunk

  ........
    r81907 | antoine.pitrou | 2010-06-11 23:42:26 +0200 (ven., 11 juin 2010) | 5 lines

    Issue #8941: decoding big endian UTF-32 data in UCS-2 builds could crash
    the interpreter with characters outside the Basic Multilingual Plane
    (higher than 0x10000).
  ........
................

Lib/test/test_codecs.py

@@ -354,6 +354,16 @@ class UTF32Test(ReadTest):
         self.check_state_handling_decode(self.encoding,
                                          "spamspam", self.spambe)
 
+    def test_issue8941(self):
+        # Issue #8941: insufficient result allocation when decoding into
+        # surrogate pairs on UCS-2 builds.
+        encoded_le = b'\xff\xfe\x00\x00' + b'\x00\x00\x01\x00' * 1024
+        self.assertEqual('\U00010000' * 1024,
+                         codecs.utf_32_decode(encoded_le)[0])
+        encoded_be = b'\x00\x00\xfe\xff' + b'\x00\x01\x00\x00' * 1024
+        self.assertEqual('\U00010000' * 1024,
+                         codecs.utf_32_decode(encoded_be)[0])
+
 class UTF32LETest(ReadTest):
     encoding = "utf-32-le"
 
@@ -387,6 +397,13 @@ class UTF32LETest(ReadTest):
         self.assertRaises(UnicodeDecodeError, codecs.utf_32_le_decode,
                           b"\xff", "strict", True)
 
+    def test_issue8941(self):
+        # Issue #8941: insufficient result allocation when decoding into
+        # surrogate pairs on UCS-2 builds.
+        encoded = b'\x00\x00\x01\x00' * 1024
+        self.assertEqual('\U00010000' * 1024,
+                         codecs.utf_32_le_decode(encoded)[0])
+
 class UTF32BETest(ReadTest):
     encoding = "utf-32-be"
 
@@ -420,6 +437,14 @@ class UTF32BETest(ReadTest):
         self.assertRaises(UnicodeDecodeError, codecs.utf_32_be_decode,
                           b"\xff", "strict", True)
 
+    def test_issue8941(self):
+        # Issue #8941: insufficient result allocation when decoding into
+        # surrogate pairs on UCS-2 builds.
+        encoded = b'\x00\x01\x00\x00' * 1024
+        self.assertEqual('\U00010000' * 1024,
+                         codecs.utf_32_be_decode(encoded)[0])
+
+
 class UTF16Test(ReadTest):
     encoding = "utf-16"
 

Misc/NEWS

@@ -12,6 +12,10 @@ What's New in Python 3.1.3?
 Core and Builtins
 -----------------
 
+- Issue #8941: decoding big endian UTF-32 data in UCS-2 builds could crash
+  the interpreter with characters outside the Basic Multilingual Plane
+  (higher than 0x10000).
+
 - In the str.format(), raise a ValueError when indexes to arguments are too
   large.
 

Objects/unicodeobject.c

@@ -2618,11 +2618,11 @@ PyUnicode_DecodeUTF32Stateful(const char *s,
     PyUnicodeObject *unicode;
     Py_UNICODE *p;
 #ifndef Py_UNICODE_WIDE
-    int i, pairs;
+    int pairs = 0;
 #else
     const int pairs = 0;
 #endif
-    const unsigned char *q, *e;
+    const unsigned char *q, *e, *qq;
     int bo = 0;       /* assume native ordering by default */
     const char *errmsg = "";
     /* Offsets from q for retrieving bytes in the right order. */
@@ -2633,23 +2633,7 @@ PyUnicode_DecodeUTF32Stateful(const char *s,
 #endif
     PyObject *errorHandler = NULL;
     PyObject *exc = NULL;
-    /* On narrow builds we split characters outside the BMP into two
-       codepoints => count how much extra space we need. */
-#ifndef Py_UNICODE_WIDE
-    for (i = pairs = 0; i < size/4; i++)
-        if (((Py_UCS4 *)s)[i] >= 0x10000)
-            pairs++;
-#endif
-
-    /* This might be one to much, because of a BOM */
-    unicode = _PyUnicode_New((size+3)/4+pairs);
-    if (!unicode)
-        return NULL;
-    if (size == 0)
-        return (PyObject *)unicode;
-
-    /* Unpack UTF-32 encoded data */
-    p = unicode->str;
+    
     q = (unsigned char *)s;
     e = q + size;
 
@@ -2701,6 +2685,24 @@ PyUnicode_DecodeUTF32Stateful(const char *s,
         iorder[3] = 0;
     }
 
+    /* On narrow builds we split characters outside the BMP into two
+       codepoints => count how much extra space we need. */
+#ifndef Py_UNICODE_WIDE
+    for (qq = q; qq < e; qq += 4)
+        if (qq[iorder[2]] != 0 || qq[iorder[3]] != 0)
+            pairs++;
+#endif
+
+    /* This might be one to much, because of a BOM */
+    unicode = _PyUnicode_New((size+3)/4+pairs);
+    if (!unicode)
+        return NULL;
+    if (size == 0)
+        return (PyObject *)unicode;
+
+    /* Unpack UTF-32 encoded data */
+    p = unicode->str;
+
     while (q < e) {
         Py_UCS4 ch;
         /* remaining bytes at the end? (size should be divisible by 4) */