Issue #15379: Fix passing of non-BMP characters as integers for the charmap...

Issue #15379: Fix passing of non-BMP characters as integers for the charmap decoder (already working as unicode strings).
Patch by Serhiy Storchaka.

Lib/test/test_codeccallbacks.py

@@ -717,7 +717,7 @@ class CodecCallbackTest(unittest.TestCase):
                 raise ValueError
         self.assertRaises(UnicodeError, codecs.charmap_decode, "\xff", "strict", {0xff: None})
         self.assertRaises(ValueError, codecs.charmap_decode, "\xff", "strict", D())
-        self.assertRaises(TypeError, codecs.charmap_decode, "\xff", "strict", {0xff: sys.maxunicode+1})
+        self.assertRaises(TypeError, codecs.charmap_decode, "\xff", "strict", {0xff: 0x110000})
 
     def test_encodehelper(self):
         # enhance coverage of:

Lib/test/test_codecs.py

@@ -1534,6 +1534,10 @@ class CharmapTest(unittest.TestCase):
             (u"ab", 3)
         )
 
+        self.assertRaises(UnicodeDecodeError,
+            codecs.charmap_decode, b"\x00\x01\x02", "strict", u"ab"
+        )
+
         self.assertEqual(
             codecs.charmap_decode("\x00\x01\x02", "ignore", u"ab\ufffe"),
             (u"ab", 3)
@@ -1545,6 +1549,107 @@ class CharmapTest(unittest.TestCase):
             (u"", len(allbytes))
         )
 
+    def test_decode_with_int2str_map(self):
+        self.assertEqual(
+            codecs.charmap_decode("\x00\x01\x02", "strict",
+                                  {0: u'a', 1: u'b', 2: u'c'}),
+            (u"abc", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode("\x00\x01\x02", "strict",
+                                  {0: u'Aa', 1: u'Bb', 2: u'Cc'}),
+            (u"AaBbCc", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode("\x00\x01\x02", "strict",
+                                  {0: u'\U0010FFFF', 1: u'b', 2: u'c'}),
+            (u"\U0010FFFFbc", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode("\x00\x01\x02", "strict",
+                                  {0: u'a', 1: u'b', 2: u''}),
+            (u"ab", 3)
+        )
+
+        self.assertRaises(UnicodeDecodeError,
+            codecs.charmap_decode, "\x00\x01\x02", "strict",
+                                   {0: u'a', 1: u'b'}
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode("\x00\x01\x02", "replace",
+                                  {0: u'a', 1: u'b'}),
+            (u"ab\ufffd", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode("\x00\x01\x02", "replace",
+                                  {0: u'a', 1: u'b', 2: None}),
+            (u"ab\ufffd", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode("\x00\x01\x02", "ignore",
+                                  {0: u'a', 1: u'b'}),
+            (u"ab", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode("\x00\x01\x02", "ignore",
+                                  {0: u'a', 1: u'b', 2: None}),
+            (u"ab", 3)
+        )
+
+        allbytes = bytes(range(256))
+        self.assertEqual(
+            codecs.charmap_decode(allbytes, "ignore", {}),
+            (u"", len(allbytes))
+        )
+
+    def test_decode_with_int2int_map(self):
+        a = ord(u'a')
+        b = ord(u'b')
+        c = ord(u'c')
+
+        self.assertEqual(
+            codecs.charmap_decode("\x00\x01\x02", "strict",
+                                  {0: a, 1: b, 2: c}),
+            (u"abc", 3)
+        )
+
+        # Issue #15379
+        self.assertEqual(
+            codecs.charmap_decode("\x00\x01\x02", "strict",
+                                  {0: 0x10FFFF, 1: b, 2: c}),
+            (u"\U0010FFFFbc", 3)
+        )
+
+        self.assertRaises(TypeError,
+            codecs.charmap_decode, "\x00\x01\x02", "strict",
+                                   {0: 0x110000, 1: b, 2: c}
+        )
+
+        self.assertRaises(UnicodeDecodeError,
+            codecs.charmap_decode, "\x00\x01\x02", "strict",
+                                   {0: a, 1: b},
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode("\x00\x01\x02", "replace",
+                                  {0: a, 1: b}),
+            (u"ab\ufffd", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode("\x00\x01\x02", "ignore",
+                                  {0: a, 1: b}),
+            (u"ab", 3)
+        )
+
+
 class WithStmtTest(unittest.TestCase):
     def test_encodedfile(self):
         f = StringIO.StringIO("\xc3\xbc")

Misc/NEWS

@@ -9,6 +9,9 @@ What's New in Python 2.7.4
 Core and Builtins
 -----------------
 
+- Issue #15379: Fix passing of non-BMP characters as integers for the charmap
+  decoder (already working as unicode strings).  Patch by Serhiy Storchaka.
+
 - Issue #16453: Fix equality testing of dead weakref objects.
 
 - Issue #9535: Fix pending signals that have been received but not yet

Objects/unicodeobject.c

@@ -4127,12 +4127,36 @@ PyObject *PyUnicode_DecodeCharmap(const char *s,
             /* Apply mapping */
             if (PyInt_Check(x)) {
                 long value = PyInt_AS_LONG(x);
-                if (value < 0 || value > 65535) {
+                if (value < 0 || value > 0x10FFFF) {
                     PyErr_SetString(PyExc_TypeError,
-                                    "character mapping must be in range(65536)");
+                                    "character mapping must be in range(0x110000)");
                     Py_DECREF(x);
                     goto onError;
                 }
+
+#ifndef Py_UNICODE_WIDE
+                if (value > 0xFFFF) {
+                    /* see the code for 1-n mapping below */
+                    if (extrachars < 2) {
+                        /* resize first */
+                        Py_ssize_t oldpos = p - PyUnicode_AS_UNICODE(v);
+                        Py_ssize_t needed = 10 - extrachars;
+                        extrachars += needed;
+                        /* XXX overflow detection missing */
+                        if (_PyUnicode_Resize(&v,
+                                              PyUnicode_GET_SIZE(v) + needed) < 0) {
+                            Py_DECREF(x);
+                            goto onError;
+                        }
+                        p = PyUnicode_AS_UNICODE(v) + oldpos;
+                    }
+                    value -= 0x10000;
+                    *p++ = 0xD800 | (value >> 10);
+                    *p++ = 0xDC00 | (value & 0x3FF);
+                    extrachars -= 2;
+                }
+                else
+#endif
                 *p++ = (Py_UNICODE)value;
             }
             else if (x == Py_None) {