Part of SF patch #1313939: Speedup charmap decoding by extending

PyUnicode_DecodeCharmap() the accept a unicode string as the mapping
argument which is used as a mapping table.

This code isn't used by any of the codecs yet.

Doc/api/concrete.tex

@@ -1322,7 +1322,12 @@ points.
                                                const char *errors}
   Create a Unicode object by decoding \var{size} bytes of the encoded
   string \var{s} using the given \var{mapping} object.  Return
-  \NULL{} if an exception was raised by the codec.
+  \NULL{} if an exception was raised by the codec. If \var{mapping} is \NULL{}
+  latin-1 decoding will be done. Else it can be a dictionary mapping byte or a
+  unicode string, which is treated as a lookup table. Byte values greater
+  that the length of the string and U+FFFE "characters" are treated as
+  "undefined mapping".
+  \versionchanged[Allowed unicode string as mapping argument]{2.4}
 \end{cfuncdesc}
 
 \begin{cfuncdesc}{PyObject*}{PyUnicode_EncodeCharmap}{const Py_UNICODE *s,

Lib/test/test_codecs.py

@@ -924,6 +924,40 @@ class BasicStrTest(unittest.TestCase):
             (chars, size) = codecs.getdecoder(encoding)(bytes)
             self.assertEqual(chars, s, "%r != %r (encoding=%r)" % (chars, s, encoding))
 
+class CharmapTest(unittest.TestCase):
+    def test_decode_with_string_map(self):
+        self.assertEquals(
+            codecs.charmap_decode("\x00\x01\x02", "strict", u"abc"),
+            (u"abc", 3)
+        )
+
+        self.assertEquals(
+            codecs.charmap_decode("\x00\x01\x02", "replace", u"ab"),
+            (u"ab\ufffd", 3)
+        )
+
+        self.assertEquals(
+            codecs.charmap_decode("\x00\x01\x02", "replace", u"ab\ufffe"),
+            (u"ab\ufffd", 3)
+        )
+
+        self.assertEquals(
+            codecs.charmap_decode("\x00\x01\x02", "ignore", u"ab"),
+            (u"ab", 3)
+        )
+
+        self.assertEquals(
+            codecs.charmap_decode("\x00\x01\x02", "ignore", u"ab\ufffe"),
+            (u"ab", 3)
+        )
+
+        allbytes = "".join(chr(i) for i in xrange(256))
+        self.assertEquals(
+            codecs.charmap_decode(allbytes, "ignore", u""),
+            (u"", len(allbytes))
+        )
+
+
 def test_main():
     test_support.run_unittest(
         UTF16Test,
@@ -940,7 +974,8 @@ def test_main():
         StreamReaderTest,
         Str2StrTest,
         BasicUnicodeTest,
-        BasicStrTest
+        BasicStrTest,
+        CharmapTest
     )
 
 

Misc/NEWS

@@ -563,6 +563,11 @@ C API
 
 - Removed PyRange_New().
 
+- Patch #1313939: PyUnicode_DecodeCharmap() accepts a unicode string as the
+  mapping argument now. This string is used as a mapping table. Byte values
+  greater than the length of the string and 0xFFFE are treated as undefined
+  mappings.
+
 
 Tests
 -----

Objects/unicodeobject.c

@@ -2833,6 +2833,8 @@ PyObject *PyUnicode_DecodeCharmap(const char *s,
     int extrachars = 0;
     PyObject *errorHandler = NULL;
     PyObject *exc = NULL;
+    Py_UNICODE *mapstring = NULL;
+    int maplen = 0;
 
     /* Default to Latin-1 */
     if (mapping == NULL)
@@ -2845,6 +2847,35 @@ PyObject *PyUnicode_DecodeCharmap(const char *s,
 	return (PyObject *)v;
     p = PyUnicode_AS_UNICODE(v);
     e = s + size;
+    if (PyUnicode_CheckExact(mapping)) {
+	mapstring = PyUnicode_AS_UNICODE(mapping);
+	maplen = PyUnicode_GET_SIZE(mapping);
+	while (s < e) {
+	    unsigned char ch = *s;
+	    Py_UNICODE x = 0xfffe; /* illegal value */
+
+	    if (ch < maplen)
+		x = mapstring[ch];
+
+	    if (x == 0xfffe) {
+		/* undefined mapping */
+		outpos = p-PyUnicode_AS_UNICODE(v);
+		startinpos = s-starts;
+		endinpos = startinpos+1;
+		if (unicode_decode_call_errorhandler(
+		     errors, &errorHandler,
+		     "charmap", "character maps to <undefined>",
+		     starts, size, &startinpos, &endinpos, &exc, &s,
+		     (PyObject **)&v, &outpos, &p)) {
+		    goto onError;
+		}
+		continue;
+	    }
+	    *p++ = x;
+	    ++s;
+	}
+    }
+    else {
 	while (s < e) {
 	    unsigned char ch = *s;
 	    PyObject *w, *x;
@@ -2931,6 +2962,7 @@ PyObject *PyUnicode_DecodeCharmap(const char *s,
 	    Py_DECREF(x);
 	    ++s;
 	}
+    }
     if (p - PyUnicode_AS_UNICODE(v) < PyUnicode_GET_SIZE(v))
 	if (_PyUnicode_Resize(&v, (int)(p - PyUnicode_AS_UNICODE(v))) < 0)
 	    goto onError;