Skip to content

C
cpython
Commit c8e58126 authored by Serhiy Storchaka's avatar Serhiy Storchaka
Browse files
Options

Issue #16979: Fix error handling bugs in the unicode-escape-decode decoder.

parent 8d15ca3c
Show whitespace changes
Inline Side-by-side
Showing with 116 additions and 53 deletions
Lib/test/test_codeccallbacks.py
View file @ c8e58126
......@@ -262,12 +262,12 @@ class CodecCallbackTest(unittest.TestCase):
self.assertEqual(
"\\u3042\u3xxx".decode("unicode-escape", "test.handler1"),
u"\u3042[<92><117><51><120>]xx"
u"\u3042[<92><117><51>]xxx"
)
self.assertEqual(
"\\u3042\u3xx".decode("unicode-escape", "test.handler1"),
u"\u3042[<92><117><51><120><120>]"
u"\u3042[<92><117><51>]xx"
)
self.assertEqual(
......
Lib/test/test_codecs.py
View file @ c8e58126
......@@ -4,6 +4,11 @@ import codecs
import locale
import sys, StringIO, _testcapi
def coding_checker(self, coder):
def check(input, expect):
self.assertEqual(coder(input), (expect, len(input)))
return check
class Queue(object):
"""
queue: write bytes at one end, read bytes from the other end
......@@ -1786,6 +1791,84 @@ class WithStmtTest(unittest.TestCase):
self.assertEqual(srw.read(), u"\xfc")
class UnicodeEscapeTest(unittest.TestCase):
def test_empty(self):
self.assertEqual(codecs.unicode_escape_encode(u""), ("", 0))
self.assertEqual(codecs.unicode_escape_decode(""), (u"", 0))
def test_raw_encode(self):
encode = codecs.unicode_escape_encode
for b in range(32, 127):
if b != ord('\\'):
self.assertEqual(encode(unichr(b)), (chr(b), 1))
def test_raw_decode(self):
decode = codecs.unicode_escape_decode
for b in range(256):
if b != ord('\\'):
self.assertEqual(decode(chr(b) + '0'), (unichr(b) + u'0', 2))
def test_escape_encode(self):
encode = codecs.unicode_escape_encode
check = coding_checker(self, encode)
check(u'\t', r'\t')
check(u'\n', r'\n')
check(u'\r', r'\r')
check(u'\\', r'\\')
for b in range(32):
if chr(b) not in '\t\n\r':
check(unichr(b), '\\x%02x' % b)
for b in range(127, 256):
check(unichr(b), '\\x%02x' % b)
check(u'\u20ac', r'\u20ac')
check(u'\U0001d120', r'\U0001d120')
def test_escape_decode(self):
decode = codecs.unicode_escape_decode
check = coding_checker(self, decode)
check("[\\\n]", u"[]")
check(r'[\"]', u'["]')
check(r"[\']", u"[']")
check(r"[\\]", ur"[\]")
check(r"[\a]", u"[\x07]")
check(r"[\b]", u"[\x08]")
check(r"[\t]", u"[\x09]")
check(r"[\n]", u"[\x0a]")
check(r"[\v]", u"[\x0b]")
check(r"[\f]", u"[\x0c]")
check(r"[\r]", u"[\x0d]")
check(r"[\7]", u"[\x07]")
check(r"[\8]", ur"[\8]")
check(r"[\78]", u"[\x078]")
check(r"[\41]", u"[!]")
check(r"[\418]", u"[!8]")
check(r"[\101]", u"[A]")
check(r"[\1010]", u"[A0]")
check(r"[\x41]", u"[A]")
check(r"[\x410]", u"[A0]")
check(r"\u20ac", u"\u20ac")
check(r"\U0001d120", u"\U0001d120")
for b in range(256):
if chr(b) not in '\n"\'\\abtnvfr01234567xuUN':
check('\\' + chr(b), u'\\' + unichr(b))
def test_decode_errors(self):
decode = codecs.unicode_escape_decode
for c, d in ('x', 2), ('u', 4), ('U', 4):
for i in range(d):
self.assertRaises(UnicodeDecodeError, decode,
"\\" + c + "0"*i)
self.assertRaises(UnicodeDecodeError, decode,
"[\\" + c + "0"*i + "]")
data = "[\\" + c + "0"*i + "]\\" + c + "0"*i
self.assertEqual(decode(data, "ignore"), (u"[]", len(data)))
self.assertEqual(decode(data, "replace"),
(u"[\ufffd]\ufffd", len(data)))
self.assertRaises(UnicodeDecodeError, decode, r"\U00110000")
self.assertEqual(decode(r"\U00110000", "ignore"), (u"", 10))
self.assertEqual(decode(r"\U00110000", "replace"), (u"\ufffd", 10))
class BomTest(unittest.TestCase):
def test_seek0(self):
data = u"1234567890"
......@@ -1871,6 +1954,7 @@ def test_main():
BasicStrTest,
CharmapTest,
WithStmtTest,
UnicodeEscapeTest,
BomTest,
)
......
Misc/NEWS
View file @ c8e58126
......@@ -200,6 +200,8 @@ Core and Builtins
Library
-------
- Issue #16979: Fix error handling bugs in the unicode-escape-decode decoder.
- Issue #17051: Fix a memory leak in os.path.isdir() on Windows. Patch by
Robert Xiao.
......
Objects/unicodeobject.c
View file @ c8e58126
......@@ -2738,7 +2738,6 @@ PyObject *PyUnicode_DecodeUnicodeEscape(const char *s,
Py_ssize_t startinpos;
Py_ssize_t endinpos;
Py_ssize_t outpos;
int i;
PyUnicodeObject *v;
Py_UNICODE *p;
const char *end;
......@@ -2824,29 +2823,19 @@ PyObject *PyUnicode_DecodeUnicodeEscape(const char *s,
message = "truncated \\UXXXXXXXX escape";
hexescape:
chr = 0;
outpos = p-PyUnicode_AS_UNICODE(v);
if (s+digits>end) {
endinpos = size;
if (unicode_decode_call_errorhandler(
errors, &errorHandler,
"unicodeescape", "end of string in escape sequence",
starts, size, &startinpos, &endinpos, &exc, &s,
&v, &outpos, &p))
goto onError;
goto nextByte;
if (end - s < digits) {
/* count only hex digits */
for (; s < end; ++s) {
c = (unsigned char)*s;
if (!Py_ISXDIGIT(c))
goto error;
}
for (i = 0; i < digits; ++i) {
c = (unsigned char) s[i];
if (!isxdigit(c)) {
endinpos = (s+i+1)-starts;
if (unicode_decode_call_errorhandler(
errors, &errorHandler,
"unicodeescape", message,
starts, size, &startinpos, &endinpos, &exc, &s,
&v, &outpos, &p))
goto onError;
goto nextByte;
goto error;
}
for (; digits--; ++s) {
c = (unsigned char)*s;
if (!Py_ISXDIGIT(c))
goto error;
chr = (chr<<4) & ~0xF;
if (c >= '0' && c <= '9')
chr += c - '0';
......@@ -2855,7 +2844,6 @@ PyObject *PyUnicode_DecodeUnicodeEscape(const char *s,
else
chr += 10 + c - 'A';
}
s += i;
if (chr == 0xffffffff && PyErr_Occurred())
/* _decoding_error will have already written into the
target buffer. */
......@@ -2876,14 +2864,8 @@ PyObject *PyUnicode_DecodeUnicodeEscape(const char *s,
*p++ = 0xDC00 + (Py_UNICODE) (chr & 0x03FF);
#endif
} else {
endinpos = s-starts;
outpos = p-PyUnicode_AS_UNICODE(v);
if (unicode_decode_call_errorhandler(
errors, &errorHandler,
"unicodeescape", "illegal Unicode character",
starts, size, &startinpos, &endinpos, &exc, &s,
&v, &outpos, &p))
goto onError;
message = "illegal Unicode character";
goto error;
}
break;
......@@ -2910,20 +2892,23 @@ PyObject *PyUnicode_DecodeUnicodeEscape(const char *s,
goto store;
}
}
endinpos = s-starts;
outpos = p-PyUnicode_AS_UNICODE(v);
if (unicode_decode_call_errorhandler(
errors, &errorHandler,
"unicodeescape", message,
starts, size, &startinpos, &endinpos, &exc, &s,
&v, &outpos, &p))
goto onError;
break;
goto error;
default:
if (s > end) {
message = "\\ at end of string";
s--;
goto error;
}
else {
*p++ = '\\';
*p++ = (unsigned char)s[-1];
}
break;
}
continue;
error:
endinpos = s-starts;
outpos = p-PyUnicode_AS_UNICODE(v);
if (unicode_decode_call_errorhandler(
......@@ -2932,15 +2917,7 @@ PyObject *PyUnicode_DecodeUnicodeEscape(const char *s,
starts, size, &startinpos, &endinpos, &exc, &s,
&v, &outpos, &p))
goto onError;
}
else {
*p++ = '\\';
*p++ = (unsigned char)s[-1];
}
break;
}
nextByte:
;
continue;
}
if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
goto onError;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7