Skip to content

C
cpython
Commit 94b9a7fe authored by Serhiy Storchaka's avatar Serhiy Storchaka
Browse files
Options

Increased coverage of standard codec error handlers.

parents c6f8ce84 7c3d7b0a
Show whitespace changes
Inline Side-by-side
Showing with 197 additions and 133 deletions
Lib/test/test_codeccallbacks.py
View file @ 94b9a7fe
...@@ -6,14 +6,6 @@ import unicodedata ...@@ -6,14 +6,6 @@ import unicodedata
import unittest import unittest
import warnings import warnings
try:
import ctypes
except ImportError:
ctypes = None
SIZEOF_WCHAR_T = -1
else:
SIZEOF_WCHAR_T = ctypes.sizeof(ctypes.c_wchar)
class PosReturn: class PosReturn:
# this can be used for configurable callbacks # this can be used for configurable callbacks
...@@ -228,14 +220,12 @@ class CodecCallbackTest(unittest.TestCase): ...@@ -228,14 +220,12 @@ class CodecCallbackTest(unittest.TestCase):
b"\x00\x00\x00\x00\x00".decode, b"\x00\x00\x00\x00\x00".decode,
"unicode-internal", "unicode-internal",
) )
if SIZEOF_WCHAR_T == 4: if len('\0'.encode('unicode-internal')) == 4:
def handler_unicodeinternal(exc): def handler_unicodeinternal(exc):
if not isinstance(exc, UnicodeDecodeError): if not isinstance(exc, UnicodeDecodeError):
raise TypeError("don't know how to handle %r" % exc) raise TypeError("don't know how to handle %r" % exc)
return ("\x01", 1) return ("\x01", 1)
with test.support.check_warnings(('unicode_internal codec has been '
'deprecated', DeprecationWarning)):
self.assertEqual( self.assertEqual(
b"\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore"), b"\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore"),
"\u0000" "\u0000"
...@@ -385,7 +375,6 @@ class CodecCallbackTest(unittest.TestCase): ...@@ -385,7 +375,6 @@ class CodecCallbackTest(unittest.TestCase):
["ascii", "\uffffx", 0, 1, "ouch"], ["ascii", "\uffffx", 0, 1, "ouch"],
"'ascii' codec can't encode character '\\uffff' in position 0: ouch" "'ascii' codec can't encode character '\\uffff' in position 0: ouch"
) )
if SIZEOF_WCHAR_T == 4:
self.check_exceptionobjectargs( self.check_exceptionobjectargs(
UnicodeEncodeError, UnicodeEncodeError,
["ascii", "\U00010000x", 0, 1, "ouch"], ["ascii", "\U00010000x", 0, 1, "ouch"],
...@@ -420,7 +409,6 @@ class CodecCallbackTest(unittest.TestCase): ...@@ -420,7 +409,6 @@ class CodecCallbackTest(unittest.TestCase):
["g\uffffrk", 1, 2, "ouch"], ["g\uffffrk", 1, 2, "ouch"],
"can't translate character '\\uffff' in position 1: ouch" "can't translate character '\\uffff' in position 1: ouch"
) )
if SIZEOF_WCHAR_T == 4:
self.check_exceptionobjectargs( self.check_exceptionobjectargs(
UnicodeTranslateError, UnicodeTranslateError,
["g\U00010000rk", 1, 2, "ouch"], ["g\U00010000rk", 1, 2, "ouch"],
...@@ -452,6 +440,16 @@ class CodecCallbackTest(unittest.TestCase): ...@@ -452,6 +440,16 @@ class CodecCallbackTest(unittest.TestCase):
codecs.strict_errors, codecs.strict_errors,
UnicodeEncodeError("ascii", "\u3042", 0, 1, "ouch") UnicodeEncodeError("ascii", "\u3042", 0, 1, "ouch")
) )
self.assertRaises(
UnicodeDecodeError,
codecs.strict_errors,
UnicodeDecodeError("ascii", bytearray(b"\xff"), 0, 1, "ouch")
)
self.assertRaises(
UnicodeTranslateError,
codecs.strict_errors,
UnicodeTranslateError("\u3042", 0, 1, "ouch")
)
def test_badandgoodignoreexceptions(self): def test_badandgoodignoreexceptions(self):
# "ignore" complains about a non-exception passed in # "ignore" complains about a non-exception passed in
...@@ -548,13 +546,15 @@ class CodecCallbackTest(unittest.TestCase): ...@@ -548,13 +546,15 @@ class CodecCallbackTest(unittest.TestCase):
UnicodeTranslateError("\u3042", 0, 1, "ouch") UnicodeTranslateError("\u3042", 0, 1, "ouch")
) )
# Use the correct exception # Use the correct exception
cs = (0, 1, 9, 10, 99, 100, 999, 1000, 9999, 10000, 0x3042) cs = (0, 1, 9, 10, 99, 100, 999, 1000, 9999, 10000, 99999, 100000,
999999, 1000000)
cs += (0xd800, 0xdfff)
s = "".join(chr(c) for c in cs) s = "".join(chr(c) for c in cs)
self.assertEqual( self.assertEqual(
codecs.xmlcharrefreplace_errors( codecs.xmlcharrefreplace_errors(
UnicodeEncodeError("ascii", s, 0, len(s), "ouch") UnicodeEncodeError("ascii", s, 0, len(s), "ouch")
), ),
("".join("&#%d;" % ord(c) for c in s), len(s)) ("".join("&#%d;" % c for c in cs), len(s))
) )
def test_badandgoodbackslashreplaceexceptions(self): def test_badandgoodbackslashreplaceexceptions(self):
...@@ -571,54 +571,45 @@ class CodecCallbackTest(unittest.TestCase): ...@@ -571,54 +571,45 @@ class CodecCallbackTest(unittest.TestCase):
UnicodeError("ouch") UnicodeError("ouch")
) )
# Use the correct exception # Use the correct exception
tests = [
("\u3042", "\\u3042"),
("\n", "\\x0a"),
("a", "\\x61"),
("\x00", "\\x00"),
("\xff", "\\xff"),
("\u0100", "\\u0100"),
("\uffff", "\\uffff"),
("\U00010000", "\\U00010000"),
("\U0010ffff", "\\U0010ffff"),
# Lone surrogates
("\ud800", "\\ud800"),
("\udfff", "\\udfff"),
("\ud800\udfff", "\\ud800\\udfff"),
]
for s, r in tests:
with self.subTest(str=s):
self.assertEqual( self.assertEqual(
codecs.backslashreplace_errors( codecs.backslashreplace_errors(
UnicodeEncodeError("ascii", "\u3042", 0, 1, "ouch")), UnicodeEncodeError("ascii", s, 0, len(s), "ouch")),
("\\u3042", 1) (r, len(s))
)
self.assertEqual(
codecs.backslashreplace_errors(
UnicodeEncodeError("ascii", "\x00", 0, 1, "ouch")),
("\\x00", 1)
)
self.assertEqual(
codecs.backslashreplace_errors(
UnicodeEncodeError("ascii", "\xff", 0, 1, "ouch")),
("\\xff", 1)
)
self.assertEqual(
codecs.backslashreplace_errors(
UnicodeEncodeError("ascii", "\u0100", 0, 1, "ouch")),
("\\u0100", 1)
)
self.assertEqual(
codecs.backslashreplace_errors(
UnicodeEncodeError("ascii", "\uffff", 0, 1, "ouch")),
("\\uffff", 1)
)
if SIZEOF_WCHAR_T > 0:
self.assertEqual(
codecs.backslashreplace_errors(
UnicodeEncodeError("ascii", "\U00010000",
0, 1, "ouch")),
("\\U00010000", 1)
)
self.assertEqual(
codecs.backslashreplace_errors(
UnicodeEncodeError("ascii", "\U0010ffff",
0, 1, "ouch")),
("\\U0010ffff", 1)
) )
# Lone surrogates (regardless of unicode width)
self.assertEqual( self.assertEqual(
codecs.backslashreplace_errors( codecs.backslashreplace_errors(
UnicodeEncodeError("ascii", "\ud800", 0, 1, "ouch")), UnicodeTranslateError(s, 0, len(s), "ouch")),
("\\ud800", 1) (r, len(s))
) )
tests = [
(b"a", "\\x61"),
(b"\n", "\\x0a"),
(b"\x00", "\\x00"),
(b"\xff", "\\xff"),
]
for b, r in tests:
with self.subTest(bytes=b):
self.assertEqual( self.assertEqual(
codecs.backslashreplace_errors( codecs.backslashreplace_errors(
UnicodeEncodeError("ascii", "\udfff", 0, 1, "ouch")), UnicodeDecodeError("ascii", bytearray(b), 0, 1, "ouch")),
("\\udfff", 1) (r, 1)
) )
def test_badandgoodnamereplaceexceptions(self): def test_badandgoodnamereplaceexceptions(self):
...@@ -646,64 +637,138 @@ class CodecCallbackTest(unittest.TestCase): ...@@ -646,64 +637,138 @@ class CodecCallbackTest(unittest.TestCase):
UnicodeTranslateError("\u3042", 0, 1, "ouch") UnicodeTranslateError("\u3042", 0, 1, "ouch")
) )
# Use the correct exception # Use the correct exception
tests = [
("\u3042", "\\N{HIRAGANA LETTER A}"),
("\x00", "\\x00"),
("\ufbf9", "\\N{ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH "
"HAMZA ABOVE WITH ALEF MAKSURA ISOLATED FORM}"),
("\U000e007f", "\\N{CANCEL TAG}"),
("\U0010ffff", "\\U0010ffff"),
# Lone surrogates
("\ud800", "\\ud800"),
("\udfff", "\\udfff"),
("\ud800\udfff", "\\ud800\\udfff"),
]
for s, r in tests:
with self.subTest(str=s):
self.assertEqual( self.assertEqual(
codecs.namereplace_errors( codecs.namereplace_errors(
UnicodeEncodeError("ascii", "\u3042", 0, 1, "ouch")), UnicodeEncodeError("ascii", s, 0, len(s), "ouch")),
("\\N{HIRAGANA LETTER A}", 1) (r, len(s))
) )
self.assertEqual(
codecs.namereplace_errors( def test_badandgoodsurrogateescapeexceptions(self):
UnicodeEncodeError("ascii", "\x00", 0, 1, "ouch")), surrogateescape_errors = codecs.lookup_error('surrogateescape')
("\\x00", 1) # "surrogateescape" complains about a non-exception passed in
self.assertRaises(
TypeError,
surrogateescape_errors,
42
) )
self.assertEqual( # "surrogateescape" complains about the wrong exception types
codecs.namereplace_errors( self.assertRaises(
UnicodeEncodeError("ascii", "\xff", 0, 1, "ouch")), TypeError,
("\\N{LATIN SMALL LETTER Y WITH DIAERESIS}", 1) surrogateescape_errors,
UnicodeError("ouch")
) )
self.assertEqual( # "surrogateescape" can not be used for translating
codecs.namereplace_errors( self.assertRaises(
UnicodeEncodeError("ascii", "\u0100", 0, 1, "ouch")), TypeError,
("\\N{LATIN CAPITAL LETTER A WITH MACRON}", 1) surrogateescape_errors,
UnicodeTranslateError("\udc80", 0, 1, "ouch")
) )
self.assertEqual( # Use the correct exception
codecs.namereplace_errors( for s in ("a", "\udc7f", "\udd00"):
UnicodeEncodeError("ascii", "\uffff", 0, 1, "ouch")), with self.subTest(str=s):
("\\uffff", 1) self.assertRaises(
UnicodeEncodeError,
surrogateescape_errors,
UnicodeEncodeError("ascii", s, 0, 1, "ouch")
) )
if SIZEOF_WCHAR_T > 0:
self.assertEqual( self.assertEqual(
codecs.namereplace_errors( surrogateescape_errors(
UnicodeEncodeError("ascii", "\U00010000", UnicodeEncodeError("ascii", "\udc80", 0, 1, "ouch")),
0, 1, "ouch")), (b"\x80", 1)
("\\N{LINEAR B SYLLABLE B008 A}", 1)
) )
self.assertEqual( self.assertRaises(
codecs.namereplace_errors( UnicodeDecodeError,
UnicodeEncodeError("ascii", "\U0010ffff", surrogateescape_errors,
0, 1, "ouch")), UnicodeDecodeError("ascii", bytearray(b"a"), 0, 1, "ouch")
("\\U0010ffff", 1)
) )
# Lone surrogates (regardless of unicode width)
self.assertEqual( self.assertEqual(
codecs.namereplace_errors( surrogateescape_errors(
UnicodeEncodeError("ascii", "\ud800", 0, 1, "ouch")), UnicodeDecodeError("ascii", bytearray(b"\x80"), 0, 1, "ouch")),
("\\ud800", 1) ("\udc80", 1)
) )
self.assertEqual(
codecs.namereplace_errors( def test_badandgoodsurrogatepassexceptions(self):
UnicodeEncodeError("ascii", "\udfff", 0, 1, "ouch")), surrogatepass_errors = codecs.lookup_error('surrogatepass')
("\\udfff", 1) # "surrogatepass" complains about a non-exception passed in
self.assertRaises(
TypeError,
surrogatepass_errors,
42
) )
# "surrogatepass" complains about the wrong exception types
self.assertRaises(
TypeError,
surrogatepass_errors,
UnicodeError("ouch")
)
# "surrogatepass" can not be used for translating
self.assertRaises(
TypeError,
surrogatepass_errors,
UnicodeTranslateError("\ud800", 0, 1, "ouch")
)
# Use the correct exception
for enc in ("utf-8", "utf-16le", "utf-16be", "utf-32le", "utf-32be"):
with self.subTest(encoding=enc):
self.assertRaises(
UnicodeEncodeError,
surrogatepass_errors,
UnicodeEncodeError(enc, "a", 0, 1, "ouch")
)
self.assertRaises(
UnicodeDecodeError,
surrogatepass_errors,
UnicodeDecodeError(enc, "a".encode(enc), 0, 1, "ouch")
)
for s in ("\ud800", "\udfff", "\ud800\udfff"):
with self.subTest(str=s):
self.assertRaises(
UnicodeEncodeError,
surrogatepass_errors,
UnicodeEncodeError("ascii", s, 0, len(s), "ouch")
)
tests = [
("utf-8", "\ud800", b'\xed\xa0\x80', 3),
("utf-16le", "\ud800", b'\x00\xd8', 2),
("utf-16be", "\ud800", b'\xd8\x00', 2),
("utf-32le", "\ud800", b'\x00\xd8\x00\x00', 4),
("utf-32be", "\ud800", b'\x00\x00\xd8\x00', 4),
("utf-8", "\udfff", b'\xed\xbf\xbf', 3),
("utf-16le", "\udfff", b'\xff\xdf', 2),
("utf-16be", "\udfff", b'\xdf\xff', 2),
("utf-32le", "\udfff", b'\xff\xdf\x00\x00', 4),
("utf-32be", "\udfff", b'\x00\x00\xdf\xff', 4),
("utf-8", "\ud800\udfff", b'\xed\xa0\x80\xed\xbf\xbf', 3),
("utf-16le", "\ud800\udfff", b'\x00\xd8\xff\xdf', 2),
("utf-16be", "\ud800\udfff", b'\xd8\x00\xdf\xff', 2),
("utf-32le", "\ud800\udfff", b'\x00\xd8\x00\x00\xff\xdf\x00\x00', 4),
("utf-32be", "\ud800\udfff", b'\x00\x00\xd8\x00\x00\x00\xdf\xff', 4),
]
for enc, s, b, n in tests:
with self.subTest(encoding=enc, str=s, bytes=b):
self.assertEqual( self.assertEqual(
codecs.backslashreplace_errors( surrogatepass_errors(
UnicodeDecodeError("ascii", bytearray(b"\xff"), 0, 1, "ouch")), UnicodeEncodeError(enc, s, 0, len(s), "ouch")),
("\\xff", 1) (b, len(s))
) )
self.assertEqual( self.assertEqual(
codecs.backslashreplace_errors( surrogatepass_errors(
UnicodeTranslateError("\u3042", 0, 1, "ouch")), UnicodeDecodeError(enc, bytearray(b[:n]), 0, n, "ouch")),
("\\u3042", 1) (s[:1], n)
) )
def test_badhandlerresults(self): def test_badhandlerresults(self):
...@@ -787,9 +852,8 @@ class CodecCallbackTest(unittest.TestCase): ...@@ -787,9 +852,8 @@ class CodecCallbackTest(unittest.TestCase):
# enhance coverage of: # enhance coverage of:
# Python/codecs.c::PyCodec_XMLCharRefReplaceErrors() # Python/codecs.c::PyCodec_XMLCharRefReplaceErrors()
# and inline implementations # and inline implementations
v = (1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 50000) v = (1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000,
if SIZEOF_WCHAR_T == 4: 500000, 1000000)
v += (100000, 500000, 1000000)
s = "".join([chr(x) for x in v]) s = "".join([chr(x) for x in v])
codecs.register_error("test.xmlcharrefreplace", codecs.xmlcharrefreplace_errors) codecs.register_error("test.xmlcharrefreplace", codecs.xmlcharrefreplace_errors)
for enc in ("ascii", "iso-8859-15"): for enc in ("ascii", "iso-8859-15"):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7