Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
C
cpython
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Kirill Smelkov
cpython
Commits
3e693810
Commit
3e693810
authored
Nov 04, 2012
by
Ezio Melotti
Browse files
Options
Browse Files
Download
Plain Diff
#8271: merge with 3.3.
parents
fff726f3
d29dec82
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
291 additions
and
37 deletions
+291
-37
Lib/test/test_unicode.py
Lib/test/test_unicode.py
+221
-1
Misc/NEWS
Misc/NEWS
+4
-0
Objects/stringlib/codecs.h
Objects/stringlib/codecs.h
+62
-30
Objects/unicodeobject.c
Objects/unicodeobject.c
+4
-6
No files found.
Lib/test/test_unicode.py
View file @
3e693810
...
...
@@ -1349,7 +1349,7 @@ class UnicodeTest(string_tests.CommonTest,
# with start byte of a 2-byte sequence
(
b'
\
xc2
'
,
FFFD
),
# only the start byte
(
b'
\
xc2
\
xc2
'
,
FFFD
*
2
),
# 2 start bytes
(
b'
\
xc2
\
xc2
\
xc2
'
,
FFFD
*
3
),
#
2
start bytes
(
b'
\
xc2
\
xc2
\
xc2
'
,
FFFD
*
3
),
#
3
start bytes
(
b'
\
xc2
\
x41
'
,
FFFD
+
'A'
),
# invalid continuation byte
# with start byte of a 3-byte sequence
(
b'
\
xe1
'
,
FFFD
),
# only the start byte
...
...
@@ -1419,6 +1419,226 @@ class UnicodeTest(string_tests.CommonTest,
self
.
assertEqual
(
seq
.
decode
(
'utf-8'
,
'ignore'
),
res
.
replace
(
'
\
uFFFD
'
,
''
))
def
to_bytestring
(
self
,
seq
):
return
bytes
(
int
(
c
,
16
)
for
c
in
seq
.
split
())
def
assertCorrectUTF8Decoding
(
self
,
seq
,
res
,
err
):
"""
Check that an invalid UTF-8 sequence raises an UnicodeDecodeError when
'strict' is used, returns res when 'replace' is used, and that doesn't
return anything when 'ignore' is used.
"""
with
self
.
assertRaises
(
UnicodeDecodeError
)
as
cm
:
seq
.
decode
(
'utf-8'
)
exc
=
cm
.
exception
self
.
assertIn
(
err
,
str
(
exc
))
self
.
assertEqual
(
seq
.
decode
(
'utf-8'
,
'replace'
),
res
)
self
.
assertEqual
((
b'aaaa'
+
seq
+
b'bbbb'
).
decode
(
'utf-8'
,
'replace'
),
'aaaa'
+
res
+
'bbbb'
)
res
=
res
.
replace
(
'
\
ufffd
'
,
''
)
self
.
assertEqual
(
seq
.
decode
(
'utf-8'
,
'ignore'
),
res
)
self
.
assertEqual
((
b'aaaa'
+
seq
+
b'bbbb'
).
decode
(
'utf-8'
,
'ignore'
),
'aaaa'
+
res
+
'bbbb'
)
def
test_invalid_start_byte
(
self
):
"""
Test that an 'invalid start byte' error is raised when the first byte
is not in the ASCII range or is not a valid start byte of a 2-, 3-, or
4-bytes sequence. The invalid start byte is replaced with a single
U+FFFD when errors='replace'.
E.g. <80> is a continuation byte and can appear only after a start byte.
"""
FFFD
=
'
\
ufffd
'
for
byte
in
b'
\
x80
\
xA0
\
x9F
\
xBF
\
xC0
\
xC1
\
xF5
\
xFF
'
:
self
.
assertCorrectUTF8Decoding
(
bytes
([
byte
]),
'
\
ufffd
'
,
'invalid start byte'
)
def
test_unexpected_end_of_data
(
self
):
"""
Test that an 'unexpected end of data' error is raised when the string
ends after a start byte of a 2-, 3-, or 4-bytes sequence without having
enough continuation bytes. The incomplete sequence is replaced with a
single U+FFFD when errors='replace'.
E.g. in the sequence <F3 80 80>, F3 is the start byte of a 4-bytes
sequence, but it's followed by only 2 valid continuation bytes and the
last continuation bytes is missing.
Note: the continuation bytes must be all valid, if one of them is
invalid another error will be raised.
"""
sequences
=
[
'C2'
,
'DF'
,
'E0 A0'
,
'E0 BF'
,
'E1 80'
,
'E1 BF'
,
'EC 80'
,
'EC BF'
,
'ED 80'
,
'ED 9F'
,
'EE 80'
,
'EE BF'
,
'EF 80'
,
'EF BF'
,
'F0 90'
,
'F0 BF'
,
'F0 90 80'
,
'F0 90 BF'
,
'F0 BF 80'
,
'F0 BF BF'
,
'F1 80'
,
'F1 BF'
,
'F1 80 80'
,
'F1 80 BF'
,
'F1 BF 80'
,
'F1 BF BF'
,
'F3 80'
,
'F3 BF'
,
'F3 80 80'
,
'F3 80 BF'
,
'F3 BF 80'
,
'F3 BF BF'
,
'F4 80'
,
'F4 8F'
,
'F4 80 80'
,
'F4 80 BF'
,
'F4 8F 80'
,
'F4 8F BF'
]
FFFD
=
'
\
ufffd
'
for
seq
in
sequences
:
self
.
assertCorrectUTF8Decoding
(
self
.
to_bytestring
(
seq
),
'
\
ufffd
'
,
'unexpected end of data'
)
def
test_invalid_cb_for_2bytes_seq
(
self
):
"""
Test that an 'invalid continuation byte' error is raised when the
continuation byte of a 2-bytes sequence is invalid. The start byte
is replaced by a single U+FFFD and the second byte is handled
separately when errors='replace'.
E.g. in the sequence <C2 41>, C2 is the start byte of a 2-bytes
sequence, but 41 is not a valid continuation byte because it's the
ASCII letter 'A'.
"""
FFFD
=
'
\
ufffd
'
FFFDx2
=
FFFD
*
2
sequences
=
[
(
'C2 00'
,
FFFD
+
'
\
x00
'
),
(
'C2 7F'
,
FFFD
+
'
\
x7f
'
),
(
'C2 C0'
,
FFFDx2
),
(
'C2 FF'
,
FFFDx2
),
(
'DF 00'
,
FFFD
+
'
\
x00
'
),
(
'DF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'DF C0'
,
FFFDx2
),
(
'DF FF'
,
FFFDx2
),
]
for
seq
,
res
in
sequences
:
self
.
assertCorrectUTF8Decoding
(
self
.
to_bytestring
(
seq
),
res
,
'invalid continuation byte'
)
def
test_invalid_cb_for_3bytes_seq
(
self
):
"""
Test that an 'invalid continuation byte' error is raised when the
continuation byte(s) of a 3-bytes sequence are invalid. When
errors='replace', if the first continuation byte is valid, the first
two bytes (start byte + 1st cb) are replaced by a single U+FFFD and the
third byte is handled separately, otherwise only the start byte is
replaced with a U+FFFD and the other continuation bytes are handled
separately.
E.g. in the sequence <E1 80 41>, E1 is the start byte of a 3-bytes
sequence, 80 is a valid continuation byte, but 41 is not a valid cb
because it's the ASCII letter 'A'.
Note: when the start byte is E0 or ED, the valid ranges for the first
continuation byte are limited to A0..BF and 80..9F respectively.
Python 2 used to consider all the bytes in range 80..BF valid when the
start byte was ED. This is fixed in Python 3.
"""
FFFD
=
'
\
ufffd
'
FFFDx2
=
FFFD
*
2
sequences
=
[
(
'E0 00'
,
FFFD
+
'
\
x00
'
),
(
'E0 7F'
,
FFFD
+
'
\
x7f
'
),
(
'E0 80'
,
FFFDx2
),
(
'E0 9F'
,
FFFDx2
),
(
'E0 C0'
,
FFFDx2
),
(
'E0 FF'
,
FFFDx2
),
(
'E0 A0 00'
,
FFFD
+
'
\
x00
'
),
(
'E0 A0 7F'
,
FFFD
+
'
\
x7f
'
),
(
'E0 A0 C0'
,
FFFDx2
),
(
'E0 A0 FF'
,
FFFDx2
),
(
'E0 BF 00'
,
FFFD
+
'
\
x00
'
),
(
'E0 BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'E0 BF C0'
,
FFFDx2
),
(
'E0 BF FF'
,
FFFDx2
),
(
'E1 00'
,
FFFD
+
'
\
x00
'
),
(
'E1 7F'
,
FFFD
+
'
\
x7f
'
),
(
'E1 C0'
,
FFFDx2
),
(
'E1 FF'
,
FFFDx2
),
(
'E1 80 00'
,
FFFD
+
'
\
x00
'
),
(
'E1 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'E1 80 C0'
,
FFFDx2
),
(
'E1 80 FF'
,
FFFDx2
),
(
'E1 BF 00'
,
FFFD
+
'
\
x00
'
),
(
'E1 BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'E1 BF C0'
,
FFFDx2
),
(
'E1 BF FF'
,
FFFDx2
),
(
'EC 00'
,
FFFD
+
'
\
x00
'
),
(
'EC 7F'
,
FFFD
+
'
\
x7f
'
),
(
'EC C0'
,
FFFDx2
),
(
'EC FF'
,
FFFDx2
),
(
'EC 80 00'
,
FFFD
+
'
\
x00
'
),
(
'EC 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'EC 80 C0'
,
FFFDx2
),
(
'EC 80 FF'
,
FFFDx2
),
(
'EC BF 00'
,
FFFD
+
'
\
x00
'
),
(
'EC BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'EC BF C0'
,
FFFDx2
),
(
'EC BF FF'
,
FFFDx2
),
(
'ED 00'
,
FFFD
+
'
\
x00
'
),
(
'ED 7F'
,
FFFD
+
'
\
x7f
'
),
(
'ED A0'
,
FFFDx2
),
(
'ED BF'
,
FFFDx2
),
# see note ^
(
'ED C0'
,
FFFDx2
),
(
'ED FF'
,
FFFDx2
),
(
'ED 80 00'
,
FFFD
+
'
\
x00
'
),
(
'ED 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'ED 80 C0'
,
FFFDx2
),
(
'ED 80 FF'
,
FFFDx2
),
(
'ED 9F 00'
,
FFFD
+
'
\
x00
'
),
(
'ED 9F 7F'
,
FFFD
+
'
\
x7f
'
),
(
'ED 9F C0'
,
FFFDx2
),
(
'ED 9F FF'
,
FFFDx2
),
(
'EE 00'
,
FFFD
+
'
\
x00
'
),
(
'EE 7F'
,
FFFD
+
'
\
x7f
'
),
(
'EE C0'
,
FFFDx2
),
(
'EE FF'
,
FFFDx2
),
(
'EE 80 00'
,
FFFD
+
'
\
x00
'
),
(
'EE 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'EE 80 C0'
,
FFFDx2
),
(
'EE 80 FF'
,
FFFDx2
),
(
'EE BF 00'
,
FFFD
+
'
\
x00
'
),
(
'EE BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'EE BF C0'
,
FFFDx2
),
(
'EE BF FF'
,
FFFDx2
),
(
'EF 00'
,
FFFD
+
'
\
x00
'
),
(
'EF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'EF C0'
,
FFFDx2
),
(
'EF FF'
,
FFFDx2
),
(
'EF 80 00'
,
FFFD
+
'
\
x00
'
),
(
'EF 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'EF 80 C0'
,
FFFDx2
),
(
'EF 80 FF'
,
FFFDx2
),
(
'EF BF 00'
,
FFFD
+
'
\
x00
'
),
(
'EF BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'EF BF C0'
,
FFFDx2
),
(
'EF BF FF'
,
FFFDx2
),
]
for
seq
,
res
in
sequences
:
self
.
assertCorrectUTF8Decoding
(
self
.
to_bytestring
(
seq
),
res
,
'invalid continuation byte'
)
def
test_invalid_cb_for_4bytes_seq
(
self
):
"""
Test that an 'invalid continuation byte' error is raised when the
continuation byte(s) of a 4-bytes sequence are invalid. When
errors='replace',the start byte and all the following valid
continuation bytes are replaced with a single U+FFFD, and all the bytes
starting from the first invalid continuation bytes (included) are
handled separately.
E.g. in the sequence <E1 80 41>, E1 is the start byte of a 3-bytes
sequence, 80 is a valid continuation byte, but 41 is not a valid cb
because it's the ASCII letter 'A'.
Note: when the start byte is E0 or ED, the valid ranges for the first
continuation byte are limited to A0..BF and 80..9F respectively.
However, when the start byte is ED, Python 2 considers all the bytes
in range 80..BF valid. This is fixed in Python 3.
"""
FFFD
=
'
\
ufffd
'
FFFDx2
=
FFFD
*
2
sequences
=
[
(
'F0 00'
,
FFFD
+
'
\
x00
'
),
(
'F0 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F0 80'
,
FFFDx2
),
(
'F0 8F'
,
FFFDx2
),
(
'F0 C0'
,
FFFDx2
),
(
'F0 FF'
,
FFFDx2
),
(
'F0 90 00'
,
FFFD
+
'
\
x00
'
),
(
'F0 90 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F0 90 C0'
,
FFFDx2
),
(
'F0 90 FF'
,
FFFDx2
),
(
'F0 BF 00'
,
FFFD
+
'
\
x00
'
),
(
'F0 BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F0 BF C0'
,
FFFDx2
),
(
'F0 BF FF'
,
FFFDx2
),
(
'F0 90 80 00'
,
FFFD
+
'
\
x00
'
),
(
'F0 90 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F0 90 80 C0'
,
FFFDx2
),
(
'F0 90 80 FF'
,
FFFDx2
),
(
'F0 90 BF 00'
,
FFFD
+
'
\
x00
'
),
(
'F0 90 BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F0 90 BF C0'
,
FFFDx2
),
(
'F0 90 BF FF'
,
FFFDx2
),
(
'F0 BF 80 00'
,
FFFD
+
'
\
x00
'
),
(
'F0 BF 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F0 BF 80 C0'
,
FFFDx2
),
(
'F0 BF 80 FF'
,
FFFDx2
),
(
'F0 BF BF 00'
,
FFFD
+
'
\
x00
'
),
(
'F0 BF BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F0 BF BF C0'
,
FFFDx2
),
(
'F0 BF BF FF'
,
FFFDx2
),
(
'F1 00'
,
FFFD
+
'
\
x00
'
),
(
'F1 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F1 C0'
,
FFFDx2
),
(
'F1 FF'
,
FFFDx2
),
(
'F1 80 00'
,
FFFD
+
'
\
x00
'
),
(
'F1 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F1 80 C0'
,
FFFDx2
),
(
'F1 80 FF'
,
FFFDx2
),
(
'F1 BF 00'
,
FFFD
+
'
\
x00
'
),
(
'F1 BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F1 BF C0'
,
FFFDx2
),
(
'F1 BF FF'
,
FFFDx2
),
(
'F1 80 80 00'
,
FFFD
+
'
\
x00
'
),
(
'F1 80 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F1 80 80 C0'
,
FFFDx2
),
(
'F1 80 80 FF'
,
FFFDx2
),
(
'F1 80 BF 00'
,
FFFD
+
'
\
x00
'
),
(
'F1 80 BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F1 80 BF C0'
,
FFFDx2
),
(
'F1 80 BF FF'
,
FFFDx2
),
(
'F1 BF 80 00'
,
FFFD
+
'
\
x00
'
),
(
'F1 BF 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F1 BF 80 C0'
,
FFFDx2
),
(
'F1 BF 80 FF'
,
FFFDx2
),
(
'F1 BF BF 00'
,
FFFD
+
'
\
x00
'
),
(
'F1 BF BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F1 BF BF C0'
,
FFFDx2
),
(
'F1 BF BF FF'
,
FFFDx2
),
(
'F3 00'
,
FFFD
+
'
\
x00
'
),
(
'F3 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F3 C0'
,
FFFDx2
),
(
'F3 FF'
,
FFFDx2
),
(
'F3 80 00'
,
FFFD
+
'
\
x00
'
),
(
'F3 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F3 80 C0'
,
FFFDx2
),
(
'F3 80 FF'
,
FFFDx2
),
(
'F3 BF 00'
,
FFFD
+
'
\
x00
'
),
(
'F3 BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F3 BF C0'
,
FFFDx2
),
(
'F3 BF FF'
,
FFFDx2
),
(
'F3 80 80 00'
,
FFFD
+
'
\
x00
'
),
(
'F3 80 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F3 80 80 C0'
,
FFFDx2
),
(
'F3 80 80 FF'
,
FFFDx2
),
(
'F3 80 BF 00'
,
FFFD
+
'
\
x00
'
),
(
'F3 80 BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F3 80 BF C0'
,
FFFDx2
),
(
'F3 80 BF FF'
,
FFFDx2
),
(
'F3 BF 80 00'
,
FFFD
+
'
\
x00
'
),
(
'F3 BF 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F3 BF 80 C0'
,
FFFDx2
),
(
'F3 BF 80 FF'
,
FFFDx2
),
(
'F3 BF BF 00'
,
FFFD
+
'
\
x00
'
),
(
'F3 BF BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F3 BF BF C0'
,
FFFDx2
),
(
'F3 BF BF FF'
,
FFFDx2
),
(
'F4 00'
,
FFFD
+
'
\
x00
'
),
(
'F4 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F4 90'
,
FFFDx2
),
(
'F4 BF'
,
FFFDx2
),
(
'F4 C0'
,
FFFDx2
),
(
'F4 FF'
,
FFFDx2
),
(
'F4 80 00'
,
FFFD
+
'
\
x00
'
),
(
'F4 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F4 80 C0'
,
FFFDx2
),
(
'F4 80 FF'
,
FFFDx2
),
(
'F4 8F 00'
,
FFFD
+
'
\
x00
'
),
(
'F4 8F 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F4 8F C0'
,
FFFDx2
),
(
'F4 8F FF'
,
FFFDx2
),
(
'F4 80 80 00'
,
FFFD
+
'
\
x00
'
),
(
'F4 80 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F4 80 80 C0'
,
FFFDx2
),
(
'F4 80 80 FF'
,
FFFDx2
),
(
'F4 80 BF 00'
,
FFFD
+
'
\
x00
'
),
(
'F4 80 BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F4 80 BF C0'
,
FFFDx2
),
(
'F4 80 BF FF'
,
FFFDx2
),
(
'F4 8F 80 00'
,
FFFD
+
'
\
x00
'
),
(
'F4 8F 80 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F4 8F 80 C0'
,
FFFDx2
),
(
'F4 8F 80 FF'
,
FFFDx2
),
(
'F4 8F BF 00'
,
FFFD
+
'
\
x00
'
),
(
'F4 8F BF 7F'
,
FFFD
+
'
\
x7f
'
),
(
'F4 8F BF C0'
,
FFFDx2
),
(
'F4 8F BF FF'
,
FFFDx2
)
]
for
seq
,
res
in
sequences
:
self
.
assertCorrectUTF8Decoding
(
self
.
to_bytestring
(
seq
),
res
,
'invalid continuation byte'
)
def
test_codecs_idna
(
self
):
# Test whether trailing dot is preserved
self
.
assertEqual
(
"www.python.org."
.
encode
(
"idna"
),
b"www.python.org."
)
...
...
Misc/NEWS
View file @
3e693810
...
...
@@ -10,6 +10,10 @@ What's New in Python 3.4.0 Alpha 1?
Core and Builtins
-----------------
- Issue #8271: the utf-8 decoder now outputs the correct number of U+FFFD
characters when used with the '
replace
' error handler on invalid utf-8
sequences. Patch by Serhiy Storchaka, tests by Ezio Melotti.
- Issue #5765: Apply a hard recursion limit in the compiler instead of
blowing the stack and segfaulting. Initial patch by Andrea Griffini.
...
...
Objects/stringlib/codecs.h
View file @
3e693810
...
...
@@ -91,15 +91,14 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end,
}
}
if
(
ch
<
0xC2
)
{
/* invalid sequence
\x80-\xBF -- continuation byte
\xC0-\xC1 -- fake 0000-007F */
goto
InvalidStart
;
}
if
(
ch
<
0xE0
)
{
/* \xC2\x80-\xDF\xBF -- 0080-07FF */
if
(
ch
<
0xC2
)
{
/* invalid sequence
\x80-\xBF -- continuation byte
\xC0-\xC1 -- fake 0000-007F */
goto
InvalidStart
;
}
Py_UCS4
ch2
;
if
(
end
-
s
<
2
)
{
/* unexpected end of data: the caller will decide whether
...
...
@@ -109,14 +108,15 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end,
ch2
=
(
unsigned
char
)
s
[
1
];
if
(
!
IS_CONTINUATION_BYTE
(
ch2
))
/* invalid continuation byte */
goto
InvalidContinuation
;
goto
InvalidContinuation
1
;
ch
=
(
ch
<<
6
)
+
ch2
-
((
0xC0
<<
6
)
+
0x80
);
assert
((
ch
>
0x007F
)
&&
(
ch
<=
0x07FF
));
s
+=
2
;
if
(
STRINGLIB_MAX_CHAR
<=
0x007F
||
(
STRINGLIB_MAX_CHAR
<
0x07FF
&&
ch
>
STRINGLIB_MAX_CHAR
))
goto
Overflow
;
/* Out-of-range */
goto
Return
;
*
p
++
=
ch
;
continue
;
}
...
...
@@ -127,28 +127,37 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end,
if
(
end
-
s
<
3
)
{
/* unexpected end of data: the caller will decide whether
it's an error or not */
if
(
end
-
s
<
2
)
break
;
ch2
=
(
unsigned
char
)
s
[
1
];
if
(
!
IS_CONTINUATION_BYTE
(
ch2
)
||
(
ch2
<
0xA0
?
ch
==
0xE0
:
ch
==
0xED
))
/* for clarification see comments below */
goto
InvalidContinuation1
;
break
;
}
ch2
=
(
unsigned
char
)
s
[
1
];
ch3
=
(
unsigned
char
)
s
[
2
];
if
(
!
IS_CONTINUATION_BYTE
(
ch2
)
||
!
IS_CONTINUATION_BYTE
(
ch3
))
{
if
(
!
IS_CONTINUATION_BYTE
(
ch2
))
{
/* invalid continuation byte */
goto
InvalidContinuation
;
goto
InvalidContinuation
1
;
}
if
(
ch
==
0xE0
)
{
if
(
ch2
<
0xA0
)
/* invalid sequence
\xE0\x80\x80-\xE0\x9F\xBF -- fake 0000-0800 */
goto
InvalidContinuation
;
}
else
if
(
ch
==
0xED
&&
ch2
>
0x9F
)
{
goto
InvalidContinuation1
;
}
else
if
(
ch
==
0xED
&&
ch2
>=
0xA0
)
{
/* Decoding UTF-8 sequences in range \xED\xA0\x80-\xED\xBF\xBF
will result in surrogates in range D800-DFFF. Surrogates are
not valid UTF-8 so they are rejected.
See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf
(table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */
goto
InvalidContinuation
;
goto
InvalidContinuation1
;
}
if
(
!
IS_CONTINUATION_BYTE
(
ch3
))
{
/* invalid continuation byte */
goto
InvalidContinuation2
;
}
ch
=
(
ch
<<
12
)
+
(
ch2
<<
6
)
+
ch3
-
((
0xE0
<<
12
)
+
(
0x80
<<
6
)
+
0x80
);
...
...
@@ -156,7 +165,8 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end,
s
+=
3
;
if
(
STRINGLIB_MAX_CHAR
<=
0x07FF
||
(
STRINGLIB_MAX_CHAR
<
0xFFFF
&&
ch
>
STRINGLIB_MAX_CHAR
))
goto
Overflow
;
/* Out-of-range */
goto
Return
;
*
p
++
=
ch
;
continue
;
}
...
...
@@ -167,27 +177,44 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end,
if
(
end
-
s
<
4
)
{
/* unexpected end of data: the caller will decide whether
it's an error or not */
if
(
end
-
s
<
2
)
break
;
ch2
=
(
unsigned
char
)
s
[
1
];
if
(
!
IS_CONTINUATION_BYTE
(
ch2
)
||
(
ch2
<
0x90
?
ch
==
0xF0
:
ch
==
0xF4
))
/* for clarification see comments below */
goto
InvalidContinuation1
;
if
(
end
-
s
<
3
)
break
;
ch3
=
(
unsigned
char
)
s
[
2
];
if
(
!
IS_CONTINUATION_BYTE
(
ch3
))
goto
InvalidContinuation2
;
break
;
}
ch2
=
(
unsigned
char
)
s
[
1
];
ch3
=
(
unsigned
char
)
s
[
2
];
ch4
=
(
unsigned
char
)
s
[
3
];
if
(
!
IS_CONTINUATION_BYTE
(
ch2
)
||
!
IS_CONTINUATION_BYTE
(
ch3
)
||
!
IS_CONTINUATION_BYTE
(
ch4
))
{
if
(
!
IS_CONTINUATION_BYTE
(
ch2
))
{
/* invalid continuation byte */
goto
InvalidContinuation
;
goto
InvalidContinuation
1
;
}
if
(
ch
==
0xF0
)
{
if
(
ch2
<
0x90
)
/* invalid sequence
\xF0\x80\x80\x80-\xF0\x80\xBF\xBF -- fake 0000-FFFF */
goto
InvalidContinuation
;
}
else
if
(
ch
==
0xF4
&&
ch2
>
0x8F
)
{
\xF0\x80\x80\x80-\xF0\x8F\xBF\xBF -- fake 0000-FFFF */
goto
InvalidContinuation1
;
}
else
if
(
ch
==
0xF4
&&
ch2
>=
0x90
)
{
/* invalid sequence
\xF4\x90\x80\80- -- 110000- overflow */
goto
InvalidContinuation
;
goto
InvalidContinuation1
;
}
if
(
!
IS_CONTINUATION_BYTE
(
ch3
))
{
/* invalid continuation byte */
goto
InvalidContinuation2
;
}
if
(
!
IS_CONTINUATION_BYTE
(
ch4
))
{
/* invalid continuation byte */
goto
InvalidContinuation3
;
}
ch
=
(
ch
<<
18
)
+
(
ch2
<<
12
)
+
(
ch3
<<
6
)
+
ch4
-
((
0xF0
<<
18
)
+
(
0x80
<<
12
)
+
(
0x80
<<
6
)
+
0x80
);
...
...
@@ -195,14 +222,14 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end,
s
+=
4
;
if
(
STRINGLIB_MAX_CHAR
<=
0xFFFF
||
(
STRINGLIB_MAX_CHAR
<
0x10FFFF
&&
ch
>
STRINGLIB_MAX_CHAR
))
goto
Overflow
;
/* Out-of-range */
goto
Return
;
*
p
++
=
ch
;
continue
;
}
goto
InvalidStart
;
}
ch
=
0
;
Overflow:
Return:
*
inptr
=
s
;
*
outpos
=
p
-
dest
;
...
...
@@ -210,13 +237,18 @@ Return:
InvalidStart:
ch
=
1
;
goto
Return
;
InvalidContinuation:
InvalidContinuation
1
:
ch
=
2
;
goto
Return
;
InvalidContinuation2:
ch
=
3
;
goto
Return
;
InvalidContinuation3:
ch
=
4
;
goto
Return
;
}
#undef ASCII_CHAR_MASK
#undef IS_CONTINUATION_BYTE
/* UTF-8 encoder specialized for a Unicode kind to avoid the slow
...
...
Objects/unicodeobject.c
View file @
3e693810
...
...
@@ -4611,9 +4611,7 @@ PyUnicode_DecodeUTF8Stateful(const char *s,
goto
End
;
errmsg
=
"unexpected end of data"
;
startinpos
=
s
-
starts
;
endinpos
=
startinpos
+
1
;
while
(
endinpos
<
size
&&
(
starts
[
endinpos
]
&
0xC0
)
==
0x80
)
endinpos
++
;
endinpos
=
end
-
starts
;
break
;
case
1
:
errmsg
=
"invalid start byte"
;
...
...
@@ -4621,11 +4619,11 @@ PyUnicode_DecodeUTF8Stateful(const char *s,
endinpos
=
startinpos
+
1
;
break
;
case
2
:
case
3
:
case
4
:
errmsg
=
"invalid continuation byte"
;
startinpos
=
s
-
starts
;
endinpos
=
startinpos
+
1
;
while
(
endinpos
<
size
&&
(
starts
[
endinpos
]
&
0xC0
)
==
0x80
)
endinpos
++
;
endinpos
=
startinpos
+
ch
-
1
;
break
;
default:
if
(
unicode_putchar
(
&
unicode
,
&
outpos
,
ch
)
<
0
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment