Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
C
cpython
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Kirill Smelkov
cpython
Commits
dafda9b0
Commit
dafda9b0
authored
Nov 26, 2013
by
Serhiy Storchaka
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Issue #11489: JSON decoder now accepts lone surrogates.
parent
60e361fe
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
84 additions
and
57 deletions
+84
-57
Lib/json/decoder.py
Lib/json/decoder.py
+18
-17
Lib/json/tests/test_scanstring.py
Lib/json/tests/test_scanstring.py
+51
-4
Misc/NEWS
Misc/NEWS
+2
-0
Modules/_json.c
Modules/_json.c
+13
-36
No files found.
Lib/json/decoder.py
View file @
dafda9b0
...
...
@@ -62,6 +62,16 @@ BACKSLASH = {
DEFAULT_ENCODING
=
"utf-8"
def
_decode_uXXXX
(
s
,
pos
):
esc
=
s
[
pos
+
1
:
pos
+
5
]
if
len
(
esc
)
==
4
and
esc
[
1
]
not
in
'xX'
:
try
:
return
int
(
esc
,
16
)
except
ValueError
:
pass
msg
=
"Invalid
\
\
uXXXX escape"
raise
ValueError
(
errmsg
(
msg
,
s
,
pos
))
def
py_scanstring
(
s
,
end
,
encoding
=
None
,
strict
=
True
,
_b
=
BACKSLASH
,
_m
=
STRINGCHUNK
.
match
):
"""Scan the string s for a JSON string. End is the index of the
...
...
@@ -116,25 +126,16 @@ def py_scanstring(s, end, encoding=None, strict=True,
end
+=
1
else
:
# Unicode escape sequence
esc
=
s
[
end
+
1
:
end
+
5
]
next_end
=
end
+
5
if
len
(
esc
)
!=
4
:
msg
=
"Invalid
\
\
uXXXX escape"
raise
ValueError
(
errmsg
(
msg
,
s
,
end
))
uni
=
int
(
esc
,
16
)
uni
=
_decode_uXXXX
(
s
,
end
)
end
+=
5
# Check for surrogate pair on UCS-4 systems
if
0xd800
<=
uni
<=
0xdbff
and
sys
.
maxunicode
>
65535
:
msg
=
"Invalid
\
\
uXXXX
\
\
uXXXX surrogate pair"
if
not
s
[
end
+
5
:
end
+
7
]
==
'
\
\
u'
:
raise
ValueError
(
errmsg
(
msg
,
s
,
end
))
esc2
=
s
[
end
+
7
:
end
+
11
]
if
len
(
esc2
)
!=
4
:
raise
ValueError
(
errmsg
(
msg
,
s
,
end
))
uni2
=
int
(
esc2
,
16
)
uni
=
0x10000
+
(((
uni
-
0xd800
)
<<
10
)
|
(
uni2
-
0xdc00
))
next_end
+=
6
if
sys
.
maxunicode
>
65535
and
\
0xd800
<=
uni
<=
0xdbff
and
s
[
end
:
end
+
2
]
==
'
\
\
u'
:
uni2
=
_decode_uXXXX
(
s
,
end
+
1
)
if
0xdc00
<=
uni2
<=
0xdfff
:
uni
=
0x10000
+
(((
uni
-
0xd800
)
<<
10
)
|
(
uni2
-
0xdc00
))
end
+=
6
char
=
unichr
(
uni
)
end
=
next_end
# Append the unescaped character
_append
(
char
)
return
u''
.
join
(
chunks
),
end
...
...
Lib/json/tests/test_scanstring.py
View file @
dafda9b0
...
...
@@ -5,10 +5,6 @@ from json.tests import PyTest, CTest
class
TestScanstring
(
object
):
def
test_scanstring
(
self
):
scanstring
=
self
.
json
.
decoder
.
scanstring
self
.
assertEqual
(
scanstring
(
'"z
\
\
ud834
\
\
udd20x"'
,
1
,
None
,
True
),
(
u'z
\
U0001d120
x'
,
16
))
if
sys
.
maxunicode
==
65535
:
self
.
assertEqual
(
scanstring
(
u'"z
\
U0001d120
x"'
,
1
,
None
,
True
),
...
...
@@ -94,6 +90,57 @@ class TestScanstring(object):
scanstring
(
'["Bad value", truth]'
,
2
,
None
,
True
),
(
u'Bad value'
,
12
))
def
test_surrogates
(
self
):
scanstring
=
self
.
json
.
decoder
.
scanstring
def
assertScan
(
given
,
expect
):
self
.
assertEqual
(
scanstring
(
given
,
1
,
None
,
True
),
(
expect
,
len
(
given
)))
if
not
isinstance
(
given
,
unicode
):
given
=
unicode
(
given
)
self
.
assertEqual
(
scanstring
(
given
,
1
,
None
,
True
),
(
expect
,
len
(
given
)))
assertScan
(
'"z
\
\
ud834
\
\
u0079x"'
,
u'z
\
ud834
yx'
)
assertScan
(
'"z
\
\
ud834
\
\
udd20x"'
,
u'z
\
U0001d120
x'
)
assertScan
(
'"z
\
\
ud834
\
\
ud834
\
\
udd20x"'
,
u'z
\
ud834
\
U0001d120
x'
)
assertScan
(
'"z
\
\
ud834x"'
,
u'z
\
ud834
x'
)
assertScan
(
u'"z
\
\
ud834
\
udd20
x12345"'
,
u'z
\
ud834
\
udd20
x12345'
)
assertScan
(
'"z
\
\
udd20x"'
,
u'z
\
udd20
x'
)
assertScan
(
u'"z
\
ud834
\
udd20
x"'
,
u'z
\
ud834
\
udd20
x'
)
assertScan
(
u'"z
\
ud834
\
\
udd20x"'
,
u'z
\
ud834
\
udd20
x'
)
assertScan
(
u'"z
\
ud834
x"'
,
u'z
\
ud834
x'
)
def
test_bad_escapes
(
self
):
scanstring
=
self
.
json
.
decoder
.
scanstring
bad_escapes
=
[
'"
\
\
"'
,
'"
\
\
x"'
,
'"
\
\
u"'
,
'"
\
\
u0"'
,
'"
\
\
u01"'
,
'"
\
\
u012"'
,
'"
\
\
uz012"'
,
'"
\
\
u0z12"'
,
'"
\
\
u01z2"'
,
'"
\
\
u012z"'
,
'"
\
\
u0x12"'
,
'"
\
\
u0X12"'
,
'"
\
\
ud834
\
\
"'
,
'"
\
\
ud834
\
\
u"'
,
'"
\
\
ud834
\
\
ud"'
,
'"
\
\
ud834
\
\
udd"'
,
'"
\
\
ud834
\
\
udd2"'
,
'"
\
\
ud834
\
\
uzdd2"'
,
'"
\
\
ud834
\
\
udzd2"'
,
'"
\
\
ud834
\
\
uddz2"'
,
'"
\
\
ud834
\
\
udd2z"'
,
'"
\
\
ud834
\
\
u0x20"'
,
'"
\
\
ud834
\
\
u0X20"'
,
]
for
s
in
bad_escapes
:
with
self
.
assertRaises
(
ValueError
):
scanstring
(
s
,
1
,
None
,
True
)
def
test_issue3623
(
self
):
self
.
assertRaises
(
ValueError
,
self
.
json
.
decoder
.
scanstring
,
b"xxx"
,
1
,
"xxx"
)
...
...
Misc/NEWS
View file @
dafda9b0
...
...
@@ -15,6 +15,8 @@ Core and Builtins
Library
-------
- Issue #11489: JSON decoder now accepts lone surrogates.
- Fix test.test_support.bind_port() to not cause an error when Python was
compiled on a system with SO_REUSEPORT defined in the headers but run on
a system with an OS kernel that does not support that new socket option.
...
...
Modules/_json.c
View file @
dafda9b0
...
...
@@ -524,16 +524,10 @@ scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_s
}
#ifdef Py_UNICODE_WIDE
/* Surrogate pair */
if
((
c
&
0xfc00
)
==
0xd800
)
{
if
((
c
&
0xfc00
)
==
0xd800
&&
end
+
6
<
len
&&
buf
[
next
++
]
==
'\\'
&&
buf
[
next
++
]
==
'u'
)
{
Py_UNICODE
c2
=
0
;
if
(
end
+
6
>=
len
)
{
raise_errmsg
(
"Unpaired high surrogate"
,
pystr
,
end
-
5
);
goto
bail
;
}
if
(
buf
[
next
++
]
!=
'\\'
||
buf
[
next
++
]
!=
'u'
)
{
raise_errmsg
(
"Unpaired high surrogate"
,
pystr
,
end
-
5
);
goto
bail
;
}
end
+=
6
;
/* Decode 4 hex digits */
for
(;
next
<
end
;
next
++
)
{
...
...
@@ -554,15 +548,10 @@ scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_s
goto
bail
;
}
}
if
((
c2
&
0xfc00
)
!=
0xdc00
)
{
raise_errmsg
(
"Unpaired high surrogate"
,
pystr
,
end
-
5
);
goto
bail
;
}
c
=
0x10000
+
(((
c
-
0xd800
)
<<
10
)
|
(
c2
-
0xdc00
));
}
else
if
((
c
&
0xfc00
)
==
0xdc00
)
{
raise_errmsg
(
"Unpaired low surrogate"
,
pystr
,
end
-
5
);
goto
bail
;
if
((
c2
&
0xfc00
)
==
0xdc00
)
c
=
0x10000
+
(((
c
-
0xd800
)
<<
10
)
|
(
c2
-
0xdc00
));
else
end
-=
6
;
}
#endif
}
...
...
@@ -703,16 +692,9 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next
}
#ifdef Py_UNICODE_WIDE
/* Surrogate pair */
if
((
c
&
0xfc00
)
==
0xd800
)
{
if
((
c
&
0xfc00
)
==
0xd800
&&
end
+
6
<
len
&&
buf
[
next
++
]
==
'\\'
&&
buf
[
next
++
]
==
'u'
)
{
Py_UNICODE
c2
=
0
;
if
(
end
+
6
>=
len
)
{
raise_errmsg
(
"Unpaired high surrogate"
,
pystr
,
end
-
5
);
goto
bail
;
}
if
(
buf
[
next
++
]
!=
'\\'
||
buf
[
next
++
]
!=
'u'
)
{
raise_errmsg
(
"Unpaired high surrogate"
,
pystr
,
end
-
5
);
goto
bail
;
}
end
+=
6
;
/* Decode 4 hex digits */
for
(;
next
<
end
;
next
++
)
{
...
...
@@ -733,15 +715,10 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next
goto
bail
;
}
}
if
((
c2
&
0xfc00
)
!=
0xdc00
)
{
raise_errmsg
(
"Unpaired high surrogate"
,
pystr
,
end
-
5
);
goto
bail
;
}
c
=
0x10000
+
(((
c
-
0xd800
)
<<
10
)
|
(
c2
-
0xdc00
));
}
else
if
((
c
&
0xfc00
)
==
0xdc00
)
{
raise_errmsg
(
"Unpaired low surrogate"
,
pystr
,
end
-
5
);
goto
bail
;
if
((
c2
&
0xfc00
)
==
0xdc00
)
c
=
0x10000
+
(((
c
-
0xd800
)
<<
10
)
|
(
c2
-
0xdc00
));
else
end
-=
6
;
}
#endif
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment