Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
C
cython
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Gwenaël Samain
cython
Commits
3f93988f
Commit
3f93988f
authored
17 years ago
by
Stefan Behnel
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
initial support for unicode literals in UTF-8
parent
79f741da
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
25 additions
and
9 deletions
+25
-9
Cython/Compiler/Lexicon.py
Cython/Compiler/Lexicon.py
+1
-1
Cython/Compiler/ModuleNode.py
Cython/Compiler/ModuleNode.py
+6
-3
Cython/Compiler/Nodes.py
Cython/Compiler/Nodes.py
+6
-2
Cython/Compiler/Parsing.py
Cython/Compiler/Parsing.py
+10
-3
Cython/Compiler/PyrexTypes.py
Cython/Compiler/PyrexTypes.py
+2
-0
No files found.
Cython/Compiler/Lexicon.py
View file @
3f93988f
...
@@ -5,7 +5,7 @@
...
@@ -5,7 +5,7 @@
# to be rebuilt next time pyrexc is run.
# to be rebuilt next time pyrexc is run.
#
#
string_prefixes
=
"cCrR"
string_prefixes
=
"cCrR
uU
"
def
make_lexicon
():
def
make_lexicon
():
from
Cython.Plex
import
\
from
Cython.Plex
import
\
...
...
This diff is collapsed.
Click to expand it.
Cython/Compiler/ModuleNode.py
View file @
3f93988f
...
@@ -1071,13 +1071,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
...
@@ -1071,13 +1071,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
"static __Pyx_StringTabEntry %s[] = {"
%
"static __Pyx_StringTabEntry %s[] = {"
%
Naming
.
stringtab_cname
)
Naming
.
stringtab_cname
)
for
entry
in
entries
:
for
entry
in
entries
:
print
repr
(
entry
.
init
),
type
(
entry
.
init
)
code
.
putln
(
code
.
putln
(
"{&%s, %s, sizeof(%s)},"
%
(
"{&%s, %s, sizeof(%s)
, %d
},"
%
(
entry
.
pystring_cname
,
entry
.
pystring_cname
,
entry
.
cname
,
entry
.
cname
,
entry
.
cname
))
entry
.
cname
,
isinstance
(
entry
.
init
,
unicode
)
))
code
.
putln
(
code
.
putln
(
"{0, 0, 0}"
)
"{0, 0, 0
, 0
}"
)
code
.
putln
(
code
.
putln
(
"};"
)
"};"
)
...
...
This diff is collapsed.
Click to expand it.
Cython/Compiler/Nodes.py
View file @
3f93988f
...
@@ -2600,7 +2600,7 @@ utility_function_predeclarations = \
...
@@ -2600,7 +2600,7 @@ utility_function_predeclarations = \
typedef struct {const char *s; const void **p;} __Pyx_CApiTabEntry; /*proto*/
typedef struct {const char *s; const void **p;} __Pyx_CApiTabEntry; /*proto*/
typedef struct {PyObject **p; char *s;} __Pyx_InternTabEntry; /*proto*/
typedef struct {PyObject **p; char *s;} __Pyx_InternTabEntry; /*proto*/
typedef struct {PyObject **p; char *s; long n;} __Pyx_StringTabEntry; /*proto*/
typedef struct {PyObject **p; char *s; long n;
int is_unicode;
} __Pyx_StringTabEntry; /*proto*/
#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False))
#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False))
static INLINE int __Pyx_PyObject_IsTrue(PyObject* x) {
static INLINE int __Pyx_PyObject_IsTrue(PyObject* x) {
...
@@ -3104,7 +3104,11 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/
...
@@ -3104,7 +3104,11 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/
"""
,
"""
"""
,
"""
static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) {
static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) {
while (t->p) {
while (t->p) {
*t->p = PyString_FromStringAndSize(t->s, t->n - 1);
if (t->is_unicode) {
*t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL);
} else {
*t->p = PyString_FromStringAndSize(t->s, t->n - 1);
}
if (!*t->p)
if (!*t->p)
return -1;
return -1;
++t;
++t;
...
...
This diff is collapsed.
Click to expand it.
Cython/Compiler/Parsing.py
View file @
3f93988f
...
@@ -493,7 +493,7 @@ def p_opt_string_literal(s):
...
@@ -493,7 +493,7 @@ def p_opt_string_literal(s):
def
p_string_literal
(
s
):
def
p_string_literal
(
s
):
# A single string or char literal.
# A single string or char literal.
# Returns (kind, value) where kind in ('', 'c', 'r')
# Returns (kind, value) where kind in ('', 'c', 'r'
, 'u'
)
if
s
.
sy
==
'STRING'
:
if
s
.
sy
==
'STRING'
:
value
=
unquote
(
s
.
systring
)
value
=
unquote
(
s
.
systring
)
s
.
next
()
s
.
next
()
...
@@ -502,7 +502,7 @@ def p_string_literal(s):
...
@@ -502,7 +502,7 @@ def p_string_literal(s):
pos
=
s
.
position
()
pos
=
s
.
position
()
#is_raw = s.systring[:1].lower() == "r"
#is_raw = s.systring[:1].lower() == "r"
kind
=
s
.
systring
[:
1
].
lower
()
kind
=
s
.
systring
[:
1
].
lower
()
if
kind
not
in
"cr"
:
if
kind
not
in
"cr
u
"
:
kind
=
''
kind
=
''
chars
=
[]
chars
=
[]
while
1
:
while
1
:
...
@@ -513,6 +513,8 @@ def p_string_literal(s):
...
@@ -513,6 +513,8 @@ def p_string_literal(s):
systr
=
s
.
systring
systr
=
s
.
systring
if
len
(
systr
)
==
1
and
systr
in
"'
\
"
\
n
"
:
if
len
(
systr
)
==
1
and
systr
in
"'
\
"
\
n
"
:
chars
.
append
(
'
\
\
'
)
chars
.
append
(
'
\
\
'
)
if
kind
==
'u'
and
not
isinstance
(
systr
,
unicode
):
systr
=
systr
.
decode
(
"UTF-8"
)
chars
.
append
(
systr
)
chars
.
append
(
systr
)
elif
sy
==
'ESCAPE'
:
elif
sy
==
'ESCAPE'
:
systr
=
s
.
systring
systr
=
s
.
systring
...
@@ -533,6 +535,8 @@ def p_string_literal(s):
...
@@ -533,6 +535,8 @@ def p_string_literal(s):
chars
.
append
(
'
\
\
x0'
+
systr
[
2
:])
chars
.
append
(
'
\
\
x0'
+
systr
[
2
:])
elif
c
==
'
\
n
'
:
elif
c
==
'
\
n
'
:
pass
pass
elif
c
==
'u'
:
chars
.
append
(
systr
)
else
:
else
:
chars
.
append
(
r'\\'
+
systr
[
1
:])
chars
.
append
(
r'\\'
+
systr
[
1
:])
elif
sy
==
'NEWLINE'
:
elif
sy
==
'NEWLINE'
:
...
@@ -546,7 +550,10 @@ def p_string_literal(s):
...
@@ -546,7 +550,10 @@ def p_string_literal(s):
"Unexpected token %r:%r in string literal"
%
"Unexpected token %r:%r in string literal"
%
(
sy
,
s
.
systring
))
(
sy
,
s
.
systring
))
s
.
next
()
s
.
next
()
value
=
join
(
chars
,
''
)
if
kind
==
'u'
:
value
=
u''
.
join
(
chars
)
else
:
value
=
''
.
join
(
chars
)
#print "p_string_literal: value =", repr(value) ###
#print "p_string_literal: value =", repr(value) ###
return
kind
,
value
return
kind
,
value
...
...
This diff is collapsed.
Click to expand it.
Cython/Compiler/PyrexTypes.py
View file @
3f93988f
...
@@ -705,6 +705,8 @@ class CStringType:
...
@@ -705,6 +705,8 @@ class CStringType:
from_py_function
=
"PyString_AsString"
from_py_function
=
"PyString_AsString"
def
literal_code
(
self
,
value
):
def
literal_code
(
self
,
value
):
if
isinstance
(
value
,
unicode
):
value
=
value
.
encode
(
"UTF-8"
)
return
'"%s"'
%
value
return
'"%s"'
%
value
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment