Commit be8d3798 authored by Stefan Behnel's avatar Stefan Behnel

support for some Python 3 (or 2.6+) syntax features (found by test_grammar.py in Py3.1.1):

- oct/bin notation: 0o12345, 0b10101
- function annotations (only pure syntax support, not currently used)
also: allow decorators on inner functions
parent 9ea6a398
......@@ -19,6 +19,7 @@ import Builtin
import Symtab
import Options
from Annotate import AnnotationItem
from Cython import Utils
from Cython.Debugging import print_call_chain
from DebugFlags import debug_disposal_code, debug_temp_alloc, \
......@@ -771,16 +772,23 @@ class IntNode(ConstNode):
self.result_code = self.get_constant_c_result_code()
def get_constant_c_result_code(self):
return str(self.value) + self.unsigned + self.longness
value = self.value
if isinstance(value, basestring) and len(value) > 2:
# must convert C-incompatible Py3 oct/bin notations
if value[1] in 'oO':
value = value[0] + value[2:] # '0o123' => '0123'
elif value[1] in 'bB':
value = int(value[2:], 2)
return str(value) + self.unsigned + self.longness
def calculate_result_code(self):
return self.result_code
def calculate_constant_result(self):
self.constant_result = int(self.value, 0)
self.constant_result = Utils.str_to_number(self.value)
def compile_time_value(self, denv):
return int(self.value, 0)
return Utils.str_to_number(self.value)
class FloatNode(ConstNode):
......@@ -966,10 +974,10 @@ class LongNode(AtomicExprNode):
type = py_object_type
def calculate_constant_result(self):
self.constant_result = long(self.value)
self.constant_result = Utils.str_to_number(self.value)
def compile_time_value(self, denv):
return long(self.value)
return Utils.str_to_number(self.value)
def analyse_types(self, env):
self.is_temp = 1
......
......@@ -17,6 +17,7 @@ def make_lexicon():
letter = Any("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_")
digit = Any("0123456789")
bindigit = Any("01")
octdigit = Any("01234567")
hexdigit = Any("0123456789ABCDEFabcdef")
indentation = Bol + Rep(Any(" \t"))
......@@ -27,7 +28,9 @@ def make_lexicon():
decimal_fract = (decimal + dot + Opt(decimal)) | (dot + decimal)
name = letter + Rep(letter | digit)
intconst = decimal | (Str("0x") + Rep1(hexdigit))
intconst = decimal | (Str("0") + ((Any("Xx") + Rep1(hexdigit)) |
(Any("Oo") + Rep1(octdigit)) |
(Any("Bb") + Rep1(bindigit)) ))
intsuffix = (Opt(Any("Uu")) + Opt(Any("Ll")) + Opt(Any("Ll"))) | (Opt(Any("Ll")) + Opt(Any("Ll")) + Opt(Any("Uu")))
intliteral = intconst + intsuffix
fltconst = (decimal_fract + Opt(exponent)) | (decimal + exponent)
......@@ -75,7 +78,7 @@ def make_lexicon():
punct = Any(":,;+-*/|&<>=.%`~^?")
diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", "//",
"+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=",
"<<=", ">>=", "**=", "//=")
"<<=", ">>=", "**=", "//=", "->")
spaces = Rep1(Any(" \t\f"))
escaped_newline = Str("\\\n")
lineterm = Eol + Opt(Str("\n"))
......
......@@ -597,6 +597,7 @@ class CArgDeclNode(Node):
# not_none boolean Tagged with 'not None'
# default ExprNode or None
# default_value PyObjectConst constant for default value
# annotation ExprNode or None Py3 function arg annotation
# is_self_arg boolean Is the "self" arg of an extension type method
# is_kw_only boolean Is a keyword-only argument
......@@ -607,6 +608,7 @@ class CArgDeclNode(Node):
type = None
name_declarator = None
default_value = None
annotation = None
def analyse(self, env, nonempty = 0):
#print "CArgDeclNode.analyse: is_self_arg =", self.is_self_arg ###
......@@ -1605,6 +1607,7 @@ class PyArgDeclNode(Node):
#
# name string
# entry Symtab.Entry
# annotation ExprNode or None Py3 argument annotation
child_attrs = []
......@@ -1626,6 +1629,8 @@ class DefNode(FuncDefNode):
# starstar_arg PyArgDeclNode or None ** argument
# doc EncodedString or None
# body StatListNode
# return_type_annotation
# ExprNode or None the Py3 return type annotation
#
# The following subnode is constructed internally
# when the def statement is inside a Python class definition.
......@@ -1641,6 +1646,7 @@ class DefNode(FuncDefNode):
reqd_kw_flags_cname = "0"
is_wrapper = 0
decorators = None
return_type_annotation = None
entry = None
acquire_gil = 0
......@@ -1685,7 +1691,7 @@ class DefNode(FuncDefNode):
cfunc_type = cfunc.type
if len(self.args) != len(cfunc_type.args) or cfunc_type.has_varargs:
error(self.pos, "wrong number of arguments")
error(declarator.pos, "previous declaration here")
error(cfunc.pos, "previous declaration here")
for formal_arg, type_arg in zip(self.args, cfunc_type.args):
name_declarator, type = formal_arg.analyse(cfunc.scope, nonempty=1)
if type is None or type is PyrexTypes.py_object_type or formal_arg.is_self:
......
......@@ -126,9 +126,9 @@ cpdef p_nogil(PyrexScanner s)
cpdef p_with_gil(PyrexScanner s)
cpdef p_exception_value_clause(PyrexScanner s)
cpdef p_c_arg_list(PyrexScanner s, ctx = *, bint in_pyfunc = *, bint cmethod_flag = *,
bint nonempty_declarators = *, bint kw_only = *)
bint nonempty_declarators = *, bint kw_only = *, bint annotated = *)
cpdef p_optional_ellipsis(PyrexScanner s)
cpdef p_c_arg_decl(PyrexScanner s, ctx, in_pyfunc, bint cmethod_flag = *, bint nonempty = *, bint kw_only = *)
cpdef p_c_arg_decl(PyrexScanner s, ctx, in_pyfunc, bint cmethod_flag = *, bint nonempty = *, bint kw_only = *, bint annotated = *)
cpdef p_api(PyrexScanner s)
cpdef p_cdef_statement(PyrexScanner s, ctx)
cpdef p_cdef_block(PyrexScanner s, ctx)
......@@ -143,7 +143,7 @@ cpdef p_c_func_or_var_declaration(PyrexScanner s, pos, ctx)
cpdef p_ctypedef_statement(PyrexScanner s, ctx)
cpdef p_decorators(PyrexScanner s)
cpdef p_def_statement(PyrexScanner s, list decorators = *)
cpdef p_varargslist(PyrexScanner s, terminator=*)
cpdef p_varargslist(PyrexScanner s, terminator=*, bint annotated = *)
cpdef p_py_arg_decl(PyrexScanner s)
cpdef p_class_statement(PyrexScanner s, decorators)
cpdef p_c_class_definition(PyrexScanner s, pos, ctx)
......
......@@ -113,7 +113,8 @@ def p_lambdef(s, allow_conditional=True):
args = []
star_arg = starstar_arg = None
else:
args, star_arg, starstar_arg = p_varargslist(s, terminator=':')
args, star_arg, starstar_arg = p_varargslist(
s, terminator=':', annotated=False)
s.expect(':')
if allow_conditional:
expr = p_simple_expr(s)
......@@ -1215,6 +1216,8 @@ def p_from_import_statement(s, first_statement = 0):
imported_names = [p_imported_name(s, is_cimport)]
while s.sy == ',':
s.next()
if is_parenthesized and s.sy == ')':
break
imported_names.append(p_imported_name(s, is_cimport))
if is_parenthesized:
s.expect(')')
......@@ -1464,7 +1467,7 @@ def p_except_clause(s):
exc_value = None
if s.sy != ':':
exc_type = p_simple_expr(s)
if s.sy == ',':
if s.sy == ',' or (s.sy == 'IDENT' and s.systring == 'as'):
s.next()
exc_value = p_simple_expr(s)
body = p_suite(s)
......@@ -1625,7 +1628,7 @@ def p_statement(s, ctx, first_statement = 0):
elif s.sy == 'IF':
return p_IF_statement(s, ctx)
elif s.sy == 'DECORATOR':
if ctx.level not in ('module', 'class', 'c_class', 'property', 'module_pxd', 'c_class_pxd'):
if ctx.level not in ('module', 'class', 'c_class', 'function', 'property', 'module_pxd', 'c_class_pxd'):
print ctx.level
s.error('decorator not allowed here')
s.level = ctx.level
......@@ -2143,14 +2146,15 @@ def p_exception_value_clause(s):
c_arg_list_terminators = ('*', '**', '.', ')')
def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,
nonempty_declarators = 0, kw_only = 0):
nonempty_declarators = 0, kw_only = 0, annotated = 1):
# Comma-separated list of C argument declarations, possibly empty.
# May have a trailing comma.
args = []
is_self_arg = cmethod_flag
while s.sy not in c_arg_list_terminators:
args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,
nonempty = nonempty_declarators, kw_only = kw_only))
nonempty = nonempty_declarators, kw_only = kw_only,
annotated = annotated))
if s.sy != ',':
break
s.next()
......@@ -2164,10 +2168,12 @@ def p_optional_ellipsis(s):
else:
return 0
def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0):
def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0,
kw_only = 0, annotated = 1):
pos = s.position()
not_none = 0
default = None
annotation = None
base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)
declarator = p_c_declarator(s, ctx, nonempty = nonempty)
if s.sy == 'not':
......@@ -2179,6 +2185,9 @@ def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0)
if not in_pyfunc:
error(pos, "'not None' only allowed in Python functions")
not_none = 1
if annotated and s.sy == ':':
s.next()
annotation = p_simple_expr(s)
if s.sy == '=':
s.next()
if 'pxd' in s.level:
......@@ -2193,6 +2202,7 @@ def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0)
declarator = declarator,
not_none = not_none,
default = default,
annotation = annotation,
kw_only = kw_only)
def p_api(s):
......@@ -2458,13 +2468,19 @@ def p_def_statement(s, decorators=None):
s.expect(')')
if p_nogil(s):
error(s.pos, "Python function cannot be declared nogil")
return_type_annotation = None
if s.sy == '->':
s.next()
return_type_annotation = p_simple_expr(s)
doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)
return Nodes.DefNode(pos, name = name, args = args,
star_arg = star_arg, starstar_arg = starstar_arg,
doc = doc, body = body, decorators = decorators)
doc = doc, body = body, decorators = decorators,
return_type_annotation = return_type_annotation)
def p_varargslist(s, terminator=')'):
args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1)
def p_varargslist(s, terminator=')', annotated=1):
args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1,
annotated = annotated)
star_arg = None
starstar_arg = None
if s.sy == '*':
......@@ -2485,7 +2501,11 @@ def p_varargslist(s, terminator=')'):
def p_py_arg_decl(s):
pos = s.position()
name = p_ident(s)
return Nodes.PyArgDeclNode(pos, name = name)
annotation = None
if s.sy == ':':
s.next()
annotation = p_simple_expr(s)
return Nodes.PyArgDeclNode(pos, name = name, annotation = annotation)
def p_class_statement(s, decorators):
# s.sy == 'class'
......
......@@ -82,16 +82,31 @@ def open_source_file(source_filename, mode="rU"):
encoding = detect_file_encoding(source_filename)
return codecs.open(source_filename, mode=mode, encoding=encoding)
def long_literal(value):
if isinstance(value, basestring):
def str_to_number(value):
# note: this expects a string as input that was accepted by the
# parser already
if len(value) < 2:
value = int(value)
elif value[0] == 0:
value = int(value, 8)
elif value[1] in 'xX':
value = int(value, 0)
elif value[0] == '0':
if value[1] in 'xX':
# hex notation ('0x1AF')
value = int(value[2:], 16)
elif value[1] in 'oO':
# Py3 octal notation ('0o136')
value = int(value[2:], 8)
elif value[1] in 'bB':
# Py3 binary notation ('0b101')
value = int(value[2:], 2)
else:
# Py2 octal notation ('0136')
value = int(value, 8)
else:
value = int(value)
value = int(value, 0)
return value
def long_literal(value):
if isinstance(value, basestring):
value = str_to_number(value)
return not -2**31 <= value < 2**31
def none_or_sub(s, data):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment