Skip to content

C
cython
Commit 2e3f542d authored by Vitja Makarov's avatar Vitja Makarov
Browse files
Options

Merge branch 'master' of github.com:vitek/cython

parents c2e89fdd 37355543
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 455 additions and 249 deletions
Cython/Compiler/CmdLine.py
View file @ 2e3f542d
......@@ -10,15 +10,15 @@ usage = """\
Cython (http://cython.org) is a compiler for code written in the
Cython language. Cython is based on Pyrex by Greg Ewing.
Usage: cython [options] sourcefile.pyx ...
Usage: cython [options] sourcefile.{pyx,py} ...
Options:
-V, --version Display version number of cython compiler
-l, --create-listing Write error messages to a listing file
-I, --include-dir <directory> Search for include files in named directory
(multiply include directories are allowed).
(multiple include directories are allowed).
-o, --output-file <filename> Specify name of generated C file
-t, --timestamps Only compile newer source files (implied with -r)
-t, --timestamps Only compile newer source files
-f, --force Compile all source files (overrides implied -t)
-q, --quiet Don't print module names in recursive mode
-v, --verbose Be verbose, print file names on multiple compilation
......@@ -30,11 +30,11 @@ Options:
are searched from)
--gdb Output debug information for cygdb
-D, --no-docstrings Remove docstrings.
-D, --no-docstrings Strip docstrings from the compiled module.
-a, --annotate Produce a colorized HTML version of the source.
--line-directives Produce #line directives pointing to the .pyx source
--cplus Output a c++ rather than c file.
--embed Embed the Python interpreter in a main() method.
--cplus Output a C++ rather than C file.
--embed Generate a main() function that embeds the Python interpreter.
-2 Compile based on Python-2 syntax and code semantics.
-3 Compile based on Python-3 syntax and code semantics.
--fast-fail Abort the compilation on the first error
......@@ -42,7 +42,7 @@ Options:
"""
# The following is broken http://trac.cython.org/cython_trac/ticket/379
# -r, --recursive Recursively find and compile dependencies
# -r, --recursive Recursively find and compile dependencies (implies -t)
#The following experimental options are supported only on MacOSX:
......@@ -143,6 +143,9 @@ def parse_command_line(args):
else:
sys.stderr.write("Unknown debug flag: %s\n" % option)
bad_usage()
elif option in ('-h', '--help'):
sys.stdout.write(usage)
sys.exit(0)
else:
sys.stderr.write("Unknown compiler flag: %s\n" % option)
sys.exit(1)
......
Cython/Compiler/Code.py
View file @ 2e3f542d
......@@ -1361,7 +1361,7 @@ class CCodeWriter(object):
return self.globalstate.lookup_filename(filename)
def put_declare_refcount_context(self):
self.putln('__Pyx_RefNannyDeclareContext')
self.putln('__Pyx_RefNannyDeclareContext;')
def put_setup_refcount_context(self, name):
self.putln('__Pyx_RefNannySetupContext("%s");' % name)
......
Cython/Compiler/ExprNodes.py
View file @ 2e3f542d
......@@ -954,7 +954,8 @@ class BytesNode(ConstNode):
#
# value BytesLiteral
type = PyrexTypes.c_char_ptr_type
# start off as Python 'bytes' to support len() in O(1)
type = bytes_type
def compile_time_value(self, denv):
return self.value
......@@ -975,11 +976,13 @@ class BytesNode(ConstNode):
return len(self.value) == 1
def coerce_to_boolean(self, env):
# This is special because we start off as a C char*. Testing
# that for truth directly would yield the wrong result.
# This is special because testing a C char* for truth directly
# would yield the wrong result.
return BoolNode(self.pos, value=bool(self.value))
def coerce_to(self, dst_type, env):
if self.type == dst_type:
return self
if dst_type.is_int:
if not self.can_coerce_to_char_literal():
error(self.pos, "Only single-character string literals can be coerced into ints.")
......@@ -990,21 +993,20 @@ class BytesNode(ConstNode):
return CharNode(self.pos, value=self.value)
node = BytesNode(self.pos, value=self.value)
if dst_type == PyrexTypes.c_char_ptr_type:
node.type = PyrexTypes.c_char_ptr_type
if dst_type.is_pyobject:
if dst_type in (py_object_type, Builtin.bytes_type):
node.type = Builtin.bytes_type
else:
self.check_for_coercion_error(dst_type, fail=True)
return node
elif dst_type == PyrexTypes.c_char_ptr_type:
node.type = dst_type
return node
elif dst_type == PyrexTypes.c_uchar_ptr_type:
node.type = PyrexTypes.c_char_ptr_type
return CastNode(node, PyrexTypes.c_uchar_ptr_type)
if not self.type.is_pyobject:
if dst_type in (py_object_type, Builtin.bytes_type):
node.type = Builtin.bytes_type
elif dst_type.is_pyobject:
self.fail_assignment(dst_type)
return self
elif dst_type.is_pyobject and dst_type is not py_object_type:
self.check_for_coercion_error(dst_type, fail=True)
elif dst_type.assignable_from(PyrexTypes.c_char_ptr_type):
node.type = dst_type
return node
# We still need to perform normal coerce_to processing on the
......@@ -1012,11 +1014,6 @@ class BytesNode(ConstNode):
# in which case a type test node will be needed.
return ConstNode.coerce_to(node, dst_type, env)
def as_py_string_node(self, env):
# Return a new BytesNode with the same value as this node
# but whose type is a Python type instead of a C type.
return BytesNode(self.pos, value = self.value, type = Builtin.bytes_type)
def generate_evaluation_code(self, code):
if self.type.is_pyobject:
self.result_code = code.get_py_string_const(self.value)
......@@ -2043,7 +2040,7 @@ class IndexNode(ExprNode):
return None
def type_dependencies(self, env):
return self.base.type_dependencies(env)
return self.base.type_dependencies(env) + self.index.type_dependencies(env)
def infer_type(self, env):
base_type = self.base.infer_type(env)
......@@ -2969,9 +2966,14 @@ class SimpleCallNode(CallNode):
arg = arg.coerce_to_temp(env)
self.args[i] = arg
for i in range(max_nargs, actual_nargs):
if self.args[i].type.is_pyobject:
error(self.args[i].pos,
"Python object cannot be passed as a varargs parameter")
arg = self.args[i]
if arg.type.is_pyobject:
arg_ctype = arg.type.default_coerced_ctype()
if arg_ctype is None:
error(self.args[i].pos,
"Python object cannot be passed as a varargs parameter")
else:
self.args[i] = arg.coerce_to(arg_ctype, env)
# Calc result type and code fragment
if isinstance(self.function, NewExprNode):
self.type = PyrexTypes.CPtrType(self.function.class_type)
......@@ -4341,15 +4343,18 @@ class DictComprehensionAppendNode(ComprehensionAppendNode):
self.value_expr.annotate(code)
class GeneratorExpressionNode(ScopedExprNode):
# A generator expression, e.g. (i for i in range(10))
#
# Result is a generator.
class InlinedGeneratorExpressionNode(ScopedExprNode):
# An inlined generator expression for which the result is
# calculated inside of the loop. This will only be created by
# transforms when replacing builtin calls on generator
# expressions.
#
# loop ForStatNode the for-loop, containing a YieldExprNode
# loop ForStatNode the for-loop, not containing any YieldExprNodes
# result_node ResultRefNode the reference to the result value temp
# orig_func String the name of the builtin function this node replaces
child_attrs = ["loop"]
loop_analysed = False
type = py_object_type
def analyse_scoped_declarations(self, env):
......@@ -4360,30 +4365,12 @@ class GeneratorExpressionNode(ScopedExprNode):
self.loop.analyse_expressions(env)
self.is_temp = True
def analyse_scoped_expressions(self, env):
if self.has_local_scope:
self.loop.analyse_expressions(env)
def may_be_none(self):
return False
def annotate(self, code):
self.loop.annotate(code)
class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
# An inlined generator expression for which the result is
# calculated inside of the loop. This will only be created by
# transforms when replacing builtin calls on generator
# expressions.
#
# loop ForStatNode the for-loop, not containing any YieldExprNodes
# result_node ResultRefNode the reference to the result value temp
# orig_func String the name of the builtin function this node replaces
child_attrs = ["loop"]
loop_analysed = False
def infer_type(self, env):
return self.result_node.infer_type(env)
......@@ -4396,7 +4383,8 @@ class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
def analyse_scoped_expressions(self, env):
self.loop_analysed = True
GeneratorExpressionNode.analyse_scoped_expressions(self, env)
if self.has_local_scope:
self.loop.analyse_expressions(env)
def coerce_to(self, dst_type, env):
if self.orig_func == 'sum' and dst_type.is_numeric and not self.loop_analysed:
......@@ -4407,7 +4395,7 @@ class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
# assignments.
self.result_node.type = self.type = dst_type
return self
return GeneratorExpressionNode.coerce_to(self, dst_type, env)
return super(InlinedGeneratorExpressionNode, self).coerce_to(dst_type, env)
def generate_result_code(self, code):
self.result_node.result_code = self.result()
......@@ -4955,6 +4943,34 @@ class LambdaNode(InnerFunctionNode):
env.add_lambda_def(self.def_node)
class GeneratorExpressionNode(LambdaNode):
# A generator expression, e.g. (i for i in range(10))
#
# Result is a generator.
#
# loop ForStatNode the for-loop, containing a YieldExprNode
# def_node DefNode the underlying generator 'def' node
name = StringEncoding.EncodedString('genexpr')
binding = False
def analyse_declarations(self, env):
# XXX: dirty hack to disable assignment synthesis
self.def_node.needs_assignment_synthesis = lambda *args, **kwargs: False
self.def_node.analyse_declarations(env)
#super(GeneratorExpressionNode, self).analyse_declarations(env)
env.add_lambda_def(self.def_node)
def generate_result_code(self, code):
code.putln(
'%s = %s(%s, NULL); %s' % (
self.result(),
self.def_node.entry.func_cname,
self.self_result_code(),
code.error_goto_if_null(self.result(), self.pos)))
code.put_gotref(self.py_result())
class YieldExprNode(ExprNode):
# Yield expression node
#
......
Cython/Compiler/ModuleNode.py
View file @ 2e3f542d
......@@ -2051,12 +2051,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
code.error_goto_if_null(type.typeptr_cname, pos))
self.use_type_import_utility_code(env)
if type.vtabptr_cname:
code.putln(
"if (__Pyx_GetVtable(%s->tp_dict, &%s) < 0) %s" % (
type.typeptr_cname,
type.vtabptr_cname,
code.error_goto(pos)))
env.use_utility_code(Nodes.get_vtable_utility_code)
code.putln("%s = (struct %s*)__Pyx_GetVtable(%s->tp_dict); %s" % (
type.vtabptr_cname,
type.vtabstruct_cname,
type.typeptr_cname,
code.error_goto_if_null(type.vtabptr_cname, pos)))
env.types_imported[type] = 1
py3_type_name_map = {'str' : 'bytes', 'unicode' : 'str'}
......@@ -2109,24 +2109,24 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
# a significant performance hit. (See trac #561.)
for func in entry.type.scope.pyfunc_entries:
if func.is_special and Options.docstrings and func.wrapperbase_cname:
code.putln("{");
code.putln("{")
code.putln(
'PyObject *wrapper = __Pyx_GetAttrString((PyObject *)&%s, "%s"); %s' % (
typeobj_cname,
func.name,
code.error_goto_if_null('wrapper', entry.pos)));
code.error_goto_if_null('wrapper', entry.pos)))
code.putln(
"if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) {");
"if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) {")
code.putln(
"%s = *((PyWrapperDescrObject *)wrapper)->d_base;" % (
func.wrapperbase_cname));
func.wrapperbase_cname))
code.putln(
"%s.doc = %s;" % (func.wrapperbase_cname, func.doc_cname));
"%s.doc = %s;" % (func.wrapperbase_cname, func.doc_cname))
code.putln(
"((PyWrapperDescrObject *)wrapper)->d_base = &%s;" % (
func.wrapperbase_cname));
code.putln("}");
code.putln("}");
func.wrapperbase_cname))
code.putln("}")
code.putln("}")
if type.vtable_cname:
code.putln(
"if (__Pyx_SetVtable(%s.tp_dict, %s) < 0) %s" % (
......
Cython/Compiler/Nodes.py
View file @ 2e3f542d
This diff is collapsed.
Cython/Compiler/Optimize.py
View file @ 2e3f542d
......@@ -1167,11 +1167,12 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform):
self.yield_nodes = []
visit_Node = Visitor.TreeVisitor.visitchildren
def visit_YieldExprNode(self, node):
# XXX: disable inlining while it's not back supported
def __visit_YieldExprNode(self, node):
self.yield_nodes.append(node)
self.visitchildren(node)
def visit_ExprStatNode(self, node):
def __visit_ExprStatNode(self, node):
self.visitchildren(node)
if node.expr in self.yield_nodes:
self.yield_stat_nodes[node.expr] = node
......
Cython/Compiler/ParseTreeTransforms.py
View file @ 2e3f542d
......@@ -183,6 +183,7 @@ class PostParse(ScopeTrackingTransform):
def visit_ModuleNode(self, node):
self.lambda_counter = 1
self.genexpr_counter = 1
return super(PostParse, self).visit_ModuleNode(node)
def visit_LambdaNode(self, node):
......@@ -190,14 +191,34 @@ class PostParse(ScopeTrackingTransform):
lambda_id = self.lambda_counter
self.lambda_counter += 1
node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
body = Nodes.ReturnStatNode(
node.result_expr.pos, value = node.result_expr)
collector = YieldNodeCollector()
collector.visitchildren(node.result_expr)
if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
body = ExprNodes.YieldExprNode(
node.result_expr.pos, arg=node.result_expr)
body = Nodes.ExprStatNode(node.result_expr.pos, expr=body)
else:
body = Nodes.ReturnStatNode(
node.result_expr.pos, value=node.result_expr)
node.def_node = Nodes.DefNode(
node.pos, name=node.name, lambda_name=node.lambda_name,
args=node.args, star_arg=node.star_arg,
starstar_arg=node.starstar_arg,
body=body)
body=body, doc=None)
self.visitchildren(node)
return node
def visit_GeneratorExpressionNode(self, node):
# unpack a generator expression into the corresponding DefNode
genexpr_id = self.genexpr_counter
self.genexpr_counter += 1
node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
node.def_node = Nodes.DefNode(node.pos, name=node.name,
doc=None,
args=[], star_arg=None,
starstar_arg=None,
body=node.loop)
self.visitchildren(node)
return node
......@@ -1317,7 +1338,7 @@ class YieldNodeCollector(TreeVisitor):
if self.has_return_value:
error(node.pos, "'yield' outside function")
self.yields.append(node)
node.label_num = len(self.yields)
self.visitchildren(node)
def visit_ReturnStatNode(self, node):
if node.value:
......@@ -1332,6 +1353,9 @@ class YieldNodeCollector(TreeVisitor):
def visit_DefNode(self, node):
pass
def visit_LambdaNode(self, node):
pass
def visit_GeneratorExpressionNode(self, node):
pass
......@@ -1352,11 +1376,22 @@ class MarkClosureVisitor(CythonTransform):
collector.visitchildren(node)
if collector.yields:
if collector.returns and not collector.has_return_value:
error(collector.returns[0].pos, "'return' inside generators not yet supported ")
node.is_generator = True
node.needs_closure = True
node.yields = collector.yields
for i, yield_expr in enumerate(collector.yields):
yield_expr.label_num = i + 1
gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
name=node.name,
body=node.body)
generator = Nodes.GeneratorDefNode(pos=node.pos,
name=node.name,
args=node.args,
star_arg=node.star_arg,
starstar_arg=node.starstar_arg,
doc=node.doc,
decorators=node.decorators,
gbody=gbody,
lambda_name=node.lambda_name)
return generator
return node
def visit_CFuncDefNode(self, node):
......@@ -1447,6 +1482,9 @@ class CreateClosureClasses(CythonTransform):
return from_closure, in_closure
def create_class_from_scope(self, node, target_module_scope, inner_node=None):
# skip generator body
if node.is_generator_body:
return
# move local variables into closure
if node.is_generator:
for entry in node.local_scope.entries.values():
......
Cython/Compiler/PyrexTypes.py
View file @ 2e3f542d
......@@ -353,6 +353,10 @@ class PyObjectType(PyrexType):
def can_coerce_to_pyobject(self, env):
return True
def default_coerced_ctype(self):
"The default C type that this Python type coerces to, or None."
return None
def assignable_from(self, src_type):
# except for pointers, conversion will be attempted
return not src_type.is_ptr or src_type.is_string
......@@ -403,7 +407,16 @@ class BuiltinObjectType(PyObjectType):
def __repr__(self):
return "<%s>"% self.cname
def default_coerced_ctype(self):
if self.name == 'bytes':
return c_char_ptr_type
elif self.name == 'bool':
return c_bint_type
elif self.name == 'float':
return c_double_type
return None
def assignable_from(self, src_type):
if isinstance(src_type, BuiltinObjectType):
return src_type.name == self.name
......
tests/bugs.txt
View file @ 2e3f542d
......@@ -10,7 +10,6 @@ cfunc_call_tuple_args_T408
compile.cpp_operators
cpp_templated_ctypedef
cpp_structs
genexpr_T491
with_statement_module_level_T536
function_as_method_T494
closure_inside_cdef_T554
......
tests/errors/e_generators.pyx
View file @ 2e3f542d
......@@ -6,10 +6,6 @@ def bar(a):
return 0
yield
def xxx():
yield
return
yield
class Foo:
......@@ -18,7 +14,6 @@ class Foo:
_ERRORS = u"""
3:4: 'return' with argument inside generator
7:4: 'yield' outside function
11:4: 'return' inside generators not yet supported
13:0: 'yield' not supported here
16:4: 'yield' not supported here
9:0: 'yield' not supported here
12:4: 'yield' not supported here
"""
tests/errors/string_assignments.pyx
View file @ 2e3f542d
......@@ -58,18 +58,18 @@ _ERRORS = u"""
30:22: Cannot convert Unicode string to 'bytes' implicitly, encoding required.
31:22: Cannot convert 'str' to 'bytes' implicitly. This is not portable.
33:17: Cannot assign type 'char *' to 'str object'
33:17: Cannot convert 'bytes' object to str implicitly. This is not portable to Py3.
34:19: Cannot convert 'bytes' object to str implicitly. This is not portable to Py3.
35:17: Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding.
36:19: Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding.
38:20: str objects do not support coercion to unicode, use a unicode string literal instead (u'')
39:22: str objects do not support coercion to unicode, use a unicode string literal instead (u'')
40:20: Cannot assign type 'char *' to 'unicode object'
40:20: Cannot convert 'bytes' object to unicode implicitly, decoding required
41:22: Cannot convert 'bytes' object to unicode implicitly, decoding required
42:22: Cannot convert 'char*' to unicode implicitly, decoding required
44:19: Cannot assign type 'str object' to 'tuple object'
45:18: Cannot assign type 'unicode object' to 'tuple object'
46:18: Cannot assign type 'char *' to 'tuple object'
46:18: Cannot assign type 'bytes object' to 'tuple object'
"""
tests/run/generators.pyx
View file @ 2e3f542d
......@@ -207,3 +207,95 @@ def generator_nonlocal():
yield x
return g
return f(1)
def test_nested(a, b, c):
"""
>>> obj = test_nested(1, 2, 3)
>>> [i() for i in obj]
[1, 2, 3, 4]
"""
def one():
return a
def two():
return b
def three():
return c
def new_closure(a, b):
def sum():
return a + b
return sum
yield one
yield two
yield three
yield new_closure(a, c)
def tolist(func):
def wrapper(*args, **kwargs):
return list(func(*args, **kwargs))
return wrapper
@tolist
def test_decorated(*args):
"""
>>> test_decorated(1, 2, 3)
[1, 2, 3]
"""
for i in args:
yield i
def test_return(a):
"""
>>> d = dict()
>>> obj = test_return(d)
>>> next(obj)
1
>>> next(obj)
Traceback (most recent call last):
StopIteration
>>> d['i_was_here']
True
"""
yield 1
a['i_was_here'] = True
return
def test_copied_yield(foo):
"""
>>> class Manager(object):
... def __enter__(self):
... return self
... def __exit__(self, type, value, tb):
... pass
>>> list(test_copied_yield(Manager()))
[1]
"""
with foo:
yield 1
def test_nested_yield():
"""
>>> obj = test_nested_yield()
>>> next(obj)
1
>>> obj.send(2)
2
>>> obj.send(3)
3
>>> obj.send(4)
Traceback (most recent call last):
StopIteration
"""
yield (yield (yield 1))
def test_inside_lambda():
"""
>>> obj = test_inside_lambda()()
>>> next(obj)
1
>>> obj.send('a')
2
>>> obj.send('b')
('a', 'b')
"""
return lambda:((yield 1), (yield 2))
tests/run/type_inference.pyx
View file @ 2e3f542d
......@@ -25,7 +25,7 @@ def simple():
xptrptr = &xptr
assert typeof(xptrptr) == "double **", typeof(xptrptr)
b = b"abc"
assert typeof(b) == "char *", typeof(b)
assert typeof(b) == "bytes object", typeof(b)
s = "abc"
assert typeof(s) == "str object", typeof(s)
u = u"xyz"
......@@ -57,7 +57,7 @@ def slicing():
>>> slicing()
"""
b = b"abc"
assert typeof(b) == "char *", typeof(b)
assert typeof(b) == "bytes object", typeof(b)
b1 = b[1:2]
assert typeof(b1) == "bytes object", typeof(b1)
b2 = b[1:2:2]
......@@ -92,9 +92,9 @@ def indexing():
>>> indexing()
"""
b = b"abc"
assert typeof(b) == "char *", typeof(b)
assert typeof(b) == "bytes object", typeof(b)
b1 = b[1]
assert typeof(b1) == "char", typeof(b1) # FIXME: Python object ??
assert typeof(b1) == "Python object", typeof(b1)
u = u"xyz"
assert typeof(u) == "unicode object", typeof(u)
u1 = u[1]
......@@ -479,3 +479,10 @@ def large_literals():
c, d = 10, 100000000000000000000000000000000
assert typeof(c) == "long", typeof(c)
assert typeof(d) == "Python object", typeof(d)
# Regression test for trac #638.
def bar(foo):
qux = foo
quux = foo[qux.baz]
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7