Skip to content

C
cython
Commit 2e3f542d authored by Vitja Makarov's avatar Vitja Makarov
Browse files
Options

Merge branch 'master' of github.com:vitek/cython

parents c2e89fdd 37355543
Show whitespace changes
Inline Side-by-side
Showing with 455 additions and 249 deletions
Cython/Compiler/CmdLine.py
View file @ 2e3f542d
......@@ -10,15 +10,15 @@ usage = """\
Cython (http://cython.org) is a compiler for code written in the
Cython language. Cython is based on Pyrex by Greg Ewing.
Usage: cython [options] sourcefile.pyx ...
Usage: cython [options] sourcefile.{pyx,py} ...
Options:
-V, --version Display version number of cython compiler
-l, --create-listing Write error messages to a listing file
-I, --include-dir <directory> Search for include files in named directory
(multiply include directories are allowed).
(multiple include directories are allowed).
-o, --output-file <filename> Specify name of generated C file
-t, --timestamps Only compile newer source files (implied with -r)
-t, --timestamps Only compile newer source files
-f, --force Compile all source files (overrides implied -t)
-q, --quiet Don't print module names in recursive mode
-v, --verbose Be verbose, print file names on multiple compilation
......@@ -30,11 +30,11 @@ Options:
are searched from)
--gdb Output debug information for cygdb
-D, --no-docstrings Remove docstrings.
-D, --no-docstrings Strip docstrings from the compiled module.
-a, --annotate Produce a colorized HTML version of the source.
--line-directives Produce #line directives pointing to the .pyx source
--cplus Output a c++ rather than c file.
--embed Embed the Python interpreter in a main() method.
--cplus Output a C++ rather than C file.
--embed Generate a main() function that embeds the Python interpreter.
-2 Compile based on Python-2 syntax and code semantics.
-3 Compile based on Python-3 syntax and code semantics.
--fast-fail Abort the compilation on the first error
......@@ -42,7 +42,7 @@ Options:
"""
# The following is broken http://trac.cython.org/cython_trac/ticket/379
# -r, --recursive Recursively find and compile dependencies
# -r, --recursive Recursively find and compile dependencies (implies -t)
#The following experimental options are supported only on MacOSX:
......@@ -143,6 +143,9 @@ def parse_command_line(args):
else:
sys.stderr.write("Unknown debug flag: %s\n" % option)
bad_usage()
elif option in ('-h', '--help'):
sys.stdout.write(usage)
sys.exit(0)
else:
sys.stderr.write("Unknown compiler flag: %s\n" % option)
sys.exit(1)
......
Cython/Compiler/Code.py
View file @ 2e3f542d
......@@ -1361,7 +1361,7 @@ class CCodeWriter(object):
return self.globalstate.lookup_filename(filename)
def put_declare_refcount_context(self):
self.putln('__Pyx_RefNannyDeclareContext')
self.putln('__Pyx_RefNannyDeclareContext;')
def put_setup_refcount_context(self, name):
self.putln('__Pyx_RefNannySetupContext("%s");' % name)
......
Cython/Compiler/ExprNodes.py
View file @ 2e3f542d
......@@ -954,7 +954,8 @@ class BytesNode(ConstNode):
#
# value BytesLiteral
type = PyrexTypes.c_char_ptr_type
# start off as Python 'bytes' to support len() in O(1)
type = bytes_type
def compile_time_value(self, denv):
return self.value
......@@ -975,11 +976,13 @@ class BytesNode(ConstNode):
return len(self.value) == 1
def coerce_to_boolean(self, env):
# This is special because we start off as a C char*. Testing
# that for truth directly would yield the wrong result.
# This is special because testing a C char* for truth directly
# would yield the wrong result.
return BoolNode(self.pos, value=bool(self.value))
def coerce_to(self, dst_type, env):
if self.type == dst_type:
return self
if dst_type.is_int:
if not self.can_coerce_to_char_literal():
error(self.pos, "Only single-character string literals can be coerced into ints.")
......@@ -990,21 +993,20 @@ class BytesNode(ConstNode):
return CharNode(self.pos, value=self.value)
node = BytesNode(self.pos, value=self.value)
if dst_type == PyrexTypes.c_char_ptr_type:
node.type = PyrexTypes.c_char_ptr_type
if dst_type.is_pyobject:
if dst_type in (py_object_type, Builtin.bytes_type):
node.type = Builtin.bytes_type
else:
self.check_for_coercion_error(dst_type, fail=True)
return node
elif dst_type == PyrexTypes.c_char_ptr_type:
node.type = dst_type
return node
elif dst_type == PyrexTypes.c_uchar_ptr_type:
node.type = PyrexTypes.c_char_ptr_type
return CastNode(node, PyrexTypes.c_uchar_ptr_type)
if not self.type.is_pyobject:
if dst_type in (py_object_type, Builtin.bytes_type):
node.type = Builtin.bytes_type
elif dst_type.is_pyobject:
self.fail_assignment(dst_type)
return self
elif dst_type.is_pyobject and dst_type is not py_object_type:
self.check_for_coercion_error(dst_type, fail=True)
elif dst_type.assignable_from(PyrexTypes.c_char_ptr_type):
node.type = dst_type
return node
# We still need to perform normal coerce_to processing on the
......@@ -1012,11 +1014,6 @@ class BytesNode(ConstNode):
# in which case a type test node will be needed.
return ConstNode.coerce_to(node, dst_type, env)
def as_py_string_node(self, env):
# Return a new BytesNode with the same value as this node
# but whose type is a Python type instead of a C type.
return BytesNode(self.pos, value = self.value, type = Builtin.bytes_type)
def generate_evaluation_code(self, code):
if self.type.is_pyobject:
self.result_code = code.get_py_string_const(self.value)
......@@ -2043,7 +2040,7 @@ class IndexNode(ExprNode):
return None
def type_dependencies(self, env):
return self.base.type_dependencies(env)
return self.base.type_dependencies(env) + self.index.type_dependencies(env)
def infer_type(self, env):
base_type = self.base.infer_type(env)
......@@ -2969,9 +2966,14 @@ class SimpleCallNode(CallNode):
arg = arg.coerce_to_temp(env)
self.args[i] = arg
for i in range(max_nargs, actual_nargs):
if self.args[i].type.is_pyobject:
arg = self.args[i]
if arg.type.is_pyobject:
arg_ctype = arg.type.default_coerced_ctype()
if arg_ctype is None:
error(self.args[i].pos,
"Python object cannot be passed as a varargs parameter")
else:
self.args[i] = arg.coerce_to(arg_ctype, env)
# Calc result type and code fragment
if isinstance(self.function, NewExprNode):
self.type = PyrexTypes.CPtrType(self.function.class_type)
......@@ -4341,15 +4343,18 @@ class DictComprehensionAppendNode(ComprehensionAppendNode):
self.value_expr.annotate(code)
class GeneratorExpressionNode(ScopedExprNode):
# A generator expression, e.g. (i for i in range(10))
#
# Result is a generator.
class InlinedGeneratorExpressionNode(ScopedExprNode):
# An inlined generator expression for which the result is
# calculated inside of the loop. This will only be created by
# transforms when replacing builtin calls on generator
# expressions.
#
# loop ForStatNode the for-loop, containing a YieldExprNode
# loop ForStatNode the for-loop, not containing any YieldExprNodes
# result_node ResultRefNode the reference to the result value temp
# orig_func String the name of the builtin function this node replaces
child_attrs = ["loop"]
loop_analysed = False
type = py_object_type
def analyse_scoped_declarations(self, env):
......@@ -4360,30 +4365,12 @@ class GeneratorExpressionNode(ScopedExprNode):
self.loop.analyse_expressions(env)
self.is_temp = True
def analyse_scoped_expressions(self, env):
if self.has_local_scope:
self.loop.analyse_expressions(env)
def may_be_none(self):
return False
def annotate(self, code):
self.loop.annotate(code)
class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
# An inlined generator expression for which the result is
# calculated inside of the loop. This will only be created by
# transforms when replacing builtin calls on generator
# expressions.
#
# loop ForStatNode the for-loop, not containing any YieldExprNodes
# result_node ResultRefNode the reference to the result value temp
# orig_func String the name of the builtin function this node replaces
child_attrs = ["loop"]
loop_analysed = False
def infer_type(self, env):
return self.result_node.infer_type(env)
......@@ -4396,7 +4383,8 @@ class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
def analyse_scoped_expressions(self, env):
self.loop_analysed = True
GeneratorExpressionNode.analyse_scoped_expressions(self, env)
if self.has_local_scope:
self.loop.analyse_expressions(env)
def coerce_to(self, dst_type, env):
if self.orig_func == 'sum' and dst_type.is_numeric and not self.loop_analysed:
......@@ -4407,7 +4395,7 @@ class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
# assignments.
self.result_node.type = self.type = dst_type
return self
return GeneratorExpressionNode.coerce_to(self, dst_type, env)
return super(InlinedGeneratorExpressionNode, self).coerce_to(dst_type, env)
def generate_result_code(self, code):
self.result_node.result_code = self.result()
......@@ -4955,6 +4943,34 @@ class LambdaNode(InnerFunctionNode):
env.add_lambda_def(self.def_node)
class GeneratorExpressionNode(LambdaNode):
# A generator expression, e.g. (i for i in range(10))
#
# Result is a generator.
#
# loop ForStatNode the for-loop, containing a YieldExprNode
# def_node DefNode the underlying generator 'def' node
name = StringEncoding.EncodedString('genexpr')
binding = False
def analyse_declarations(self, env):
# XXX: dirty hack to disable assignment synthesis
self.def_node.needs_assignment_synthesis = lambda *args, **kwargs: False
self.def_node.analyse_declarations(env)
#super(GeneratorExpressionNode, self).analyse_declarations(env)
env.add_lambda_def(self.def_node)
def generate_result_code(self, code):
code.putln(
'%s = %s(%s, NULL); %s' % (
self.result(),
self.def_node.entry.func_cname,
self.self_result_code(),
code.error_goto_if_null(self.result(), self.pos)))
code.put_gotref(self.py_result())
class YieldExprNode(ExprNode):
# Yield expression node
#
......
Cython/Compiler/ModuleNode.py
View file @ 2e3f542d
......@@ -2051,12 +2051,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
code.error_goto_if_null(type.typeptr_cname, pos))
self.use_type_import_utility_code(env)
if type.vtabptr_cname:
code.putln(
"if (__Pyx_GetVtable(%s->tp_dict, &%s) < 0) %s" % (
type.typeptr_cname,
type.vtabptr_cname,
code.error_goto(pos)))
env.use_utility_code(Nodes.get_vtable_utility_code)
code.putln("%s = (struct %s*)__Pyx_GetVtable(%s->tp_dict); %s" % (
type.vtabptr_cname,
type.vtabstruct_cname,
type.typeptr_cname,
code.error_goto_if_null(type.vtabptr_cname, pos)))
env.types_imported[type] = 1
py3_type_name_map = {'str' : 'bytes', 'unicode' : 'str'}
......@@ -2109,24 +2109,24 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
# a significant performance hit. (See trac #561.)
for func in entry.type.scope.pyfunc_entries:
if func.is_special and Options.docstrings and func.wrapperbase_cname:
code.putln("{");
code.putln("{")
code.putln(
'PyObject *wrapper = __Pyx_GetAttrString((PyObject *)&%s, "%s"); %s' % (
typeobj_cname,
func.name,
code.error_goto_if_null('wrapper', entry.pos)));
code.error_goto_if_null('wrapper', entry.pos)))
code.putln(
"if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) {");
"if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) {")
code.putln(
"%s = *((PyWrapperDescrObject *)wrapper)->d_base;" % (
func.wrapperbase_cname));
func.wrapperbase_cname))
code.putln(
"%s.doc = %s;" % (func.wrapperbase_cname, func.doc_cname));
"%s.doc = %s;" % (func.wrapperbase_cname, func.doc_cname))
code.putln(
"((PyWrapperDescrObject *)wrapper)->d_base = &%s;" % (
func.wrapperbase_cname));
code.putln("}");
code.putln("}");
func.wrapperbase_cname))
code.putln("}")
code.putln("}")
if type.vtable_cname:
code.putln(
"if (__Pyx_SetVtable(%s.tp_dict, %s) < 0) %s" % (
......
Cython/Compiler/Nodes.py
View file @ 2e3f542d
......@@ -1167,6 +1167,7 @@ class FuncDefNode(StatNode, BlockNode):
needs_closure = False
needs_outer_scope = False
is_generator = False
is_generator_body = False
modifiers = []
def analyse_default_values(self, env):
......@@ -1210,6 +1211,9 @@ class FuncDefNode(StatNode, BlockNode):
lenv.directives = env.directives
return lenv
def generate_function_body(self, env, code):
self.body.generate_execution_code(code)
def generate_function_definitions(self, env, code):
import Buffer
......@@ -1297,7 +1301,6 @@ class FuncDefNode(StatNode, BlockNode):
init))
tempvardecl_code = code.insertion_point()
code.put_declare_refcount_context()
if not self.is_generator:
self.generate_keyword_list(code)
if profile:
code.put_trace_declarations()
......@@ -1317,14 +1320,7 @@ class FuncDefNode(StatNode, BlockNode):
if is_getbuffer_slot:
self.getbuffer_init(code)
# ----- Create closure scope object
if self.is_generator:
code.putln("%s = (%s) %s;" % (
Naming.cur_scope_cname,
lenv.scope_class.type.declaration_code(''),
Naming.self_cname))
gotref_code = code.insertion_point()
elif self.needs_closure:
if self.needs_closure:
code.putln("%s = (%s)%s->tp_new(%s, %s, NULL);" % (
Naming.cur_scope_cname,
lenv.scope_class.type.declaration_code(''),
......@@ -1341,7 +1337,7 @@ class FuncDefNode(StatNode, BlockNode):
code.putln("}")
code.put_gotref(Naming.cur_scope_cname)
# Note that it is unsafe to decref the scope at this point.
if self.needs_outer_scope and not self.is_generator:
if self.needs_outer_scope:
code.putln("%s = (%s)%s;" % (
outer_scope_cname,
cenv.scope_class.type.declaration_code(''),
......@@ -1358,23 +1354,33 @@ class FuncDefNode(StatNode, BlockNode):
# fatal error before hand, it's not really worth tracing
code.put_trace_call(self.entry.name, self.pos)
# ----- Fetch arguments
if not self.is_generator:
self.generate_preamble(env, code)
if self.is_generator:
code.funcstate.init_closure_temps(lenv.scope_class.type.scope)
resume_code = code.insertion_point()
first_run_label = code.new_label('first_run')
code.use_label(first_run_label)
code.put_label(first_run_label)
code.putln('%s' %
(code.error_goto_if_null(Naming.sent_value_cname, self.pos)))
self.generate_argument_parsing_code(env, code)
# If an argument is assigned to in the body, we must
# incref it to properly keep track of refcounts.
for entry in lenv.arg_entries:
if entry.type.is_pyobject:
if (acquire_gil or entry.assignments) and not entry.in_closure:
code.put_var_incref(entry)
# ----- Initialise local variables
for entry in lenv.var_entries:
if entry.type.is_pyobject and entry.init_to_none and entry.used:
code.put_init_var_to_py_none(entry)
# ----- Initialise local buffer auxiliary variables
for entry in lenv.var_entries + lenv.arg_entries:
if entry.type.is_buffer and entry.buffer_aux.buffer_info_var.used:
code.putln("%s.buf = NULL;" %
entry.buffer_aux.buffer_info_var.cname)
# ----- Check and convert arguments
self.generate_argument_type_tests(code)
# ----- Acquire buffer arguments
for entry in lenv.arg_entries:
if entry.type.is_buffer:
Buffer.put_acquire_arg_buffer(entry, code, self.pos)
# -------------------------
# ----- Function body -----
# -------------------------
self.body.generate_execution_code(code)
if self.is_generator:
code.putln('PyErr_SetNone(PyExc_StopIteration); %s' % code.error_goto(self.pos))
self.generate_function_body(env, code)
# ----- Default return value
code.putln("")
......@@ -1461,10 +1467,8 @@ class FuncDefNode(StatNode, BlockNode):
if entry.type.is_pyobject:
if (acquire_gil or entry.assignments) and not entry.in_closure:
code.put_var_decref(entry)
if self.needs_closure and not self.is_generator:
if self.needs_closure:
code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
if self.is_generator:
code.putln('%s->%s.resume_label = -1;' % (Naming.cur_scope_cname, Naming.obj_base_cname))
# ----- Return
# This code is duplicated in ModuleNode.generate_module_init_func
......@@ -1502,56 +1506,15 @@ class FuncDefNode(StatNode, BlockNode):
if preprocessor_guard:
code.putln("#endif /*!(%s)*/" % preprocessor_guard)
# ----- Go back and insert temp variable declarations
tempvardecl_code.put_temp_declarations(code.funcstate)
# ----- Generator resume code
if self.is_generator:
resume_code.putln("switch (%s->%s.resume_label) {" % (Naming.cur_scope_cname, Naming.obj_base_cname));
resume_code.putln("case 0: goto %s;" % first_run_label)
for yield_expr in self.yields:
resume_code.putln("case %d: goto %s;" % (yield_expr.label_num, yield_expr.label_name));
resume_code.putln("default: /* CPython raises the right error here */");
resume_code.putln("return NULL;");
resume_code.putln("}");
# ----- Python version
code.exit_cfunc_scope()
if self.py_func:
self.py_func.generate_function_definitions(env, code)
self.generate_wrapper_functions(code)
if self.is_generator:
self.generator.generate_function_body(self.local_scope, code)
def generate_preamble(self, env, code):
"""Parse arguments and prepare scope"""
import Buffer
lenv = self.local_scope
acquire_gil = self.acquire_gil
self.generate_argument_parsing_code(env, code)
# If an argument is assigned to in the body, we must
# incref it to properly keep track of refcounts.
for entry in lenv.arg_entries:
if entry.type.is_pyobject:
if (acquire_gil or entry.assignments) and not entry.in_closure:
code.put_var_incref(entry)
# ----- Initialise local variables
for entry in lenv.var_entries:
if entry.type.is_pyobject and entry.init_to_none and entry.used:
code.put_init_var_to_py_none(entry)
# ----- Initialise local buffer auxiliary variables
for entry in lenv.var_entries + lenv.arg_entries:
if entry.type.is_buffer and entry.buffer_aux.buffer_info_var.used:
code.putln("%s.buf = NULL;" %
entry.buffer_aux.buffer_info_var.cname)
# ----- Check and convert arguments
self.generate_argument_type_tests(code)
# ----- Acquire buffer arguments
for entry in lenv.arg_entries:
if entry.type.is_buffer:
Buffer.put_acquire_arg_buffer(entry, code, self.pos)
def declare_argument(self, env, arg):
if arg.type.is_void:
error(arg.pos, "Invalid use of 'void'")
......@@ -1902,61 +1865,6 @@ class DecoratorNode(Node):
child_attrs = ['decorator']
class GeneratorWrapperNode(object):
# Wrapper
def __init__(self, def_node, func_cname=None, body_cname=None, header=None):
self.def_node = def_node
self.func_cname = func_cname
self.body_cname = body_cname
self.header = header
def generate_function_body(self, env, code):
code.mark_pos(self.def_node.pos)
cenv = env.outer_scope # XXX: correct?
while cenv.is_py_class_scope or cenv.is_c_class_scope:
cenv = cenv.outer_scope
lenv = self.def_node.local_scope
code.enter_cfunc_scope()
code.putln()
code.putln('%s {' % self.header)
code.put_declare_refcount_context()
self.def_node.generate_keyword_list(code)
code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
code.putln(";")
code.put_setup_refcount_context(self.def_node.entry.name)
code.putln("%s = (%s)%s->tp_new(%s, %s, NULL);" % (
Naming.cur_scope_cname,
lenv.scope_class.type.declaration_code(''),
lenv.scope_class.type.typeptr_cname,
lenv.scope_class.type.typeptr_cname,
Naming.empty_tuple))
code.putln("if (unlikely(!%s)) {" % Naming.cur_scope_cname)
code.put_finish_refcount_context()
code.putln("return NULL;");
code.putln("}");
code.put_gotref(Naming.cur_scope_cname)
if self.def_node.needs_outer_scope:
outer_scope_cname = '%s->%s' % (Naming.cur_scope_cname, Naming.outer_scope_cname)
code.putln("%s = (%s)%s;" % (
outer_scope_cname,
cenv.scope_class.type.declaration_code(''),
Naming.self_cname))
code.put_incref(outer_scope_cname, cenv.scope_class.type)
code.put_giveref(outer_scope_cname)
self.def_node.generate_preamble(env, code)
generator_cname = '%s->%s' % (Naming.cur_scope_cname, Naming.obj_base_cname)
code.putln('%s.resume_label = 0;' % generator_cname)
code.putln('%s.body = %s;' % (generator_cname, self.body_cname))
code.put_giveref(Naming.cur_scope_cname)
code.put_finish_refcount_context()
code.putln("return (PyObject *) %s;" % Naming.cur_scope_cname);
code.putln('}\n')
code.exit_cfunc_scope()
class DefNode(FuncDefNode):
# A Python function definition.
#
......@@ -2250,10 +2158,6 @@ class DefNode(FuncDefNode):
Naming.pyfunc_prefix + prefix + name
entry.pymethdef_cname = \
Naming.pymethdef_prefix + prefix + name
if self.is_generator:
self.generator_body_cname = Naming.genbody_prefix + env.next_id(env.scope_prefix) + name
if Options.docstrings:
entry.doc = embed_position(self.pos, self.doc)
entry.doc_cname = \
......@@ -2401,14 +2305,6 @@ class DefNode(FuncDefNode):
"static PyMethodDef %s = " %
self.entry.pymethdef_cname)
code.put_pymethoddef(self.entry, ";", allow_skip=False)
if self.is_generator:
code.putln("static PyObject *%s(PyObject *%s, PyObject *%s) /* generator body */\n{" %
(self.generator_body_cname, Naming.self_cname, Naming.sent_value_cname))
self.generator = GeneratorWrapperNode(self,
func_cname=self.entry.func_cname,
body_cname=self.generator_body_cname,
header=header)
else:
code.putln("%s {" % header)
def generate_argument_declarations(self, env, code):
......@@ -2556,8 +2452,6 @@ class DefNode(FuncDefNode):
self.name, Naming.args_cname, self.error_value()))
code.putln("}")
code.globalstate.use_utility_code(keyword_string_check_utility_code)
if self.starstar_arg:
if self.star_arg:
kwarg_check = "unlikely(%s)" % Naming.kwds_cname
......@@ -2566,6 +2460,7 @@ class DefNode(FuncDefNode):
else:
kwarg_check = "unlikely(%s) && unlikely(PyDict_Size(%s) > 0)" % (
Naming.kwds_cname, Naming.kwds_cname)
code.globalstate.use_utility_code(keyword_string_check_utility_code)
code.putln(
"if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, \"%s\", %d))) return %s;" % (
kwarg_check, Naming.kwds_cname, self.name,
......@@ -2629,8 +2524,6 @@ class DefNode(FuncDefNode):
has_fixed_positional_count = not self.star_arg and \
min_positional_args == max_positional_args
code.globalstate.use_utility_code(raise_double_keywords_utility_code)
code.globalstate.use_utility_code(raise_argtuple_invalid_utility_code)
if self.num_required_kw_args:
code.globalstate.use_utility_code(raise_keyword_required_utility_code)
......@@ -2721,6 +2614,7 @@ class DefNode(FuncDefNode):
if code.label_used(argtuple_error_label):
code.put_goto(success_label)
code.put_label(argtuple_error_label)
code.globalstate.use_utility_code(raise_argtuple_invalid_utility_code)
code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, PyTuple_GET_SIZE(%s)); ' % (
self.name, has_fixed_positional_count,
min_positional_args, max_positional_args,
......@@ -2842,6 +2736,7 @@ class DefNode(FuncDefNode):
# kwargs) that were passed into positional
# arguments up to this point
code.putln('else {')
code.globalstate.use_utility_code(raise_argtuple_invalid_utility_code)
code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, %d); ' % (
self.name, has_fixed_positional_count,
min_positional_args, max_positional_args, i))
......@@ -3003,6 +2898,151 @@ class DefNode(FuncDefNode):
def caller_will_check_exceptions(self):
return 1
class GeneratorDefNode(DefNode):
# Generator DefNode.
#
# gbody GeneratorBodyDefNode
#
is_generator = True
needs_closure = True
child_attrs = ["args", "star_arg", "starstar_arg", "body", "decorators", "gbody"]
def __init__(self, **kwargs):
# XXX: don't actually needs a body
kwargs['body'] = StatListNode(kwargs['pos'], stats=[])
super(GeneratorDefNode, self).__init__(**kwargs)
def analyse_declarations(self, env):
super(GeneratorDefNode, self).analyse_declarations(env)
self.gbody.local_scope = self.local_scope
self.gbody.analyse_declarations(env)
def generate_function_body(self, env, code):
body_cname = self.gbody.entry.func_cname
generator_cname = '%s->%s' % (Naming.cur_scope_cname, Naming.obj_base_cname)
code.putln('%s.resume_label = 0;' % generator_cname)
code.putln('%s.body = (__pyx_generator_body_t) %s;' % (generator_cname, body_cname))
code.put_giveref(Naming.cur_scope_cname)
code.put_finish_refcount_context()
code.putln("return (PyObject *) %s;" % Naming.cur_scope_cname);
def generate_function_definitions(self, env, code):
self.gbody.generate_function_header(code, proto=True)
super(GeneratorDefNode, self).generate_function_definitions(env, code)
self.gbody.generate_function_definitions(env, code)
class GeneratorBodyDefNode(DefNode):
# Generator body DefNode.
#
is_generator_body = True
child_attrs = ["args", "star_arg", "starstar_arg", "body", "decorators"]
def __init__(self, pos=None, name=None, body=None):
super(GeneratorBodyDefNode, self).__init__(pos=pos, body=body, name=name, doc=None,
args=[],
star_arg=None, starstar_arg=None)
def declare_generator_body(self, env):
prefix = env.next_id(env.scope_prefix)
name = env.next_id('generator')
entry = env.declare_var(prefix + name, py_object_type, self.pos, visibility='private')
entry.func_cname = Naming.genbody_prefix + prefix + name
entry.qualified_name = EncodedString(self.name)
self.entry = entry
def analyse_declarations(self, env):
self.analyse_argument_types(env)
self.declare_generator_body(env)
def generate_function_header(self, code, proto=False):
header = "static PyObject *%s(%s, PyObject *%s)" % (
self.entry.func_cname,
self.local_scope.scope_class.type.declaration_code(Naming.cur_scope_cname),
Naming.sent_value_cname)
if proto:
code.putln('%s; /* proto */' % header)
else:
code.putln('%s /* generator body */\n{' % header);
def generate_function_definitions(self, env, code):
lenv = self.local_scope
# Generate closure function definitions
self.body.generate_function_definitions(lenv, code)
# generate lambda function definitions
self.generate_lambda_definitions(lenv, code)
# Generate C code for header and body of function
code.enter_cfunc_scope()
code.return_from_error_cleanup_label = code.new_label()
# ----- Top-level constants used by this function
code.mark_pos(self.pos)
self.generate_cached_builtins_decls(lenv, code)
# ----- Function header
code.putln("")
self.generate_function_header(code)
# ----- Local variables
code.putln("PyObject *%s = NULL;" % Naming.retval_cname)
tempvardecl_code = code.insertion_point()
code.put_declare_refcount_context()
code.put_setup_refcount_context(self.entry.name)
# ----- Resume switch point.
code.funcstate.init_closure_temps(lenv.scope_class.type.scope)
resume_code = code.insertion_point()
first_run_label = code.new_label('first_run')
code.use_label(first_run_label)
code.put_label(first_run_label)
code.putln('%s' %
(code.error_goto_if_null(Naming.sent_value_cname, self.pos)))
# ----- Function body
self.generate_function_body(env, code)
code.putln('PyErr_SetNone(PyExc_StopIteration); %s' % code.error_goto(self.pos))
# ----- Error cleanup
if code.error_label in code.labels_used:
code.put_goto(code.return_label)
code.put_label(code.error_label)
for cname, type in code.funcstate.all_managed_temps():
code.put_xdecref(cname, type)
code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
# XXX: ^^^ is this enough?
# ----- Non-error return cleanup
code.put_label(code.return_label)
code.put_xdecref(Naming.retval_cname, py_object_type)
code.putln('%s->%s.resume_label = -1;' % (Naming.cur_scope_cname, Naming.obj_base_cname))
code.put_finish_refcount_context()
code.putln('return NULL;');
code.putln("}")
# ----- Go back and insert temp variable declarations
tempvardecl_code.put_temp_declarations(code.funcstate)
# ----- Generator resume code
resume_code.putln("switch (%s->%s.resume_label) {" % (Naming.cur_scope_cname, Naming.obj_base_cname));
resume_code.putln("case 0: goto %s;" % first_run_label)
from ParseTreeTransforms import YieldNodeCollector
collector = YieldNodeCollector()
collector.visitchildren(self)
for yield_expr in collector.yields:
resume_code.putln("case %d: goto %s;" % (yield_expr.label_num, yield_expr.label_name));
resume_code.putln("default: /* CPython raises the right error here */");
resume_code.put_finish_refcount_context()
resume_code.putln("return NULL;");
resume_code.putln("}");
code.exit_cfunc_scope()
class OverrideCheckNode(StatNode):
# A Node for dispatching to the def method if it
# is overriden.
......@@ -6237,7 +6277,8 @@ invalid_keyword:
bad:
return -1;
}
""")
""",
requires=[raise_double_keywords_utility_code])
#------------------------------------------------------------------------------------
......@@ -6381,25 +6422,26 @@ bad:
get_vtable_utility_code = UtilityCode(
proto = """
static int __Pyx_GetVtable(PyObject *dict, void *vtabptr); /*proto*/
static void* __Pyx_GetVtable(PyObject *dict); /*proto*/
""",
impl = r"""
static int __Pyx_GetVtable(PyObject *dict, void *vtabptr) {
static void* __Pyx_GetVtable(PyObject *dict) {
void* ptr;
PyObject *ob = PyMapping_GetItemString(dict, (char *)"__pyx_vtable__");
if (!ob)
goto bad;
#if PY_VERSION_HEX >= 0x02070000 && !(PY_MAJOR_VERSION==3&&PY_MINOR_VERSION==0)
*(void **)vtabptr = PyCapsule_GetPointer(ob, 0);
ptr = PyCapsule_GetPointer(ob, 0);
#else
*(void **)vtabptr = PyCObject_AsVoidPtr(ob);
ptr = PyCObject_AsVoidPtr(ob);
#endif
if (!*(void **)vtabptr)
goto bad;
if (!ptr && !PyErr_Occurred())
PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type");
Py_DECREF(ob);
return 0;
return ptr;
bad:
Py_XDECREF(ob);
return -1;
return NULL;
}
""")
......
Cython/Compiler/Optimize.py
View file @ 2e3f542d
......@@ -1167,11 +1167,12 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform):
self.yield_nodes = []
visit_Node = Visitor.TreeVisitor.visitchildren
def visit_YieldExprNode(self, node):
# XXX: disable inlining while it's not back supported
def __visit_YieldExprNode(self, node):
self.yield_nodes.append(node)
self.visitchildren(node)
def visit_ExprStatNode(self, node):
def __visit_ExprStatNode(self, node):
self.visitchildren(node)
if node.expr in self.yield_nodes:
self.yield_stat_nodes[node.expr] = node
......
Cython/Compiler/ParseTreeTransforms.py
View file @ 2e3f542d
......@@ -183,6 +183,7 @@ class PostParse(ScopeTrackingTransform):
def visit_ModuleNode(self, node):
self.lambda_counter = 1
self.genexpr_counter = 1
return super(PostParse, self).visit_ModuleNode(node)
def visit_LambdaNode(self, node):
......@@ -190,14 +191,34 @@ class PostParse(ScopeTrackingTransform):
lambda_id = self.lambda_counter
self.lambda_counter += 1
node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
collector = YieldNodeCollector()
collector.visitchildren(node.result_expr)
if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
body = ExprNodes.YieldExprNode(
node.result_expr.pos, arg=node.result_expr)
body = Nodes.ExprStatNode(node.result_expr.pos, expr=body)
else:
body = Nodes.ReturnStatNode(
node.result_expr.pos, value = node.result_expr)
node.result_expr.pos, value=node.result_expr)
node.def_node = Nodes.DefNode(
node.pos, name=node.name, lambda_name=node.lambda_name,
args=node.args, star_arg=node.star_arg,
starstar_arg=node.starstar_arg,
body=body)
body=body, doc=None)
self.visitchildren(node)
return node
def visit_GeneratorExpressionNode(self, node):
# unpack a generator expression into the corresponding DefNode
genexpr_id = self.genexpr_counter
self.genexpr_counter += 1
node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
node.def_node = Nodes.DefNode(node.pos, name=node.name,
doc=None,
args=[], star_arg=None,
starstar_arg=None,
body=node.loop)
self.visitchildren(node)
return node
......@@ -1317,7 +1338,7 @@ class YieldNodeCollector(TreeVisitor):
if self.has_return_value:
error(node.pos, "'yield' outside function")
self.yields.append(node)
node.label_num = len(self.yields)
self.visitchildren(node)
def visit_ReturnStatNode(self, node):
if node.value:
......@@ -1332,6 +1353,9 @@ class YieldNodeCollector(TreeVisitor):
def visit_DefNode(self, node):
pass
def visit_LambdaNode(self, node):
pass
def visit_GeneratorExpressionNode(self, node):
pass
......@@ -1352,11 +1376,22 @@ class MarkClosureVisitor(CythonTransform):
collector.visitchildren(node)
if collector.yields:
if collector.returns and not collector.has_return_value:
error(collector.returns[0].pos, "'return' inside generators not yet supported ")
node.is_generator = True
node.needs_closure = True
node.yields = collector.yields
for i, yield_expr in enumerate(collector.yields):
yield_expr.label_num = i + 1
gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
name=node.name,
body=node.body)
generator = Nodes.GeneratorDefNode(pos=node.pos,
name=node.name,
args=node.args,
star_arg=node.star_arg,
starstar_arg=node.starstar_arg,
doc=node.doc,
decorators=node.decorators,
gbody=gbody,
lambda_name=node.lambda_name)
return generator
return node
def visit_CFuncDefNode(self, node):
......@@ -1447,6 +1482,9 @@ class CreateClosureClasses(CythonTransform):
return from_closure, in_closure
def create_class_from_scope(self, node, target_module_scope, inner_node=None):
# skip generator body
if node.is_generator_body:
return
# move local variables into closure
if node.is_generator:
for entry in node.local_scope.entries.values():
......
Cython/Compiler/PyrexTypes.py
View file @ 2e3f542d
......@@ -353,6 +353,10 @@ class PyObjectType(PyrexType):
def can_coerce_to_pyobject(self, env):
return True
def default_coerced_ctype(self):
"The default C type that this Python type coerces to, or None."
return None
def assignable_from(self, src_type):
# except for pointers, conversion will be attempted
return not src_type.is_ptr or src_type.is_string
......@@ -404,6 +408,15 @@ class BuiltinObjectType(PyObjectType):
def __repr__(self):
return "<%s>"% self.cname
def default_coerced_ctype(self):
if self.name == 'bytes':
return c_char_ptr_type
elif self.name == 'bool':
return c_bint_type
elif self.name == 'float':
return c_double_type
return None
def assignable_from(self, src_type):
if isinstance(src_type, BuiltinObjectType):
return src_type.name == self.name
......
tests/bugs.txt
View file @ 2e3f542d
......@@ -10,7 +10,6 @@ cfunc_call_tuple_args_T408
compile.cpp_operators
cpp_templated_ctypedef
cpp_structs
genexpr_T491
with_statement_module_level_T536
function_as_method_T494
closure_inside_cdef_T554
......
tests/errors/e_generators.pyx
View file @ 2e3f542d
......@@ -6,10 +6,6 @@ def bar(a):
return 0
yield
def xxx():
yield
return
yield
class Foo:
......@@ -18,7 +14,6 @@ class Foo:
_ERRORS = u"""
3:4: 'return' with argument inside generator
7:4: 'yield' outside function
11:4: 'return' inside generators not yet supported
13:0: 'yield' not supported here
16:4: 'yield' not supported here
9:0: 'yield' not supported here
12:4: 'yield' not supported here
"""
tests/errors/string_assignments.pyx
View file @ 2e3f542d
......@@ -58,18 +58,18 @@ _ERRORS = u"""
30:22: Cannot convert Unicode string to 'bytes' implicitly, encoding required.
31:22: Cannot convert 'str' to 'bytes' implicitly. This is not portable.
33:17: Cannot assign type 'char *' to 'str object'
33:17: Cannot convert 'bytes' object to str implicitly. This is not portable to Py3.
34:19: Cannot convert 'bytes' object to str implicitly. This is not portable to Py3.
35:17: Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding.
36:19: Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding.
38:20: str objects do not support coercion to unicode, use a unicode string literal instead (u'')
39:22: str objects do not support coercion to unicode, use a unicode string literal instead (u'')
40:20: Cannot assign type 'char *' to 'unicode object'
40:20: Cannot convert 'bytes' object to unicode implicitly, decoding required
41:22: Cannot convert 'bytes' object to unicode implicitly, decoding required
42:22: Cannot convert 'char*' to unicode implicitly, decoding required
44:19: Cannot assign type 'str object' to 'tuple object'
45:18: Cannot assign type 'unicode object' to 'tuple object'
46:18: Cannot assign type 'char *' to 'tuple object'
46:18: Cannot assign type 'bytes object' to 'tuple object'
"""
tests/run/generators.pyx
View file @ 2e3f542d
......@@ -207,3 +207,95 @@ def generator_nonlocal():
yield x
return g
return f(1)
def test_nested(a, b, c):
"""
>>> obj = test_nested(1, 2, 3)
>>> [i() for i in obj]
[1, 2, 3, 4]
"""
def one():
return a
def two():
return b
def three():
return c
def new_closure(a, b):
def sum():
return a + b
return sum
yield one
yield two
yield three
yield new_closure(a, c)
def tolist(func):
def wrapper(*args, **kwargs):
return list(func(*args, **kwargs))
return wrapper
@tolist
def test_decorated(*args):
"""
>>> test_decorated(1, 2, 3)
[1, 2, 3]
"""
for i in args:
yield i
def test_return(a):
"""
>>> d = dict()
>>> obj = test_return(d)
>>> next(obj)
1
>>> next(obj)
Traceback (most recent call last):
StopIteration
>>> d['i_was_here']
True
"""
yield 1
a['i_was_here'] = True
return
def test_copied_yield(foo):
"""
>>> class Manager(object):
... def __enter__(self):
... return self
... def __exit__(self, type, value, tb):
... pass
>>> list(test_copied_yield(Manager()))
[1]
"""
with foo:
yield 1
def test_nested_yield():
"""
>>> obj = test_nested_yield()
>>> next(obj)
1
>>> obj.send(2)
2
>>> obj.send(3)
3
>>> obj.send(4)
Traceback (most recent call last):
StopIteration
"""
yield (yield (yield 1))
def test_inside_lambda():
"""
>>> obj = test_inside_lambda()()
>>> next(obj)
1
>>> obj.send('a')
2
>>> obj.send('b')
('a', 'b')
"""
return lambda:((yield 1), (yield 2))
tests/run/type_inference.pyx
View file @ 2e3f542d
......@@ -25,7 +25,7 @@ def simple():
xptrptr = &xptr
assert typeof(xptrptr) == "double **", typeof(xptrptr)
b = b"abc"
assert typeof(b) == "char *", typeof(b)
assert typeof(b) == "bytes object", typeof(b)
s = "abc"
assert typeof(s) == "str object", typeof(s)
u = u"xyz"
......@@ -57,7 +57,7 @@ def slicing():
>>> slicing()
"""
b = b"abc"
assert typeof(b) == "char *", typeof(b)
assert typeof(b) == "bytes object", typeof(b)
b1 = b[1:2]
assert typeof(b1) == "bytes object", typeof(b1)
b2 = b[1:2:2]
......@@ -92,9 +92,9 @@ def indexing():
>>> indexing()
"""
b = b"abc"
assert typeof(b) == "char *", typeof(b)
assert typeof(b) == "bytes object", typeof(b)
b1 = b[1]
assert typeof(b1) == "char", typeof(b1) # FIXME: Python object ??
assert typeof(b1) == "Python object", typeof(b1)
u = u"xyz"
assert typeof(u) == "unicode object", typeof(u)
u1 = u[1]
......@@ -479,3 +479,10 @@ def large_literals():
c, d = 10, 100000000000000000000000000000000
assert typeof(c) == "long", typeof(c)
assert typeof(d) == "Python object", typeof(d)
# Regression test for trac #638.
def bar(foo):
qux = foo
quux = foo[qux.baz]
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7