Commit 7ef3e52c authored by Stefan Behnel's avatar Stefan Behnel

merged in Vitek's generators branch

parents b2b337b2 fb3ac076
......@@ -34,6 +34,8 @@ cdef class FunctionState:
cdef public dict temps_used_type
cdef public size_t temp_counter
cdef public object closure_temps
@cython.locals(n=size_t)
cpdef new_label(self, name=*)
cpdef tuple get_loop_labels(self)
......
......@@ -117,6 +117,7 @@ class FunctionState(object):
self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status
self.temps_used_type = {} # name -> (type, manage_ref)
self.temp_counter = 0
self.closure_temps = None
# labels
......@@ -270,6 +271,9 @@ class FunctionState(object):
if manage_ref
for cname in freelist]
def init_closure_temps(self, scope):
self.closure_temps = ClosureTempAllocator(scope)
class IntConst(object):
"""Global info about a Python integer constant held by GlobalState.
......@@ -475,6 +479,7 @@ class GlobalState(object):
w.enter_cfunc_scope()
w.putln("")
w.putln("static int __Pyx_InitCachedConstants(void) {")
w.put_declare_refcount_context()
w.put_setup_refcount_context("__Pyx_InitCachedConstants")
w = self.parts['init_globals']
......@@ -1297,6 +1302,8 @@ class CCodeWriter(object):
#if entry.type.is_extension_type:
# code = "((PyObject*)%s)" % code
self.put_init_to_py_none(code, entry.type, nanny)
if entry.in_closure:
self.put_giveref('Py_None')
def put_pymethoddef(self, entry, term, allow_skip=True):
if entry.is_special or entry.name == '__getattribute__':
......@@ -1366,6 +1373,9 @@ class CCodeWriter(object):
def lookup_filename(self, filename):
return self.globalstate.lookup_filename(filename)
def put_declare_refcount_context(self):
self.putln('__Pyx_RefNannyDeclareContext;')
def put_setup_refcount_context(self, name):
self.putln('__Pyx_RefNannySetupContext("%s");' % name)
......@@ -1402,3 +1412,26 @@ class PyrexCodeWriter(object):
def dedent(self):
self.level -= 1
class ClosureTempAllocator(object):
def __init__(self, klass):
self.klass = klass
self.temps_allocated = {}
self.temps_free = {}
self.temps_count = 0
def reset(self):
for type, cnames in self.temps_allocated.items():
self.temps_free[type] = list(cnames)
def allocate_temp(self, type):
if not type in self.temps_allocated:
self.temps_allocated[type] = []
self.temps_free[type] = []
elif self.temps_free[type]:
return self.temps_free[type].pop(0)
cname = '%s%d' % (Naming.codewriter_temp_prefix, self.temps_count)
self.klass.declare_var(pos=None, name=cname, cname=cname, type=type, is_cdef=True)
self.temps_allocated[type].append(cname)
self.temps_count += 1
return cname
......@@ -77,12 +77,15 @@ class ExprNode(Node):
# [ExprNode or [ExprNode or None] or None]
# Cached result of subexpr_nodes()
# use_managed_ref boolean use ref-counted temps/assignments/etc.
# result_is_used boolean indicates that the result will be dropped and the
# result_code/temp_result can safely be set to None
result_ctype = None
type = None
temp_code = None
old_temp = None # error checker for multiple frees etc.
use_managed_ref = True # can be set by optimisation transforms
result_is_used = True
# The Analyse Expressions phase for expressions is split
# into two sub-phases:
......@@ -452,6 +455,9 @@ class ExprNode(Node):
def release_temp_result(self, code):
if not self.temp_code:
if not self.result_is_used:
# not used anyway, so ignore if not set up
return
if self.old_temp:
raise RuntimeError("temp %s released multiple times in %s" % (
self.old_temp, self.__class__.__name__))
......@@ -497,7 +503,7 @@ class ExprNode(Node):
def generate_disposal_code(self, code):
if self.is_temp:
if self.type.is_pyobject:
if self.type.is_pyobject and self.result():
code.put_decref_clear(self.result(), self.ctype())
else:
# Already done if self.is_temp
......@@ -1628,9 +1634,10 @@ class NameNode(AtomicExprNode):
#print "...RHS type", rhs.type, "ctype", rhs.ctype() ###
if self.use_managed_ref:
rhs.make_owned_reference(code)
if entry.is_cglobal:
code.put_gotref(self.py_result())
is_external_ref = entry.is_cglobal or self.entry.in_closure or self.entry.from_closure
if not self.lhs_of_first_assignment:
if is_external_ref:
code.put_gotref(self.py_result())
if entry.is_local and not Options.init_local_none:
initialized = entry.scope.control_flow.get_state((entry.name, 'initialized'), self.pos)
if initialized is True:
......@@ -1639,7 +1646,7 @@ class NameNode(AtomicExprNode):
code.put_xdecref(self.result(), self.ctype())
else:
code.put_decref(self.result(), self.ctype())
if entry.is_cglobal:
if is_external_ref:
code.put_giveref(rhs.py_result())
code.putln('%s = %s;' % (self.result(),
......@@ -4442,15 +4449,18 @@ class DictComprehensionAppendNode(ComprehensionAppendNode):
self.value_expr.annotate(code)
class GeneratorExpressionNode(ScopedExprNode):
# A generator expression, e.g. (i for i in range(10))
#
# Result is a generator.
class InlinedGeneratorExpressionNode(ScopedExprNode):
# An inlined generator expression for which the result is
# calculated inside of the loop. This will only be created by
# transforms when replacing builtin calls on generator
# expressions.
#
# loop ForStatNode the for-loop, containing a YieldExprNode
# loop ForStatNode the for-loop, not containing any YieldExprNodes
# result_node ResultRefNode the reference to the result value temp
# orig_func String the name of the builtin function this node replaces
child_attrs = ["loop"]
loop_analysed = False
type = py_object_type
def analyse_scoped_declarations(self, env):
......@@ -4461,30 +4471,12 @@ class GeneratorExpressionNode(ScopedExprNode):
self.loop.analyse_expressions(env)
self.is_temp = True
def analyse_scoped_expressions(self, env):
if self.has_local_scope:
self.loop.analyse_expressions(env)
def may_be_none(self):
return False
def annotate(self, code):
self.loop.annotate(code)
class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
# An inlined generator expression for which the result is
# calculated inside of the loop. This will only be created by
# transforms when replacing builtin calls on generator
# expressions.
#
# loop ForStatNode the for-loop, not containing any YieldExprNodes
# result_node ResultRefNode the reference to the result value temp
# orig_func String the name of the builtin function this node replaces
child_attrs = ["loop"]
loop_analysed = False
def infer_type(self, env):
return self.result_node.infer_type(env)
......@@ -4497,7 +4489,8 @@ class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
def analyse_scoped_expressions(self, env):
self.loop_analysed = True
GeneratorExpressionNode.analyse_scoped_expressions(self, env)
if self.has_local_scope:
self.loop.analyse_expressions(env)
def coerce_to(self, dst_type, env):
if self.orig_func == 'sum' and dst_type.is_numeric and not self.loop_analysed:
......@@ -4508,7 +4501,7 @@ class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
# assignments.
self.result_node.type = self.type = dst_type
return self
return GeneratorExpressionNode.coerce_to(self, dst_type, env)
return super(InlinedGeneratorExpressionNode, self).coerce_to(dst_type, env)
def generate_result_code(self, code):
self.result_node.result_code = self.result()
......@@ -5055,32 +5048,98 @@ class LambdaNode(InnerFunctionNode):
self.pymethdef_cname = self.def_node.entry.pymethdef_cname
env.add_lambda_def(self.def_node)
class GeneratorExpressionNode(LambdaNode):
# A generator expression, e.g. (i for i in range(10))
#
# Result is a generator.
#
# loop ForStatNode the for-loop, containing a YieldExprNode
# def_node DefNode the underlying generator 'def' node
name = StringEncoding.EncodedString('genexpr')
binding = False
def analyse_declarations(self, env):
self.def_node.no_assignment_synthesis = True
self.def_node.analyse_declarations(env)
env.add_lambda_def(self.def_node)
def generate_result_code(self, code):
code.putln(
'%s = %s(%s, NULL); %s' % (
self.result(),
self.def_node.entry.func_cname,
self.self_result_code(),
code.error_goto_if_null(self.result(), self.pos)))
code.put_gotref(self.py_result())
class YieldExprNode(ExprNode):
# Yield expression node
#
# arg ExprNode the value to return from the generator
# label_name string name of the C label used for this yield
# label_num integer yield label number
subexprs = ['arg']
type = py_object_type
label_num = 0
def analyse_types(self, env):
if not self.label_num:
error(self.pos, "'yield' not supported here")
self.is_temp = 1
if self.arg is not None:
self.arg.analyse_types(env)
if not self.arg.type.is_pyobject:
self.arg = self.arg.coerce_to_pyobject(env)
error(self.pos, "Generators are not supported")
env.use_utility_code(generator_utility_code)
def generate_result_code(self, code):
def generate_evaluation_code(self, code):
self.label_name = code.new_label('resume_from_yield')
code.use_label(self.label_name)
code.putln("/* FIXME: save temporary variables */")
code.putln("/* FIXME: return from function, yielding value */")
if self.arg:
self.arg.generate_evaluation_code(code)
self.arg.make_owned_reference(code)
code.putln(
"%s = %s;" % (
Naming.retval_cname,
self.arg.result_as(py_object_type)))
self.arg.generate_post_assignment_code(code)
#self.arg.generate_disposal_code(code)
self.arg.free_temps(code)
else:
code.put_init_to_py_none(Naming.retval_cname, py_object_type)
saved = []
code.funcstate.closure_temps.reset()
for cname, type, manage_ref in code.funcstate.temps_in_use():
save_cname = code.funcstate.closure_temps.allocate_temp(type)
saved.append((cname, save_cname, type))
if type.is_pyobject:
code.put_xgiveref(cname)
code.putln('%s->%s = %s;' % (Naming.cur_scope_cname, save_cname, cname))
code.put_xgiveref(Naming.retval_cname)
code.put_finish_refcount_context()
code.putln("/* return from generator, yielding value */")
code.putln("%s->%s.resume_label = %d;" % (Naming.cur_scope_cname, Naming.obj_base_cname, self.label_num))
code.putln("return %s;" % Naming.retval_cname);
code.put_label(self.label_name)
code.putln("/* FIXME: restore temporary variables and */")
code.putln("/* FIXME: extract sent value from closure */")
for cname, save_cname, type in saved:
code.putln('%s = %s->%s;' % (cname, Naming.cur_scope_cname, save_cname))
if type.is_pyobject:
code.putln('%s->%s = 0;' % (Naming.cur_scope_cname, save_cname))
if type.is_pyobject:
code.put_xgotref(cname)
if self.result_is_used:
self.allocate_temp_result(code)
code.putln('%s = %s; %s' %
(self.result(), Naming.sent_value_cname,
code.error_goto_if_null(self.result(), self.pos)))
code.put_incref(self.result(), py_object_type)
else:
code.putln(code.error_goto_if_null(Naming.sent_value_cname, self.pos))
#-------------------------------------------------------------------
#
......@@ -8403,3 +8462,101 @@ static int %(binding_cfunc)s_init(void) {
}
""" % Naming.__dict__)
generator_utility_code = UtilityCode(
proto="""
static PyObject *__Pyx_Generator_Next(PyObject *self);
static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value);
static PyObject *__Pyx_Generator_Close(PyObject *self);
static PyObject *__Pyx_Generator_Throw(PyObject *gen, PyObject *args, CYTHON_UNUSED PyObject *kwds);
typedef PyObject *(*__pyx_generator_body_t)(PyObject *, PyObject *);
""",
impl="""
static CYTHON_INLINE PyObject *__Pyx_Generator_SendEx(struct __pyx_Generator_object *self, PyObject *value)
{
PyObject *retval;
if (self->is_running) {
PyErr_SetString(PyExc_ValueError,
"generator already executing");
return NULL;
}
if (self->resume_label == 0) {
if (value && value != Py_None) {
PyErr_SetString(PyExc_TypeError,
"can't send non-None value to a "
"just-started generator");
return NULL;
}
}
if (self->resume_label == -1) {
PyErr_SetNone(PyExc_StopIteration);
return NULL;
}
self->is_running = 1;
retval = self->body((PyObject *) self, value);
self->is_running = 0;
return retval;
}
static PyObject *__Pyx_Generator_Next(PyObject *self)
{
return __Pyx_Generator_SendEx((struct __pyx_Generator_object *) self, Py_None);
}
static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value)
{
return __Pyx_Generator_SendEx((struct __pyx_Generator_object *) self, value);
}
static PyObject *__Pyx_Generator_Close(PyObject *self)
{
struct __pyx_Generator_object *generator = (struct __pyx_Generator_object *) self;
PyObject *retval;
#if PY_VERSION_HEX < 0x02050000
PyErr_SetNone(PyExc_StopIteration);
#else
PyErr_SetNone(PyExc_GeneratorExit);
#endif
retval = __Pyx_Generator_SendEx(generator, NULL);
if (retval) {
Py_DECREF(retval);
PyErr_SetString(PyExc_RuntimeError,
"generator ignored GeneratorExit");
return NULL;
}
#if PY_VERSION_HEX < 0x02050000
if (PyErr_ExceptionMatches(PyExc_StopIteration))
#else
if (PyErr_ExceptionMatches(PyExc_StopIteration)
|| PyErr_ExceptionMatches(PyExc_GeneratorExit))
#endif
{
PyErr_Clear(); /* ignore these errors */
Py_INCREF(Py_None);
return Py_None;
}
return NULL;
}
static PyObject *__Pyx_Generator_Throw(PyObject *self, PyObject *args, CYTHON_UNUSED PyObject *kwds)
{
struct __pyx_Generator_object *generator = (struct __pyx_Generator_object *) self;
PyObject *typ;
PyObject *tb = NULL;
PyObject *val = NULL;
if (!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb))
return NULL;
__Pyx_Raise(typ, val, tb);
return __Pyx_Generator_SendEx(generator, NULL);
}
""",
proto_block='utility_code_proto_before_types',
requires=[Nodes.raise_utility_code],
)
......@@ -964,9 +964,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
type.vtabstruct_cname,
type.vtabslot_cname))
for attr in type.scope.var_entries:
if attr.is_declared_generic:
attr_type = py_object_type
else:
attr_type = attr.type
code.putln(
"%s;" %
attr.type.declaration_code(attr.cname))
attr_type.declaration_code(attr.cname))
code.putln(footer)
if type.objtypedef_cname is not None:
# Only for exposing public typedef name.
......@@ -1265,7 +1269,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
for entry in py_attrs:
name = "p->%s" % entry.cname
code.putln("tmp = ((PyObject*)%s);" % name)
code.put_init_to_py_none(name, entry.type, nanny=False)
if entry.is_declared_generic:
code.put_init_to_py_none(name, py_object_type, nanny=False)
else:
code.put_init_to_py_none(name, entry.type, nanny=False)
code.putln("Py_XDECREF(tmp);")
code.putln(
"return 0;")
......@@ -2762,8 +2769,9 @@ refnanny_utility_code = UtilityCode(proto="""
Py_XDECREF(m);
return (__Pyx_RefNannyAPIStruct *)r;
}
#define __Pyx_RefNannyDeclareContext void *__pyx_refnanny;
#define __Pyx_RefNannySetupContext(name) \
void *__pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__)
__pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__)
#define __Pyx_RefNannyFinishContext() \
__Pyx_RefNanny->FinishContext(&__pyx_refnanny)
#define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
......@@ -2772,6 +2780,7 @@ refnanny_utility_code = UtilityCode(proto="""
#define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
#define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r);} } while(0)
#else
#define __Pyx_RefNannyDeclareContext
#define __Pyx_RefNannySetupContext(name)
#define __Pyx_RefNannyFinishContext()
#define __Pyx_INCREF(r) Py_INCREF(r)
......
......@@ -19,6 +19,7 @@ funcdoc_prefix = pyrex_prefix + "doc_"
enum_prefix = pyrex_prefix + "e_"
func_prefix = pyrex_prefix + "f_"
pyfunc_prefix = pyrex_prefix + "pf_"
genbody_prefix = pyrex_prefix + "gb_"
gstab_prefix = pyrex_prefix + "getsets_"
prop_get_prefix = pyrex_prefix + "getprop_"
const_prefix = pyrex_prefix + "k_"
......@@ -51,6 +52,7 @@ lambda_func_prefix = pyrex_prefix + "lambda_"
module_is_main = pyrex_prefix + "module_is_main_"
args_cname = pyrex_prefix + "args"
sent_value_cname = pyrex_prefix + "sent_value"
pykwdlist_cname = pyrex_prefix + "pyargnames"
obj_base_cname = pyrex_prefix + "base"
builtins_cname = pyrex_prefix + "b"
......@@ -107,10 +109,6 @@ exc_lineno_name = pyrex_prefix + "exc_lineno"
exc_vars = (exc_type_name, exc_value_name, exc_tb_name)
exc_save_vars = (pyrex_prefix + 'save_exc_type',
pyrex_prefix + 'save_exc_value',
pyrex_prefix + 'save_exc_tb')
api_name = pyrex_prefix + "capi__"
h_guard_prefix = "__PYX_HAVE__"
......
......@@ -23,7 +23,7 @@ from PyrexTypes import py_object_type, error_type, CFuncType
from Symtab import ModuleScope, LocalScope, ClosureScope, \
StructOrUnionScope, PyClassScope, CClassScope, CppClassScope
from Cython.Utils import open_new_file, replace_suffix
from Code import UtilityCode
from Code import UtilityCode, ClosureTempAllocator
from StringEncoding import EncodedString, escape_byte_string, split_string_literal
import Options
import ControlFlow
......@@ -1177,6 +1177,8 @@ class FuncDefNode(StatNode, BlockNode):
assmt = None
needs_closure = False
needs_outer_scope = False
is_generator = False
is_generator_body = False
modifiers = []
def analyse_default_values(self, env):
......@@ -1236,6 +1238,9 @@ class FuncDefNode(StatNode, BlockNode):
lenv.directives = env.directives
return lenv
def generate_function_body(self, env, code):
self.body.generate_execution_code(code)
def generate_function_definitions(self, env, code):
import Buffer
......@@ -1323,6 +1328,8 @@ class FuncDefNode(StatNode, BlockNode):
(self.return_type.declaration_code(Naming.retval_cname),
init))
tempvardecl_code = code.insertion_point()
if not lenv.nogil:
code.put_declare_refcount_context()
self.generate_keyword_list(code)
if profile:
code.put_trace_declarations()
......@@ -1402,7 +1409,7 @@ class FuncDefNode(StatNode, BlockNode):
# -------------------------
# ----- Function body -----
# -------------------------
self.body.generate_execution_code(code)
self.generate_function_body(env, code)
# ----- Default return value
code.putln("")
......@@ -1484,14 +1491,10 @@ class FuncDefNode(StatNode, BlockNode):
if entry.type.is_pyobject:
if entry.used and not entry.in_closure:
code.put_var_decref(entry)
elif entry.in_closure and self.needs_closure:
code.put_giveref(entry.cname)
# Decref any increfed args
for entry in lenv.arg_entries:
if entry.type.is_pyobject:
if entry.in_closure:
code.put_var_giveref(entry)
elif acquire_gil or entry.assignments:
if (acquire_gil or entry.assignments) and not entry.in_closure:
code.put_var_decref(entry)
if self.needs_closure:
code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
......@@ -1920,6 +1923,7 @@ class DefNode(FuncDefNode):
num_required_kw_args = 0
reqd_kw_flags_cname = "0"
is_wrapper = 0
no_assignment_synthesis = 0
decorators = None
return_type_annotation = None
entry = None
......@@ -2250,6 +2254,8 @@ class DefNode(FuncDefNode):
self.synthesize_assignment_node(env)
def needs_assignment_synthesis(self, env, code=None):
if self.no_assignment_synthesis:
return False
# Should enable for module level as well, that will require more testing...
if self.entry.is_anonymous:
return True
......@@ -2353,8 +2359,8 @@ class DefNode(FuncDefNode):
code.putln("0};")
def generate_argument_parsing_code(self, env, code):
# Generate PyArg_ParseTuple call for generic
# arguments, if any.
# Generate fast equivalent of PyArg_ParseTuple call for
# generic arguments, if any, including args/kwargs
if self.entry.signature.has_dummy_arg and not self.self_in_stararg:
# get rid of unused argument warning
code.putln("%s = %s;" % (Naming.self_cname, Naming.self_cname))
......@@ -2431,14 +2437,24 @@ class DefNode(FuncDefNode):
if code.label_used(end_label):
code.put_label(end_label)
# fix refnanny view on closure variables here, instead of
# doing it separately for each arg parsing special case
if self.star_arg and self.star_arg.entry.in_closure:
code.put_var_giveref(self.star_arg.entry)
if self.starstar_arg and self.starstar_arg.entry.in_closure:
code.put_var_giveref(self.starstar_arg.entry)
for arg in self.args:
if arg.type.is_pyobject and arg.entry.in_closure:
code.put_var_giveref(arg.entry)
def generate_arg_assignment(self, arg, item, code):
if arg.type.is_pyobject:
if arg.is_generic:
item = PyrexTypes.typecast(arg.type, PyrexTypes.py_object_type, item)
entry = arg.entry
code.putln("%s = %s;" % (entry.cname, item))
if entry.in_closure:
code.put_var_incref(entry)
code.put_incref(item, PyrexTypes.py_object_type)
code.putln("%s = %s;" % (entry.cname, item))
else:
func = arg.type.from_py_function
if func:
......@@ -2657,19 +2673,18 @@ class DefNode(FuncDefNode):
code.putln('if (PyTuple_GET_SIZE(%s) > %d) {' % (
Naming.args_cname,
max_positional_args))
code.put('%s = PyTuple_GetSlice(%s, %d, PyTuple_GET_SIZE(%s)); ' % (
code.putln('%s = PyTuple_GetSlice(%s, %d, PyTuple_GET_SIZE(%s));' % (
self.star_arg.entry.cname, Naming.args_cname,
max_positional_args, Naming.args_cname))
code.put_gotref(self.star_arg.entry.cname)
code.putln("if (unlikely(!%s)) {" % self.star_arg.entry.cname)
if self.starstar_arg:
code.putln("")
code.putln("if (unlikely(!%s)) {" % self.star_arg.entry.cname)
code.put_decref_clear(self.starstar_arg.entry.cname, py_object_type)
code.putln('return %s;' % self.error_value())
code.putln('}')
else:
code.putln("if (unlikely(!%s)) return %s;" % (
self.star_arg.entry.cname, self.error_value()))
if self.needs_closure:
code.put_decref(Naming.cur_scope_cname, self.local_scope.scope_class.type)
code.put_finish_refcount_context()
code.putln('return %s;' % self.error_value())
code.putln('}')
code.put_gotref(self.star_arg.entry.cname)
code.putln('} else {')
code.put("%s = %s; " % (self.star_arg.entry.cname, Naming.empty_tuple))
code.put_incref(Naming.empty_tuple, py_object_type)
......@@ -2839,9 +2854,9 @@ class DefNode(FuncDefNode):
if arg.needs_conversion:
self.generate_arg_conversion(arg, code)
elif arg.entry.in_closure:
code.putln('%s = %s;' % (arg.entry.cname, arg.hdr_cname))
if arg.type.is_pyobject:
code.put_var_incref(arg.entry)
code.put_incref(arg.hdr_cname, py_object_type)
code.putln('%s = %s;' % (arg.entry.cname, arg.hdr_cname))
def generate_arg_conversion(self, arg, code):
# Generate conversion code for one argument.
......@@ -2914,6 +2929,146 @@ class DefNode(FuncDefNode):
def caller_will_check_exceptions(self):
return 1
class GeneratorDefNode(DefNode):
# Generator DefNode.
#
# gbody GeneratorBodyDefNode
#
is_generator = True
needs_closure = True
child_attrs = DefNode.child_attrs + ["gbody"]
def __init__(self, **kwargs):
# XXX: don't actually needs a body
kwargs['body'] = StatListNode(kwargs['pos'], stats=[])
super(GeneratorDefNode, self).__init__(**kwargs)
def analyse_declarations(self, env):
super(GeneratorDefNode, self).analyse_declarations(env)
self.gbody.local_scope = self.local_scope
self.gbody.analyse_declarations(env)
def generate_function_body(self, env, code):
body_cname = self.gbody.entry.func_cname
generator_cname = '%s->%s' % (Naming.cur_scope_cname, Naming.obj_base_cname)
code.putln('%s.resume_label = 0;' % generator_cname)
code.putln('%s.body = (__pyx_generator_body_t) %s;' % (generator_cname, body_cname))
code.put_giveref(Naming.cur_scope_cname)
code.put_finish_refcount_context()
code.putln("return (PyObject *) %s;" % Naming.cur_scope_cname);
def generate_function_definitions(self, env, code):
self.gbody.generate_function_header(code, proto=True)
super(GeneratorDefNode, self).generate_function_definitions(env, code)
self.gbody.generate_function_definitions(env, code)
class GeneratorBodyDefNode(DefNode):
# Generator body DefNode.
#
is_generator_body = True
def __init__(self, pos=None, name=None, body=None):
super(GeneratorBodyDefNode, self).__init__(pos=pos, body=body, name=name, doc=None,
args=[],
star_arg=None, starstar_arg=None)
def declare_generator_body(self, env):
prefix = env.next_id(env.scope_prefix)
name = env.next_id('generator')
entry = env.declare_var(prefix + name, py_object_type, self.pos, visibility='private')
entry.func_cname = Naming.genbody_prefix + prefix + name
entry.qualified_name = EncodedString(self.name)
self.entry = entry
def analyse_declarations(self, env):
self.analyse_argument_types(env)
self.declare_generator_body(env)
def generate_function_header(self, code, proto=False):
header = "static PyObject *%s(%s, PyObject *%s)" % (
self.entry.func_cname,
self.local_scope.scope_class.type.declaration_code(Naming.cur_scope_cname),
Naming.sent_value_cname)
if proto:
code.putln('%s; /* proto */' % header)
else:
code.putln('%s /* generator body */\n{' % header);
def generate_function_definitions(self, env, code):
lenv = self.local_scope
# Generate closure function definitions
self.body.generate_function_definitions(lenv, code)
# Generate C code for header and body of function
code.enter_cfunc_scope()
code.return_from_error_cleanup_label = code.new_label()
# ----- Top-level constants used by this function
code.mark_pos(self.pos)
self.generate_cached_builtins_decls(lenv, code)
# ----- Function header
code.putln("")
self.generate_function_header(code)
# ----- Local variables
code.putln("PyObject *%s = NULL;" % Naming.retval_cname)
tempvardecl_code = code.insertion_point()
code.put_declare_refcount_context()
code.put_setup_refcount_context(self.entry.name)
# ----- Resume switch point.
code.funcstate.init_closure_temps(lenv.scope_class.type.scope)
resume_code = code.insertion_point()
first_run_label = code.new_label('first_run')
code.use_label(first_run_label)
code.put_label(first_run_label)
code.putln('%s' %
(code.error_goto_if_null(Naming.sent_value_cname, self.pos)))
# ----- Function body
self.generate_function_body(env, code)
code.putln('PyErr_SetNone(PyExc_StopIteration); %s' % code.error_goto(self.pos))
# ----- Error cleanup
if code.error_label in code.labels_used:
code.put_goto(code.return_label)
code.put_label(code.error_label)
for cname, type in code.funcstate.all_managed_temps():
code.put_xdecref(cname, type)
code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
# ----- Non-error return cleanup
code.put_label(code.return_label)
code.put_xdecref(Naming.retval_cname, py_object_type)
code.putln('%s->%s.resume_label = -1;' % (Naming.cur_scope_cname, Naming.obj_base_cname))
code.put_finish_refcount_context()
code.putln('return NULL;');
code.putln("}")
# ----- Go back and insert temp variable declarations
tempvardecl_code.put_temp_declarations(code.funcstate)
# ----- Generator resume code
resume_code.putln("switch (%s->%s.resume_label) {" % (Naming.cur_scope_cname, Naming.obj_base_cname));
resume_code.putln("case 0: goto %s;" % first_run_label)
from ParseTreeTransforms import YieldNodeCollector
collector = YieldNodeCollector()
collector.visitchildren(self)
for yield_expr in collector.yields:
resume_code.putln("case %d: goto %s;" % (yield_expr.label_num, yield_expr.label_name));
resume_code.putln("default: /* CPython raises the right error here */");
resume_code.put_finish_refcount_context()
resume_code.putln("return NULL;");
resume_code.putln("}");
code.exit_cfunc_scope()
class OverrideCheckNode(StatNode):
# A Node for dispatching to the def method if it
# is overriden.
......@@ -3352,6 +3507,24 @@ class GlobalNode(StatNode):
pass
class NonlocalNode(StatNode):
# Nonlocal variable declaration via the 'nonlocal' keyword.
#
# names [string]
child_attrs = []
def analyse_declarations(self, env):
for name in self.names:
env.declare_nonlocal(name, self.pos)
def analyse_expressions(self, env):
pass
def generate_execution_code(self, code):
pass
class ExprStatNode(StatNode):
# Expression used as a statement.
#
......@@ -3376,6 +3549,7 @@ class ExprStatNode(StatNode):
self.__class__ = PassStatNode
def analyse_expressions(self, env):
self.expr.result_is_used = False # hint that .result() may safely be left empty
self.expr.analyse_expressions(env)
def nogil_check(self, env):
......@@ -4705,12 +4879,12 @@ class TryExceptStatNode(StatNode):
try_continue_label = code.new_label('try_continue')
try_end_label = code.new_label('try_end')
exc_save_vars = [code.funcstate.allocate_temp(py_object_type, False)
for i in xrange(3)]
code.putln("{")
code.putln("PyObject %s;" %
', '.join(['*%s' % var for var in Naming.exc_save_vars]))
code.putln("__Pyx_ExceptionSave(%s);" %
', '.join(['&%s' % var for var in Naming.exc_save_vars]))
for var in Naming.exc_save_vars:
', '.join(['&%s' % var for var in exc_save_vars]))
for var in exc_save_vars:
code.put_xgotref(var)
code.putln(
"/*try:*/ {")
......@@ -4729,14 +4903,15 @@ class TryExceptStatNode(StatNode):
self.else_clause.generate_execution_code(code)
code.putln(
"}")
for var in Naming.exc_save_vars:
for var in exc_save_vars:
code.put_xdecref_clear(var, py_object_type)
code.put_goto(try_end_label)
if code.label_used(try_return_label):
code.put_label(try_return_label)
for var in Naming.exc_save_vars: code.put_xgiveref(var)
for var in exc_save_vars:
code.put_xgiveref(var)
code.putln("__Pyx_ExceptionReset(%s);" %
', '.join(Naming.exc_save_vars))
', '.join(exc_save_vars))
code.put_goto(old_return_label)
code.put_label(our_error_label)
for temp_name, type in temps_to_clean_up:
......@@ -4748,9 +4923,10 @@ class TryExceptStatNode(StatNode):
if error_label_used or not self.has_default_clause:
if error_label_used:
code.put_label(except_error_label)
for var in Naming.exc_save_vars: code.put_xgiveref(var)
for var in exc_save_vars:
code.put_xgiveref(var)
code.putln("__Pyx_ExceptionReset(%s);" %
', '.join(Naming.exc_save_vars))
', '.join(exc_save_vars))
code.put_goto(old_error_label)
for exit_label, old_label in zip(
......@@ -4759,19 +4935,24 @@ class TryExceptStatNode(StatNode):
if code.label_used(exit_label):
code.put_label(exit_label)
for var in Naming.exc_save_vars: code.put_xgiveref(var)
for var in exc_save_vars:
code.put_xgiveref(var)
code.putln("__Pyx_ExceptionReset(%s);" %
', '.join(Naming.exc_save_vars))
', '.join(exc_save_vars))
code.put_goto(old_label)
if code.label_used(except_end_label):
code.put_label(except_end_label)
for var in Naming.exc_save_vars: code.put_xgiveref(var)
for var in exc_save_vars:
code.put_xgiveref(var)
code.putln("__Pyx_ExceptionReset(%s);" %
', '.join(Naming.exc_save_vars))
', '.join(exc_save_vars))
code.put_label(try_end_label)
code.putln("}")
for cname in exc_save_vars:
code.funcstate.release_temp(cname)
code.return_label = old_return_label
code.break_label = old_break_label
code.continue_label = old_continue_label
......
......@@ -1170,11 +1170,12 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform):
self.yield_nodes = []
visit_Node = Visitor.TreeVisitor.visitchildren
def visit_YieldExprNode(self, node):
# XXX: disable inlining while it's not back supported
def __visit_YieldExprNode(self, node):
self.yield_nodes.append(node)
self.visitchildren(node)
def visit_ExprStatNode(self, node):
def __visit_ExprStatNode(self, node):
self.visitchildren(node)
if node.expr in self.yield_nodes:
self.yield_stat_nodes[node.expr] = node
......
......@@ -19,6 +19,7 @@ cdef class NormalizeTree(CythonTransform):
cdef class PostParse(ScopeTrackingTransform):
cdef dict specialattribute_handlers
cdef size_t lambda_counter
cdef size_t genexpr_counter
cdef _visit_assignment_node(self, node, list expr_list)
......@@ -45,6 +46,11 @@ cdef class AlignFunctionDefinitions(CythonTransform):
cdef dict directives
cdef scope
cdef class YieldNodeCollector(TreeVisitor):
cdef public list yields
cdef public list returns
cdef public bint has_return_value
cdef class MarkClosureVisitor(CythonTransform):
cdef bint needs_closure
......@@ -52,6 +58,7 @@ cdef class CreateClosureClasses(CythonTransform):
cdef list path
cdef bint in_lambda
cdef module_scope
cdef generator_class
cdef class GilCheck(VisitorTransform):
cdef list env_stack
......
......@@ -182,6 +182,7 @@ class PostParse(ScopeTrackingTransform):
def visit_ModuleNode(self, node):
self.lambda_counter = 1
self.genexpr_counter = 1
return super(PostParse, self).visit_ModuleNode(node)
def visit_LambdaNode(self, node):
......@@ -189,14 +190,34 @@ class PostParse(ScopeTrackingTransform):
lambda_id = self.lambda_counter
self.lambda_counter += 1
node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
body = Nodes.ReturnStatNode(
node.result_expr.pos, value = node.result_expr)
collector = YieldNodeCollector()
collector.visitchildren(node.result_expr)
if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
body = ExprNodes.YieldExprNode(
node.result_expr.pos, arg=node.result_expr)
body = Nodes.ExprStatNode(node.result_expr.pos, expr=body)
else:
body = Nodes.ReturnStatNode(
node.result_expr.pos, value=node.result_expr)
node.def_node = Nodes.DefNode(
node.pos, name=node.name, lambda_name=node.lambda_name,
args=node.args, star_arg=node.star_arg,
starstar_arg=node.starstar_arg,
body=body)
body=body, doc=None)
self.visitchildren(node)
return node
def visit_GeneratorExpressionNode(self, node):
# unpack a generator expression into the corresponding DefNode
genexpr_id = self.genexpr_counter
self.genexpr_counter += 1
node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
node.def_node = Nodes.DefNode(node.pos, name=node.name,
doc=None,
args=[], star_arg=None,
starstar_arg=None,
body=node.loop)
self.visitchildren(node)
return node
......@@ -1408,6 +1429,42 @@ class AlignFunctionDefinitions(CythonTransform):
return node
class YieldNodeCollector(TreeVisitor):
def __init__(self):
super(YieldNodeCollector, self).__init__()
self.yields = []
self.returns = []
self.has_return_value = False
def visit_Node(self, node):
return self.visitchildren(node)
def visit_YieldExprNode(self, node):
if self.has_return_value:
error(node.pos, "'yield' outside function")
self.yields.append(node)
self.visitchildren(node)
def visit_ReturnStatNode(self, node):
if node.value:
self.has_return_value = True
if self.yields:
error(node.pos, "'return' with argument inside generator")
self.returns.append(node)
def visit_ClassDefNode(self, node):
pass
def visit_DefNode(self, node):
pass
def visit_LambdaNode(self, node):
pass
def visit_GeneratorExpressionNode(self, node):
pass
class MarkClosureVisitor(CythonTransform):
def visit_ModuleNode(self, node):
......@@ -1420,6 +1477,27 @@ class MarkClosureVisitor(CythonTransform):
self.visitchildren(node)
node.needs_closure = self.needs_closure
self.needs_closure = True
collector = YieldNodeCollector()
collector.visitchildren(node)
if collector.yields:
for i, yield_expr in enumerate(collector.yields):
yield_expr.label_num = i + 1
gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
name=node.name,
body=node.body)
generator = Nodes.GeneratorDefNode(pos=node.pos,
name=node.name,
args=node.args,
star_arg=node.star_arg,
starstar_arg=node.starstar_arg,
doc=node.doc,
decorators=node.decorators,
gbody=gbody,
lambda_name=node.lambda_name)
return generator
return node
def visit_CFuncDefNode(self, node):
......@@ -1440,7 +1518,6 @@ class MarkClosureVisitor(CythonTransform):
self.needs_closure = True
return node
class CreateClosureClasses(CythonTransform):
# Output closure classes in module scope for all functions
# that really need it.
......@@ -1449,24 +1526,78 @@ class CreateClosureClasses(CythonTransform):
super(CreateClosureClasses, self).__init__(context)
self.path = []
self.in_lambda = False
self.generator_class = None
def visit_ModuleNode(self, node):
self.module_scope = node.scope
self.visitchildren(node)
return node
def get_scope_use(self, node):
def create_generator_class(self, target_module_scope, pos):
if self.generator_class:
return self.generator_class
# XXX: make generator class creation cleaner
entry = target_module_scope.declare_c_class(name='__pyx_Generator',
objstruct_cname='__pyx_Generator_object',
typeobj_cname='__pyx_Generator_type',
pos=pos, defining=True, implementing=True)
klass = entry.type.scope
klass.is_internal = True
klass.directives = {'final': True}
body_type = PyrexTypes.create_typedef_type('generator_body',
PyrexTypes.c_void_ptr_type,
'__pyx_generator_body_t')
klass.declare_var(pos=pos, name='body', cname='body',
type=body_type, is_cdef=True)
klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
is_cdef=True)
klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
is_cdef=True)
import TypeSlots
e = klass.declare_pyfunction('send', pos)
e.func_cname = '__Pyx_Generator_Send'
e.signature = TypeSlots.binaryfunc
e = klass.declare_pyfunction('close', pos)
e.func_cname = '__Pyx_Generator_Close'
e.signature = TypeSlots.unaryfunc
e = klass.declare_pyfunction('throw', pos)
e.func_cname = '__Pyx_Generator_Throw'
e.signature = TypeSlots.pyfunction_signature
e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
e.func_cname = 'PyObject_SelfIter'
e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
e.func_cname = '__Pyx_Generator_Next'
self.generator_class = entry.type
return self.generator_class
def find_entries_used_in_closures(self, node):
from_closure = []
in_closure = []
for name, entry in node.local_scope.entries.items():
if entry.from_closure:
from_closure.append((name, entry))
elif entry.in_closure and not entry.from_closure:
elif entry.in_closure:
in_closure.append((name, entry))
return from_closure, in_closure
def create_class_from_scope(self, node, target_module_scope, inner_node=None):
from_closure, in_closure = self.get_scope_use(node)
# skip generator body
if node.is_generator_body:
return
# move local variables into closure
if node.is_generator:
for entry in node.local_scope.entries.values():
if not entry.from_closure:
entry.in_closure = True
from_closure, in_closure = self.find_entries_used_in_closures(node)
in_closure.sort()
# Now from the begining
......@@ -1485,8 +1616,11 @@ class CreateClosureClasses(CythonTransform):
inner_node = node.assmt.rhs
inner_node.needs_self_code = False
node.needs_outer_scope = False
# Simple cases
if not in_closure and not from_closure:
base_type = None
if node.is_generator:
base_type = self.create_generator_class(target_module_scope, node.pos)
elif not in_closure and not from_closure:
return
elif not in_closure:
func_scope.is_passthrough = True
......@@ -1496,8 +1630,10 @@ class CreateClosureClasses(CythonTransform):
as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
entry = target_module_scope.declare_c_class(name = as_name,
pos = node.pos, defining = True, implementing = True)
entry = target_module_scope.declare_c_class(
name=as_name, pos=node.pos, defining=True,
implementing=True, base_type=base_type)
func_scope.scope_class = entry
class_scope = entry.type.scope
class_scope.is_internal = True
......@@ -1512,11 +1648,13 @@ class CreateClosureClasses(CythonTransform):
is_cdef=True)
node.needs_outer_scope = True
for name, entry in in_closure:
class_scope.declare_var(pos=entry.pos,
closure_entry = class_scope.declare_var(pos=entry.pos,
name=entry.name,
cname=entry.cname,
type=entry.type,
is_cdef=True)
if entry.is_declared_generic:
closure_entry.is_declared_generic = 1
node.needs_closure = True
# Do it here because other classes are already checked
target_module_scope.check_c_class(func_scope.scope_class)
......
......@@ -84,6 +84,7 @@ cdef p_genexp(PyrexScanner s, expr)
#-------------------------------------------------------
cdef p_global_statement(PyrexScanner s)
cdef p_nonlocal_statement(PyrexScanner s)
cdef p_expression_or_assignment(PyrexScanner s)
cdef p_print_statement(PyrexScanner s)
cdef p_exec_statement(PyrexScanner s)
......
......@@ -1045,6 +1045,12 @@ def p_global_statement(s):
names = p_ident_list(s)
return Nodes.GlobalNode(pos, names = names)
def p_nonlocal_statement(s):
pos = s.position()
s.next()
names = p_ident_list(s)
return Nodes.NonlocalNode(pos, names = names)
def p_expression_or_assignment(s):
expr_list = [p_testlist_star_expr(s)]
while s.sy == '=':
......@@ -1598,6 +1604,8 @@ def p_simple_statement(s, first_statement = 0):
#print "p_simple_statement:", s.sy, s.systring ###
if s.sy == 'global':
node = p_global_statement(s)
elif s.sy == 'nonlocal':
node = p_nonlocal_statement(s)
elif s.sy == 'print':
node = p_print_statement(s)
elif s.sy == 'exec':
......
......@@ -36,7 +36,7 @@ def get_lexicon():
#------------------------------------------------------------------
py_reserved_words = [
"global", "def", "class", "print", "del", "pass", "break",
"global", "nonlocal", "def", "class", "print", "del", "pass", "break",
"continue", "return", "raise", "import", "exec", "try",
"except", "finally", "while", "if", "elif", "else", "for",
"in", "assert", "and", "or", "not", "is", "in", "lambda",
......
......@@ -1309,6 +1309,16 @@ class LocalScope(Scope):
entry = self.global_scope().lookup_target(name)
self.entries[name] = entry
def declare_nonlocal(self, name, pos):
# Pull entry from outer scope into local scope
orig_entry = self.lookup_here(name)
if orig_entry and orig_entry.scope is self and not orig_entry.from_closure:
error(pos, "'%s' redeclared as nonlocal" % name)
else:
entry = self.lookup(name)
if entry is None or not entry.from_closure:
error(pos, "no binding for nonlocal '%s' found" % name)
def lookup(self, name):
# Look up name in this scope or an enclosing one.
# Return None if not found.
......@@ -1326,6 +1336,7 @@ class LocalScope(Scope):
inner_entry.is_variable = True
inner_entry.outer_entry = entry
inner_entry.from_closure = True
inner_entry.is_declared_generic = entry.is_declared_generic
self.entries[name] = inner_entry
return inner_entry
return entry
......@@ -1479,6 +1490,20 @@ class PyClassScope(ClassScope):
entry.is_pyclass_attr = 1
return entry
def declare_nonlocal(self, name, pos):
# Pull entry from outer scope into local scope
orig_entry = self.lookup_here(name)
if orig_entry and orig_entry.scope is self and not orig_entry.from_closure:
error(pos, "'%s' redeclared as nonlocal" % name)
else:
entry = self.lookup(name)
if entry is None:
error(pos, "no binding for nonlocal '%s' found" % name)
else:
# FIXME: this works, but it's unclear if it's the
# right thing to do
self.entries[name] = entry
def add_default_value(self, type):
return self.outer_scope.add_default_value(type)
......
......@@ -221,7 +221,8 @@ class SimpleAssignmentTypeInferer(object):
# TODO: Implement a real type inference algorithm.
# (Something more powerful than just extending this one...)
def infer_types(self, scope):
enabled = not scope.is_closure_scope and scope.directives['infer_types']
closure_or_inner = scope.is_closure_scope or (scope.outer_scope and scope.outer_scope.is_closure_scope)
enabled = not closure_or_inner and scope.directives['infer_types']
verbose = scope.directives['infer_types.verbose']
if enabled == True:
spanning_type = aggressive_spanning_type
......
......@@ -10,7 +10,6 @@ cfunc_call_tuple_args_T408
compile.cpp_operators
cpp_templated_ctypedef
cpp_structs
genexpr_T491
with_statement_module_level_T536
function_as_method_T494
closure_inside_cdef_T554
......@@ -19,6 +18,7 @@ genexpr_iterable_lookup_T600
for_from_pyvar_loop_T601
decorators_T593
temp_sideeffects_T654
generator_type_inference
# CPython regression tests that don't current work:
pyregr.test_threadsignals
......
def foo():
yield
return 0
def bar(a):
return 0
yield
yield
class Foo:
yield
_ERRORS = u"""
3:4: 'return' with argument inside generator
7:4: 'yield' outside function
9:0: 'yield' not supported here
12:4: 'yield' not supported here
"""
def test_non_existant():
nonlocal no_such_name
no_such_name = 1
def redef():
x = 1
def f():
x = 2
nonlocal x
global_name = 5
def ref_to_global():
nonlocal global_name
global_name = 6
def global_in_class_scope():
class Test():
nonlocal global_name
global_name = 6
def redef_in_class_scope():
x = 1
class Test():
x = 2
nonlocal x
_ERRORS = u"""
3:4: no binding for nonlocal 'no_such_name' found
10:8: 'x' redeclared as nonlocal
15:4: no binding for nonlocal 'global_name' found
27:8: 'x' redeclared as nonlocal
"""
cdef class Test:
cdef int x
cdef class SelfInClosure(object):
cdef Test _t
cdef int x
def plain(self):
"""
>>> o = SelfInClosure()
>>> o.plain()
1
"""
self.x = 1
return self.x
def closure_method(self):
"""
>>> o = SelfInClosure()
>>> o.closure_method()() == o
True
"""
def nested():
return self
return nested
def closure_method_cdef_attr(self, Test t):
"""
>>> o = SelfInClosure()
>>> o.closure_method_cdef_attr(Test())()
(1, 2)
"""
t.x = 2
self._t = t
self.x = 1
def nested():
return self.x, t.x
return nested
# mode: run
# tag: typeinference, generators
cimport cython
def test_type_inference():
"""
>>> [ item for item in test_type_inference() ]
[(2.0, 'double'), (2.0, 'double'), (2.0, 'double')]
"""
x = 1.0
for i in range(3):
yield x * 2.0, cython.typeof(x)
try:
from builtins import next # Py3k
except ImportError:
def next(it):
return it.next()
if hasattr(__builtins__, 'GeneratorExit'):
GeneratorExit = __builtins__.GeneratorExit
else: # < 2.5
GeneratorExit = StopIteration
def very_simple():
"""
>>> x = very_simple()
>>> next(x)
1
>>> next(x)
Traceback (most recent call last):
StopIteration
>>> next(x)
Traceback (most recent call last):
StopIteration
>>> x = very_simple()
>>> x.send(1)
Traceback (most recent call last):
TypeError: can't send non-None value to a just-started generator
"""
yield 1
def simple():
"""
>>> x = simple()
>>> list(x)
[1, 2, 3]
"""
yield 1
yield 2
yield 3
def simple_seq(seq):
"""
>>> x = simple_seq("abc")
>>> list(x)
['a', 'b', 'c']
"""
for i in seq:
yield i
def simple_send():
"""
>>> x = simple_send()
>>> next(x)
>>> x.send(1)
1
>>> x.send(2)
2
>>> x.send(3)
3
"""
i = None
while True:
i = yield i
def raising():
"""
>>> x = raising()
>>> next(x)
Traceback (most recent call last):
KeyError: 'foo'
>>> next(x)
Traceback (most recent call last):
StopIteration
"""
yield {}['foo']
def with_outer(*args):
"""
>>> x = with_outer(1, 2, 3)
>>> list(x())
[1, 2, 3]
"""
def generator():
for i in args:
yield i
return generator
def with_outer_raising(*args):
"""
>>> x = with_outer_raising(1, 2, 3)
>>> list(x())
[1, 2, 3]
"""
def generator():
for i in args:
yield i
raise StopIteration
return generator
def test_close():
"""
>>> x = test_close()
>>> x.close()
>>> x = test_close()
>>> next(x)
>>> x.close()
>>> next(x)
Traceback (most recent call last):
StopIteration
"""
while True:
yield
def test_ignore_close():
"""
>>> x = test_ignore_close()
>>> x.close()
>>> x = test_ignore_close()
>>> next(x)
>>> x.close()
Traceback (most recent call last):
RuntimeError: generator ignored GeneratorExit
"""
try:
yield
except GeneratorExit:
yield
def check_throw():
"""
>>> x = check_throw()
>>> x.throw(ValueError)
Traceback (most recent call last):
ValueError
>>> next(x)
Traceback (most recent call last):
StopIteration
>>> x = check_throw()
>>> next(x)
>>> x.throw(ValueError)
>>> next(x)
>>> x.throw(IndexError, "oops")
Traceback (most recent call last):
IndexError: oops
>>> next(x)
Traceback (most recent call last):
StopIteration
"""
while True:
try:
yield
except ValueError:
pass
def test_first_assignment():
"""
>>> gen = test_first_assignment()
>>> next(gen)
5
>>> next(gen)
10
>>> next(gen)
(5, 10)
"""
cdef x = 5 # first
yield x
cdef y = 10 # first
yield y
yield (x,y)
def test_swap_assignment():
"""
>>> gen = test_swap_assignment()
>>> next(gen)
(5, 10)
>>> next(gen)
(10, 5)
"""
x,y = 5,10
yield (x,y)
x,y = y,x # no ref-counting here
yield (x,y)
class Foo(object):
"""
>>> obj = Foo()
>>> list(obj.simple(1, 2, 3))
[1, 2, 3]
"""
def simple(self, *args):
for i in args:
yield i
def generator_nonlocal():
"""
>>> g = generator_nonlocal()
>>> list(g(5))
[2, 3, 4, 5, 6]
"""
def f(x):
def g(y):
nonlocal x
for i in range(y):
x += 1
yield x
return g
return f(1)
def test_nested(a, b, c):
"""
>>> obj = test_nested(1, 2, 3)
>>> [i() for i in obj]
[1, 2, 3, 4]
"""
def one():
return a
def two():
return b
def three():
return c
def new_closure(a, b):
def sum():
return a + b
return sum
yield one
yield two
yield three
yield new_closure(a, c)
def tolist(func):
def wrapper(*args, **kwargs):
return list(func(*args, **kwargs))
return wrapper
@tolist
def test_decorated(*args):
"""
>>> test_decorated(1, 2, 3)
[1, 2, 3]
"""
for i in args:
yield i
def test_return(a):
"""
>>> d = dict()
>>> obj = test_return(d)
>>> next(obj)
1
>>> next(obj)
Traceback (most recent call last):
StopIteration
>>> d['i_was_here']
True
"""
yield 1
a['i_was_here'] = True
return
def test_copied_yield(foo):
"""
>>> class Manager(object):
... def __enter__(self):
... return self
... def __exit__(self, type, value, tb):
... pass
>>> list(test_copied_yield(Manager()))
[1]
"""
with foo:
yield 1
def test_nested_yield():
"""
>>> obj = test_nested_yield()
>>> next(obj)
1
>>> obj.send(2)
2
>>> obj.send(3)
3
>>> obj.send(4)
Traceback (most recent call last):
StopIteration
"""
yield (yield (yield 1))
def test_inside_lambda():
"""
>>> obj = test_inside_lambda()()
>>> next(obj)
1
>>> obj.send('a')
2
>>> obj.send('b')
('a', 'b')
"""
return lambda:((yield 1), (yield 2))
def test_nested_gen(int n):
"""
>>> [list(a) for a in test_nested_gen(5)]
[[], [0], [0, 1], [0, 1, 2], [0, 1, 2, 3]]
"""
for a in range(n):
yield (b for b in range(a))
def test_lambda(n):
"""
>>> [i() for i in test_lambda(3)]
[0, 1, 2]
"""
for i in range(n):
yield lambda : i
def simple():
"""
>>> simple()
1
2
"""
x = 1
y = 2
def f():
nonlocal x
nonlocal x, y
print(x)
print(y)
f()
def assign():
"""
>>> assign()
1
"""
xx = 0
def ff():
nonlocal xx
xx += 1
print(xx)
ff()
def nested():
"""
>>> nested()
1
"""
x = 0
def fx():
def gx():
nonlocal x
x=1
print(x)
return gx
fx()()
def arg(x):
"""
>>> arg('x')
xyy
"""
def appendy():
nonlocal x
x += 'y'
x+='y'
appendy()
print x
return
def argtype(int n):
"""
>>> argtype(0)
1
"""
def inc():
nonlocal n
n += 1
inc()
print n
return
def ping_pong():
"""
>>> f = ping_pong()
>>> inc, dec = f(0)
>>> inc()
1
>>> inc()
2
>>> dec()
1
>>> inc()
2
>>> dec()
1
>>> dec()
0
"""
def f(x):
def inc():
nonlocal x
x += 1
return x
def dec():
nonlocal x
x -= 1
return x
return inc, dec
return f
def methods():
"""
>>> f = methods()
>>> c = f(0)
>>> c.inc()
1
>>> c.inc()
2
>>> c.dec()
1
>>> c.dec()
0
"""
def f(x):
class c:
def inc(self):
nonlocal x
x += 1
return x
def dec(self):
nonlocal x
x -= 1
return x
return c()
return f
def class_body(int x, y):
"""
>>> c = class_body(2,99)
>>> c.z
(3, 2)
>>> c.x #doctest: +ELLIPSIS
Traceback (most recent call last):
AttributeError: ...
>>> c.y #doctest: +ELLIPSIS
Traceback (most recent call last):
AttributeError: ...
"""
class c(object):
nonlocal x
nonlocal y
y = 2
x += 1
z = x,y
return c()
def nested_nonlocals(x):
"""
>>> g = nested_nonlocals(1)
>>> h = g()
>>> h()
3
"""
def g():
nonlocal x
x -= 2
def h():
nonlocal x
x += 4
return x
return h
return g
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment