Commit a4798d9d authored by Vitja Makarov's avatar Vitja Makarov

Move generators implementation to Cython/Utility and make closure

generator's attribute.
parent a6ba00ae
......@@ -5828,7 +5828,7 @@ class ClassCellNode(ExprNode):
Naming.self_cname))
else:
code.putln('%s = %s->classobj;' % (
self.result(), Naming.cur_scope_cname))
self.result(), Naming.generator_cname))
code.putln(
'if (!%s) { PyErr_SetString(PyExc_SystemError, '
'"super(): empty __class__ cell"); %s }' % (
......@@ -6250,7 +6250,8 @@ class YieldExprNode(ExprNode):
code.put_xgiveref(Naming.retval_cname)
code.put_finish_refcount_context()
code.putln("/* return from generator, yielding value */")
code.putln("%s->%s.resume_label = %d;" % (Naming.cur_scope_cname, Naming.obj_base_cname, self.label_num))
code.putln("%s->resume_label = %d;" % (
Naming.generator_cname, self.label_num))
code.putln("return %s;" % Naming.retval_cname);
code.put_label(self.label_name)
for cname, save_cname, type in saved:
......@@ -9979,122 +9980,8 @@ cyfunction_class_cell_utility_code = UtilityCode.load(
"CythonFunction.c",
requires=[binding_cfunc_utility_code])
generator_utility_code = UtilityCode(
proto="""
static PyObject *__Pyx_Generator_Next(PyObject *self);
static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value);
static PyObject *__Pyx_Generator_Close(PyObject *self);
static PyObject *__Pyx_Generator_Throw(PyObject *gen, PyObject *args, CYTHON_UNUSED PyObject *kwds);
typedef PyObject *(*__pyx_generator_body_t)(PyObject *, PyObject *);
""",
impl="""
static CYTHON_INLINE void __Pyx_Generator_ExceptionClear(struct __pyx_Generator_object *self)
{
Py_XDECREF(self->exc_type);
Py_XDECREF(self->exc_value);
Py_XDECREF(self->exc_traceback);
self->exc_type = NULL;
self->exc_value = NULL;
self->exc_traceback = NULL;
}
static CYTHON_INLINE PyObject *__Pyx_Generator_SendEx(struct __pyx_Generator_object *self, PyObject *value)
{
PyObject *retval;
if (self->is_running) {
PyErr_SetString(PyExc_ValueError,
"generator already executing");
return NULL;
}
if (self->resume_label == 0) {
if (value && value != Py_None) {
PyErr_SetString(PyExc_TypeError,
"can't send non-None value to a "
"just-started generator");
return NULL;
}
}
if (self->resume_label == -1) {
PyErr_SetNone(PyExc_StopIteration);
return NULL;
}
if (value)
__Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback);
else
__Pyx_Generator_ExceptionClear(self);
self->is_running = 1;
retval = self->body((PyObject *) self, value);
self->is_running = 0;
if (retval)
__Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback);
else
__Pyx_Generator_ExceptionClear(self);
return retval;
}
static PyObject *__Pyx_Generator_Next(PyObject *self)
{
return __Pyx_Generator_SendEx((struct __pyx_Generator_object *) self, Py_None);
}
static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value)
{
return __Pyx_Generator_SendEx((struct __pyx_Generator_object *) self, value);
}
static PyObject *__Pyx_Generator_Close(PyObject *self)
{
struct __pyx_Generator_object *generator = (struct __pyx_Generator_object *) self;
PyObject *retval;
#if PY_VERSION_HEX < 0x02050000
PyErr_SetNone(PyExc_StopIteration);
#else
PyErr_SetNone(PyExc_GeneratorExit);
#endif
retval = __Pyx_Generator_SendEx(generator, NULL);
if (retval) {
Py_DECREF(retval);
PyErr_SetString(PyExc_RuntimeError,
"generator ignored GeneratorExit");
return NULL;
}
#if PY_VERSION_HEX < 0x02050000
if (PyErr_ExceptionMatches(PyExc_StopIteration))
#else
if (PyErr_ExceptionMatches(PyExc_StopIteration)
|| PyErr_ExceptionMatches(PyExc_GeneratorExit))
#endif
{
PyErr_Clear(); /* ignore these errors */
Py_INCREF(Py_None);
return Py_None;
}
return NULL;
}
static PyObject *__Pyx_Generator_Throw(PyObject *self, PyObject *args, CYTHON_UNUSED PyObject *kwds)
{
struct __pyx_Generator_object *generator = (struct __pyx_Generator_object *) self;
PyObject *typ;
PyObject *tb = NULL;
PyObject *val = NULL;
if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb))
return NULL;
__Pyx_Raise(typ, val, tb, NULL);
return __Pyx_Generator_SendEx(generator, NULL);
}
""",
proto_block='utility_code_proto_before_types',
requires=[Nodes.raise_utility_code, Nodes.swap_exception_utility_code],
generator_utility_code = UtilityCode.load(
"Generator",
"Generator.c",
requires=[Nodes.raise_utility_code, Nodes.swap_exception_utility_code],
)
......@@ -1767,6 +1767,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
code.putln("if (__pyx_FusedFunction_init() < 0) %s" % code.error_goto(self.pos))
code.putln("#endif")
code.putln("#ifdef __Pyx_Generator_USED")
code.putln("if (__pyx_Generator_init() < 0) %s" % code.error_goto(self.pos))
code.putln("#endif")
code.putln("/*--- Library function declarations ---*/")
env.generate_library_function_declarations(code)
......
......@@ -51,6 +51,7 @@ lambda_func_prefix = pyrex_prefix + "lambda_"
module_is_main = pyrex_prefix + "module_is_main_"
args_cname = pyrex_prefix + "args"
generator_cname = pyrex_prefix + "generator"
sent_value_cname = pyrex_prefix + "sent_value"
pykwdlist_cname = pyrex_prefix + "pyargnames"
obj_base_cname = pyrex_prefix + "base"
......
......@@ -3770,19 +3770,22 @@ class GeneratorDefNode(DefNode):
def generate_function_body(self, env, code):
body_cname = self.gbody.entry.func_cname
generator_cname = '%s->%s' % (Naming.cur_scope_cname, Naming.obj_base_cname)
code.putln('%s.resume_label = 0;' % generator_cname)
code.putln('%s.body = (__pyx_generator_body_t) %s;' % (generator_cname, body_cname))
code.put_giveref(Naming.cur_scope_cname)
code.putln('{')
code.putln('__pyx_GeneratorObject *gen = __Pyx_Generator_New('
'(__pyx_generator_body_t) %s, (PyObject *) %s); %s' % (
body_cname, Naming.cur_scope_cname,
code.error_goto_if_null('gen', self.pos)))
code.put_decref(Naming.cur_scope_cname, py_object_type)
if self.requires_classobj:
classobj_cname = '%s->classobj' % Naming.cur_scope_cname
classobj_cname = 'gen->classobj'
code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % (
classobj_cname, Naming.self_cname))
code.put_incref(classobj_cname, py_object_type)
code.put_giveref(classobj_cname)
code.put_finish_refcount_context()
code.putln("return (PyObject *) %s;" % Naming.cur_scope_cname);
code.putln('return (PyObject *) gen;');
code.putln('}')
def generate_function_definitions(self, env, code):
from ExprNodes import generator_utility_code
......@@ -3800,9 +3803,9 @@ class GeneratorBodyDefNode(DefNode):
is_generator_body = True
def __init__(self, pos=None, name=None, body=None):
super(GeneratorBodyDefNode, self).__init__(pos=pos, body=body, name=name, doc=None,
args=[],
star_arg=None, starstar_arg=None)
super(GeneratorBodyDefNode, self).__init__(
pos=pos, body=body, name=name, doc=None,
args=[], star_arg=None, starstar_arg=None)
def declare_generator_body(self, env):
prefix = env.next_id(env.scope_prefix)
......@@ -3819,9 +3822,9 @@ class GeneratorBodyDefNode(DefNode):
self.declare_generator_body(env)
def generate_function_header(self, code, proto=False):
header = "static PyObject *%s(%s, PyObject *%s)" % (
header = "static PyObject *%s(__pyx_GeneratorObject *%s, PyObject *%s)" % (
self.entry.func_cname,
self.local_scope.scope_class.type.declaration_code(Naming.cur_scope_cname),
Naming.generator_cname,
Naming.sent_value_cname)
if proto:
code.putln('%s; /* proto */' % header)
......@@ -3844,6 +3847,7 @@ class GeneratorBodyDefNode(DefNode):
# ----- Function header
code.putln("")
self.generate_function_header(code)
closure_init_code = code.insertion_point()
# ----- Local variables
code.putln("PyObject *%s = NULL;" % Naming.retval_cname)
tempvardecl_code = code.insertion_point()
......@@ -3861,6 +3865,12 @@ class GeneratorBodyDefNode(DefNode):
# ----- Function body
self.generate_function_body(env, code)
# ----- Closure initialization
if lenv.scope_class.type.scope.entries:
closure_init_code.putln('%s = %s;' % (
lenv.scope_class.type.declaration_code(Naming.cur_scope_cname),
lenv.scope_class.type.cast_code('%s->closure' %
Naming.generator_cname)))
code.putln('PyErr_SetNone(PyExc_StopIteration); %s' % code.error_goto(self.pos))
# ----- Error cleanup
if code.error_label in code.labels_used:
......@@ -3873,7 +3883,7 @@ class GeneratorBodyDefNode(DefNode):
# ----- Non-error return cleanup
code.put_label(code.return_label)
code.put_xdecref(Naming.retval_cname, py_object_type)
code.putln('%s->%s.resume_label = -1;' % (Naming.cur_scope_cname, Naming.obj_base_cname))
code.putln('%s->resume_label = -1;' % Naming.generator_cname)
code.put_finish_refcount_context()
code.putln('return NULL;');
code.putln("}")
......@@ -3881,14 +3891,16 @@ class GeneratorBodyDefNode(DefNode):
# ----- Go back and insert temp variable declarations
tempvardecl_code.put_temp_declarations(code.funcstate)
# ----- Generator resume code
resume_code.putln("switch (%s->%s.resume_label) {" % (Naming.cur_scope_cname, Naming.obj_base_cname));
resume_code.putln("switch (%s->resume_label) {" % (
Naming.generator_cname));
resume_code.putln("case 0: goto %s;" % first_run_label)
from ParseTreeTransforms import YieldNodeCollector
collector = YieldNodeCollector()
collector.visitchildren(self)
for yield_expr in collector.yields:
resume_code.putln("case %d: goto %s;" % (yield_expr.label_num, yield_expr.label_name));
resume_code.putln("case %d: goto %s;" % (
yield_expr.label_num, yield_expr.label_name));
resume_code.putln("default: /* CPython raises the right error here */");
resume_code.put_finish_refcount_context()
resume_code.putln("return NULL;");
......
......@@ -2043,63 +2043,12 @@ class CreateClosureClasses(CythonTransform):
super(CreateClosureClasses, self).__init__(context)
self.path = []
self.in_lambda = False
self.generator_class = None
def visit_ModuleNode(self, node):
self.module_scope = node.scope
self.visitchildren(node)
return node
def create_generator_class(self, target_module_scope, pos):
if self.generator_class:
return self.generator_class
# XXX: make generator class creation cleaner
entry = target_module_scope.declare_c_class(name='__pyx_Generator',
objstruct_cname='__pyx_Generator_object',
typeobj_cname='__pyx_Generator_type',
pos=pos, defining=True, implementing=True)
entry.type.is_final_type = True
klass = entry.type.scope
klass.is_internal = True
body_type = PyrexTypes.create_typedef_type('generator_body',
PyrexTypes.c_void_ptr_type,
'__pyx_generator_body_t')
klass.declare_var(pos=pos, name='body', cname='body',
type=body_type, is_cdef=True)
klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
is_cdef=True)
klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
is_cdef=True)
klass.declare_var(pos=pos, name='exc_type', cname='exc_type',
type=PyrexTypes.py_object_type, is_cdef=True)
klass.declare_var(pos=pos, name='exc_value', cname='exc_value',
type=PyrexTypes.py_object_type, is_cdef=True)
klass.declare_var(pos=pos, name='exc_traceback', cname='exc_traceback',
type=PyrexTypes.py_object_type, is_cdef=True)
import TypeSlots
e = klass.declare_pyfunction('send', pos)
e.func_cname = '__Pyx_Generator_Send'
e.signature = TypeSlots.binaryfunc
e = klass.declare_pyfunction('close', pos)
e.func_cname = '__Pyx_Generator_Close'
e.signature = TypeSlots.unaryfunc
e = klass.declare_pyfunction('throw', pos)
e.func_cname = '__Pyx_Generator_Throw'
e.signature = TypeSlots.pyfunction_signature
e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
e.func_cname = 'PyObject_SelfIter'
e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
e.func_cname = '__Pyx_Generator_Next'
self.generator_class = entry.type
return self.generator_class
def find_entries_used_in_closures(self, node):
from_closure = []
in_closure = []
......@@ -2140,9 +2089,8 @@ class CreateClosureClasses(CythonTransform):
inner_node.needs_self_code = False
node.needs_outer_scope = False
base_type = None
if node.is_generator:
base_type = self.create_generator_class(target_module_scope, node.pos)
pass
elif not in_closure and not from_closure:
return
elif not in_closure:
......@@ -2151,23 +2099,19 @@ class CreateClosureClasses(CythonTransform):
node.needs_outer_scope = True
return
as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
as_name = '%s_%s' % (
target_module_scope.next_id(Naming.closure_class_prefix),
node.entry.cname)
entry = target_module_scope.declare_c_class(
name=as_name, pos=node.pos, defining=True,
implementing=True, base_type=base_type)
implementing=True)
entry.type.is_final_type = True
func_scope.scope_class = entry
class_scope = entry.type.scope
class_scope.is_internal = True
if node.is_generator and node.requires_classobj:
class_scope.declare_var(pos=node.pos, name='classobj',
cname='classobj',
type=PyrexTypes.py_object_type,
is_cdef=True)
if from_closure:
assert cscope.is_closure_scope
class_scope.declare_var(pos=node.pos,
......
//////////////////// Generator.proto ////////////////////
#define __Pyx_Generator_USED
#include <structmember.h>
typedef PyObject *(*__pyx_generator_body_t)(PyObject *, PyObject *);
typedef struct {
PyObject_HEAD
__pyx_generator_body_t body;
PyObject *closure;
int is_running;
int resume_label;
PyObject *exc_type;
PyObject *exc_value;
PyObject *exc_traceback;
PyObject *gi_weakreflist;
PyObject *classobj;
} __pyx_GeneratorObject;
static __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body,
PyObject *closure);
static int __pyx_Generator_init(void);
//////////////////// Generator ////////////////////
static PyObject *__Pyx_Generator_Next(PyObject *self);
static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value);
static PyObject *__Pyx_Generator_Close(PyObject *self);
static PyObject *__Pyx_Generator_Throw(PyObject *gen, PyObject *args);
static CYTHON_INLINE
void __Pyx_Generator_ExceptionClear(__pyx_GeneratorObject *self)
{
Py_XDECREF(self->exc_type);
Py_XDECREF(self->exc_value);
Py_XDECREF(self->exc_traceback);
self->exc_type = NULL;
self->exc_value = NULL;
self->exc_traceback = NULL;
}
static CYTHON_INLINE
PyObject *__Pyx_Generator_SendEx(__pyx_GeneratorObject *self, PyObject *value)
{
PyObject *retval;
if (self->is_running) {
PyErr_SetString(PyExc_ValueError,
"generator already executing");
return NULL;
}
if (self->resume_label == 0) {
if (value && value != Py_None) {
PyErr_SetString(PyExc_TypeError,
"can't send non-None value to a "
"just-started generator");
return NULL;
}
}
if (self->resume_label == -1) {
PyErr_SetNone(PyExc_StopIteration);
return NULL;
}
if (value)
__Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback);
else
__Pyx_Generator_ExceptionClear(self);
self->is_running = 1;
retval = self->body((PyObject *) self, value);
self->is_running = 0;
if (retval)
__Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback);
else
__Pyx_Generator_ExceptionClear(self);
return retval;
}
static PyObject *__Pyx_Generator_Next(PyObject *self)
{
return __Pyx_Generator_SendEx((__pyx_GeneratorObject *) self, Py_None);
}
static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value)
{
return __Pyx_Generator_SendEx((__pyx_GeneratorObject *) self, value);
}
static PyObject *__Pyx_Generator_Close(PyObject *self)
{
__pyx_GeneratorObject *generator = (__pyx_GeneratorObject *) self;
PyObject *retval;
#if PY_VERSION_HEX < 0x02050000
PyErr_SetNone(PyExc_StopIteration);
#else
PyErr_SetNone(PyExc_GeneratorExit);
#endif
retval = __Pyx_Generator_SendEx(generator, NULL);
if (retval) {
Py_DECREF(retval);
PyErr_SetString(PyExc_RuntimeError,
"generator ignored GeneratorExit");
return NULL;
}
#if PY_VERSION_HEX < 0x02050000
if (PyErr_ExceptionMatches(PyExc_StopIteration))
#else
if (PyErr_ExceptionMatches(PyExc_StopIteration)
|| PyErr_ExceptionMatches(PyExc_GeneratorExit))
#endif
{
PyErr_Clear(); /* ignore these errors */
Py_INCREF(Py_None);
return Py_None;
}
return NULL;
}
static PyObject *__Pyx_Generator_Throw(PyObject *self, PyObject *args)
{
__pyx_GeneratorObject *generator = (__pyx_GeneratorObject *) self;
PyObject *typ;
PyObject *tb = NULL;
PyObject *val = NULL;
if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb))
return NULL;
__Pyx_Raise(typ, val, tb, NULL);
return __Pyx_Generator_SendEx(generator, NULL);
}
static int
__Pyx_Generator_traverse(PyObject *self, visitproc visit, void *arg)
{
__pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self;
Py_VISIT(gen->closure);
Py_VISIT(gen->classobj);
Py_VISIT(gen->exc_type);
Py_VISIT(gen->exc_value);
Py_VISIT(gen->exc_traceback);
return 0;
}
static void
__Pyx_Generator_dealloc(PyObject *self)
{
__pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self;
PyObject_GC_UnTrack(self);
Py_CLEAR(gen->closure);
Py_CLEAR(gen->classobj);
Py_CLEAR(gen->exc_type);
Py_CLEAR(gen->exc_value);
Py_CLEAR(gen->exc_traceback);
PyObject_GC_Del(gen);
}
static PyMethodDef __pyx_Generator_methods[] = {
{__Pyx_NAMESTR("send"), (PyCFunction) __Pyx_Generator_Send, METH_O, 0},
{__Pyx_NAMESTR("throw"), (PyCFunction) __Pyx_Generator_Throw, METH_VARARGS, 0},
{__Pyx_NAMESTR("close"), (PyCFunction) __Pyx_Generator_Close, METH_NOARGS, 0},
{0, 0, 0, 0}
};
static PyTypeObject __pyx_GeneratorType = {
PyVarObject_HEAD_INIT(0, 0)
__Pyx_NAMESTR("generator"), /*tp_name*/
sizeof(__pyx_GeneratorObject), /*tp_basicsize*/
0, /*tp_itemsize*/
(destructor) __Pyx_Generator_dealloc,/*tp_dealloc*/
0, /*tp_print*/
0, /*tp_getattr*/
0, /*tp_setattr*/
#if PY_MAJOR_VERSION < 3
0, /*tp_compare*/
#else
0, /*reserved*/
#endif
0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
0, /*tp_as_mapping*/
0, /*tp_hash*/
0, /*tp_call*/
0, /*tp_str*/
PyObject_GenericGetAttr, /*tp_getattro*/
0, /*tp_setattro*/
0, /*tp_as_buffer*/
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags*/
0, /*tp_doc*/
(traverseproc) __Pyx_Generator_traverse, /*tp_traverse*/
0, /*tp_clear*/
0, /*tp_richcompare*/
offsetof(__pyx_GeneratorObject, gi_weakreflist), /* tp_weaklistoffse */
PyObject_SelfIter, /*tp_iter*/
(iternextfunc) __Pyx_Generator_Next, /*tp_iternext*/
__pyx_Generator_methods, /*tp_methods*/
0, /*tp_members*/
0, /*tp_getset*/
0, /*tp_base*/
0, /*tp_dict*/
0, /*tp_descr_get*/
0, /*tp_descr_set*/
0, /*tp_dictoffset*/
0, /*tp_init*/
0, /*tp_alloc*/
0, /*tp_new*/
0, /*tp_free*/
0, /*tp_is_gc*/
0, /*tp_bases*/
0, /*tp_mro*/
0, /*tp_cache*/
0, /*tp_subclasses*/
0, /*tp_weaklist*/
0, /*tp_del*/
#if PY_VERSION_HEX >= 0x02060000
0, /*tp_version_tag*/
#endif
};
static
__pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body,
PyObject *closure)
{
__pyx_GeneratorObject *gen =
PyObject_GC_New(__pyx_GeneratorObject, &__pyx_GeneratorType);
if (gen == NULL)
return NULL;
gen->body = body;
gen->closure = closure;
Py_XINCREF(closure);
gen->is_running = 0;
gen->resume_label = 0;
gen->classobj = NULL;
gen->exc_type = NULL;
gen->exc_value = NULL;
gen->exc_traceback = NULL;
gen->gi_weakreflist = NULL;
PyObject_GC_Track(gen);
return gen;
}
static int __pyx_Generator_init(void)
{
return PyType_Ready(&__pyx_GeneratorType);
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment