Skip to content

C
cython
Commit 3e310ce3 authored by scoder's avatar scoder
Browse files
Options

Merge pull request #96 from scoder/_yield_from 

implementation of PEP 380 (yield from)
parents f98829fb 123f1c85
Show whitespace changes
Inline Side-by-side
Showing with 1436 additions and 65 deletions
Cython/Compiler/ExprNodes.py
View file @ 3e310ce3
......@@ -6538,10 +6538,12 @@ class YieldExprNode(ExprNode):
# arg ExprNode the value to return from the generator
# label_name string name of the C label used for this yield
# label_num integer yield label number
# is_yield_from boolean is a YieldFromExprNode to delegate to another generator
subexprs = ['arg']
type = py_object_type
label_num = 0
is_yield_from = False
def analyse_types(self, env):
if not self.label_num:
......@@ -6550,11 +6552,12 @@ class YieldExprNode(ExprNode):
if self.arg is not None:
self.arg.analyse_types(env)
if not self.arg.type.is_pyobject:
self.coerce_yield_argument(env)
def coerce_yield_argument(self, env):
self.arg = self.arg.coerce_to_pyobject(env)
def generate_evaluation_code(self, code):
self.label_name = code.new_label('resume_from_yield')
code.use_label(self.label_name)
if self.arg:
self.arg.generate_evaluation_code(code)
self.arg.make_owned_reference(code)
......@@ -6563,10 +6566,19 @@ class YieldExprNode(ExprNode):
Naming.retval_cname,
self.arg.result_as(py_object_type)))
self.arg.generate_post_assignment_code(code)
#self.arg.generate_disposal_code(code)
self.arg.free_temps(code)
else:
code.put_init_to_py_none(Naming.retval_cname, py_object_type)
self.generate_yield_code(code)
def generate_yield_code(self, code):
"""
Generate the code to return the argument in 'Naming.retval_cname'
and to continue at the yield label.
"""
self.label_name = code.new_label('resume_from_yield')
code.use_label(self.label_name)
saved = []
code.funcstate.closure_temps.reset()
for cname, type, manage_ref in code.funcstate.temps_in_use():
......@@ -6582,6 +6594,7 @@ class YieldExprNode(ExprNode):
code.putln("%s->resume_label = %d;" % (
Naming.generator_cname, self.label_num))
code.putln("return %s;" % Naming.retval_cname);
code.put_label(self.label_name)
for cname, save_cname, type in saved:
code.putln('%s = %s->%s;' % (cname, Naming.cur_scope_cname, save_cname))
......@@ -6599,6 +6612,46 @@ class YieldExprNode(ExprNode):
code.putln(code.error_goto_if_null(Naming.sent_value_cname, self.pos))
class YieldFromExprNode(YieldExprNode):
# "yield from GEN" expression
is_yield_from = True
def coerce_yield_argument(self, env):
if not self.arg.type.is_string:
# FIXME: support C arrays and C++ iterators?
error(self.pos, "yielding from non-Python object not supported")
self.arg = self.arg.coerce_to_pyobject(env)
def generate_evaluation_code(self, code):
code.globalstate.use_utility_code(UtilityCode.load_cached("YieldFrom", "Generator.c"))
self.arg.generate_evaluation_code(code)
code.putln("%s = __Pyx_Generator_Yield_From(%s, %s);" % (
Naming.retval_cname,
Naming.generator_cname,
self.arg.result_as(py_object_type)))
self.arg.generate_disposal_code(code)
self.arg.free_temps(code)
code.put_xgotref(Naming.retval_cname)
code.putln("if (likely(%s)) {" % Naming.retval_cname)
self.generate_yield_code(code)
code.putln("} else {")
# either error or sub-generator has normally terminated: return value => node result
if self.result_is_used:
# YieldExprNode has allocated the result temp for us
code.putln("if (__Pyx_PyGen_FetchStopIterationValue(&%s) < 0) %s" % (
self.result(),
code.error_goto(self.pos)))
else:
code.putln("PyObject* exc_type = PyErr_Occurred();")
code.putln("if (exc_type) {")
code.putln("if (!PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)) %s" %
code.error_goto(self.pos))
code.putln("PyErr_Clear();")
code.putln("}")
code.putln("}")
class GlobalsExprNode(AtomicExprNode):
type = dict_type
is_temp = 1
......
Cython/Compiler/Nodes.py
View file @ 3e310ce3
......@@ -4182,6 +4182,8 @@ class GeneratorBodyDefNode(DefNode):
code.put_label(code.return_label)
code.put_xdecref(Naming.retval_cname, py_object_type)
code.putln('%s->resume_label = -1;' % Naming.generator_cname)
# clean up as early as possible to help breaking any reference cycles
code.putln('__Pyx_Generator_clear((PyObject*)%s);' % Naming.generator_cname)
code.put_finish_refcount_context()
code.putln('return NULL;')
code.putln("}")
......@@ -5297,9 +5299,11 @@ class ReturnStatNode(StatNode):
#
# value ExprNode or None
# return_type PyrexType
# in_generator return inside of generator => raise StopIteration
child_attrs = ["value"]
is_terminator = True
in_generator = False
# Whether we are in a parallel section
in_parallel = False
......@@ -5349,6 +5353,13 @@ class ReturnStatNode(StatNode):
rhs=self.value,
code=code,
have_gil=self.in_nogil_context)
elif self.in_generator:
# return value == raise StopIteration(value), but uncatchable
code.putln(
"%s = NULL; PyErr_SetObject(PyExc_StopIteration, %s);" % (
Naming.retval_cname,
self.value.result_as(self.return_type)))
self.value.generate_disposal_code(code)
else:
self.value.make_owned_reference(code)
code.putln(
......
Cython/Compiler/ParseTreeTransforms.py
View file @ 3e310ce3
......@@ -2025,16 +2025,12 @@ class YieldNodeCollector(TreeVisitor):
return self.visitchildren(node)
def visit_YieldExprNode(self, node):
if self.has_return_value:
error(node.pos, "'yield' outside function")
self.yields.append(node)
self.visitchildren(node)
def visit_ReturnStatNode(self, node):
if node.value:
self.has_return_value = True
if self.yields:
error(node.pos, "'return' with argument inside generator")
self.returns.append(node)
def visit_ClassDefNode(self, node):
......@@ -2071,6 +2067,8 @@ class MarkClosureVisitor(CythonTransform):
return node
for i, yield_expr in enumerate(collector.yields):
yield_expr.label_num = i + 1
for retnode in collector.returns:
retnode.in_generator = True
gbody = Nodes.GeneratorBodyDefNode(
pos=node.pos, name=node.name, body=node.body)
......
Cython/Compiler/Parsing.py
View file @ 3e310ce3
......@@ -340,10 +340,19 @@ def p_yield_expression(s):
# s.sy == "yield"
pos = s.position()
s.next()
is_yield_from = False
if s.sy == 'from':
is_yield_from = True
s.next()
if s.sy != ')' and s.sy not in statement_terminators:
arg = p_testlist(s)
else:
if is_yield_from:
s.error("'yield from' requires a source argument", pos=pos)
arg = None
if is_yield_from:
return ExprNodes.YieldFromExprNode(pos, arg=arg)
else:
return ExprNodes.YieldExprNode(pos, arg=arg)
def p_yield_statement(s):
......
Cython/Utility/Generator.c
View file @ 3e310ce3
//////////////////// YieldFrom.proto ////////////////////
static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_GeneratorObject *gen, PyObject *source);
//////////////////// YieldFrom ////////////////////
//@requires: Generator
static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_GeneratorObject *gen, PyObject *source) {
PyObject *source_gen, *retval;
source_gen = PyObject_GetIter(source);
if (unlikely(!source_gen))
return NULL;
/* source_gen is now the iterator, make the first next() call */
retval = Py_TYPE(source_gen)->tp_iternext(source_gen);
if (likely(retval)) {
gen->yieldfrom = source_gen;
return retval;
}
Py_DECREF(source_gen);
return NULL;
}
//////////////////// Generator.proto ////////////////////
#define __Pyx_Generator_USED
#include <structmember.h>
......@@ -15,21 +37,110 @@ typedef struct {
PyObject *exc_traceback;
PyObject *gi_weakreflist;
PyObject *classobj;
PyObject *yieldfrom;
} __pyx_GeneratorObject;
static __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body,
PyObject *closure);
static int __pyx_Generator_init(void);
static int __Pyx_Generator_clear(PyObject* self);
#if 1 || PY_VERSION_HEX < 0x030300B0
static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue);
#else
#define __Pyx_PyGen_FetchStopIterationValue(pvalue) PyGen_FetchStopIterationValue(pvalue)
#endif
//////////////////// Generator ////////////////////
//@requires: Exceptions.c::PyErrFetchRestore
//@requires: Exceptions.c::SwapException
//@requires: Exceptions.c::RaiseException
static PyObject *__Pyx_Generator_Next(PyObject *self);
static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value);
static PyObject *__Pyx_Generator_Close(PyObject *self);
static PyObject *__Pyx_Generator_Throw(PyObject *gen, PyObject *args);
static PyTypeObject __pyx_GeneratorType;
#define __Pyx_Generator_CheckExact(obj) (Py_TYPE(obj) == &__pyx_GeneratorType)
#define __Pyx_Generator_Undelegate(gen) Py_CLEAR((gen)->yieldfrom)
// If StopIteration exception is set, fetches its 'value'
// attribute if any, otherwise sets pvalue to None.
//
// Returns 0 if no exception or StopIteration is set.
// If any other exception is set, returns -1 and leaves
// pvalue unchanged.
#if 1 || PY_VERSION_HEX < 0x030300B0
static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) {
PyObject *et, *ev, *tb;
PyObject *value = NULL;
__Pyx_ErrFetch(&et, &ev, &tb);
if (!et) {
Py_XDECREF(tb);
Py_XDECREF(ev);
Py_INCREF(Py_None);
*pvalue = Py_None;
return 0;
}
if (unlikely(et != PyExc_StopIteration) &&
unlikely(!PyErr_GivenExceptionMatches(et, PyExc_StopIteration))) {
__Pyx_ErrRestore(et, ev, tb);
return -1;
}
// most common case: plain StopIteration without or with separate argument
if (likely(et == PyExc_StopIteration)) {
if (likely(!ev) || !PyObject_IsInstance(ev, PyExc_StopIteration)) {
// PyErr_SetObject() and friends put the value directly into ev
if (!ev) {
Py_INCREF(Py_None);
ev = Py_None;
}
Py_XDECREF(tb);
Py_DECREF(et);
*pvalue = ev;
return 0;
}
}
// otherwise: normalise and check what that gives us
PyErr_NormalizeException(&et, &ev, &tb);
if (unlikely(!PyObject_IsInstance(ev, PyExc_StopIteration))) {
// looks like normalisation failed - raise the new exception
__Pyx_ErrRestore(et, ev, tb);
return -1;
}
Py_XDECREF(tb);
Py_DECREF(et);
#if PY_VERSION_HEX >= 0x030300A0
value = ((PyStopIterationObject *)ev)->value;
Py_INCREF(value);
Py_DECREF(ev);
#else
{
PyObject* args = PyObject_GetAttrString(ev, "args");
Py_DECREF(ev);
if (likely(args)) {
value = PyObject_GetItem(args, 0);
Py_DECREF(args);
}
if (unlikely(!value)) {
__Pyx_ErrRestore(NULL, NULL, NULL);
Py_INCREF(Py_None);
value = Py_None;
}
}
#endif
*pvalue = value;
return 0;
}
#endif
static CYTHON_INLINE
void __Pyx_Generator_ExceptionClear(__pyx_GeneratorObject *self)
{
void __Pyx_Generator_ExceptionClear(__pyx_GeneratorObject *self) {
PyObject *exc_type = self->exc_type;
PyObject *exc_value = self->exc_value;
PyObject *exc_traceback = self->exc_traceback;
......@@ -44,15 +155,20 @@ void __Pyx_Generator_ExceptionClear(__pyx_GeneratorObject *self)
}
static CYTHON_INLINE
PyObject *__Pyx_Generator_SendEx(__pyx_GeneratorObject *self, PyObject *value)
{
PyObject *retval;
if (unlikely(self->is_running)) {
int __Pyx_Generator_CheckRunning(__pyx_GeneratorObject *gen) {
if (unlikely(gen->is_running)) {
PyErr_SetString(PyExc_ValueError,
"generator already executing");
return NULL;
return 1;
}
return 0;
}
static CYTHON_INLINE
PyObject *__Pyx_Generator_SendEx(__pyx_GeneratorObject *self, PyObject *value) {
PyObject *retval;
assert(!self->is_running);
if (unlikely(self->resume_label == 0)) {
if (unlikely(value && value != Py_None)) {
......@@ -86,75 +202,225 @@ PyObject *__Pyx_Generator_SendEx(__pyx_GeneratorObject *self, PyObject *value)
return retval;
}
static PyObject *__Pyx_Generator_Next(PyObject *self)
{
return __Pyx_Generator_SendEx((__pyx_GeneratorObject *) self, Py_None);
static CYTHON_INLINE
PyObject *__Pyx_Generator_FinishDelegation(__pyx_GeneratorObject *gen) {
PyObject *ret;
PyObject *val = NULL;
__Pyx_Generator_Undelegate(gen);
__Pyx_PyGen_FetchStopIterationValue(&val);
// val == NULL on failure => pass on exception
ret = __Pyx_Generator_SendEx(gen, val);
Py_XDECREF(val);
return ret;
}
static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value)
{
return __Pyx_Generator_SendEx((__pyx_GeneratorObject *) self, value);
static PyObject *__Pyx_Generator_Next(PyObject *self) {
__pyx_GeneratorObject *gen = (__pyx_GeneratorObject*) self;
PyObject *yf = gen->yieldfrom;
if (unlikely(__Pyx_Generator_CheckRunning(gen)))
return NULL;
if (yf) {
PyObject *ret;
// FIXME: does this really need an INCREF() ?
//Py_INCREF(yf);
/* YieldFrom code ensures that yf is an iterator */
gen->is_running = 1;
ret = Py_TYPE(yf)->tp_iternext(yf);
gen->is_running = 0;
//Py_DECREF(yf);
if (likely(ret)) {
return ret;
}
return __Pyx_Generator_FinishDelegation(gen);
}
return __Pyx_Generator_SendEx(gen, Py_None);
}
static PyObject *__Pyx_Generator_Close(PyObject *self)
{
__pyx_GeneratorObject *generator = (__pyx_GeneratorObject *) self;
PyObject *retval;
static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value) {
__pyx_GeneratorObject *gen = (__pyx_GeneratorObject*) self;
PyObject *yf = gen->yieldfrom;
if (unlikely(__Pyx_Generator_CheckRunning(gen)))
return NULL;
if (yf) {
PyObject *ret;
// FIXME: does this really need an INCREF() ?
//Py_INCREF(yf);
gen->is_running = 1;
if (__Pyx_Generator_CheckExact(yf)) {
ret = __Pyx_Generator_Send(yf, value);
} else {
if (value == Py_None)
ret = PyIter_Next(yf);
else
ret = PyObject_CallMethod(yf, "send", "O", value);
}
gen->is_running = 0;
//Py_DECREF(yf);
if (likely(ret)) {
return ret;
}
return __Pyx_Generator_FinishDelegation(gen);
}
return __Pyx_Generator_SendEx(gen, value);
}
// This helper function is used by gen_close and gen_throw to
// close a subiterator being delegated to by yield-from.
static int __Pyx_Generator_CloseIter(__pyx_GeneratorObject *gen, PyObject *yf) {
PyObject *retval = NULL;
int err = 0;
if (__Pyx_Generator_CheckExact(yf)) {
retval = __Pyx_Generator_Close(yf);
if (!retval)
return -1;
} else {
PyObject *meth;
gen->is_running = 1;
meth = PyObject_GetAttrString(yf, "close");
if (unlikely(!meth)) {
if (!PyErr_ExceptionMatches(PyExc_AttributeError)) {
PyErr_WriteUnraisable(yf);
}
PyErr_Clear();
} else {
retval = PyObject_CallFunction(meth, "");
Py_DECREF(meth);
if (!retval)
err = -1;
}
gen->is_running = 0;
}
Py_XDECREF(retval);
return err;
}
static PyObject *__Pyx_Generator_Close(PyObject *self) {
__pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self;
PyObject *retval, *raised_exception;
PyObject *yf = gen->yieldfrom;
int err = 0;
if (unlikely(__Pyx_Generator_CheckRunning(gen)))
return NULL;
if (yf) {
Py_INCREF(yf);
err = __Pyx_Generator_CloseIter(gen, yf);
__Pyx_Generator_Undelegate(gen);
Py_DECREF(yf);
}
if (err == 0)
#if PY_VERSION_HEX < 0x02050000
PyErr_SetNone(PyExc_StopIteration);
#else
PyErr_SetNone(PyExc_GeneratorExit);
#endif
retval = __Pyx_Generator_SendEx(generator, NULL);
retval = __Pyx_Generator_SendEx(gen, NULL);
if (retval) {
Py_DECREF(retval);
PyErr_SetString(PyExc_RuntimeError,
"generator ignored GeneratorExit");
return NULL;
}
#if PY_VERSION_HEX < 0x02050000
if (PyErr_ExceptionMatches(PyExc_StopIteration))
#else
if (PyErr_ExceptionMatches(PyExc_StopIteration)
|| PyErr_ExceptionMatches(PyExc_GeneratorExit))
raised_exception = PyErr_Occurred();
if (!raised_exception
|| raised_exception == PyExc_StopIteration
#if PY_VERSION_HEX >= 0x02050000
|| raised_exception == PyExc_GeneratorExit
|| PyErr_GivenExceptionMatches(raised_exception, PyExc_GeneratorExit)
#endif
|| PyErr_GivenExceptionMatches(raised_exception, PyExc_StopIteration))
{
PyErr_Clear(); /* ignore these errors */
if (raised_exception) PyErr_Clear(); /* ignore these errors */
Py_INCREF(Py_None);
return Py_None;
}
return NULL;
}
static PyObject *__Pyx_Generator_Throw(PyObject *self, PyObject *args)
{
__pyx_GeneratorObject *generator = (__pyx_GeneratorObject *) self;
static PyObject *__Pyx_Generator_Throw(PyObject *self, PyObject *args) {
__pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self;
PyObject *typ;
PyObject *tb = NULL;
PyObject *val = NULL;
PyObject *yf = gen->yieldfrom;
if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb))
return NULL;
if (unlikely(__Pyx_Generator_CheckRunning(gen)))
return NULL;
if (yf) {
PyObject *ret;
Py_INCREF(yf);
#if PY_VERSION_HEX >= 0x02050000
if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) {
int err = __Pyx_Generator_CloseIter(gen, yf);
Py_DECREF(yf);
__Pyx_Generator_Undelegate(gen);
if (err < 0)
return __Pyx_Generator_SendEx(gen, NULL);
goto throw_here;
}
#endif
gen->is_running = 1;
if (__Pyx_Generator_CheckExact(yf)) {
ret = __Pyx_Generator_Throw(yf, args);
} else {
PyObject *meth = PyObject_GetAttrString(yf, "throw");
if (unlikely(!meth)) {
Py_DECREF(yf);
if (!PyErr_ExceptionMatches(PyExc_AttributeError)) {
gen->is_running = 0;
return NULL;
}
PyErr_Clear();
__Pyx_Generator_Undelegate(gen);
gen->is_running = 0;
goto throw_here;
}
ret = PyObject_CallObject(meth, args);
Py_DECREF(meth);
}
gen->is_running = 0;
Py_DECREF(yf);
if (!ret) {
ret = __Pyx_Generator_FinishDelegation(gen);
}
return ret;
}
throw_here:
__Pyx_Raise(typ, val, tb, NULL);
return __Pyx_Generator_SendEx(generator, NULL);
return __Pyx_Generator_SendEx(gen, NULL);
}
static int
__Pyx_Generator_traverse(PyObject *self, visitproc visit, void *arg)
{
static int __Pyx_Generator_traverse(PyObject *self, visitproc visit, void *arg) {
__pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self;
Py_VISIT(gen->closure);
Py_VISIT(gen->classobj);
Py_VISIT(gen->yieldfrom);
Py_VISIT(gen->exc_type);
Py_VISIT(gen->exc_value);
Py_VISIT(gen->exc_traceback);
return 0;
}
static void
__Pyx_Generator_dealloc(PyObject *self)
{
static int __Pyx_Generator_clear(PyObject *self) {
__pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self;
Py_CLEAR(gen->closure);
Py_CLEAR(gen->classobj);
Py_CLEAR(gen->yieldfrom);
Py_CLEAR(gen->exc_type);
Py_CLEAR(gen->exc_value);
Py_CLEAR(gen->exc_traceback);
return 0;
}
static void __Pyx_Generator_dealloc(PyObject *self) {
__pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self;
PyObject_GC_UnTrack(gen);
......@@ -170,17 +436,11 @@ __Pyx_Generator_dealloc(PyObject *self)
}
PyObject_GC_UnTrack(self);
Py_CLEAR(gen->closure);
Py_CLEAR(gen->classobj);
Py_CLEAR(gen->exc_type);
Py_CLEAR(gen->exc_value);
Py_CLEAR(gen->exc_traceback);
__Pyx_Generator_clear(self);
PyObject_GC_Del(gen);
}
static void
__Pyx_Generator_del(PyObject *self)
{
static void __Pyx_Generator_del(PyObject *self) {
PyObject *res;
PyObject *error_type, *error_value, *error_traceback;
__pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self;
......@@ -233,14 +493,18 @@ __Pyx_Generator_del(PyObject *self)
* undone.
*/
#ifdef COUNT_ALLOCS
--self->ob_type->tp_frees;
--self->ob_type->tp_allocs;
--Py_TYPE(self)->tp_frees;
--Py_TYPE(self)->tp_allocs;
#endif
}
static PyMemberDef __pyx_Generator_memberlist[] = {
{(char *) "gi_running",
#if PY_VERSION_HEX >= 0x02060000
T_BOOL,
#else
T_INT,
#endif
offsetof(__pyx_GeneratorObject, is_running),
READONLY,
NULL},
......@@ -281,7 +545,7 @@ static PyTypeObject __pyx_GeneratorType = {
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags*/
0, /*tp_doc*/
(traverseproc) __Pyx_Generator_traverse, /*tp_traverse*/
0, /*tp_clear*/
(inquiry) __Pyx_Generator_clear, /*tp_clear*/
0, /*tp_richcompare*/
offsetof(__pyx_GeneratorObject, gi_weakreflist), /* tp_weaklistoffse */
PyObject_SelfIter, /*tp_iter*/
......@@ -310,10 +574,8 @@ static PyTypeObject __pyx_GeneratorType = {
#endif
};
static
__pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body,
PyObject *closure)
{
static __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body,
PyObject *closure) {
__pyx_GeneratorObject *gen =
PyObject_GC_New(__pyx_GeneratorObject, &__pyx_GeneratorType);
......@@ -326,6 +588,7 @@ __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body,
gen->is_running = 0;
gen->resume_label = 0;
gen->classobj = NULL;
gen->yieldfrom = NULL;
gen->exc_type = NULL;
gen->exc_value = NULL;
gen->exc_traceback = NULL;
......@@ -335,7 +598,6 @@ __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body,
return gen;
}
static int __pyx_Generator_init(void)
{
static int __pyx_Generator_init(void) {
return PyType_Ready(&__pyx_GeneratorType);
}
runtests.py
View file @ 3e310ce3
......@@ -166,6 +166,7 @@ VER_DEP_MODULES = {
]),
(2,5) : (operator.lt, lambda x: x in ['run.any',
'run.all',
'run.yield_from_pep380', # GeneratorExit
'run.relativeimport_T542',
'run.relativeimport_star_T542',
]),
......
tests/errors/e_generators.pyx
View file @ 3e310ce3
......@@ -14,8 +14,8 @@ class Foo:
yield
_ERRORS = u"""
5:4: 'return' with argument inside generator
9:4: 'yield' outside function
#5:4: 'return' with argument inside generator
#9:4: 'yield' outside function
11:0: 'yield' not supported here
14:4: 'yield' not supported here
"""
tests/run/generators_in_refcycles.pyx 0 → 100644
View file @ 3e310ce3
def test_reference_cycle_cleanup():
"""
>>> import gc
>>> delegator, gen, next, deleted = test_reference_cycle_cleanup()
>>> next(delegator(gen()))
123
>>> _ = gc.collect(); print(sorted(deleted))
['bar', 'foo']
"""
deleted = []
class Destructed(object):
def __init__(self, name):
self.name = name
def __del__(self):
deleted.append(self.name)
def delegator(c):
d = Destructed('foo')
yield from c
def gen():
d = Destructed('bar')
while True:
yield 123
return delegator, gen, next, deleted
tests/run/yield_from_pep380.pyx 0 → 100644
View file @ 3e310ce3
# -*- coding: utf-8 -*-
"""
Test suite for PEP 380 implementation
adapted from original tests written by Greg Ewing
see <http://www.cosc.canterbury.ac.nz/greg.ewing/python/yield-from/YieldFrom-Python3.1.2-rev5.zip>
"""
import sys
IS_PY3 = sys.version_info[0] >= 3
def _lines(trace):
for line in trace:
print(line)
def test_delegation_of_initial_next_to_subgenerator():
"""
>>> _lines(test_delegation_of_initial_next_to_subgenerator())
Starting g1
Starting g2
Yielded 42
Finishing g2
Finishing g1
"""
trace = []
def g1():
trace.append("Starting g1")
yield from g2()
trace.append("Finishing g1")
def g2():
trace.append("Starting g2")
yield 42
trace.append("Finishing g2")
for x in g1():
trace.append("Yielded %s" % (x,))
return trace
def test_raising_exception_in_initial_next_call():
"""
>>> _lines(test_raising_exception_in_initial_next_call())
Starting g1
Starting g2
Finishing g2
Finishing g1
"""
trace = []
def g1():
try:
trace.append("Starting g1")
yield from g2()
finally:
trace.append("Finishing g1")
def g2():
try:
trace.append("Starting g2")
raise ValueError("spanish inquisition occurred")
finally:
trace.append("Finishing g2")
try:
for x in g1():
trace.append("Yielded %s" % (x,))
except ValueError as e:
pass
else:
trace.append("subgenerator failed to raise ValueError")
return trace
def test_delegation_of_next_call_to_subgenerator():
"""
>>> _lines(test_delegation_of_next_call_to_subgenerator())
Starting g1
Yielded g1 ham
Starting g2
Yielded g2 spam
Yielded g2 more spam
Finishing g2
Yielded g1 eggs
Finishing g1
"""
trace = []
def g1():
trace.append("Starting g1")
yield "g1 ham"
yield from g2()
yield "g1 eggs"
trace.append("Finishing g1")
def g2():
trace.append("Starting g2")
yield "g2 spam"
yield "g2 more spam"
trace.append("Finishing g2")
for x in g1():
trace.append("Yielded %s" % (x,))
return trace
def test_raising_exception_in_delegated_next_call():
"""
>>> _lines(test_raising_exception_in_delegated_next_call())
Starting g1
Yielded g1 ham
Starting g2
Yielded g2 spam
Finishing g2
Finishing g1
"""
trace = []
def g1():
try:
trace.append("Starting g1")
yield "g1 ham"
yield from g2()
yield "g1 eggs"
finally:
trace.append("Finishing g1")
def g2():
try:
trace.append("Starting g2")
yield "g2 spam"
raise ValueError("hovercraft is full of eels")
yield "g2 more spam"
finally:
trace.append("Finishing g2")
try:
for x in g1():
trace.append("Yielded %s" % (x,))
except ValueError:
pass
else:
trace.append("subgenerator failed to raise ValueError")
return trace
def test_delegation_of_send():
"""
>>> _lines(test_delegation_of_send())
Starting g1
g1 received 1
Starting g2
Yielded g2 spam
g2 received 2
Yielded g2 more spam
g2 received 3
Finishing g2
Yielded g1 eggs
g1 received 4
Finishing g1
"""
trace = []
def g1():
trace.append("Starting g1")
x = yield "g1 ham"
trace.append("g1 received %s" % (x,))
yield from g2()
x = yield "g1 eggs"
trace.append("g1 received %s" % (x,))
trace.append("Finishing g1")
def g2():
trace.append("Starting g2")
x = yield "g2 spam"
trace.append("g2 received %s" % (x,))
x = yield "g2 more spam"
trace.append("g2 received %s" % (x,))
trace.append("Finishing g2")
g = g1()
y = next(g)
x = 1
try:
while 1:
y = g.send(x)
trace.append("Yielded %s" % (y,))
x += 1
except StopIteration:
pass
return trace
def test_handling_exception_while_delegating_send():
"""
>>> _lines(test_handling_exception_while_delegating_send())
Starting g1
g1 received 1
Starting g2
Yielded g2 spam
g2 received 2
"""
trace = []
def g1():
trace.append("Starting g1")
x = yield "g1 ham"
trace.append("g1 received %s" % (x,))
yield from g2()
x = yield "g1 eggs"
trace.append("g1 received %s" % (x,))
trace.append("Finishing g1")
def g2():
trace.append("Starting g2")
x = yield "g2 spam"
trace.append("g2 received %s" % (x,))
raise ValueError("hovercraft is full of eels")
x = yield "g2 more spam"
trace.append("g2 received %s" % (x,))
trace.append("Finishing g2")
def run():
g = g1()
y = next(g)
x = 1
try:
while 1:
y = g.send(x)
trace.append("Yielded %s" % (y,))
x += 1
except StopIteration:
trace.append("StopIteration")
try:
run()
except ValueError:
pass # ok
else:
trace.append("no ValueError")
return trace
def test_delegating_close():
"""
>>> _lines(test_delegating_close())
Starting g1
Yielded g1 ham
Starting g2
Yielded g2 spam
Finishing g2
Finishing g1
"""
trace = []
def g1():
try:
trace.append("Starting g1")
yield "g1 ham"
yield from g2()
yield "g1 eggs"
finally:
trace.append("Finishing g1")
def g2():
try:
trace.append("Starting g2")
yield "g2 spam"
yield "g2 more spam"
finally:
trace.append("Finishing g2")
g = g1()
for i in range(2):
x = next(g)
trace.append("Yielded %s" % (x,))
g.close()
return trace
def test_handing_exception_while_delegating_close():
"""
>>> _lines(test_handing_exception_while_delegating_close())
Starting g1
Yielded g1 ham
Starting g2
Yielded g2 spam
Finishing g2
Finishing g1
"""
trace = []
def g1():
try:
trace.append("Starting g1")
yield "g1 ham"
yield from g2()
yield "g1 eggs"
finally:
trace.append("Finishing g1")
def g2():
try:
trace.append("Starting g2")
yield "g2 spam"
yield "g2 more spam"
finally:
trace.append("Finishing g2")
raise ValueError("nybbles have exploded with delight")
try:
g = g1()
for i in range(2):
x = next(g)
trace.append("Yielded %s" % (x,))
g.close()
except ValueError:
pass
else:
trace.append("subgenerator failed to raise ValueError")
return trace
def test_delegating_throw():
"""
>>> _lines(test_delegating_throw())
Starting g1
Yielded g1 ham
Starting g2
Yielded g2 spam
Finishing g2
Finishing g1
"""
trace = []
def g1():
try:
trace.append("Starting g1")
yield "g1 ham"
yield from g2()
yield "g1 eggs"
finally:
trace.append("Finishing g1")
def g2():
try:
trace.append("Starting g2")
yield "g2 spam"
yield "g2 more spam"
finally:
trace.append("Finishing g2")
try:
g = g1()
for i in range(2):
x = next(g)
trace.append("Yielded %s" % (x,))
e = ValueError("tomato ejected")
g.throw(e)
except ValueError:
pass
else:
trace.append("subgenerator failed to raise ValueError")
return trace
def __test_value_attribute_of_StopIteration_exception():
"""
StopIteration:
value = None
StopIteration: spam
value = spam
StopIteration: spam
value = eggs
"""
trace = []
def pex(e):
trace.append("%s: %s" % (e.__class__.__name__, e))
trace.append("value = %s" % (e.value,))
e = StopIteration()
pex(e)
e = StopIteration("spam")
pex(e)
e.value = "eggs"
pex(e)
return trace
def test_exception_value_crash():
"""
>>> test_exception_value_crash()
['g2']
"""
# There used to be a refcount error in CPython when the return value
# stored in the StopIteration has a refcount of 1.
def g1():
yield from g2()
def g2():
yield "g2"
return [42]
return list(g1())
def test_generator_return_value():
"""
>>> _lines(test_generator_return_value())
Starting g1
Yielded g1 ham
Starting g2
Yielded g2 spam
Yielded g2 more spam
Finishing g2
g2 returned None
Starting g2
Yielded g2 spam
Yielded g2 more spam
Finishing g2
g2 returned 42
Yielded g1 eggs
Finishing g1
"""
trace = []
def g1():
trace.append("Starting g1")
yield "g1 ham"
ret = yield from g2()
trace.append("g2 returned %s" % (ret,))
ret = yield from g2(42)
trace.append("g2 returned %s" % (ret,))
yield "g1 eggs"
trace.append("Finishing g1")
def g2(v = None):
trace.append("Starting g2")
yield "g2 spam"
yield "g2 more spam"
trace.append("Finishing g2")
if v:
return v
for x in g1():
trace.append("Yielded %s" % (x,))
return trace
def test_delegation_of_next_to_non_generator():
"""
>>> _lines(test_delegation_of_next_to_non_generator())
Yielded 0
Yielded 1
Yielded 2
"""
trace = []
def g():
yield from range(3)
for x in g():
trace.append("Yielded %s" % (x,))
return trace
def test_conversion_of_sendNone_to_next():
"""
>>> _lines(test_conversion_of_sendNone_to_next())
Yielded: 0
Yielded: 1
Yielded: 2
"""
trace = []
def g():
yield from range(3)
gi = g()
for x in range(3):
y = gi.send(None)
trace.append("Yielded: %s" % (y,))
return trace
def test_delegation_of_close_to_non_generator():
"""
>>> _lines(test_delegation_of_close_to_non_generator())
starting g
finishing g
"""
trace = []
def g():
try:
trace.append("starting g")
yield from range(3)
trace.append("g should not be here")
finally:
trace.append("finishing g")
gi = g()
next(gi)
gi.close()
return trace
def test_delegating_throw_to_non_generator():
"""
>>> _lines(test_delegating_throw_to_non_generator())
Starting g
Yielded 0
Yielded 1
Yielded 2
Yielded 3
Yielded 4
Finishing g
"""
trace = []
def g():
try:
trace.append("Starting g")
yield from range(10)
finally:
trace.append("Finishing g")
try:
gi = g()
for i in range(5):
x = next(gi)
trace.append("Yielded %s" % (x,))
e = ValueError("tomato ejected")
gi.throw(e)
except ValueError:
pass
else:
trace.append("subgenerator failed to raise ValueError")
return trace
def test_attempting_to_send_to_non_generator():
"""
>>> _lines(test_attempting_to_send_to_non_generator())
starting g
finishing g
"""
trace = []
def g():
try:
trace.append("starting g")
yield from range(3)
trace.append("g should not be here")
finally:
trace.append("finishing g")
try:
gi = g()
next(gi)
for x in range(3):
y = gi.send(42)
trace.append("Should not have yielded:", y)
except AttributeError:
pass
else:
trace.append("was able to send into non-generator")
return trace
def test_broken_getattr_handling():
"""
>>> test_broken_getattr_handling()
[]
"""
class Broken:
def __iter__(self):
return self
def __next__(self):
return 1
next = __next__
def __getattr__(self, attr):
1/0
if IS_PY3:
expected_exception = ZeroDivisionError
else:
expected_exception = AttributeError
def g():
yield from Broken()
not_raised = []
try:
gi = g()
assert next(gi) == 1
gi.send(1)
except expected_exception:
pass
else:
not_raised.append(1)
try:
gi = g()
assert next(gi) == 1
gi.throw(AttributeError)
except ZeroDivisionError:
pass
else:
not_raised.append(2)
"""
# this currently only calls PyErr_WriteUnraisable() and doesn't raise ...
try:
gi = g()
assert next(gi) == 1
gi.close()
except ZeroDivisionError:
pass
else:
not_raised.append(3)
"""
gi = g()
assert next(gi) == 1
gi.close()
return not_raised
def test_exception_in_initial_next_call():
"""
>>> _lines(test_exception_in_initial_next_call())
g1 about to yield from g2
"""
trace = []
def g1():
trace.append("g1 about to yield from g2")
yield from g2()
trace.append("g1 should not be here")
def g2():
yield 1/0
def run():
gi = g1()
next(gi)
try:
run()
except ZeroDivisionError:
pass
else:
trace.append("ZeroDivisionError not raised")
return trace
def test_attempted_yield_from_loop():
"""
>>> _lines(test_attempted_yield_from_loop())
g1: starting
Yielded: y1
g1: about to yield from g2
g2: starting
Yielded: y2
g2: about to yield from g1
"""
trace = []
def g1():
trace.append("g1: starting")
yield "y1"
trace.append("g1: about to yield from g2")
yield from g2()
trace.append("g1 should not be here")
def g2():
trace.append("g2: starting")
yield "y2"
trace.append("g2: about to yield from g1")
yield from gi
trace.append("g2 should not be here")
try:
gi = g1()
for y in gi:
trace.append("Yielded: %s" % (y,))
except ValueError:
pass # "generator already executing"
else:
trace.append("subgenerator didn't raise ValueError")
return trace
def test_attempted_reentry():
"""
>>> _lines(test_attempted_reentry())
g1: starting
Yielded: y1
g1: about to yield from g2
g2: starting
Yielded: y2
g2: about to yield from g1
g2: caught ValueError
Yielded: y3
g1: after delegating to g2
Yielded: y4
"""
trace = []
def g1():
trace.append("g1: starting")
yield "y1"
trace.append("g1: about to yield from g2")
yield from g2()
trace.append("g1: after delegating to g2")
yield "y4"
def g2():
trace.append("g2: starting")
yield "y2"
trace.append("g2: about to yield from g1")
try:
yield from gi
except ValueError:
trace.append("g2: caught ValueError")
else:
trace.append("g1 did not raise ValueError on reentry")
yield "y3"
gi = g1()
for y in gi:
trace.append("Yielded: %s" % (y,))
return trace
def test_returning_value_from_delegated_throw():
"""
>>> _lines(test_returning_value_from_delegated_throw())
Starting g1
Yielded g1 ham
Starting g2
Yielded g2 spam
Caught LunchError in g2
Yielded g2 yet more spam
Yielded g1 eggs
Finishing g1
"""
trace = []
def g1():
try:
trace.append("Starting g1")
yield "g1 ham"
yield from g2()
yield "g1 eggs"
finally:
trace.append("Finishing g1")
def g2():
try:
trace.append("Starting g2")
yield "g2 spam"
yield "g2 more spam"
except LunchError:
trace.append("Caught LunchError in g2")
yield "g2 lunch saved"
yield "g2 yet more spam"
class LunchError(Exception):
pass
g = g1()
for i in range(2):
x = next(g)
trace.append("Yielded %s" % (x,))
e = LunchError("tomato ejected")
g.throw(e)
for x in g:
trace.append("Yielded %s" % (x,))
return trace
def test_next_and_return_with_value():
"""
>>> _lines(test_next_and_return_with_value())
g starting
f resuming g
g returning None
f caught StopIteration
g starting
f resuming g
g returning 42
f caught StopIteration
"""
trace = []
def f(r):
gi = g(r)
next(gi)
try:
trace.append("f resuming g")
next(gi)
trace.append("f SHOULD NOT BE HERE")
except StopIteration:
trace.append("f caught StopIteration")
def g(r):
trace.append("g starting")
yield
trace.append("g returning %s" % (r,))
return r
f(None)
f(42)
return trace
def test_send_and_return_with_value():
"""
>>> _lines(test_send_and_return_with_value())
g starting
f sending spam to g
g received spam
g returning None
f caught StopIteration
g starting
f sending spam to g
g received spam
g returning 42
f caught StopIteration
"""
trace = []
def f(r):
gi = g(r)
next(gi)
try:
trace.append("f sending spam to g")
gi.send("spam")
trace.append("f SHOULD NOT BE HERE")
except StopIteration:
trace.append("f caught StopIteration")
def g(r):
trace.append("g starting")
x = yield
trace.append("g received %s" % (x,))
trace.append("g returning %s" % (r,))
return r
f(None)
f(42)
return trace
def test_catching_exception_from_subgen_and_returning():
"""
Test catching an exception thrown into a
subgenerator and returning a value
>>> _lines(test_catching_exception_from_subgen_and_returning())
1
inner caught ValueError
inner returned 2 to outer
2
"""
trace = []
def inner():
try:
yield 1
except ValueError:
trace.append("inner caught ValueError")
return 2
def outer():
v = yield from inner()
trace.append("inner returned %r to outer" % v)
yield v
g = outer()
trace.append(next(g))
trace.append(g.throw(ValueError))
return trace
def test_throwing_GeneratorExit_into_subgen_that_returns():
"""
Test throwing GeneratorExit into a subgenerator that
catches it and returns normally.
>>> _lines(test_throwing_GeneratorExit_into_subgen_that_returns())
Enter g
Enter f
"""
trace = []
def f():
try:
trace.append("Enter f")
yield
trace.append("Exit f")
except GeneratorExit:
return
def g():
trace.append("Enter g")
yield from f()
trace.append("Exit g")
try:
gi = g()
next(gi)
gi.throw(GeneratorExit)
except GeneratorExit:
pass
else:
trace.append("subgenerator failed to raise GeneratorExit")
return trace
def test_throwing_GeneratorExit_into_subgenerator_that_yields():
"""
Test throwing GeneratorExit into a subgenerator that
catches it and yields.
>>> _lines(test_throwing_GeneratorExit_into_subgenerator_that_yields())
Enter g
Enter f
"""
trace = []
def f():
try:
trace.append("Enter f")
yield
trace.append("Exit f")
except GeneratorExit:
yield
def g():
trace.append("Enter g")
yield from f()
trace.append("Exit g")
try:
gi = g()
next(gi)
gi.throw(GeneratorExit)
except RuntimeError:
pass # "generator ignored GeneratorExit"
else:
trace.append("subgenerator failed to raise GeneratorExit")
return trace
def test_throwing_GeneratorExit_into_subgen_that_raises():
"""
Test throwing GeneratorExit into a subgenerator that
catches it and raises a different exception.
>>> _lines(test_throwing_GeneratorExit_into_subgen_that_raises())
Enter g
Enter f
"""
trace = []
def f():
try:
trace.append("Enter f")
yield
trace.append("Exit f")
except GeneratorExit:
raise ValueError("Vorpal bunny encountered")
def g():
trace.append("Enter g")
yield from f()
trace.append("Exit g")
try:
gi = g()
next(gi)
gi.throw(GeneratorExit)
except ValueError:
pass # "Vorpal bunny encountered"
else:
trace.append("subgenerator failed to raise ValueError")
return trace
def test_yield_from_empty():
"""
>>> test_yield_from_empty()
"""
def g():
yield from ()
try:
next(g())
except StopIteration:
pass
else:
return "FAILED"
# test re-entry guards
def _reentering_gen():
def one():
yield 0
yield from two()
yield 3
def two():
yield 1
try:
yield from g1
except ValueError:
pass
yield 2
g1 = one()
return g1
def test_delegating_generators_claim_to_be_running_next():
"""
>>> test_delegating_generators_claim_to_be_running_next()
[0, 1, 2, 3]
"""
return list(_reentering_gen())
def test_delegating_generators_claim_to_be_running_send():
"""
>>> test_delegating_generators_claim_to_be_running_send()
[0, 1, 2, 3]
"""
g1 = _reentering_gen()
res = [next(g1)]
try:
while True:
res.append(g1.send(42))
except StopIteration:
pass
return res
def test_delegating_generators_claim_to_be_running_throw():
"""
>>> test_delegating_generators_claim_to_be_running_throw()
[0, 1, 2, 3]
"""
class MyErr(Exception):
pass
def one():
try:
yield 0
except MyErr:
pass
yield from two()
try:
yield 3
except MyErr:
pass
def two():
try:
yield 1
except MyErr:
pass
try:
yield from g1
except ValueError:
pass
try:
yield 2
except MyErr:
pass
g1 = one()
res = [next(g1)]
try:
while True:
res.append(g1.throw(MyErr))
except StopIteration:
pass
return res
def test_delegating_generators_claim_to_be_running_close():
"""
>>> test_delegating_generators_claim_to_be_running_close()
42
"""
class MyIt(object):
def __iter__(self):
return self
def __next__(self):
return 42
next = __next__
def close(self_):
assert g1.gi_running
try:
next(g1)
except ValueError:
pass # guard worked
else:
assert False, "re-entry guard failed to bark"
def one():
yield from MyIt()
g1 = one()
ret = next(g1)
g1.close()
return ret
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7