PEP 342 implementation. Per Guido's comments, the generator throw()

method still needs to support string exceptions, and allow None for the
third argument.  Documentation updates are needed, too.

Grammar/Grammar

@@ -39,7 +39,7 @@ fplist: fpdef (',' fpdef)* [',']
 stmt: simple_stmt | compound_stmt
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | exec_stmt | assert_stmt
-expr_stmt: testlist (augassign testlist | ('=' testlist)*)
+expr_stmt: testlist (augassign (yield_expr|testlist) | ('=' (yield_expr|testlist))*)
 augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//='
 # For normal assignments, additional restrictions enforced by the interpreter
 print_stmt: 'print' ( [ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ] )
@@ -49,7 +49,7 @@ flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
 break_stmt: 'break'
 continue_stmt: 'continue'
 return_stmt: 'return' [testlist]
-yield_stmt: 'yield' testlist
+yield_stmt: yield_expr
 raise_stmt: 'raise' [test [',' test [',' test]]]
 import_stmt: import_name | import_from
 import_name: 'import' dotted_as_names
@@ -86,7 +86,7 @@ arith_expr: term (('+'|'-') term)*
 term: factor (('*'|'/'|'%'|'//') factor)*
 factor: ('+'|'-'|'~') factor | power
 power: atom trailer* ['**' factor]
-atom: '(' [testlist_gexp] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+
+atom: '(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+
 listmaker: test ( list_for | (',' test)* [','] )
 testlist_gexp: test ( gen_for | (',' test)* [','] )
 lambdef: 'lambda' [varargslist] ':' test
@@ -116,3 +116,6 @@ testlist1: test (',' test)*
 
 # not used in grammar, but may appear in "node" passed from Parser to Compiler
 encoding_decl: NAME
+
+yield_expr: 'yield' [testlist]
+

Include/ceval.h

@@ -65,6 +65,7 @@ PyAPI_FUNC(char *) PyEval_GetFuncDesc(PyObject *);
 
 PyAPI_FUNC(PyObject *) PyEval_GetCallStats(PyObject *);
 PyAPI_FUNC(PyObject *) PyEval_EvalFrame(struct _frame *);
+PyAPI_FUNC(PyObject *) PyEval_EvalFrameEx(struct _frame *f, int exc);
 
 /* this used to be handled on a per-thread basis - now just two globals */
 PyAPI_DATA(volatile int) _Py_Ticker;

Include/graminit.h

@@ -76,3 +76,4 @@
 #define gen_if 331
 #define testlist1 332
 #define encoding_decl 333
+#define yield_expr 334

Include/pyerrors.h

@@ -25,6 +25,7 @@ PyAPI_FUNC(void) PyErr_NormalizeException(PyObject**, PyObject**, PyObject**);
 
 PyAPI_DATA(PyObject *) PyExc_Exception;
 PyAPI_DATA(PyObject *) PyExc_StopIteration;
+PyAPI_DATA(PyObject *) PyExc_GeneratorExit;
 PyAPI_DATA(PyObject *) PyExc_StandardError;
 PyAPI_DATA(PyObject *) PyExc_ArithmeticError;
 PyAPI_DATA(PyObject *) PyExc_LookupError;

Lib/compiler/transformer.py

@@ -403,7 +403,15 @@ class Transformer:
         return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2])
 
     def yield_stmt(self, nodelist):
-        return Yield(self.com_node(nodelist[1]), lineno=nodelist[0][2])
+        expr = self.com_node(nodelist[0])
+        return Discard(expr, lineno=expr.lineno)
+
+    def yield_expr(self, nodelist):
+        if len(nodelist)>1:
+            value = nodelist[1]
+        else:
+            value = Const(None)
+        return Yield(self.com_node(value), lineno=nodelist[0][2])
 
     def raise_stmt(self, nodelist):
         # raise: [test [',' test [',' test]]]
@@ -1402,6 +1410,8 @@ _legal_node_types = [
 
 if hasattr(symbol, 'yield_stmt'):
     _legal_node_types.append(symbol.yield_stmt)
+if hasattr(symbol, 'yield_expr'):
+    _legal_node_types.append(symbol.yield_expr)
 
 _assign_types = [
     symbol.test,

Lib/symbol.py

@@ -88,6 +88,7 @@ gen_for = 330
 gen_if = 331
 testlist1 = 332
 encoding_decl = 333
+yield_expr = 334
 #--end constants--
 
 sym_name = {}

Lib/test/test_generators.py

@@ -382,7 +382,7 @@ From the Iterators list, about the types of these things.
 >>> type(i)
 <type 'generator'>
 >>> [s for s in dir(i) if not s.startswith('_')]
-['gi_frame', 'gi_running', 'next']
+['close', 'gi_frame', 'gi_running', 'next', 'send', 'throw']
 >>> print i.next.__doc__
 x.next() -> the next value, or raise StopIteration
 >>> iter(i) is i
@@ -421,6 +421,7 @@ Subject: Re: PEP 255: Simple Generators
 ...         self.name = name
 ...         self.parent = None
 ...         self.generator = self.generate()
+...         self.close = self.generator.close
 ...
 ...     def generate(self):
 ...         while not self.parent:
@@ -482,6 +483,9 @@ merged F into A
 A->A B->G C->A D->G E->G F->A G->G H->G I->A J->G K->A L->A M->G
 merged A into G
 A->G B->G C->G D->G E->G F->G G->G H->G I->G J->G K->G L->G M->G
+
+>>> for s in sets: s.close()	# break cycles
+
 """
 # Emacs turd '
 
@@ -589,6 +593,7 @@ arguments are iterable -- a LazyList is the same as a generator to times().
 ...     def __init__(self, g):
 ...         self.sofar = []
 ...         self.fetch = g.next
+...         self.close = g.close
 ...
 ...     def __getitem__(self, i):
 ...         sofar, fetch = self.sofar, self.fetch
@@ -619,6 +624,7 @@ efficient.
 [200, 216, 225, 240, 243, 250, 256, 270, 288, 300, 320, 324, 360, 375, 384]
 [400, 405, 432, 450, 480, 486, 500, 512, 540, 576, 600, 625, 640, 648, 675]
 
+>>> m235.close()
 
 Ye olde Fibonacci generator, LazyList style.
 
@@ -642,6 +648,7 @@ Ye olde Fibonacci generator, LazyList style.
 >>> fib = LazyList(fibgen(1, 2))
 >>> firstn(iter(fib), 17)
 [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987, 1597, 2584]
+>>> fib.close()
 """
 
 # syntax_tests mostly provokes SyntaxErrors.  Also fiddling with #if 0
@@ -672,7 +679,7 @@ Traceback (most recent call last):
   ..
 SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.syntax[2]>, line 3)
 
-This one is fine:
+These are fine:
 
 >>> def f():
 ...     yield 1
@@ -683,9 +690,6 @@ This one is fine:
 ...         yield 1
 ...     finally:
 ...         pass
-Traceback (most recent call last):
-  ..
-SyntaxError: 'yield' not allowed in a 'try' block with a 'finally' clause (<doctest test.test_generators.__test__.syntax[4]>, line 3)
 
 >>> def f():
 ...     try:
@@ -697,11 +701,6 @@ SyntaxError: 'yield' not allowed in a 'try' block with a 'finally' clause (<doct
 ...             pass
 ...     finally:
 ...         pass
-Traceback (most recent call last):
-  ...
-SyntaxError: 'yield' not allowed in a 'try' block with a 'finally' clause (<doctest test.test_generators.__test__.syntax[5]>, line 6)
-
-But this is fine:
 
 >>> def f():
 ...     try:
@@ -722,14 +721,16 @@ But this is fine:
 
 >>> def f():
 ...    yield
-Traceback (most recent call last):
-SyntaxError: invalid syntax
+>>> type(f())
+<type 'generator'>
+
 
 >>> def f():
 ...    if 0:
 ...        yield
-Traceback (most recent call last):
-SyntaxError: invalid syntax
+>>> type(f())
+<type 'generator'>
+
 
 >>> def f():
 ...     if 0:
@@ -805,7 +806,7 @@ SyntaxError: invalid syntax
 ...     if 0:
 ...         yield 2             # because it's a generator
 Traceback (most recent call last):
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.syntax[22]>, line 8)
+SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.syntax[24]>, line 8)
 
 This one caused a crash (see SF bug 567538):
 
@@ -1383,6 +1384,250 @@ True
 
 """
 
+coroutine_tests = """\
+Sending a value into a started generator:
+
+>>> def f():
+...     print (yield 1)
+...     yield 2
+>>> g = f()
+>>> g.next()
+1
+>>> g.send(42)
+42
+2
+
+Sending a value into a new generator produces a TypeError:
+
+>>> f().send("foo")
+Traceback (most recent call last):
+...
+TypeError: can't send non-None value to a just-started generator
+
+
+Yield by itself yields None:
+
+>>> def f(): yield
+>>> list(f())
+[None]
+
+
+
+An obscene abuse of a yield expression within a generator expression:
+
+>>> list((yield 21) for i in range(4))
+[21, None, 21, None, 21, None, 21, None]
+
+And a more sane, but still weird usage:
+
+>>> def f(): list(i for i in [(yield 26)])
+>>> type(f())
+<type 'generator'>
+
+
+Check some syntax errors for yield expressions:
+
+>>> f=lambda: (yield 1),(yield 2)
+Traceback (most recent call last):
+  ...
+SyntaxError: 'yield' outside function (<doctest test.test_generators.__test__.coroutine[10]>, line 1)
+
+>>> def f(): return lambda x=(yield): 1
+Traceback (most recent call last):
+  ...
+SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.coroutine[11]>, line 1)
+
+>>> def f(): x = yield = y
+Traceback (most recent call last):
+  ...
+SyntaxError: assignment to yield expression not possible (<doctest test.test_generators.__test__.coroutine[12]>, line 1)
+
+
+Now check some throw() conditions:
+
+>>> def f():
+...     while True:
+...         try:
+...             print (yield)
+...         except ValueError,v:
+...             print "caught ValueError (%s)" % (v),
+>>> import sys
+>>> g = f()
+>>> g.next()
+
+>>> g.throw(ValueError) # type only
+caught ValueError ()
+
+>>> g.throw(ValueError("xyz"))  # value only
+caught ValueError (xyz)
+
+>>> g.throw(ValueError, ValueError(1))   # value+matching type
+caught ValueError (1)
+
+>>> g.throw(ValueError, TypeError(1))  # mismatched type, rewrapped
+caught ValueError (1)
+
+>>> g.throw(ValueError(1), "foo")	# bad args
+Traceback (most recent call last):
+  ...
+TypeError: instance exception may not have a separate value
+
+>>> g.throw(ValueError, "foo", 23)	# bad args
+Traceback (most recent call last):
+  ...
+TypeError: throw() third argument must be a traceback object
+
+>>> def throw(g,exc):
+...     try:
+...         raise exc
+...     except:
+...         g.throw(*sys.exc_info())
+>>> throw(g,ValueError)	# do it with traceback included
+caught ValueError ()
+
+>>> g.send(1)
+1
+
+>>> throw(g,TypeError)	# terminate the generator
+Traceback (most recent call last):
+  ...
+TypeError
+
+>>> print g.gi_frame
+None
+
+>>> g.send(2)
+Traceback (most recent call last):
+  ...
+StopIteration
+
+>>> g.throw(ValueError,6)	# throw on closed generator
+Traceback (most recent call last):
+  ...
+ValueError: 6
+
+>>> f().throw(ValueError,7)	# throw on just-opened generator
+Traceback (most recent call last):
+  ...
+ValueError: 7
+
+
+Now let's try closing a generator:
+
+>>> def f():
+...     try: yield
+...     except GeneratorExit:
+...         print "exiting"
+
+>>> g = f()
+>>> g.next()
+>>> g.close()
+exiting
+>>> g.close()  # should be no-op now
+
+>>> f().close()  # close on just-opened generator should be fine
+
+>>> def f(): yield	# an even simpler generator
+>>> f().close()		# close before opening
+>>> g = f()
+>>> g.next()
+>>> g.close()		# close normally
+
+And finalization:
+
+>>> def f():
+...     try: yield
+...     finally:
+...         print "exiting"
+
+>>> g = f()
+>>> g.next()
+>>> del g
+exiting
+
+
+Now let's try some ill-behaved generators:
+
+>>> def f():
+...     try: yield
+...     except GeneratorExit:
+...         yield "foo!"
+>>> g = f()
+>>> g.next()
+>>> g.close()
+Traceback (most recent call last):
+  ...
+RuntimeError: generator ignored GeneratorExit
+>>> g.close()
+
+
+Our ill-behaved code should be invoked during GC:
+
+>>> import sys, StringIO
+>>> old, sys.stderr = sys.stderr, StringIO.StringIO()
+>>> g = f()
+>>> g.next()
+>>> del g
+>>> sys.stderr.getvalue().startswith(
+...     "Exception exceptions.RuntimeError: 'generator ignored GeneratorExit' in "
+... )
+True
+>>> sys.stderr = old
+
+
+And errors thrown during closing should propagate:
+
+>>> def f():
+...     try: yield
+...     except GeneratorExit:
+...         raise TypeError("fie!")
+>>> g = f()
+>>> g.next()
+>>> g.close()
+Traceback (most recent call last):
+  ...
+TypeError: fie!
+
+
+Ensure that various yield expression constructs make their
+enclosing function a generator:
+
+>>> def f(): x += yield
+>>> type(f())
+<type 'generator'>
+
+>>> def f(): x = yield
+>>> type(f())
+<type 'generator'>
+
+>>> def f(): lambda x=(yield): 1
+>>> type(f())
+<type 'generator'>
+
+>>> def f(): x=(i for i in (yield) if (yield))
+>>> type(f())
+<type 'generator'>
+
+>>> def f(d): d[(yield "a")] = d[(yield "b")] = 27
+>>> data = [1,2]
+>>> g = f(data)
+>>> type(g)
+<type 'generator'>
+>>> g.send(None)
+'a'
+>>> data
+[1, 2]
+>>> g.send(0)
+'b'
+>>> data
+[27, 2]
+>>> try: g.send(1)
+... except StopIteration: pass
+>>> data
+[27, 27]
+
+"""
+
 __test__ = {"tut":      tutorial_tests,
             "pep":      pep_tests,
             "email":    email_tests,
@@ -1390,6 +1635,7 @@ __test__ = {"tut":      tutorial_tests,
             "syntax":   syntax_tests,
             "conjoin":  conjoin_tests,
             "weakref":  weakref_tests,
+            "coroutine":  coroutine_tests,
             }
 
 # Magic test name that regrtest.py invokes *after* importing this module.

Lib/test/test_genexps.py

@@ -130,7 +130,7 @@ Verify that syntax error's are raised for genexps used as lvalues
     >>> (y for y in (1,2)) += 10
     Traceback (most recent call last):
        ...
-    SyntaxError: augmented assign to tuple literal or generator expression not possible
+    SyntaxError: augmented assign to tuple literal, yield, or generator expression not possible
 
 
 

Lib/test/test_parser.py

@@ -29,11 +29,22 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase):
 
     def test_yield_statement(self):
         self.check_suite("def f(): yield 1")
+        self.check_suite("def f(): yield")
+        self.check_suite("def f(): x += yield")
+        self.check_suite("def f(): x = yield 1")
+        self.check_suite("def f(): x = y = yield 1")
+        self.check_suite("def f(): x = yield")
+        self.check_suite("def f(): x = y = yield")
+        self.check_suite("def f(): 1 + (yield)*2")
+        self.check_suite("def f(): (yield 1)*2")
         self.check_suite("def f(): return; yield 1")
         self.check_suite("def f(): yield 1; return")
         self.check_suite("def f():\n"
                          "    for x in range(30):\n"
                          "        yield x\n")
+        self.check_suite("def f():\n"
+                         "    if (yield):\n"
+                         "        yield x\n")
 
     def test_expressions(self):
         self.check_expr("foo(1)")

Modules/gcmodule.c

@@ -413,10 +413,8 @@ has_finalizer(PyObject *op)
 		assert(delstr != NULL);
 		return _PyInstance_Lookup(op, delstr) != NULL;
 	}
-	else if (PyType_HasFeature(op->ob_type, Py_TPFLAGS_HEAPTYPE))
+	else 
 		return op->ob_type->tp_del != NULL;
-	else
-		return 0;
 }
 
 /* Move the objects in unreachable with __del__ methods into `finalizers`.

Modules/parsermodule.c

@@ -859,7 +859,8 @@ VALIDATER(arglist);             VALIDATER(argument);
 VALIDATER(listmaker);           VALIDATER(yield_stmt);
 VALIDATER(testlist1);           VALIDATER(gen_for);
 VALIDATER(gen_iter);            VALIDATER(gen_if);
-VALIDATER(testlist_gexp);
+VALIDATER(testlist_gexp);	VALIDATER(yield_expr);
+VALIDATER(yield_or_testlist);	
 
 #undef VALIDATER
 
@@ -1506,6 +1507,15 @@ validate_compound_stmt(node *tree)
 }
 
 
+static int
+validate_yield_or_testlist(node *tree)
+{
+	if (TYPE(tree) == yield_expr) 
+		return validate_yield_expr(tree);
+	else
+		return validate_testlist(tree);
+}
+
 static int
 validate_expr_stmt(node *tree)
 {
@@ -1517,8 +1527,8 @@ validate_expr_stmt(node *tree)
 
     if (res && nch == 3
         && TYPE(CHILD(tree, 1)) == augassign) {
-        res = (validate_numnodes(CHILD(tree, 1), 1, "augassign")
-               && validate_testlist(CHILD(tree, 2)));
+        res = validate_numnodes(CHILD(tree, 1), 1, "augassign")
+		&& validate_yield_or_testlist(CHILD(tree, 2));
 
         if (res) {
             char *s = STR(CHILD(CHILD(tree, 1), 0));
@@ -1541,8 +1551,8 @@ validate_expr_stmt(node *tree)
     }
     else {
         for (j = 1; res && (j < nch); j += 2)
-            res = (validate_equal(CHILD(tree, j))
-                   && validate_testlist(CHILD(tree, j + 1)));
+            res = validate_equal(CHILD(tree, j))
+                   && validate_yield_or_testlist(CHILD(tree, j + 1));
     }
     return (res);
 }
@@ -1649,15 +1659,31 @@ validate_raise_stmt(node *tree)
 }
 
 
-/* yield_stmt: 'yield' testlist
+/* yield_expr: 'yield' [testlist]
+ */
+static int
+validate_yield_expr(node *tree)
+{
+    int nch = NCH(tree);
+    int res = (validate_ntype(tree, yield_expr)
+               && ((nch == 1) || (nch == 2))
+               && validate_name(CHILD(tree, 0), "yield"));
+
+    if (res && (nch == 2))
+        res = validate_testlist(CHILD(tree, 1));
+
+    return (res);
+}
+
+
+/* yield_stmt: yield_expr
  */
 static int
 validate_yield_stmt(node *tree)
 {
     return (validate_ntype(tree, yield_stmt)
-            && validate_numnodes(tree, 2, "yield_stmt")
-            && validate_name(CHILD(tree, 0), "yield")
-            && validate_testlist(CHILD(tree, 1)));
+            && validate_numnodes(tree, 1, "yield_stmt")
+            && validate_yield_expr(CHILD(tree, 0)));
 }
 
 
@@ -2300,8 +2326,12 @@ validate_atom(node *tree)
             res = ((nch <= 3)
                    && (validate_rparen(CHILD(tree, nch - 1))));
 
-            if (res && (nch == 3))
-                res = validate_testlist_gexp(CHILD(tree, 1));
+            if (res && (nch == 3)) {
+		if (TYPE(CHILD(tree, 1))==yield_expr)
+			res = validate_yield_expr(CHILD(tree, 1));
+		else
+                	res = validate_testlist_gexp(CHILD(tree, 1));
+	    }
             break;
           case LSQB:
             if (nch == 2)
@@ -2914,6 +2944,9 @@ validate_node(node *tree)
           case testlist:
             res = validate_testlist(tree);
             break;
+          case yield_expr:
+            res = validate_yield_expr(tree);
+            break;
           case testlist1:
             res = validate_testlist1(tree);
             break;

Objects/genobject.c

@@ -15,15 +15,31 @@ gen_traverse(PyGenObject *gen, visitproc visit, void *arg)
 static void
 gen_dealloc(PyGenObject *gen)
 {
+	PyObject *self = (PyObject *) gen;
+
 	_PyObject_GC_UNTRACK(gen);
+
 	if (gen->gi_weakreflist != NULL)
 		PyObject_ClearWeakRefs((PyObject *) gen);
-	Py_DECREF(gen->gi_frame);
+
+
+	_PyObject_GC_TRACK(self);
+
+	if (gen->gi_frame->f_stacktop!=NULL) {
+		/* Generator is paused, so we need to close */
+		gen->ob_type->tp_del(self);
+		if (self->ob_refcnt > 0)
+			return;		/* resurrected.  :( */
+	}
+
+	_PyObject_GC_UNTRACK(self);
+	Py_XDECREF(gen->gi_frame);
 	PyObject_GC_Del(gen);
 }
 
+
 static PyObject *
-gen_iternext(PyGenObject *gen)
+gen_send_ex(PyGenObject *gen, PyObject *arg, int exc)
 {
 	PyThreadState *tstate = PyThreadState_GET();
 	PyFrameObject *f = gen->gi_frame;
@@ -34,8 +50,24 @@ gen_iternext(PyGenObject *gen)
 				"generator already executing");
 		return NULL;
 	}
-	if (f->f_stacktop == NULL)
+	if ((PyObject *)f == Py_None || f->f_stacktop == NULL) {
+		/* Only set exception if called from send() */
+		if (arg && !exc) PyErr_SetNone(PyExc_StopIteration);
 		return NULL;
+	}
+
+	if (f->f_lasti == -1) {
+		if (arg && arg != Py_None) {
+			PyErr_SetString(PyExc_TypeError,
+				"can't send non-None value to a just-started generator");
+			return NULL;
+		}
+	} else {
+		/* Push arg onto the frame's value stack */
+		result = arg ? arg : Py_None;
+	        Py_INCREF(result);
+	        *(f->f_stacktop++) = result;
+	}
 
 	/* Generators always return to their most recent caller, not
 	 * necessarily their creator. */
@@ -44,7 +76,7 @@ gen_iternext(PyGenObject *gen)
 	f->f_back = tstate->frame;
 
 	gen->gi_running = 1;
-	result = PyEval_EvalFrame(f);
+	result = PyEval_EvalFrameEx(f, exc);
 	gen->gi_running = 0;
 
 	/* Don't keep the reference to f_back any longer than necessary.  It
@@ -58,17 +90,199 @@ gen_iternext(PyGenObject *gen)
 	if (result == Py_None && f->f_stacktop == NULL) {
 		Py_DECREF(result);
 		result = NULL;
+		/* Set exception if not called by gen_iternext() */
+		if (arg) PyErr_SetNone(PyExc_StopIteration);
+	}
+
+	if (!result || f->f_stacktop == NULL) {
+		/* generator can't be rerun, so release the frame */
+		Py_DECREF(f);
+		gen->gi_frame = (PyFrameObject *)Py_None;
+		Py_INCREF(Py_None);
 	}
 
 	return result;
 }
 
+PyDoc_STRVAR(send_doc,
+"send(arg) -> send 'arg' into generator, return next yielded value or raise StopIteration.");
+
+static PyObject *
+gen_send(PyGenObject *gen, PyObject *arg)
+{
+	return gen_send_ex(gen, arg, 0);
+}
+
+PyDoc_STRVAR(close_doc,
+"close(arg) -> raise GeneratorExit inside generator.");
+
+static PyObject *
+gen_close(PyGenObject *gen, PyObject *args)
+{
+	PyObject *retval;
+	PyErr_SetNone(PyExc_GeneratorExit);
+	retval = gen_send_ex(gen, Py_None, 1);
+	if (retval) {
+		Py_DECREF(retval);
+		PyErr_SetString(PyExc_RuntimeError,
+			"generator ignored GeneratorExit");
+		return NULL;
+	}
+	if ( PyErr_ExceptionMatches(PyExc_StopIteration) 
+	     || PyErr_ExceptionMatches(PyExc_GeneratorExit) ) 
+	{
+		PyErr_Clear();	/* ignore these errors */
+		Py_INCREF(Py_None);
+		return Py_None;
+	}
+	return NULL;
+}
+
+static void
+gen_del(PyObject *self)
+{
+        PyObject *res;
+        PyObject *error_type, *error_value, *error_traceback;
+	PyGenObject *gen = (PyGenObject *)self;
+
+	if ((PyObject *)gen->gi_frame == Py_None || gen->gi_frame->f_stacktop==NULL)
+		/* Generator isn't paused, so no need to close */
+		return;
+
+        /* Temporarily resurrect the object. */
+        assert(self->ob_refcnt == 0);
+        self->ob_refcnt = 1;
+
+        /* Save the current exception, if any. */
+        PyErr_Fetch(&error_type, &error_value, &error_traceback);
+
+	res = gen_close((PyGenObject *)self, NULL);
+
+	if (res == NULL)
+		PyErr_WriteUnraisable((PyObject *)self);
+	else
+		Py_DECREF(res);
+
+        /* Restore the saved exception. */
+        PyErr_Restore(error_type, error_value, error_traceback);
+
+        /* Undo the temporary resurrection; can't use DECREF here, it would
+         * cause a recursive call.
+         */
+        assert(self->ob_refcnt > 0);
+        if (--self->ob_refcnt == 0)
+                return; /* this is the normal path out */
+
+        /* close() resurrected it!  Make it look like the original Py_DECREF
+         * never happened.
+         */
+        {
+                int refcnt = self->ob_refcnt;
+                _Py_NewReference(self);
+                self->ob_refcnt = refcnt;
+        }
+        assert(!PyType_IS_GC(self->ob_type) ||
+               _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED);
+
+        /* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so
+         * we need to undo that. */
+        _Py_DEC_REFTOTAL;
+        /* If Py_TRACE_REFS, _Py_NewReference re-added self to the object
+         * chain, so no more to do there.
+         * If COUNT_ALLOCS, the original decref bumped tp_frees, and
+         * _Py_NewReference bumped tp_allocs:  both of those need to be
+         * undone.
+         */
+#ifdef COUNT_ALLOCS
+        --self->ob_type->tp_frees;
+        --self->ob_type->tp_allocs;
+#endif
+}
+
+
+
+PyDoc_STRVAR(throw_doc,
+"throw(typ[,val[,tb]]) -> raise exception in generator, return next yielded value or raise StopIteration.");
+
+static PyObject *
+gen_throw(PyGenObject *gen, PyObject *args) 
+{
+	PyObject *typ;
+	PyObject *tb = NULL;
+	PyObject *val = NULL;
+
+	if (!PyArg_ParseTuple(args, "O|OO:throw", &typ, &val, &tb))
+		return NULL;
+
+	if (tb && !PyTraceBack_Check(tb)) {
+		PyErr_SetString(PyExc_TypeError,
+			"throw() third argument must be a traceback object");
+		return NULL;
+	}
+
+	Py_INCREF(typ);
+	Py_XINCREF(val);
+	Py_XINCREF(tb);
+
+	if (PyClass_Check(typ)) {
+		PyErr_NormalizeException(&typ, &val, &tb);
+	}
+
+	else if (PyInstance_Check(typ)) {
+		/* Raising an instance.  The value should be a dummy. */
+		if (val && val != Py_None) {
+			PyErr_SetString(PyExc_TypeError,
+			  "instance exception may not have a separate value");
+			goto failed_throw;
+		}
+		else {
+			/* Normalize to raise <class>, <instance> */
+			val = typ;
+			typ = (PyObject*) ((PyInstanceObject*)typ)->in_class;
+			Py_INCREF(typ);
+		}
+	}
+	else {
+		/* Not something you can raise.  You get an exception
+		   anyway, just not what you specified :-) */
+		PyErr_Format(PyExc_TypeError,
+			     "exceptions must be classes, or instances, not %s",
+			     typ->ob_type->tp_name);
+			goto failed_throw;
+	}
+
+	PyErr_Restore(typ,val,tb);
+	return gen_send_ex(gen, Py_None, 1);
+
+failed_throw:
+	/* Didn't use our arguments, so restore their original refcounts */
+	Py_DECREF(typ);
+	Py_XDECREF(val);
+	Py_XDECREF(tb);
+	return NULL;
+}
+
+
+static PyObject *
+gen_iternext(PyGenObject *gen)
+{
+	return gen_send_ex(gen, NULL, 0);
+}
+
+
 static PyMemberDef gen_memberlist[] = {
 	{"gi_frame",	T_OBJECT, offsetof(PyGenObject, gi_frame),	RO},
 	{"gi_running",	T_INT,    offsetof(PyGenObject, gi_running),	RO},
 	{NULL}	/* Sentinel */
 };
 
+static PyMethodDef gen_methods[] = {
+	{"send",(PyCFunction)gen_send, METH_O, send_doc},
+	{"throw",(PyCFunction)gen_throw, METH_VARARGS, throw_doc},
+	{"close",(PyCFunction)gen_close, METH_NOARGS, close_doc},
+	{NULL, NULL}	/* Sentinel */
+};
+
 PyTypeObject PyGen_Type = {
 	PyObject_HEAD_INIT(&PyType_Type)
 	0,					/* ob_size */
@@ -99,11 +313,26 @@ PyTypeObject PyGen_Type = {
 	offsetof(PyGenObject, gi_weakreflist),	/* tp_weaklistoffset */
 	PyObject_SelfIter,			/* tp_iter */
 	(iternextfunc)gen_iternext,		/* tp_iternext */
-	0,					/* tp_methods */
+	gen_methods,				/* tp_methods */
 	gen_memberlist,				/* tp_members */
 	0,					/* tp_getset */
 	0,					/* tp_base */
 	0,					/* tp_dict */
+        
+	0,					/* tp_descr_get */
+	0,					/* tp_descr_set */
+	0,					/* tp_dictoffset */
+	0,					/* tp_init */
+	0,					/* tp_alloc */
+	0,					/* tp_new */
+	0,					/* tp_free */
+	0,					/* tp_is_gc */
+	0,					/* tp_bases */
+	0,					/* tp_mro */
+	0,					/* tp_cache */
+	0,					/* tp_subclasses */
+	0,					/* tp_weaklist */
+	gen_del,				/* tp_del */
 };
 
 PyObject *

Python/ceval.c

@@ -499,7 +499,14 @@ PyEval_EvalCode(PyCodeObject *co, PyObject *globals, PyObject *locals)
 /* Interpreter main loop */
 
 PyObject *
-PyEval_EvalFrame(PyFrameObject *f)
+PyEval_EvalFrame(PyFrameObject *f) {
+	/* This is for backward compatibility with extension modules that
+           used this API; core interpreter code should call PyEval_EvalFrameEx() */
+	return PyEval_EvalFrameEx(f, 0);
+}
+
+PyObject *
+PyEval_EvalFrameEx(PyFrameObject *f, int throw)
 {
 #ifdef DXPAIRS
 	int lastopcode = 0;
@@ -747,6 +754,11 @@ PyEval_EvalFrame(PyFrameObject *f)
 	x = Py_None;	/* Not a reference, just anything non-NULL */
 	w = NULL;
 
+	if (throw) { /* support for generator.throw() */
+		why = WHY_EXCEPTION;
+		goto on_error;
+	}
+		
 	for (;;) {
 #ifdef WITH_TSC
 		if (inst1 == 0) {
@@ -2733,7 +2745,7 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
 		return PyGen_New(f);
 	}
 
-        retval = PyEval_EvalFrame(f);
+        retval = PyEval_EvalFrameEx(f,0);
 
   fail: /* Jump here from prelude on failure */
 
@@ -3636,7 +3648,7 @@ fast_function(PyObject *func, PyObject ***pp_stack, int n, int na, int nk)
 			Py_INCREF(*stack);
 			fastlocals[i] = *stack++;
 		}
-		retval = PyEval_EvalFrame(f);
+		retval = PyEval_EvalFrameEx(f,0);
 		assert(tstate != NULL);
 		++tstate->recursion_depth;
 		Py_DECREF(f);

Python/compile.c

@@ -2145,6 +2145,7 @@ com_gen_for(struct compiling *c, node *n, node *t, int is_outmost)
 	else {
 		com_test(c, t);
 		com_addbyte(c, YIELD_VALUE);
+		com_addbyte(c, POP_TOP);
 		com_pop(c, 1);
 	}
 
@@ -2193,6 +2194,7 @@ com_gen_if(struct compiling *c, node *n, node *t)
 	else {
 		com_test(c, t);
 		com_addbyte(c, YIELD_VALUE);
+		com_addbyte(c, POP_TOP);
 		com_pop(c, 1);
 	}
 	com_addfwref(c, JUMP_FORWARD, &anchor);
@@ -2354,6 +2356,10 @@ com_dictmaker(struct compiling *c, node *n)
 	}
 }
 
+
+/* forward reference */
+static void com_yield_expr(struct compiling *c, node *n);
+
 static void
 com_atom(struct compiling *c, node *n)
 {
@@ -2369,7 +2375,10 @@ com_atom(struct compiling *c, node *n)
 			com_push(c, 1);
 		}
 		else
-			com_testlist_gexp(c, CHILD(n, 1));
+			if (TYPE(CHILD(n, 1)) == yield_expr)
+				com_yield_expr(c, CHILD(n, 1));
+			else
+				com_testlist_gexp(c, CHILD(n, 1));
 		break;
 	case LSQB: /* '[' [listmaker] ']' */
 		if (TYPE(CHILD(n, 1)) == RSQB) {
@@ -3436,7 +3445,11 @@ com_assign(struct compiling *c, node *n, int assigning, node *augn)
 			}
 			n = CHILD(n, 0);
 			break;
-		
+		case yield_expr:
+			com_error(c, PyExc_SyntaxError,
+			  "assignment to yield expression not possible");
+			return;
+					
 		case test:
 		case and_test:
 		case not_test:
@@ -3493,7 +3506,7 @@ com_assign(struct compiling *c, node *n, int assigning, node *augn)
 				}
 				if (assigning > OP_APPLY) {
 					com_error(c, PyExc_SyntaxError,
-				  "augmented assign to tuple literal or generator expression not possible");
+				  "augmented assign to tuple literal, yield, or generator expression not possible");
 					return;
 				}
 				break;
@@ -3729,27 +3742,42 @@ com_return_stmt(struct compiling *c, node *n)
 }
 
 static void
-com_yield_stmt(struct compiling *c, node *n)
+com_yield_expr(struct compiling *c, node *n)
 {
 	int i;
-	REQ(n, yield_stmt); /* 'yield' testlist */
+	REQ(n, yield_expr); /* 'yield' testlist */
 	if (!c->c_infunction) {
 		com_error(c, PyExc_SyntaxError, "'yield' outside function");
 	}
 	
-	for (i = 0; i < c->c_nblocks; ++i) {
+	/* for (i = 0; i < c->c_nblocks; ++i) {
 		if (c->c_block[i] == SETUP_FINALLY) {
 			com_error(c, PyExc_SyntaxError,
 				  "'yield' not allowed in a 'try' block "
 				  "with a 'finally' clause");
 			return;
 		}
+	} */
+
+	if (NCH(n) < 2) {
+		com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None));
+		com_push(c, 1);
 	}
-	com_node(c, CHILD(n, 1));
+	else
+		com_node(c, CHILD(n, 1));
 	com_addbyte(c, YIELD_VALUE);
+}
+
+static void
+com_yield_stmt(struct compiling *c, node *n)
+{
+	REQ(n, yield_stmt); /* yield_expr */
+	com_node(c, CHILD(n, 0));
+	com_addbyte(c, POP_TOP);
 	com_pop(c, 1);
 }
 
+
 static void
 com_raise_stmt(struct compiling *c, node *n)
 {
@@ -4768,6 +4796,10 @@ com_node(struct compiling *c, node *n)
 	
 	/* Expression nodes */
 	
+	case yield_expr:
+		com_yield_expr(c, n);
+		break;
+
 	case testlist:
 	case testlist1:
 	case testlist_safe:
@@ -5027,7 +5059,9 @@ compile_generator_expression(struct compiling *c, node *n)
 	REQ(CHILD(n, 1), gen_for); 
 
 	c->c_name = "<generator expression>";
+	c->c_infunction = 1;
 	com_gen_for(c, CHILD(n, 1), CHILD(n, 0), 1);
+	c->c_infunction = 0;
 
 	com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None));
 	com_push(c, 1);
@@ -6115,7 +6149,7 @@ symtable_add_def_o(struct symtable *st, PyObject *dict,
 
 #define symtable_add_use(ST, NAME) symtable_add_def((ST), (NAME), USE)
 
-/* Look for a yield stmt under n.  Return 1 if found, else 0.
+/* Look for a yield stmt or expr under n.  Return 1 if found, else 0.
    This hack is used to look inside "if 0:" blocks (which are normally
    ignored) in case those are the only places a yield occurs (so that this
    function is a generator). */
@@ -6137,6 +6171,7 @@ look_for_yield(node *n)
 			return 0;
 
 		case yield_stmt:
+		case yield_expr:
 			return GENERATOR;
 
 		default:
@@ -6247,8 +6282,10 @@ symtable_node(struct symtable *st, node *n)
 	case del_stmt:
 		symtable_assign(st, CHILD(n, 1), 0);
 		break;
-	case yield_stmt:
+	case yield_expr:
 		st->st_cur->ste_generator = 1;
+		if (NCH(n)==1) 
+			break;
 		n = CHILD(n, 1);
 		goto loop;
 	case expr_stmt:
@@ -6341,9 +6378,15 @@ symtable_node(struct symtable *st, node *n)
 		/* fall through */
 
 	case atom:
-		if (TYPE(n) == atom && TYPE(CHILD(n, 0)) == NAME) {
-			symtable_add_use(st, STR(CHILD(n, 0)));
-			break;
+		if (TYPE(n) == atom) {
+			if (TYPE(CHILD(n, 0)) == NAME) {
+				symtable_add_use(st, STR(CHILD(n, 0)));
+				break;
+			}
+			else if (TYPE(CHILD(n,0)) == LPAR) {
+				n = CHILD(n,1);
+				goto loop;
+			}
 		}
 		/* fall through */
 	default:
@@ -6739,6 +6782,15 @@ symtable_assign(struct symtable *st, node *n, int def_flag)
 			symtable_add_def(st, STR(tmp), DEF_LOCAL | def_flag);
 		}
 		return;
+
+	case yield_expr:
+		st->st_cur->ste_generator = 1;
+		if (NCH(n)==2) {
+			n = CHILD(n, 1);
+			goto loop;
+		}
+		return;
+
 	case dotted_as_name:
 		if (NCH(n) == 3)
 			symtable_add_def(st, STR(CHILD(n, 2)),

Python/exceptions.c

@@ -57,6 +57,7 @@ Exception\n\
  |\n\
  +-- SystemExit\n\
  +-- StopIteration\n\
+ +-- GeneratorExit\n\
  +-- StandardError\n\
  |    |\n\
  |    +-- KeyboardInterrupt\n\
@@ -394,6 +395,7 @@ PyDoc_STRVAR(StandardError__doc__,
 PyDoc_STRVAR(TypeError__doc__, "Inappropriate argument type.");
 
 PyDoc_STRVAR(StopIteration__doc__, "Signal the end from iterator.next().");
+PyDoc_STRVAR(GeneratorExit__doc__, "Request that a generator exit.");
 
 
 
@@ -1583,6 +1585,7 @@ static PyMethodDef functions[] = {
 
 PyObject *PyExc_Exception;
 PyObject *PyExc_StopIteration;
+PyObject *PyExc_GeneratorExit;
 PyObject *PyExc_StandardError;
 PyObject *PyExc_ArithmeticError;
 PyObject *PyExc_LookupError;
@@ -1657,6 +1660,8 @@ static struct {
  {"Exception", &PyExc_Exception},
  {"StopIteration", &PyExc_StopIteration, &PyExc_Exception,
   StopIteration__doc__},
+ {"GeneratorExit", &PyExc_GeneratorExit, &PyExc_Exception,
+  GeneratorExit__doc__},
  {"StandardError", &PyExc_StandardError, &PyExc_Exception,
   StandardError__doc__},
  {"TypeError", &PyExc_TypeError, 0, TypeError__doc__},