Add support for inspect.isgenerator and move is_generator from BoxedFunction to CLFunction

from_cpython/Lib/inspect.py

@@ -44,6 +44,7 @@ from collections import namedtuple
 # Pyston change: we don't support all of these flags yet, so just enable the ones that we do:
 CO_VARARGS = 0x4
 CO_VARKEYWORDS = 0x8
+CO_GENERATOR = 0x20
 # These constants are from Include/code.h.
 # CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS = 0x1, 0x2, 0x4, 0x8
 # CO_NESTED, CO_GENERATOR, CO_NOFREE = 0x10, 0x20, 0x40

src/codegen/ast_interpreter.cpp

@@ -759,8 +759,6 @@ Box* ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::v
         takes_closure = source_info->scoping->getScopeInfoForNode(node)->takesClosure();
     }
 
-    bool is_generator = cl->source->is_generator;
-
     BoxedClosure* closure = 0;
     if (takes_closure) {
         if (scope_info->createsClosure()) {
@@ -775,7 +773,7 @@ Box* ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::v
     Box* passed_globals = NULL;
     if (!getCF()->clfunc->source->scoping->areGlobalsFromModule())
         passed_globals = globals;
-    return boxCLFunction(cl, closure, is_generator, passed_globals, u.il);
+    return boxCLFunction(cl, closure, passed_globals, u.il);
 }
 
 Value ASTInterpreter::visit_makeFunction(AST_MakeFunction* mkfn) {
@@ -815,7 +813,7 @@ Value ASTInterpreter::visit_makeClass(AST_MakeClass* mkclass) {
     Box* passed_globals = NULL;
     if (!getCF()->clfunc->source->scoping->areGlobalsFromModule())
         passed_globals = globals;
-    Box* attrDict = runtimeCall(boxCLFunction(cl, closure, false, passed_globals, {}), ArgPassSpec(0), 0, 0, 0, 0, 0);
+    Box* attrDict = runtimeCall(boxCLFunction(cl, closure, passed_globals, {}), ArgPassSpec(0), 0, 0, 0, 0, 0);
 
     Box* classobj = createUserClass(&node->name.str(), basesTuple, attrDict);
 

src/codegen/compvars.cpp

@@ -723,8 +723,8 @@ ConcreteCompilerVariable* UnknownType::hasnext(IREmitter& emitter, const OpInfo&
     return boolFromI1(emitter, rtn_val);
 }
 
-CompilerVariable* makeFunction(IREmitter& emitter, CLFunction* f, CompilerVariable* closure, bool isGenerator,
-                               Box* globals, const std::vector<ConcreteCompilerVariable*>& defaults) {
+CompilerVariable* makeFunction(IREmitter& emitter, CLFunction* f, CompilerVariable* closure, Box* globals,
+                               const std::vector<ConcreteCompilerVariable*>& defaults) {
     // Unlike the CLFunction*, which can be shared between recompilations, the Box* around it
     // should be created anew every time the functiondef is encountered
 
@@ -752,17 +752,14 @@ CompilerVariable* makeFunction(IREmitter& emitter, CLFunction* f, CompilerVariab
         scratch = getNullPtr(g.llvm_value_type_ptr_ptr);
     }
 
-    llvm::Value* isGenerator_v = llvm::ConstantInt::get(g.i1, isGenerator, false);
-
     assert(globals == NULL);
     llvm::Value* globals_v = getNullPtr(g.llvm_value_type_ptr);
 
     // We know this function call can't throw, so it's safe to use emitter.getBuilder()->CreateCall() rather than
     // emitter.createCall().
     llvm::Value* boxed = emitter.getBuilder()->CreateCall(
-        g.funcs.boxCLFunction,
-        std::vector<llvm::Value*>{ embedRelocatablePtr(f, g.llvm_clfunction_type_ptr), closure_v, isGenerator_v,
-                                   globals_v, scratch, getConstantInt(defaults.size(), g.i64) });
+        g.funcs.boxCLFunction, std::vector<llvm::Value*>{ embedRelocatablePtr(f, g.llvm_clfunction_type_ptr), closure_v,
+                                                          globals_v, scratch, getConstantInt(defaults.size(), g.i64) });
 
     if (convertedClosure)
         convertedClosure->decvref(emitter);

src/codegen/compvars.h

@@ -398,8 +398,8 @@ ConcreteCompilerVariable* makeLong(IREmitter& emitter, std::string&);
 ConcreteCompilerVariable* makePureImaginary(IREmitter& emitter, double imag);
 CompilerVariable* makeStr(const std::string*);
 CompilerVariable* makeUnicode(IREmitter& emitter, const std::string*);
-CompilerVariable* makeFunction(IREmitter& emitter, CLFunction*, CompilerVariable* closure, bool isGenerator,
-                               Box* globals, const std::vector<ConcreteCompilerVariable*>& defaults);
+CompilerVariable* makeFunction(IREmitter& emitter, CLFunction*, CompilerVariable* closure, Box* globals,
+                               const std::vector<ConcreteCompilerVariable*>& defaults);
 ConcreteCompilerVariable* undefVariable();
 CompilerVariable* makeTuple(const std::vector<CompilerVariable*>& elts);
 

src/codegen/irgen/irgenerator.cpp

@@ -1269,7 +1269,7 @@ private:
         // but since the classdef can't create its own closure, shouldn't need to explicitly
         // create that scope to pass the closure through.
         assert(irstate->getSourceInfo()->scoping->areGlobalsFromModule());
-        CompilerVariable* func = makeFunction(emitter, cl, created_closure, false, NULL, {});
+        CompilerVariable* func = makeFunction(emitter, cl, created_closure, NULL, {});
 
         CompilerVariable* attr_dict = func->call(emitter, getEmptyOpInfo(unw_info), ArgPassSpec(0), {}, NULL);
 
@@ -1333,7 +1333,7 @@ private:
         }
 
         assert(irstate->getSourceInfo()->scoping->areGlobalsFromModule());
-        CompilerVariable* func = makeFunction(emitter, cl, created_closure, is_generator, NULL, defaults);
+        CompilerVariable* func = makeFunction(emitter, cl, created_closure, NULL, defaults);
 
         for (auto d : defaults) {
             d->decvref(emitter);

src/core/types.h

@@ -318,6 +318,12 @@ public:
             osr_versions[compiled->entry_descriptor] = compiled;
         }
     }
+
+    bool isGenerator() const {
+        if (source)
+            return source->is_generator;
+        return false;
+    }
 };
 
 CLFunction* createRTFunction(int num_args, int num_defaults, bool takes_varargs, bool takes_kwargs,

src/runtime/code.cpp

@@ -98,6 +98,8 @@ public:
             flags |= CO_VARARGS;
         if (code->f->param_names.kwarg.size())
             flags |= CO_VARKEYWORDS;
+        if (code->f->isGenerator())
+            flags |= CO_GENERATOR;
         return boxInt(flags);
     }
 };

src/runtime/objmodel.cpp

@@ -2844,7 +2844,7 @@ Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpe
     int num_output_args = f->numReceivedArgs();
     int num_passed_args = argspec.totalPassed();
 
-    if (argspec.has_starargs || argspec.has_kwargs || func->isGenerator) {
+    if (argspec.has_starargs || argspec.has_kwargs || f->isGenerator()) {
         rewrite_args = NULL;
         REWRITE_ABORTED("");
     }
@@ -2878,7 +2878,7 @@ Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpe
 
     // Fast path: if it's a simple-enough call, we don't have to do anything special.  On a simple
     // django-admin test this covers something like 93% of all calls to callFunc.
-    if (!func->isGenerator) {
+    if (!f->isGenerator()) {
         if (argspec.num_keywords == 0 && !argspec.has_starargs && !argspec.has_kwargs && argspec.num_args == f->num_args
             && !f->takes_varargs && !f->takes_kwargs) {
             return callCLFunc(f, rewrite_args, argspec.num_args, closure, NULL, func->globals, arg1, arg2, arg3, args);
@@ -3165,7 +3165,7 @@ Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpe
     // special handling for generators:
     // the call to function containing a yield should just create a new generator object.
     Box* res;
-    if (func->isGenerator) {
+    if (f->isGenerator()) {
         res = createGenerator(func, oarg1, oarg2, oarg3, oargs);
     } else {
         res = callCLFunc(f, rewrite_args, num_output_args, closure, NULL, func->globals, oarg1, oarg2, oarg3, oargs);

src/runtime/types.cpp

@@ -287,8 +287,7 @@ Box* Box::nextIC() {
 std::string builtinStr("__builtin__");
 
 extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f)
-    : in_weakreflist(NULL), f(f), closure(NULL), isGenerator(false), ndefaults(0), defaults(NULL), modname(NULL),
-      name(NULL), doc(NULL) {
+    : in_weakreflist(NULL), f(f), closure(NULL), ndefaults(0), defaults(NULL), modname(NULL), name(NULL), doc(NULL) {
     if (f->source) {
         this->modname = PyDict_GetItemString(getGlobalsDict(), "__name__");
         this->doc = f->source->getDocString();
@@ -301,9 +300,8 @@ extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f)
 }
 
 extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f, std::initializer_list<Box*> defaults,
-                                                BoxedClosure* closure, bool isGenerator)
-    : in_weakreflist(NULL), f(f), closure(closure), isGenerator(isGenerator), ndefaults(0), defaults(NULL),
-      modname(NULL), name(NULL), doc(NULL) {
+                                                BoxedClosure* closure)
+    : in_weakreflist(NULL), f(f), closure(closure), ndefaults(0), defaults(NULL), modname(NULL), name(NULL), doc(NULL) {
     if (defaults.size()) {
         // make sure to initialize defaults first, since the GC behavior is triggered by ndefaults,
         // and a GC can happen within this constructor:
@@ -326,9 +324,8 @@ extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f, std::initializer_
 BoxedFunction::BoxedFunction(CLFunction* f) : BoxedFunction(f, {}) {
 }
 
-BoxedFunction::BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure,
-                             bool isGenerator, Box* globals)
-    : BoxedFunctionBase(f, defaults, closure, isGenerator) {
+BoxedFunction::BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure, Box* globals)
+    : BoxedFunctionBase(f, defaults, closure) {
 
     assert((!globals) == (!f->source || f->source->scoping->areGlobalsFromModule()));
     this->globals = globals;
@@ -349,8 +346,8 @@ BoxedBuiltinFunctionOrMethod::BoxedBuiltinFunctionOrMethod(CLFunction* f, const
 
 BoxedBuiltinFunctionOrMethod::BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name,
                                                            std::initializer_list<Box*> defaults, BoxedClosure* closure,
-                                                           bool isGenerator, const char* doc)
-    : BoxedFunctionBase(f, defaults, closure, isGenerator) {
+                                                           const char* doc)
+    : BoxedFunctionBase(f, defaults, closure) {
 
     assert(name);
     this->name = static_cast<BoxedString*>(boxString(name));
@@ -437,12 +434,12 @@ extern "C" void moduleGCHandler(GCVisitor* v, Box* b) {
 // This mustn't throw; our IR generator generates calls to it without "invoke" even when there are exception handlers /
 // finally-blocks in scope.
 // TODO: should we use C++11 `noexcept' here?
-extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator, Box* globals,
+extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, Box* globals,
                               std::initializer_list<Box*> defaults) {
     if (closure)
         assert(closure->cls == closure_cls);
 
-    return new BoxedFunction(f, defaults, closure, isGenerator, globals);
+    return new BoxedFunction(f, defaults, closure, globals);
 }
 
 extern "C" CLFunction* unboxCLFunction(Box* b) {

src/runtime/types.h

@@ -129,8 +129,7 @@ char* getWriteableStringContents(BoxedString* s);
 
 extern "C" void listAppendInternal(Box* self, Box* v);
 extern "C" void listAppendArrayInternal(Box* self, Box** v, int nelts);
-extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator, Box* globals,
-                              std::initializer_list<Box*> defaults);
+extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, Box* globals, std::initializer_list<Box*> defaults);
 extern "C" CLFunction* unboxCLFunction(Box* b);
 extern "C" Box* createUserClass(const std::string* name, Box* base, Box* attr_dict);
 extern "C" double unboxFloat(Box* b);
@@ -663,7 +662,6 @@ public:
     BoxedClosure* closure;
     Box* globals;
 
-    bool isGenerator;
     int ndefaults;
     GCdArray* defaults;
 
@@ -675,8 +673,7 @@ public:
     Box* doc;          // __doc__
 
     BoxedFunctionBase(CLFunction* f);
-    BoxedFunctionBase(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure = NULL,
-                      bool isGenerator = false);
+    BoxedFunctionBase(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure = NULL);
 };
 
 class BoxedFunction : public BoxedFunctionBase {
@@ -685,7 +682,7 @@ public:
 
     BoxedFunction(CLFunction* f);
     BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure = NULL,
-                  bool isGenerator = false, Box* globals = NULL);
+                  Box* globals = NULL);
 
     DEFAULT_CLASS(function_cls);
 };
@@ -694,7 +691,7 @@ class BoxedBuiltinFunctionOrMethod : public BoxedFunctionBase {
 public:
     BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name, const char* doc = NULL);
     BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name, std::initializer_list<Box*> defaults,
-                                 BoxedClosure* closure = NULL, bool isGenerator = false, const char* doc = NULL);
+                                 BoxedClosure* closure = NULL, const char* doc = NULL);
 
     DEFAULT_CLASS(builtin_function_or_method_cls);
 };

test/tests/inspect_test.py

@@ -8,3 +8,9 @@ def f2():
 print inspect.getargspec(f1)
 print inspect.getargspec(f2)
 
+def G():
+    yield 1
+print inspect.isgenerator(f1)
+print inspect.isgenerator(G)
+print inspect.isgenerator(G())
+