Merge pull request #592 from undingen/generator_gc

free generator even if the don't exit

src/codegen/ast_interpreter.h

@@ -33,6 +33,7 @@ struct LineInfo;
 
 extern const void* interpreter_instr_addr;
 
+void setupInterpreter();
 Box* astInterpretFunction(CompiledFunction* f, int nargs, Box* closure, Box* generator, Box* globals, Box* arg1,
                           Box* arg2, Box* arg3, Box** args);
 Box* astInterpretFunctionEval(CompiledFunction* cf, Box* globals, Box* boxedLocals);
@@ -46,7 +47,6 @@ struct FrameInfo;
 FrameInfo* getFrameInfoForInterpretedFrame(void* frame_ptr);
 BoxedClosure* passedClosureForInterpretedFrame(void* frame_ptr);
 
-void gatherInterpreterRoots(gc::GCVisitor* visitor);
 BoxedDict* localsForInterpretedFrame(void* frame_ptr, bool only_user_visible);
 }
 

src/codegen/irgen/irgenerator.cpp

@@ -117,8 +117,7 @@ static llvm::Value* getBoxedLocalsGep(llvm::IRBuilder<true>& builder, llvm::Valu
 
 static llvm::Value* getExcinfoGep(llvm::IRBuilder<true>& builder, llvm::Value* v) {
     static_assert(offsetof(FrameInfo, exc) == 0, "");
-    static_assert(offsetof(ExcInfo, type) == 0, "");
-    return builder.CreateConstInBoundsGEP2_32(builder.CreateConstInBoundsGEP2_32(v, 0, 0), 0, 0);
+    return builder.CreateConstInBoundsGEP2_32(v, 0, 0);
 }
 
 static llvm::Value* getFrameObjGep(llvm::IRBuilder<true>& builder, llvm::Value* v) {
@@ -180,8 +179,16 @@ llvm::Value* IRGenState::getFrameInfoVar() {
             // The "normal" case
 
             // frame_info.exc.type = NULL
-            builder.CreateStore(getNullPtr(g.llvm_value_type_ptr), getExcinfoGep(builder, al));
-
+            // frame_info.exc.value = NULL
+            // frame_info.exc.traceback = NULL
+            llvm::Constant* null_value = getNullPtr(g.llvm_value_type_ptr);
+            llvm::Value* exc_info = getExcinfoGep(builder, al);
+            builder.CreateStore(
+                null_value, builder.CreateConstInBoundsGEP2_32(exc_info, 0, offsetof(ExcInfo, type) / sizeof(Box*)));
+            builder.CreateStore(
+                null_value, builder.CreateConstInBoundsGEP2_32(exc_info, 0, offsetof(ExcInfo, value) / sizeof(Box*)));
+            builder.CreateStore(null_value, builder.CreateConstInBoundsGEP2_32(exc_info, 0, offsetof(ExcInfo, traceback)
+                                                                                                / sizeof(Box*)));
             // frame_info.boxedLocals = NULL
             llvm::Value* boxed_locals_gep = getBoxedLocalsGep(builder, al);
             builder.CreateStore(getNullPtr(g.llvm_value_type_ptr), boxed_locals_gep);

src/core/types.h

@@ -706,6 +706,8 @@ struct FrameInfo {
     BoxedFrame* frame_obj;
 
     FrameInfo(ExcInfo exc) : exc(exc), boxedLocals(NULL), frame_obj(0) {}
+
+    void gcVisit(GCVisitor* visitor);
 };
 
 struct CallattrFlags {

src/gc/collector.cpp

@@ -277,7 +277,6 @@ void markPhase() {
     GCVisitor visitor(&stack);
 
     threading::visitAllStacks(&visitor);
-    gatherInterpreterRoots(&visitor);
 
     for (auto h : *getRootHandles()) {
         visitor.visit(h->value);

src/runtime/types.cpp

@@ -24,6 +24,7 @@
 
 #include "capi/typeobject.h"
 #include "capi/types.h"
+#include "codegen/ast_interpreter.h"
 #include "codegen/unwinding.h"
 #include "core/options.h"
 #include "core/stats.h"
@@ -91,6 +92,14 @@ bool IN_SHUTDOWN = false;
 #define SLICE_STOP_OFFSET ((char*)&(((BoxedSlice*)0x01)->stop) - (char*)0x1)
 #define SLICE_STEP_OFFSET ((char*)&(((BoxedSlice*)0x01)->step) - (char*)0x1)
 
+void FrameInfo::gcVisit(GCVisitor* visitor) {
+    visitor->visit(boxedLocals);
+    visitor->visit(exc.traceback);
+    visitor->visit(exc.type);
+    visitor->visit(exc.value);
+    visitor->visit(frame_obj);
+}
+
 // Analogue of PyType_GenericAlloc (default tp_alloc), but should only be used for Pyston classes!
 extern "C" PyObject* PystonType_GenericAlloc(BoxedClass* cls, Py_ssize_t nitems) noexcept {
     assert(cls);
@@ -2637,6 +2646,7 @@ void setupRuntime() {
 
     closure_cls->freeze();
 
+    setupInterpreter();
     setupCAPI();
 
     // Can't set up object methods until we set up CAPI support:

test/tests/generator_collection_finally.py

+# expected: fail
+# We currently don't call finalizers when destroying a generator.
+def G():
+    try:
+        yield 0
+        yield 1
+        print "end"
+    except Exception as e:
+        print e
+    finally:
+        print "finally"
+
+def foo():
+    g = G()
+    print g.next()
+    print g.next()
+foo()

test/tests/generator_collection_running.py

+# This test checks if generators which get started but haven't yet stopped (=not raisen a StopIteration exc, etc)
+# get freed when there aren't any references to the generators left.
+
+import gc
+import weakref
+
+class C(object):
+    val = 42
+
+def G():
+    l = range(100)
+    yield weakref.ref(C())
+    while True:
+        yield 1
+
+def get_weakrefs(num=5):
+    wr = []
+    for i in range(num):
+        g = G()
+        w = g.next()
+        wr.append(w)
+    return wr
+
+def recurse(f, n):
+     if n:
+        return recurse(f, n-1)
+     return f()
+
+wr = recurse(get_weakrefs, 100)
+gc.collect()
+for w in wr:
+    try:
+        print w.__hash__()
+        print w().val
+    except TypeError as e:
+        print e