Tests for finalization code.

New testing helper function ensures that objects are garbage
collected and their finalizers tested in a more strict order. This
reduces the chances that an object isn't collected because a reference
remains in the stack, but it's hard to fully solve this problem as long
as we have conservative stack scanning.

test/tests/class_finalization.py

+from testing_helpers import test_gc
+
+unordered_finalize = {}
+
+class ObjWithFinalizer(object):
+    def __init__(self, index):
+        self.index = index
+    def __del__(self):
+        unordered_finalize[self.index] = True
+
+class ObjWithFinalizerAndRef(object):
+    def __init__(self, index, append_list):
+        self.index = index
+        self.ref = None
+        self.append_list = append_list
+    def __del__(self):
+        self.append_list.append(self.index)
+
+def scope1():
+    # No ordering guarantees.
+    objs1 = [ObjWithFinalizer(i) for i in xrange(20)]
+
+items_in_list = 8
+
+# We run several attempts in parallel and check that at least one of the works - because
+# this test requires that a large number of objects is finalized, it's hard to make sure
+# that none of them get retained for longer than they should due to conservative collection.
+number_of_attempts = 10
+
+def scope2():
+    increasing_lists = []
+
+    for _ in xrange(number_of_attempts):
+        increasing_list = []
+        increasing_lists.append(increasing_list)
+        objs = [ObjWithFinalizerAndRef(i, increasing_list) for i in xrange(items_in_list)]
+        for i in xrange(items_in_list - 1):
+            objs[i].ref = objs[i+1]
+
+    return increasing_lists
+
+def scope3():
+    decreasing_lists = []
+
+    for _ in xrange(number_of_attempts):
+        decreasing_list = []
+        decreasing_lists.append(decreasing_list)
+        objs = [ObjWithFinalizerAndRef(i, decreasing_list) for i in xrange(items_in_list)]
+        for i in xrange(items_in_list - 1):
+            objs[i+1].ref = objs[i]
+
+    return decreasing_lists
+
+test_gc(scope1)
+print sorted(unordered_finalize.keys())
+
+increasing_lists = test_gc(scope2, 25)
+decreasing_lists = test_gc(scope3, 25)
+
+for increasing_list in increasing_lists:
+    if increasing_list == range(items_in_list):
+        print "success! got "
+        print increasing_list
+        print "at least once"
+        break
+for decreasing_list in decreasing_lists:
+    decreasing_list.reverse()
+    if decreasing_list == range(items_in_list):
+        print "success! got "
+        print decreasing_list
+        print "at least once"
+        break

test/tests/finalization_cycles.expected

-[1, 2]

test/tests/finalization_cycles.py

-# expected: fail
-#  - finalization not implemented yet
-
-# Finalizers should be called before any objects are deallocated
-# Note: the behavior here will differ from cPython and maybe PyPy
-
-finalizers_run = []
-class C(object):
-    def __init__(self, n):
-        self.n = n
-        self.x = None
-
-    def __del__(self):
-        finalizers_run.append((self.n, self.x.n if self.x else None))
-
-def f():
-    x1 = C(1)
-    x2 = C(2)
-    x1.x = x2
-    x2.x = x1
-f()
-
-finalizers_run.sort()
-print finalizers_run

test/tests/finalizer_cycle.expected

+finished

test/tests/finalizer_cycle.py

+import gc
+
+finalized_at_least_once = False
+
+class ObjWithFinalizerAndRef(object):
+    def __init__(self, index):
+        self.index = index
+        self.ref = None
+    def __del__(self):
+        global finalized_at_least_once
+        finalized_at_least_once = True
+
+items_in_list = 100
+
+# Make a lot of cycles
+for _ in xrange(100):
+    # Create a finalizer cycle. We should break those arbitrarily.
+    objs = [ObjWithFinalizerAndRef(i) for i in xrange(items_in_list)]
+    for i in xrange(items_in_list):
+        objs[i].ref = objs[(i+1) % items_in_list]
+    gc.collect()
+
+print "finished"
+if not finalized_at_least_once:
+    raise Exception("should gc at least once - consider creating more cycles?")

test/tests/finalizer_exception.py

+import sys
+from testing_helpers import test_gc
+
+class Writer(object):
+    def write(self, data):
+        print "something printed to stderr"
+
+sys.stderr = Writer()
+
+strs = []
+
+class C(object):
+    def __init__(self, index):
+        self.index = index
+
+    def __del__(self):
+        strs.append("never do this %d" % self.index)
+        raise Exception("it's a bad idea")
+
+def test():
+    cs = [C(i) for i in range(10)]
+
+test_gc(test, 10)
+
+print sorted(strs)
+
+print "done"

test/tests/finalizer_fallback.py

+from testing_helpers import test_gc
+
+# __del__ does not get called because it doesn't fallback to getattr
+
+# Note that this is an old-style class.
+class C:
+    def __getattr__(self, name):
+        def foo():
+            return 0
+        print name
+        return foo
+
+def foo():
+    c = C()
+    l = range(10)
+
+    # This should cause __index__ to be printed because it fallbacks to getattr
+    l[c] = 1
+
+    # Here, c goes out of scope.
+    return
+
+test_gc(foo)

test/tests/finalizers_inheritance.py

+from testing_helpers import test_gc
+
+class C(object):
+    def __del__(self):
+        print "C del"
+
+class D(C):
+    def __del__(self):
+        print "D del"
+
+class E(C):
+    def __del__(self):
+        print "E del"
+
+class F(D, E):
+    def __del__(self):
+        print "F del"
+
+class G(D, E):
+    pass
+
+class H(C):
+    pass
+
+class I(H, E):
+    pass
+
+def scopeC():
+    c = C()
+def scopeD():
+    d = D()
+def scopeE():
+    e = E()
+def scopeF():
+    f = F()
+def scopeG():
+    g = G()
+def scopeH():
+    h = H()
+def scopeI():
+    i = I()
+
+test_gc(scopeC)
+test_gc(scopeD)
+test_gc(scopeE)
+test_gc(scopeF)
+test_gc(scopeG)
+test_gc(scopeH)
+test_gc(scopeI)

test/tests/gc_type_object.py

@@ -4,21 +4,9 @@ import gc
 # that both the class object and the instance object will be freed in the same
 # garbage collection pass. Hope that this doesn't cause any problems.
 def generateClassAndInstances():
-    for i in xrange(5000):
-        def method(self, x):
-            return x + self.i
-        NewType1 = type("Class1_" + str(i), (),
-                dict(a={}, b=range(10), i=1, f=method))
-        NewType2 = type("Class2_" + str(i), (object,),
-                dict(a={}, b=range(10), i=2, f=method))
-        NewType3 = type("Class3_" + str(i), (NewType2,), {})
-        NewType4 = type("Class4_" + str(i), (NewType3,), {})
-        NewType5 = type("Class5_" + str(i), (NewType4,), {})
-        obj1 = NewType1()
-        obj2 = NewType2()
-        obj3 = NewType3()
-        obj4 = NewType4()
-        obj5 = NewType5()
+    for i in xrange(12000):
+        NewType = type("Class" + str(i), (), {})
+        obj = NewType()
 
 generateClassAndInstances()
 gc.collect()

test/tests/nonself_resurrection.py

-# expected: fail
-#  - finalization (let alone resurrection) not implemented yet
-
 # Objects are allowed to resurrect other objects too, I guess
+from testing_helpers import test_gc
 
 class C(object):
     def __init__(self, x):
@@ -12,10 +10,10 @@ class C(object):
         x = self.x
 
 x = None
-c = C([])
-del c
 
-import gc
-gc.collect()
+def test():
+    c = C([])
+
+test_gc(test)
 
 print x

test/tests/testing_helpers.py

+# This file isn't really meant to be run as a test, though it won't really
+# make a difference.
+
+import gc
+
+# Sometimes pointer objects from popped stack frames remain up the stack
+# and end up being marked when the GC conservatively scans the stack, but
+# this causes flaky tests because we really want the object to be collected.
+# By having a deep recursive function, we ensure that the object we want to
+# collect is really far in the stack and won't get scanned.
+def call_function_far_up_the_stack(fn, num_calls_left=200):
+    if num_calls_left == 0:
+        return fn()
+    else:
+        return call_function_far_up_the_stack(fn, num_calls_left - 1)
+
+# It's useful to call the GC at different locations in the stack in case that it's the
+# call to the GC itself that left a lingering pointer (e.g. the pointer could be the
+# __del__ attribute of an object we'd like to collect).
+def call_gc_throughout_the_stack(number_of_gc_calls, num_calls_left=100):
+    if num_calls_left > 0:
+        call_gc_throughout_the_stack(number_of_gc_calls, num_calls_left - 1)
+        if number_of_gc_calls >= num_calls_left:
+            gc.collect()
+
+# test_gc takes in a function fn that presumably allocations some objects and
+# attempts to collect those objects in order to trigger a call to the finalizers.
+#
+# The problem is that it's actually quite hard to guarantee finalizer calls
+# because with conservative scanning, there can always be lingering pointers
+# on the stack. This function has a bunch of hacks to attempt to clear those
+# lingering pointers.
+def test_gc(fn, number_of_gc_calls=3):
+    class DummyNewObject(object):
+        pass
+    class DummyOldObject():
+        pass
+
+    def dummyFunctionThatDoesSomeAllocation():
+        # Allocating a few objects on the heap seems to be helpful.
+        for _ in xrange(100):
+            n, o = DummyNewObject(), DummyOldObject()
+        objs = [DummyNewObject() for _ in xrange(100)]
+
+    # Call fn after a few recursive calls to get those allocations.
+    val = call_function_far_up_the_stack(fn)
+
+    # Call a dummy function in the same way as fn. By following the same
+    # code path, there is a better chance of clearing lingering references.
+    call_function_far_up_the_stack(dummyFunctionThatDoesSomeAllocation)
+
+    # Force garbage collection.
+    call_gc_throughout_the_stack(number_of_gc_calls - 1)
+    gc.collect()
+    return val
+

test/tests/weakref1.py

@@ -15,5 +15,5 @@ def doStuff():
 
 l = [doStuff() for i in xrange(5)]
 gc.collect()
-
+gc.collect()
 assert num_destroyed >= 1

test/tests/weakref_finalizers.py

+# test to ensure that weakref callbacks and finalizers get called in the
+# right order
+
+import weakref
+from testing_helpers import test_gc
+
+def callback(wr):
+    print "object was destroyed", wr()
+
+def retainer(ref):
+    def cb(wr):
+        print "object was destroyed", ref, wr()
+    return cb
+
+class OldStyle():
+    def __init__(self, index):
+        self.index = index
+    def __del__(self):
+        print "deleted", self.index
+
+class NewStyle(object):
+    def __init__(self, index):
+        self.index = index
+    def __del__(self):
+        print "deleted", self.index
+
+def scope_old1():
+    c1 = OldStyle(1)
+    return weakref.ref(c1, callback)
+
+def scope_old2():
+    c2 = OldStyle(2)
+    return (weakref.ref(c2, callback), weakref.ref(c2, callback))
+
+def scope_old3():
+    c3 = OldStyle(3)
+    adverserial_weakref = weakref.ref(c3, retainer(c3))
+
+def scope_new1():
+    c1 = NewStyle(1)
+    return weakref.ref(c1, callback)
+
+def scope_new2():
+    c2 = NewStyle(2)
+    return (weakref.ref(c2, callback), weakref.ref(c2, callback))
+
+def scope_new3():
+    c3 = NewStyle(3)
+    adverserial_weakref = weakref.ref(c3, retainer(c3))
+
+print ">> Test old style"
+test_gc(scope_old1)
+test_gc(scope_old2)
+test_gc(scope_old3, 3)
+
+print ">> Test new style"
+test_gc(scope_new1)
+test_gc(scope_new2)
+test_gc(scope_new3, 3)

test/tests/weakref_proxy.py

 import weakref
-import gc
+from testing_helpers import test_gc
 
 class C(object):
     def foo(self):
         print "inside foo()"
 
-def fact(n):
-    if n <= 1:
-        return n
-    return n * fact(n-1)
-
 def getWR():
     c = C()
     wr = weakref.proxy(c)
@@ -19,15 +14,7 @@ def getWR():
     del c
     return wr
 
-wr = getWR()
-fact(100) # try to clear some memory
-
-def recurse(f, n):
-    if n:
-        return recurse(f, n - 1)
-    return f()
-recurse(gc.collect, 50)
-gc.collect()
+wr = test_gc(getWR)
 
 try:
     wr.foo()

test/tests/weakref_resurrection.py

+# expected: fail
+# It's hard to guarantee the order of weakref callbacks being called
+# when we have a GC
+import weakref
+from testing_helpers import test_gc
+
+def callback(wr):
+    print "object was destroyed", wr()
+
+class C(object):
+    def __init__(self, index):
+        self.index = index
+
+saved_wrs = []
+def weak_retainer(to_be_resurrected):
+    def cb(wr):
+        global saved_wr
+        saved_wrs.append(to_be_resurrected())
+        print "staying alive~", wr, to_be_resurrected
+    return cb
+
+def foo1():
+    c1 = C(1)
+    c2 = C(2)
+    wr1 = weakref.ref(c1, callback)
+    wr2 = weakref.ref(c2, weak_retainer(wr1))
+    return (wr1, wr2)
+
+def foo2():
+    c3 = C(3)
+    c4 = C(4)
+    wr4 = weakref.ref(c4, callback)
+    wr3 = weakref.ref(c3, weak_retainer(wr4))
+    return (wr3, wr4)
+
+wr1, wr2 = test_gc(foo1, 5)
+wr3, wr4 = test_gc(foo2, 5)
+
+print wr1(), wr2()
+print wr3(), wr4()