Refcount nanny framework in place; smaller fixes and better build needed

Cython/Compiler/Code.py

@@ -673,50 +673,52 @@ class CCodeWriter(object):
         return typecast(py_object_type, type, cname)
     
     def put_gotref(self, cname):
-        if DebugFlags.debug_ref_check_code:
-            self.putln("__Pyx_GOTREF(%s);" % cname)
+        self.putln("__Pyx_GOTREF(%s);" % cname)
+    
+    def put_giveref(self, cname):
+        self.putln("__Pyx_GIVEREF(%s);" % cname)
     
     def put_incref(self, cname, type):
-        self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type))
+        self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type))
     
     def put_decref(self, cname, type):
-        self.putln("Py_DECREF(%s);" % self.as_pyobject(cname, type))
+        self.putln("__Pyx_DECREF(%s);" % self.as_pyobject(cname, type))
     
     def put_var_incref(self, entry):
         if entry.type.is_pyobject:
-            self.putln("Py_INCREF(%s);" % self.entry_as_pyobject(entry))
+            self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry))
     
     def put_decref_clear(self, cname, type):
-        self.putln("Py_DECREF(%s); %s = 0;" % (
+        self.putln("__Pyx_DECREF(%s); %s = 0;" % (
             typecast(py_object_type, type, cname), cname))
             #self.as_pyobject(cname, type), cname))
     
     def put_xdecref(self, cname, type):
-        self.putln("Py_XDECREF(%s);" % self.as_pyobject(cname, type))
+        self.putln("__Pyx_XDECREF(%s);" % self.as_pyobject(cname, type))
     
     def put_xdecref_clear(self, cname, type):
-        self.putln("Py_XDECREF(%s); %s = 0;" % (
+        self.putln("__Pyx_XDECREF(%s); %s = 0;" % (
             self.as_pyobject(cname, type), cname))
 
     def put_var_decref(self, entry):
         if entry.type.is_pyobject:
             if entry.init_to_none is False:
-                self.putln("Py_XDECREF(%s);" % self.entry_as_pyobject(entry))
+                self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
             else:
-                self.putln("Py_DECREF(%s);" % self.entry_as_pyobject(entry))
+                self.putln("__Pyx_DECREF(%s);" % self.entry_as_pyobject(entry))
     
     def put_var_decref_clear(self, entry):
         if entry.type.is_pyobject:
-            self.putln("Py_DECREF(%s); %s = 0;" % (
+            self.putln("__Pyx_DECREF(%s); %s = 0;" % (
                 self.entry_as_pyobject(entry), entry.cname))
     
     def put_var_xdecref(self, entry):
         if entry.type.is_pyobject:
-            self.putln("Py_XDECREF(%s);" % self.entry_as_pyobject(entry))
+            self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
     
     def put_var_xdecref_clear(self, entry):
         if entry.type.is_pyobject:
-            self.putln("Py_XDECREF(%s); %s = 0;" % (
+            self.putln("__Pyx_XDECREF(%s); %s = 0;" % (
                 self.entry_as_pyobject(entry), entry.cname))
     
     def put_var_decrefs(self, entries, used_only = 0):
@@ -737,7 +739,7 @@ class CCodeWriter(object):
     
     def put_init_to_py_none(self, cname, type):
         py_none = typecast(type, py_object_type, "Py_None")
-        self.putln("%s = %s; Py_INCREF(Py_None);" % (cname, py_none))
+        self.putln("%s = %s; __Pyx_INCREF(Py_None);" % (cname, py_none))
     
     def put_init_var_to_py_none(self, entry, template = "%s"):
         code = template % entry.cname
@@ -775,21 +777,25 @@ class CCodeWriter(object):
             return 'unlikely(%s)' % cond
         else:
             return cond
-        
-    def error_goto(self, pos):
-        lbl = self.funcstate.error_label
-        self.funcstate.use_label(lbl)
+
+    def set_error_info(self, pos):
         if Options.c_line_in_traceback:
             cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro)
         else:
             cinfo = ""
-        return "{%s = %s[%s]; %s = %s;%s goto %s;}" % (
+        return "%s = %s[%s]; %s = %s;%s" % (
             Naming.filename_cname,
             Naming.filetable_cname,
             self.lookup_filename(pos[0]),
             Naming.lineno_cname,
             pos[1],
-            cinfo,
+            cinfo)
+        
+    def error_goto(self, pos):
+        lbl = self.funcstate.error_label
+        self.funcstate.use_label(lbl)
+        return "{%s goto %s;}" % (
+            self.set_error_info(pos),
             lbl)
 
     def error_goto_if(self, cond, pos):

Cython/Compiler/DebugFlags.py

@@ -2,4 +2,4 @@ debug_disposal_code = 0
 debug_temp_alloc = 0
 debug_coercion = 0
 
-debug_ref_check_code = 1
+debug_refnanny = 1

Cython/Compiler/ExprNodes.py

@@ -1678,7 +1678,6 @@ class IndexNode(ExprNode):
                 value_code,
                 self.index_unsigned_parameter(),
                 code.error_goto(self.pos)))
-        code.put_gotref(self.base.py_result())
 
     def generate_buffer_setitem_code(self, rhs, code, op=""):
         # Used from generate_assignment_code and InPlaceAssignmentNode
@@ -2785,6 +2784,7 @@ class TupleNode(SequenceNode):
                     self.result(),
                     i,
                     arg.py_result()))
+            code.put_giveref(arg.py_result())
     
     def generate_subexpr_disposal_code(self, code):
         # We call generate_post_assignment_code here instead
@@ -2864,6 +2864,7 @@ class ListNode(SequenceNode):
                 (self.result(),
                 len(self.args),
                 code.error_goto_if_null(self.result(), self.pos)))
+            code.put_gotref(self.result())
             for i in range(len(self.args)):
                 arg = self.args[i]
                 #if not arg.is_temp:
@@ -2873,6 +2874,7 @@ class ListNode(SequenceNode):
                     (self.result(),
                     i,
                     arg.py_result()))
+                code.put_giveref(arg.py_result())
         elif self.type.is_array:
             for i, arg in enumerate(self.args):
                 code.putln("%s[%s] = %s;" % (

Cython/Compiler/ModuleNode.py

@@ -19,6 +19,7 @@ import Options
 import PyrexTypes
 import TypeSlots
 import Version
+import DebugFlags
 
 from Errors import error, warning
 from PyrexTypes import py_object_type
@@ -250,6 +251,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.globalstate.module_pos = self.pos
         code.globalstate.directives = self.directives
 
+        code.globalstate.use_utility_code(refcount_utility_code)
+
         code.putln("")
         code.putln("/* Implementation of %s */" % env.qualified_name)
         self.generate_const_definitions(env, code)
@@ -535,7 +538,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code.putln('')
             code.putln('static char %s[] = "%s";' % (
                     env.doc_cname, escape_byte_string(docstr)))
-    
+
     def generate_extern_c_macro_definition(self, code):
         name = Naming.extern_c_macro
         code.putln("#ifdef __cplusplus")
@@ -1559,6 +1562,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("#endif")
         code.putln("{")
         tempdecl_code = code.insertion_point()
+
+        code.putln('__Pyx_SetupRefcountContext("%s");' % header3)
+
         code.putln("%s = PyTuple_New(0); %s" % (Naming.empty_tuple, code.error_goto_if_null(Naming.empty_tuple, self.pos)));
 
         code.putln("/*--- Library function declarations ---*/")
@@ -2274,3 +2280,36 @@ bad:
 }
 """ % {'IMPORT_STAR'     : Naming.import_star,
        'IMPORT_STAR_SET' : Naming.import_star_set }
+        
+refcount_utility_code = UtilityCode(proto="""
+#ifdef CYTHON_REFNANNY
+
+void __Pyx_Refnanny_INCREF(void*, PyObject*, int);
+void __Pyx_Refnanny_GOTREF(void*, PyObject*, int);
+void __Pyx_Refnanny_GIVEREF(void*, PyObject*, int);
+void __Pyx_Refnanny_INCREF(void*, PyObject*, int);
+void __Pyx_Refnanny_DECREF(void*, PyObject*, int);
+void* __Pyx_Refnanny_NewContext(char*, int);
+int __Pyx_Refnanny_FinishContext(void*);
+
+#define __Pyx_INCREF(r) __Pyx_Refnanny_INCREF(__pyx_refchk, r, __LINE__)
+#define __Pyx_GOTREF(r) __Pyx_Refnanny_GOTREF(__pyx_refchk, r, __LINE__)
+#define __Pyx_GIVEREF(r) __Pyx_Refnanny_GIVEREF(__pyx_refchk, r, __LINE__)
+#define __Pyx_DECREF(r) __Pyx_Refnanny_DECREF(__pyx_refchk, r, __LINE__)
+#define __Pyx_XDECREF(r) (r ? __Pyx_Refnanny_DECREF(__pyx_refchk, r, __LINE__) : 0)
+#define __Pyx_SetupRefcountContext(name) \
+  void* __pyx_refchk = __Pyx_Refnanny_NewContext(name, __LINE__)
+#define __Pyx_FinishRefcountContext() __Pyx_Refnanny_FinishContext(__pyx_refchk)
+
+#else
+
+#define __Pyx_INCREF(r) Py_INCREF(r)
+#define __Pyx_GOTREF(r)
+#define __Pyx_GIVEREF(r)
+#define __Pyx_DECREF(r) Py_DECREF(r)
+#define __Pyx_XDECREF(r) Py_XDECREF(r)
+#define __Pyx_SetupRefcountContext(name)
+#define __Pyx_FinishRefcountContext() 0
+
+#endif /* CYTHON_REFNANNY */
+""")

Cython/Compiler/Nodes.py

@@ -16,6 +16,7 @@ from Cython.Utils import open_new_file, replace_suffix, UtilityCode
 from StringEncoding import EncodedString, escape_byte_string, split_docstring
 import Options
 import ControlFlow
+import DebugFlags
 
 from DebugFlags import debug_disposal_code
 
@@ -1010,6 +1011,7 @@ class FuncDefNode(StatNode, BlockNode):
         # ----- Automatic lead-ins for certain special functions
         if is_getbuffer_slot:
             self.getbuffer_init(code)
+        code.putln('__Pyx_SetupRefcountContext("%s");' % self.entry.name)
         # ----- Fetch arguments
         self.generate_argument_parsing_code(env, code)
         # If an argument is assigned to in the body, we must 
@@ -1119,8 +1121,24 @@ class FuncDefNode(StatNode, BlockNode):
             code.putln("PyGILState_Release(_save);")
         # code.putln("/* TODO: decref scope object */")
         # ----- Return
+        default_retval = self.return_type.default_value
+        err_val = self.error_value()
+        if err_val is None and default_retval:
+            err_val = default_retval
+        if self.return_type.is_pyobject:
+            code.put_giveref(Naming.retval_cname)
+        if err_val is None:
+            code.putln('__Pyx_FinishRefcountContext();')
+        else:
+            code.putln('if (__Pyx_FinishRefcountContext() == -1) {')
+            code.putln(code.set_error_info(self.pos))
+            code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
+            code.putln('%s = %s;' % (Naming.retval_cname, err_val))
+            code.putln('}')
+
         if not self.return_type.is_void:
             code.putln("return %s;" % Naming.retval_cname)
+            
         code.putln("}")
         # ----- Go back and insert temp variable declarations
         tempvardecl_code.put_var_declarations(lenv.temp_entries)
@@ -1171,16 +1189,16 @@ class FuncDefNode(StatNode, BlockNode):
         # getbuffer with a NULL parameter. For now we work around this;
         # the following line should be removed when this bug is fixed.
         code.putln("if (%s == NULL) return 0;" % info) 
-        code.putln("%s->obj = Py_None; Py_INCREF(Py_None);" % info)
+        code.putln("%s->obj = Py_None; __Pyx_INCREF(Py_None);" % info)
 
     def getbuffer_error_cleanup(self, code):
         info = self.local_scope.arg_entries[1].cname
-        code.putln("Py_DECREF(%s->obj); %s->obj = NULL;" %
+        code.putln("__Pyx_DECREF(%s->obj); %s->obj = NULL;" %
                    (info, info))
 
     def getbuffer_normal_cleanup(self, code):
         info = self.local_scope.arg_entries[1].cname
-        code.putln("if (%s->obj == Py_None) { Py_DECREF(Py_None); %s->obj = NULL; }" %
+        code.putln("if (%s->obj == Py_None) { __Pyx_DECREF(Py_None); %s->obj = NULL; }" %
                    (info, info))
 
 class CFuncDefNode(FuncDefNode):
@@ -4040,7 +4058,7 @@ class ExceptClauseNode(Node):
         self.body.generate_execution_code(code)
         code.funcstate.exc_vars = old_exc_vars
         for var in self.exc_vars:
-            code.putln("Py_DECREF(%s); %s = 0;" % (var, var))
+            code.putln("__Pyx_DECREF(%s); %s = 0;" % (var, var))
         code.put_goto(end_label)
         code.putln(
             "}")

Cython/Runtime/build.sh

+cat <<EOF > ../../Cython/Compiler/DebugFlags.py
+debug_disposal_code = 0
+debug_temp_alloc = 0
+debug_coercion = 0
+
+debug_refnanny = 0
+EOF
+
+
+python ../../cython.py refnanny.pyx
+gcc -shared -pthread -fPIC -fwrapv -O2 -Wall \
+  -fno-strict-aliasing -I/local/include/python2.5 \
+  -o refnanny.so -I. refnanny.c
+
+cat <<EOF > ../../Cython/Compiler/DebugFlags.py
+debug_disposal_code = 0
+debug_temp_alloc = 0
+debug_coercion = 0
+
+debug_refnanny = 1
+EOF

Cython/Runtime/refnanny.pyx

+from python_ref cimport Py_INCREF, Py_DECREF
+
+loglevel = 0
+reflog = []
+
+cdef log(level, action, obj, lineno):
+    if loglevel >= level:
+        reflog.append((lineno, action, id(obj)))
+
+LOG_NONE, LOG_ALL = range(2)
+
+class RefnannyException(Exception):
+    pass
+
+class RefnannyContext(object):
+    def __init__(self):
+        self.refs = {} # id -> (count, [lineno])
+        self.errors = []
+
+    def regref(self, obj, lineno):
+        log(LOG_ALL, 'regref', obj, lineno)
+        id_ = id(obj)
+        count, linenumbers = self.refs.get(id_, (0, []))
+        self.refs[id_] = (count + 1, linenumbers)
+        linenumbers.append(lineno)
+
+    def delref(self, obj, lineno):
+        log(LOG_ALL, 'delref', obj, lineno)
+        id_ = id(obj)
+        count, linenumbers = self.refs.get(id_, (0, []))
+        if count == 0:
+            self.errors.append("Too many decrefs on line %d, reference acquired on lines %r" %
+                (lineno, linenumbers))
+        elif count == 1:
+            del self.refs[id_]
+        else:
+            self.refs[id_] = (count - 1, linenumbers)
+
+    def end(self):
+        if len(self.refs) > 0:
+            msg = ""
+            for count, linenos in self.refs.itervalues():
+                msg += "\n  Acquired on lines: " + ", ".join(["%d" % x for x in linenos])
+            self.errors.append("References leaked: %s" % msg)
+        if self.errors:
+            raise RefnannyException("\n".join(self.errors))
+
+cdef public void* __Pyx_Refnanny_NewContext(char* funcname, int lineno) except NULL:
+    ctx = RefnannyContext()
+    Py_INCREF(ctx)
+    return <void*>ctx
+
+cdef public void __Pyx_Refnanny_GOTREF(void* ctx, object obj, int lineno):
+    if ctx == NULL: return
+    (<object>ctx).regref(obj, lineno)
+
+cdef public void __Pyx_Refnanny_GIVEREF(void* ctx, object obj, int lineno):
+    if ctx == NULL: return
+    (<object>ctx).delref(obj, lineno)
+
+cdef public void __Pyx_Refnanny_INCREF(void* ctx, object obj, int lineno):
+    Py_INCREF(obj)
+    __Pyx_Refnanny_GOTREF(ctx, obj, lineno)
+    
+cdef public void __Pyx_Refnanny_DECREF(void* ctx, object obj, int lineno):
+    # GIVEREF raises exception if we hit 0
+    # 
+    __Pyx_Refnanny_GIVEREF(ctx, obj, lineno)
+    Py_DECREF(obj)
+    
+cdef public int __Pyx_Refnanny_FinishContext(void* ctx) except -1:
+    obj = <object>ctx
+    try:
+        obj.end()
+    finally:
+        Py_DECREF(obj)
+    return 0
+
+    

runtests.py

 #!/usr/bin/python
 
-import os, sys, re, shutil, unittest, doctest
+import os, sys, re, shutil, unittest, doctest, ctypes
 
 WITH_CYTHON = True
 
@@ -26,6 +26,10 @@ VER_DEP_MODULES = {
 INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ]
 CFLAGS = os.getenv('CFLAGS', '').split()
 
+ctypes.PyDLL("Cython/Runtime/refnanny.so", mode=ctypes.RTLD_GLOBAL)
+sys.path.append("Cython/Runtime")
+import refnanny
+#CFLAGS.append("-DCYTHON_REFNANNY")
 
 class ErrorWriter(object):
     match_error = re.compile('(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match
@@ -594,3 +598,5 @@ if __name__ == '__main__':
         sys.stderr.write("Following tests excluded because of missing dependencies on your system:\n")
         for test in missing_dep_excluder.tests_missing_deps:
             sys.stderr.write("   %s\n" % test)
+
+print "\n".join([repr(x) for x in refnanny.reflog])