Commit 026931cf authored by Dag Sverre Seljebotn's avatar Dag Sverre Seljebotn

Refcount nanny framework in place; smaller fixes and better build needed

parent d444a436
...@@ -673,50 +673,52 @@ class CCodeWriter(object): ...@@ -673,50 +673,52 @@ class CCodeWriter(object):
return typecast(py_object_type, type, cname) return typecast(py_object_type, type, cname)
def put_gotref(self, cname): def put_gotref(self, cname):
if DebugFlags.debug_ref_check_code:
self.putln("__Pyx_GOTREF(%s);" % cname) self.putln("__Pyx_GOTREF(%s);" % cname)
def put_giveref(self, cname):
self.putln("__Pyx_GIVEREF(%s);" % cname)
def put_incref(self, cname, type): def put_incref(self, cname, type):
self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type)) self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type))
def put_decref(self, cname, type): def put_decref(self, cname, type):
self.putln("Py_DECREF(%s);" % self.as_pyobject(cname, type)) self.putln("__Pyx_DECREF(%s);" % self.as_pyobject(cname, type))
def put_var_incref(self, entry): def put_var_incref(self, entry):
if entry.type.is_pyobject: if entry.type.is_pyobject:
self.putln("Py_INCREF(%s);" % self.entry_as_pyobject(entry)) self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry))
def put_decref_clear(self, cname, type): def put_decref_clear(self, cname, type):
self.putln("Py_DECREF(%s); %s = 0;" % ( self.putln("__Pyx_DECREF(%s); %s = 0;" % (
typecast(py_object_type, type, cname), cname)) typecast(py_object_type, type, cname), cname))
#self.as_pyobject(cname, type), cname)) #self.as_pyobject(cname, type), cname))
def put_xdecref(self, cname, type): def put_xdecref(self, cname, type):
self.putln("Py_XDECREF(%s);" % self.as_pyobject(cname, type)) self.putln("__Pyx_XDECREF(%s);" % self.as_pyobject(cname, type))
def put_xdecref_clear(self, cname, type): def put_xdecref_clear(self, cname, type):
self.putln("Py_XDECREF(%s); %s = 0;" % ( self.putln("__Pyx_XDECREF(%s); %s = 0;" % (
self.as_pyobject(cname, type), cname)) self.as_pyobject(cname, type), cname))
def put_var_decref(self, entry): def put_var_decref(self, entry):
if entry.type.is_pyobject: if entry.type.is_pyobject:
if entry.init_to_none is False: if entry.init_to_none is False:
self.putln("Py_XDECREF(%s);" % self.entry_as_pyobject(entry)) self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
else: else:
self.putln("Py_DECREF(%s);" % self.entry_as_pyobject(entry)) self.putln("__Pyx_DECREF(%s);" % self.entry_as_pyobject(entry))
def put_var_decref_clear(self, entry): def put_var_decref_clear(self, entry):
if entry.type.is_pyobject: if entry.type.is_pyobject:
self.putln("Py_DECREF(%s); %s = 0;" % ( self.putln("__Pyx_DECREF(%s); %s = 0;" % (
self.entry_as_pyobject(entry), entry.cname)) self.entry_as_pyobject(entry), entry.cname))
def put_var_xdecref(self, entry): def put_var_xdecref(self, entry):
if entry.type.is_pyobject: if entry.type.is_pyobject:
self.putln("Py_XDECREF(%s);" % self.entry_as_pyobject(entry)) self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
def put_var_xdecref_clear(self, entry): def put_var_xdecref_clear(self, entry):
if entry.type.is_pyobject: if entry.type.is_pyobject:
self.putln("Py_XDECREF(%s); %s = 0;" % ( self.putln("__Pyx_XDECREF(%s); %s = 0;" % (
self.entry_as_pyobject(entry), entry.cname)) self.entry_as_pyobject(entry), entry.cname))
def put_var_decrefs(self, entries, used_only = 0): def put_var_decrefs(self, entries, used_only = 0):
...@@ -737,7 +739,7 @@ class CCodeWriter(object): ...@@ -737,7 +739,7 @@ class CCodeWriter(object):
def put_init_to_py_none(self, cname, type): def put_init_to_py_none(self, cname, type):
py_none = typecast(type, py_object_type, "Py_None") py_none = typecast(type, py_object_type, "Py_None")
self.putln("%s = %s; Py_INCREF(Py_None);" % (cname, py_none)) self.putln("%s = %s; __Pyx_INCREF(Py_None);" % (cname, py_none))
def put_init_var_to_py_none(self, entry, template = "%s"): def put_init_var_to_py_none(self, entry, template = "%s"):
code = template % entry.cname code = template % entry.cname
...@@ -776,20 +778,24 @@ class CCodeWriter(object): ...@@ -776,20 +778,24 @@ class CCodeWriter(object):
else: else:
return cond return cond
def error_goto(self, pos): def set_error_info(self, pos):
lbl = self.funcstate.error_label
self.funcstate.use_label(lbl)
if Options.c_line_in_traceback: if Options.c_line_in_traceback:
cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro) cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro)
else: else:
cinfo = "" cinfo = ""
return "{%s = %s[%s]; %s = %s;%s goto %s;}" % ( return "%s = %s[%s]; %s = %s;%s" % (
Naming.filename_cname, Naming.filename_cname,
Naming.filetable_cname, Naming.filetable_cname,
self.lookup_filename(pos[0]), self.lookup_filename(pos[0]),
Naming.lineno_cname, Naming.lineno_cname,
pos[1], pos[1],
cinfo, cinfo)
def error_goto(self, pos):
lbl = self.funcstate.error_label
self.funcstate.use_label(lbl)
return "{%s goto %s;}" % (
self.set_error_info(pos),
lbl) lbl)
def error_goto_if(self, cond, pos): def error_goto_if(self, cond, pos):
......
...@@ -2,4 +2,4 @@ debug_disposal_code = 0 ...@@ -2,4 +2,4 @@ debug_disposal_code = 0
debug_temp_alloc = 0 debug_temp_alloc = 0
debug_coercion = 0 debug_coercion = 0
debug_ref_check_code = 1 debug_refnanny = 1
...@@ -1678,7 +1678,6 @@ class IndexNode(ExprNode): ...@@ -1678,7 +1678,6 @@ class IndexNode(ExprNode):
value_code, value_code,
self.index_unsigned_parameter(), self.index_unsigned_parameter(),
code.error_goto(self.pos))) code.error_goto(self.pos)))
code.put_gotref(self.base.py_result())
def generate_buffer_setitem_code(self, rhs, code, op=""): def generate_buffer_setitem_code(self, rhs, code, op=""):
# Used from generate_assignment_code and InPlaceAssignmentNode # Used from generate_assignment_code and InPlaceAssignmentNode
...@@ -2785,6 +2784,7 @@ class TupleNode(SequenceNode): ...@@ -2785,6 +2784,7 @@ class TupleNode(SequenceNode):
self.result(), self.result(),
i, i,
arg.py_result())) arg.py_result()))
code.put_giveref(arg.py_result())
def generate_subexpr_disposal_code(self, code): def generate_subexpr_disposal_code(self, code):
# We call generate_post_assignment_code here instead # We call generate_post_assignment_code here instead
...@@ -2864,6 +2864,7 @@ class ListNode(SequenceNode): ...@@ -2864,6 +2864,7 @@ class ListNode(SequenceNode):
(self.result(), (self.result(),
len(self.args), len(self.args),
code.error_goto_if_null(self.result(), self.pos))) code.error_goto_if_null(self.result(), self.pos)))
code.put_gotref(self.result())
for i in range(len(self.args)): for i in range(len(self.args)):
arg = self.args[i] arg = self.args[i]
#if not arg.is_temp: #if not arg.is_temp:
...@@ -2873,6 +2874,7 @@ class ListNode(SequenceNode): ...@@ -2873,6 +2874,7 @@ class ListNode(SequenceNode):
(self.result(), (self.result(),
i, i,
arg.py_result())) arg.py_result()))
code.put_giveref(arg.py_result())
elif self.type.is_array: elif self.type.is_array:
for i, arg in enumerate(self.args): for i, arg in enumerate(self.args):
code.putln("%s[%s] = %s;" % ( code.putln("%s[%s] = %s;" % (
......
...@@ -19,6 +19,7 @@ import Options ...@@ -19,6 +19,7 @@ import Options
import PyrexTypes import PyrexTypes
import TypeSlots import TypeSlots
import Version import Version
import DebugFlags
from Errors import error, warning from Errors import error, warning
from PyrexTypes import py_object_type from PyrexTypes import py_object_type
...@@ -250,6 +251,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -250,6 +251,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
code.globalstate.module_pos = self.pos code.globalstate.module_pos = self.pos
code.globalstate.directives = self.directives code.globalstate.directives = self.directives
code.globalstate.use_utility_code(refcount_utility_code)
code.putln("") code.putln("")
code.putln("/* Implementation of %s */" % env.qualified_name) code.putln("/* Implementation of %s */" % env.qualified_name)
self.generate_const_definitions(env, code) self.generate_const_definitions(env, code)
...@@ -1559,6 +1562,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -1559,6 +1562,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
code.putln("#endif") code.putln("#endif")
code.putln("{") code.putln("{")
tempdecl_code = code.insertion_point() tempdecl_code = code.insertion_point()
code.putln('__Pyx_SetupRefcountContext("%s");' % header3)
code.putln("%s = PyTuple_New(0); %s" % (Naming.empty_tuple, code.error_goto_if_null(Naming.empty_tuple, self.pos))); code.putln("%s = PyTuple_New(0); %s" % (Naming.empty_tuple, code.error_goto_if_null(Naming.empty_tuple, self.pos)));
code.putln("/*--- Library function declarations ---*/") code.putln("/*--- Library function declarations ---*/")
...@@ -2274,3 +2280,36 @@ bad: ...@@ -2274,3 +2280,36 @@ bad:
} }
""" % {'IMPORT_STAR' : Naming.import_star, """ % {'IMPORT_STAR' : Naming.import_star,
'IMPORT_STAR_SET' : Naming.import_star_set } 'IMPORT_STAR_SET' : Naming.import_star_set }
refcount_utility_code = UtilityCode(proto="""
#ifdef CYTHON_REFNANNY
void __Pyx_Refnanny_INCREF(void*, PyObject*, int);
void __Pyx_Refnanny_GOTREF(void*, PyObject*, int);
void __Pyx_Refnanny_GIVEREF(void*, PyObject*, int);
void __Pyx_Refnanny_INCREF(void*, PyObject*, int);
void __Pyx_Refnanny_DECREF(void*, PyObject*, int);
void* __Pyx_Refnanny_NewContext(char*, int);
int __Pyx_Refnanny_FinishContext(void*);
#define __Pyx_INCREF(r) __Pyx_Refnanny_INCREF(__pyx_refchk, r, __LINE__)
#define __Pyx_GOTREF(r) __Pyx_Refnanny_GOTREF(__pyx_refchk, r, __LINE__)
#define __Pyx_GIVEREF(r) __Pyx_Refnanny_GIVEREF(__pyx_refchk, r, __LINE__)
#define __Pyx_DECREF(r) __Pyx_Refnanny_DECREF(__pyx_refchk, r, __LINE__)
#define __Pyx_XDECREF(r) (r ? __Pyx_Refnanny_DECREF(__pyx_refchk, r, __LINE__) : 0)
#define __Pyx_SetupRefcountContext(name) \
void* __pyx_refchk = __Pyx_Refnanny_NewContext(name, __LINE__)
#define __Pyx_FinishRefcountContext() __Pyx_Refnanny_FinishContext(__pyx_refchk)
#else
#define __Pyx_INCREF(r) Py_INCREF(r)
#define __Pyx_GOTREF(r)
#define __Pyx_GIVEREF(r)
#define __Pyx_DECREF(r) Py_DECREF(r)
#define __Pyx_XDECREF(r) Py_XDECREF(r)
#define __Pyx_SetupRefcountContext(name)
#define __Pyx_FinishRefcountContext() 0
#endif /* CYTHON_REFNANNY */
""")
...@@ -16,6 +16,7 @@ from Cython.Utils import open_new_file, replace_suffix, UtilityCode ...@@ -16,6 +16,7 @@ from Cython.Utils import open_new_file, replace_suffix, UtilityCode
from StringEncoding import EncodedString, escape_byte_string, split_docstring from StringEncoding import EncodedString, escape_byte_string, split_docstring
import Options import Options
import ControlFlow import ControlFlow
import DebugFlags
from DebugFlags import debug_disposal_code from DebugFlags import debug_disposal_code
...@@ -1010,6 +1011,7 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1010,6 +1011,7 @@ class FuncDefNode(StatNode, BlockNode):
# ----- Automatic lead-ins for certain special functions # ----- Automatic lead-ins for certain special functions
if is_getbuffer_slot: if is_getbuffer_slot:
self.getbuffer_init(code) self.getbuffer_init(code)
code.putln('__Pyx_SetupRefcountContext("%s");' % self.entry.name)
# ----- Fetch arguments # ----- Fetch arguments
self.generate_argument_parsing_code(env, code) self.generate_argument_parsing_code(env, code)
# If an argument is assigned to in the body, we must # If an argument is assigned to in the body, we must
...@@ -1119,8 +1121,24 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1119,8 +1121,24 @@ class FuncDefNode(StatNode, BlockNode):
code.putln("PyGILState_Release(_save);") code.putln("PyGILState_Release(_save);")
# code.putln("/* TODO: decref scope object */") # code.putln("/* TODO: decref scope object */")
# ----- Return # ----- Return
default_retval = self.return_type.default_value
err_val = self.error_value()
if err_val is None and default_retval:
err_val = default_retval
if self.return_type.is_pyobject:
code.put_giveref(Naming.retval_cname)
if err_val is None:
code.putln('__Pyx_FinishRefcountContext();')
else:
code.putln('if (__Pyx_FinishRefcountContext() == -1) {')
code.putln(code.set_error_info(self.pos))
code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
code.putln('%s = %s;' % (Naming.retval_cname, err_val))
code.putln('}')
if not self.return_type.is_void: if not self.return_type.is_void:
code.putln("return %s;" % Naming.retval_cname) code.putln("return %s;" % Naming.retval_cname)
code.putln("}") code.putln("}")
# ----- Go back and insert temp variable declarations # ----- Go back and insert temp variable declarations
tempvardecl_code.put_var_declarations(lenv.temp_entries) tempvardecl_code.put_var_declarations(lenv.temp_entries)
...@@ -1171,16 +1189,16 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1171,16 +1189,16 @@ class FuncDefNode(StatNode, BlockNode):
# getbuffer with a NULL parameter. For now we work around this; # getbuffer with a NULL parameter. For now we work around this;
# the following line should be removed when this bug is fixed. # the following line should be removed when this bug is fixed.
code.putln("if (%s == NULL) return 0;" % info) code.putln("if (%s == NULL) return 0;" % info)
code.putln("%s->obj = Py_None; Py_INCREF(Py_None);" % info) code.putln("%s->obj = Py_None; __Pyx_INCREF(Py_None);" % info)
def getbuffer_error_cleanup(self, code): def getbuffer_error_cleanup(self, code):
info = self.local_scope.arg_entries[1].cname info = self.local_scope.arg_entries[1].cname
code.putln("Py_DECREF(%s->obj); %s->obj = NULL;" % code.putln("__Pyx_DECREF(%s->obj); %s->obj = NULL;" %
(info, info)) (info, info))
def getbuffer_normal_cleanup(self, code): def getbuffer_normal_cleanup(self, code):
info = self.local_scope.arg_entries[1].cname info = self.local_scope.arg_entries[1].cname
code.putln("if (%s->obj == Py_None) { Py_DECREF(Py_None); %s->obj = NULL; }" % code.putln("if (%s->obj == Py_None) { __Pyx_DECREF(Py_None); %s->obj = NULL; }" %
(info, info)) (info, info))
class CFuncDefNode(FuncDefNode): class CFuncDefNode(FuncDefNode):
...@@ -4040,7 +4058,7 @@ class ExceptClauseNode(Node): ...@@ -4040,7 +4058,7 @@ class ExceptClauseNode(Node):
self.body.generate_execution_code(code) self.body.generate_execution_code(code)
code.funcstate.exc_vars = old_exc_vars code.funcstate.exc_vars = old_exc_vars
for var in self.exc_vars: for var in self.exc_vars:
code.putln("Py_DECREF(%s); %s = 0;" % (var, var)) code.putln("__Pyx_DECREF(%s); %s = 0;" % (var, var))
code.put_goto(end_label) code.put_goto(end_label)
code.putln( code.putln(
"}") "}")
......
cat <<EOF > ../../Cython/Compiler/DebugFlags.py
debug_disposal_code = 0
debug_temp_alloc = 0
debug_coercion = 0
debug_refnanny = 0
EOF
python ../../cython.py refnanny.pyx
gcc -shared -pthread -fPIC -fwrapv -O2 -Wall \
-fno-strict-aliasing -I/local/include/python2.5 \
-o refnanny.so -I. refnanny.c
cat <<EOF > ../../Cython/Compiler/DebugFlags.py
debug_disposal_code = 0
debug_temp_alloc = 0
debug_coercion = 0
debug_refnanny = 1
EOF
from python_ref cimport Py_INCREF, Py_DECREF
loglevel = 0
reflog = []
cdef log(level, action, obj, lineno):
if loglevel >= level:
reflog.append((lineno, action, id(obj)))
LOG_NONE, LOG_ALL = range(2)
class RefnannyException(Exception):
pass
class RefnannyContext(object):
def __init__(self):
self.refs = {} # id -> (count, [lineno])
self.errors = []
def regref(self, obj, lineno):
log(LOG_ALL, 'regref', obj, lineno)
id_ = id(obj)
count, linenumbers = self.refs.get(id_, (0, []))
self.refs[id_] = (count + 1, linenumbers)
linenumbers.append(lineno)
def delref(self, obj, lineno):
log(LOG_ALL, 'delref', obj, lineno)
id_ = id(obj)
count, linenumbers = self.refs.get(id_, (0, []))
if count == 0:
self.errors.append("Too many decrefs on line %d, reference acquired on lines %r" %
(lineno, linenumbers))
elif count == 1:
del self.refs[id_]
else:
self.refs[id_] = (count - 1, linenumbers)
def end(self):
if len(self.refs) > 0:
msg = ""
for count, linenos in self.refs.itervalues():
msg += "\n Acquired on lines: " + ", ".join(["%d" % x for x in linenos])
self.errors.append("References leaked: %s" % msg)
if self.errors:
raise RefnannyException("\n".join(self.errors))
cdef public void* __Pyx_Refnanny_NewContext(char* funcname, int lineno) except NULL:
ctx = RefnannyContext()
Py_INCREF(ctx)
return <void*>ctx
cdef public void __Pyx_Refnanny_GOTREF(void* ctx, object obj, int lineno):
if ctx == NULL: return
(<object>ctx).regref(obj, lineno)
cdef public void __Pyx_Refnanny_GIVEREF(void* ctx, object obj, int lineno):
if ctx == NULL: return
(<object>ctx).delref(obj, lineno)
cdef public void __Pyx_Refnanny_INCREF(void* ctx, object obj, int lineno):
Py_INCREF(obj)
__Pyx_Refnanny_GOTREF(ctx, obj, lineno)
cdef public void __Pyx_Refnanny_DECREF(void* ctx, object obj, int lineno):
# GIVEREF raises exception if we hit 0
#
__Pyx_Refnanny_GIVEREF(ctx, obj, lineno)
Py_DECREF(obj)
cdef public int __Pyx_Refnanny_FinishContext(void* ctx) except -1:
obj = <object>ctx
try:
obj.end()
finally:
Py_DECREF(obj)
return 0
#!/usr/bin/python #!/usr/bin/python
import os, sys, re, shutil, unittest, doctest import os, sys, re, shutil, unittest, doctest, ctypes
WITH_CYTHON = True WITH_CYTHON = True
...@@ -26,6 +26,10 @@ VER_DEP_MODULES = { ...@@ -26,6 +26,10 @@ VER_DEP_MODULES = {
INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ] INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ]
CFLAGS = os.getenv('CFLAGS', '').split() CFLAGS = os.getenv('CFLAGS', '').split()
ctypes.PyDLL("Cython/Runtime/refnanny.so", mode=ctypes.RTLD_GLOBAL)
sys.path.append("Cython/Runtime")
import refnanny
#CFLAGS.append("-DCYTHON_REFNANNY")
class ErrorWriter(object): class ErrorWriter(object):
match_error = re.compile('(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match match_error = re.compile('(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match
...@@ -594,3 +598,5 @@ if __name__ == '__main__': ...@@ -594,3 +598,5 @@ if __name__ == '__main__':
sys.stderr.write("Following tests excluded because of missing dependencies on your system:\n") sys.stderr.write("Following tests excluded because of missing dependencies on your system:\n")
for test in missing_dep_excluder.tests_missing_deps: for test in missing_dep_excluder.tests_missing_deps:
sys.stderr.write(" %s\n" % test) sys.stderr.write(" %s\n" % test)
print "\n".join([repr(x) for x in refnanny.reflog])
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment