Commit 20b41a5a by Kirill Smelkov

bigfile/py: Dump pybuf referrers if pybuf->ob_refcnt != 1 before dying in loadblk epilogue

Instead of only printing "BUG" let's print information about objects
which still refer to pybuf - to help debugging.

For example with the following artificial pybuf leak

```
diff --git a/bigfile/tests/test_basic.py b/bigfile/tests/test_basic.py
index c737621..f5e057a 100644
--- a/bigfile/tests/test_basic.py
+++ b/bigfile/tests/test_basic.py
@@ -126,6 +126,7 @@ def test_basic():

 # test that python exception state is preserved across pagefaulting
 def test_pagefault_savestate():
+    zzz = []
     class BadFile(BigFile):
         def loadblk(self, blk, buf):
             # simulate some errors in-between to overwrite thread exception
@@ -154,6 +155,7 @@ def loadblk(self, blk, buf):
             # which result in holding additional ref to buf, but loadblk caller
             # will detect and handle this situation via garbage-collecting
             # above cycle.
+            zzz.append(buf)

             self.loadblk_run = 1
```

it dies this way:

    bigfile/_bigfile.c:567 pybigfile_loadblk WARN: pybuf->ob_refcnt != 1 even after GC:
    pybuf (ob_refcnt=2):    <read-write buffer ptr 0x7f08d3e88000, size 2097152 at 0x7f08d48b7070>
    pybuf referrers:        [[<read-write buffer ptr 0x7f08d3e88000, size 2097152 at 0x7f08d48b7070>]]
    bigfile/_bigfile.c:573 pybigfile_loadblk        BUG!
1 parent 9aa6a5d7
......@@ -36,6 +36,8 @@ typedef struct _frame PyFrameObject;
#include <wendelin/bug.h>
#include <wendelin/compat_py2.h>
static PyObject *gcmodule;
/* whether to pass old buffer instead of memoryview to .loadblk() / .storeblk()
*
* on python2 < 2.7.10 memoreview object is not accepted in a lot of
......@@ -118,6 +120,9 @@ static void XPyErr_SetFromErrno(void);
* everything else related to exception state */
static void XPyErr_FullClear(void);
/* print objects that refer to obj */
static void XPyObject_PrintReferrers(PyObject *obj, FILE *fp);
/************
* PyVMA *
......@@ -558,7 +563,15 @@ out:
}
/* now it is real bug if pybuf remains referenced from somewhere */
BUG_ON(pybuf->ob_refcnt != 1);
if (pybuf->ob_refcnt != 1) {
WARN("pybuf->ob_refcnt != 1 even after GC:");
fprintf(stderr, "pybuf (ob_refcnt=%ld):\t", (long)pybuf->ob_refcnt);
PyObject_Print(pybuf, stderr, 0);
fprintf(stderr, "\npybuf referrers:\t");
XPyObject_PrintReferrers(pybuf, stderr);
fprintf(stderr, "\n");
BUG();
}
}
/* drop pybuf
......@@ -895,6 +908,11 @@ _init_bigfile(void)
CSTi(WRITEOUT_STORE);
CSTi(WRITEOUT_MARKSTORED);
/* import gc */
gcmodule = PyImport_ImportModule("gc");
if (!gcmodule)
return NULL;
return m;
}
......@@ -957,3 +975,12 @@ XPyErr_FullClear(void)
PySys_SetObject("exc_value", Py_None);
PySys_SetObject("exc_traceback", Py_None);
}
static void
XPyObject_PrintReferrers(PyObject *obj, FILE *fp)
{
PyObject *obj_referrers = PyObject_CallMethod(gcmodule, "get_referrers", "O", obj);
BUG_ON(!obj_referrers);
PyObject_Print(obj_referrers, fp, 0);
Py_DECREF(obj_referrers);
}
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!