futex-pylat: print py-level tracebacks that trigger long-blocked futex syscall

futex-pylat

+#!/usr/bin/env python
+"""print py-level tracebacks that trigger long-blocked futex syscall
+
+usage: futex-pylat <tid> <too_long_ms>
+
+FIXME traces only one thread instead of whole process.
+"""
+from __future__ import print_function
+
+from ptrace.debugger import PtraceDebugger, ProcessSignal
+from ptrace.func_call import FunctionCallOptions
+
+import os, sys
+from time import time
+
+def main():
+    dbg = PtraceDebugger()
+    try:
+        _main(dbg)
+    finally:
+        dbg.quit()
+
+def _main(dbg):
+    tid = int(sys.argv[1])
+    too_long_ms = float(sys.argv[2])
+    dbg.traceFork()  # forks
+    dbg.traceExec()
+    dbg.traceClone() # threads
+
+    # NOTE we trace only one thread
+    # TODO -> attach all threads from /proc/<pid>/tasks
+    print("Attaching to tid%d" % tid)
+    proc = dbg.addProcess(tid, is_attached=False) 
+
+    syscall_options = FunctionCallOptions()
+
+    # setup to break at first syscall
+    proc.syscall()
+    def _(syscall):
+        return (not syscall.name.startswith("futex"))
+    proc.syscall_state.ignore_callback = _
+
+    sys_start = 0.
+    while 1:
+        if not dbg:
+            break # proc exited
+
+        # wait for next syscall event (enter/exit)
+        try:
+            event = dbg.waitSyscall()
+        except ProcessSignal as event:
+            event.display()
+            event.process.syscall(event.signum)
+            continue
+
+        assert event.process is proc
+
+        state = proc.syscall_state
+        syscall = state.event(syscall_options)
+        if syscall:
+            if state.syscall is not None:
+                sys_start = time()
+                #print("enter %s" % syscall)
+            else:
+                sys_end = time()
+                lat = sys_end - sys_start
+                if lat > too_long_ms*1E-3:
+                    #print("exit  %s" % syscall)
+                    print(">>> %s\tlat: %.3fms" % (syscall.name, lat * 1E3))
+
+                    # a syscall was detected to block > too_long
+                    # -> dump state of python threads
+                    #
+                    # --nonblocking because are we already holding the thread in stopped state.
+                    os.system("py-spy dump --nonblocking --pid %d" % tid)
+
+
+        # break at next syscall
+        proc.syscall()
+
+
+if __name__ == '__main__':
+    main()