Merge pull request #674 from markdrayton/offcputime

offcputime improvements: use less RAM, add PID/TID support

man/man8/offcputime.8

@@ -40,6 +40,9 @@ Print output in folded stack format.
 \-p PID
 Trace this process ID only (filtered in-kernel).
 .TP
+\-t TID
+Trace this thread ID only (filtered in-kernel).
+.TP
 \-u
 Only trace user threads (no kernel threads).
 .TP
@@ -52,6 +55,15 @@ Show stacks from user space only (no kernel space stacks).
 \-K
 Show stacks from kernel space only (no user space stacks).
 .TP
+\-d
+Insert delimiter between kernel/user stacks.
+.TP
+\-f
+Output folded format.
+.TP
+\-\-stack-storage-size STACK_STORAGE_SIZE
+Change the number of unique stack traces that can be stored and displayed.
+.TP
 duration
 Duration to trace, in seconds.
 .SH EXAMPLES

tools/offcputime.py

@@ -41,6 +41,7 @@ examples = """examples:
     ./offcputime 5           # trace for 5 seconds only
     ./offcputime -f 5        # 5 seconds, and output in folded format
     ./offcputime -p 185      # only trace threads for PID 185
+    ./offcputime -t 188      # only trace thread 188
     ./offcputime -u          # only trace user threads (no kernel)
     ./offcputime -k          # only trace kernel threads (no user)
     ./offcputime -U          # only show user space stacks (no kernel)
@@ -51,8 +52,12 @@ parser = argparse.ArgumentParser(
     formatter_class=argparse.RawDescriptionHelpFormatter,
     epilog=examples)
 thread_group = parser.add_mutually_exclusive_group()
-thread_group.add_argument("-p", "--pid", type=positive_int,
-    help="trace this PID only")
+# Note: this script provides --pid and --tid flags but their arguments are
+# referred to internally using kernel nomenclature: TGID and PID.
+thread_group.add_argument("-p", "--pid", metavar="PID", dest="tgid",
+    help="trace this PID only", type=positive_int)
+thread_group.add_argument("-t", "--tid", metavar="TID", dest="pid",
+    help="trace this TID only", type=positive_int)
 thread_group.add_argument("-u", "--user-threads-only", action="store_true",
     help="user threads only (no kernel threads)")
 thread_group.add_argument("-k", "--kernel-threads-only", action="store_true",
@@ -68,12 +73,14 @@ parser.add_argument("-f", "--folded", action="store_true",
     help="output folded format")
 parser.add_argument("--stack-storage-size", default=1024,
     type=positive_nonzero_int,
-    help="the number of unique stack traces that can be stored and " \
-        "displayed (default 1024)")
+    help="the number of unique stack traces that can be stored and "
+         "displayed (default 1024)")
 parser.add_argument("duration", nargs="?", default=99999999,
     type=positive_nonzero_int,
     help="duration of trace, in seconds")
 args = parser.parse_args()
+if args.pid and args.tgid:
+    parser.error("specify only one of -p and -t")
 folded = args.folded
 duration = int(args.duration)
 
@@ -90,6 +97,7 @@ bpf_text = """
 
 struct key_t {
     u32 pid;
+    u32 tgid;
     int user_stack_id;
     int kernel_stack_id;
     char name[TASK_COMM_LEN];
@@ -100,6 +108,7 @@ BPF_STACK_TRACE(stack_traces, STACK_STORAGE_SIZE)
 
 int oncpu(struct pt_regs *ctx, struct task_struct *prev) {
     u32 pid = prev->pid;
+    u32 tgid = prev->tgid;
     u64 ts, *tsp;
 
     // record previous thread sleep time
@@ -110,6 +119,7 @@ int oncpu(struct pt_regs *ctx, struct task_struct *prev) {
 
     // get the current thread's start time
     pid = bpf_get_current_pid_tgid();
+    tgid = bpf_get_current_pid_tgid() >> 32;
     tsp = start.lookup(&pid);
     if (tsp == 0) {
         return 0;        // missed start or filtered
@@ -128,6 +138,7 @@ int oncpu(struct pt_regs *ctx, struct task_struct *prev) {
     struct key_t key = {};
 
     key.pid = pid;
+    key.tgid = tgid;
     key.user_stack_id = USER_STACK_GET;
     key.kernel_stack_id = KERNEL_STACK_GET;
     bpf_get_current_comm(&key.name, sizeof(key.name));
@@ -140,9 +151,12 @@ int oncpu(struct pt_regs *ctx, struct task_struct *prev) {
 
 # set thread filter
 thread_context = ""
-if args.pid is not None:
-    thread_context = "PID %s" % args.pid
-    thread_filter = 'pid == %s' % args.pid
+if args.tgid is not None:
+    thread_context = "PID %d" % args.tgid
+    thread_filter = 'tgid == %d' % args.tgid
+elif args.pid is not None:
+    thread_context = "TID %d" % args.pid
+    thread_filter = 'pid == %d' % args.pid
 elif args.user_threads_only:
     thread_context = "user threads"
     thread_filter = '!(prev->flags & PF_KTHREAD)'
@@ -224,6 +238,8 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
             has_enomem = True
         continue
 
+    # user stacks will be symbolized by tgid, not pid, to avoid the overhead
+    # of one symbol resolver per thread
     user_stack = [] if k.user_stack_id < 0 else \
         stack_traces.walk(k.user_stack_id)
     kernel_stack = [] if k.kernel_stack_id < 0 else \
@@ -234,7 +250,7 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
         user_stack = list(user_stack)
         kernel_stack = list(kernel_stack)
         line = [k.name.decode()] + \
-            [b.sym(addr, k.pid) for addr in reversed(user_stack)] + \
+            [b.sym(addr, k.tgid) for addr in reversed(user_stack)] + \
             (need_delimiter and ["-"] or []) + \
             [b.ksym(addr) for addr in reversed(kernel_stack)]
         print("%s %d" % (";".join(line), v.value))
@@ -245,7 +261,7 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
         if need_delimiter:
             print("    --")
         for addr in user_stack:
-            print("    %016x %s" % (addr, b.sym(addr, k.pid)))
+            print("    %016x %s" % (addr, b.sym(addr, k.tgid)))
         print("    %-16s %s (%d)" % ("-", k.name, k.pid))
         print("        %d\n" % v.value)
 

tools/offcputime_example.txt

@@ -719,7 +719,7 @@ creating your "off-CPU time flame graphs".
 USAGE message:
 
 # ./offcputime -h
-usage: offcputime.py [-h] [-p PID | -k | -u] [-K | -U] [-f]
+usage: offcputime.py [-h] [-p PID | -t TID | -u | -k] [-U | -K] [-d] [-f]
                      [--stack-storage-size STACK_STORAGE_SIZE]
                      [duration]
 
@@ -731,6 +731,7 @@ positional arguments:
 optional arguments:
   -h, --help            show this help message and exit
   -p PID, --pid PID     trace this PID only
+  -t TID, --tid TID     trace this TID only
   -u, --user-threads-only
                         user threads only (no kernel threads)
   -k, --kernel-threads-only
@@ -741,6 +742,7 @@ optional arguments:
   -K, --kernel-stacks-only
                         show stacks from kernel space only (no user space
                         stacks)
+  -d, --delimited       insert delimiter between kernel/user stacks
   -f, --folded          output folded format
   --stack-storage-size STACK_STORAGE_SIZE
                         the number of unique stack traces that can be stored
@@ -751,6 +753,7 @@ examples:
     ./offcputime 5           # trace for 5 seconds only
     ./offcputime -f 5        # 5 seconds, and output in folded format
     ./offcputime -p 185      # only trace threads for PID 185
+    ./offcputime -t 188      # only trace thread 188
     ./offcputime -u          # only trace user threads (no kernel)
     ./offcputime -k          # only trace kernel threads (no user)
     ./offcputime -U          # only show user space stacks (no kernel)