Merge pull request #316 from brendangregg/master

offcputime improvements

man/man8/offcputime.8

@@ -2,7 +2,7 @@
 .SH NAME
 offcputime \- Summarize off-CPU time by kernel stack trace. Uses Linux eBPF/bcc.
 .SH SYNOPSIS
-.B offcputime [\-h] [\-p PID] [\-i INTERVAL] [\-T] [duration]
+.B offcputime [\-h] [\-u] [\-p PID] [\-i INTERVAL] [\-T] [duration]
 .SH DESCRIPTION
 This program shows kernel stack traces and task names that were blocked and
 "off-CPU", and the total duration they were blocked: their "off-CPU time".
@@ -33,6 +33,9 @@ CONFIG_BPF and bcc.
 \-h
 Print usage message.
 .TP
+\-u
+Only trace user threads (not kernel threads).
+.TP
 \-v
 Show raw addresses.
 .TP

tools/offcputime

@@ -3,7 +3,7 @@
 # offcputime    Summarize off-CPU time by kernel stack trace
 #               For Linux, uses BCC, eBPF.
 #
-# USAGE: offcputime [-h] [-p PID] [-i INTERVAL] [-T] [duration]
+# USAGE: offcputime [-h] [-u] [-p PID] [-i INTERVAL] [-T] [duration]
 #
 # The current implementation uses an unrolled loop for x86_64, and was written
 # as a proof of concept. This implementation should be replaced in the future
@@ -27,12 +27,15 @@ examples = """examples:
     ./offcputime             # trace off-CPU stack time until Ctrl-C
     ./offcputime 5           # trace for 5 seconds only
     ./offcputime -f 5        # 5 seconds, and output in folded format
+    ./offcputime -u          # don't include kernel threads (user only)
     ./offcputime -p 185      # trace fo PID 185 only
 """
 parser = argparse.ArgumentParser(
     description="Summarize off-CPU time by kernel stack trace",
     formatter_class=argparse.RawDescriptionHelpFormatter,
     epilog=examples)
+parser.add_argument("-u", "--useronly", action="store_true",
+    help="user threads only (no kernel threads)")
 parser.add_argument("-p", "--pid",
     help="trace this PID only")
 parser.add_argument("-v", "--verbose", action="store_true",
@@ -46,12 +49,15 @@ folded = args.folded
 duration = int(args.duration)
 debug = 0
 maxdepth = 20    # and MAXDEPTH
+if args.pid and args.useronly:
+    print("ERROR: use either -p or -u.")
+    exit()
 
 # signal handler
 def signal_ignore(signal, frame):
     print()
 
-# load BPF program
+# define BPF program
 bpf_text = """
 #include <uapi/linux/ptrace.h>
 #include <linux/sched.h>
@@ -82,32 +88,32 @@ static u64 get_frame(u64 *bp) {
     return 0;
 }
 
-int offcpu(struct pt_regs *ctx) {
-    u32 pid = bpf_get_current_pid_tgid();
-    u64 ts = bpf_ktime_get_ns();
-    FILTER
-    start.update(&pid, &ts);
-    return 0;
-}
+int oncpu(struct pt_regs *ctx, struct task_struct *prev) {
+    u32 pid;
+    u64 ts, *tsp;
 
-int oncpu(struct pt_regs *ctx) {
-    u32 pid = bpf_get_current_pid_tgid();
-    FILTER
-    u64 ts = bpf_ktime_get_ns();
-    struct key_t key = {};
-    u64 zero = 0, *val, bp = 0, *tsp, delta;
-    int depth = 0;
+    // record previous thread sleep time
+    if (FILTER) {
+        pid = prev->pid;
+        ts = bpf_ktime_get_ns();
+        start.update(&pid, &ts);
+    }
 
-    // calculate delta time
+    // calculate current thread's delta time
+    pid = bpf_get_current_pid_tgid();
     tsp = start.lookup(&pid);
     if (tsp == 0)
-        return 0;        // missed start
-    delta = bpf_ktime_get_ns() - *tsp;
+        return 0;        // missed start or filtered
+    u64 delta = bpf_ktime_get_ns() - *tsp;
     start.delete(&pid);
     delta = delta / 1000;
     if (delta < MINBLOCK_US)
         return 0;
 
+    // create map key
+    u64 zero = 0, *val, bp = 0;
+    int depth = 0;
+    struct key_t key = {};
     bpf_get_current_comm(&key.name, sizeof(key.name));
     bp = ctx->bp;
 
@@ -141,14 +147,17 @@ out:
 }
 """
 if args.pid:
-    bpf_text = bpf_text.replace('FILTER',
-        'if (pid != %s) { return 0; }' % (args.pid))
+    filter = 'pid == %s' % args.pid
+elif args.useronly:
+    filter = '!(prev->flags & PF_KTHREAD)'
 else:
-    bpf_text = bpf_text.replace('FILTER', '')
+    filter = '1'
+bpf_text = bpf_text.replace('FILTER', filter)
 if debug:
     print(bpf_text)
+
+# initialize BPF
 b = BPF(text=bpf_text)
-b.attach_kprobe(event="schedule", fn_name="offcpu")
 b.attach_kprobe(event="finish_task_switch", fn_name="oncpu")
 matched = b.num_open_kprobes()
 if matched == 0:

tools/offcputime_example.txt

@@ -723,8 +723,8 @@ creating your "off-CPU time flame graphs".
 
 USAGE message:
 
-./offcputime --help
-usage: offcputime [-h] [-p PID] [-v] [-f] [duration]
+# ./offcputime -h
+usage: offcputime [-h] [-u] [-p PID] [-v] [-f] [duration]
 
 Summarize off-CPU time by kernel stack trace
 
@@ -733,6 +733,7 @@ positional arguments:
 
 optional arguments:
   -h, --help         show this help message and exit
+  -u, --useronly     user threads only (no kernel threads)
   -p PID, --pid PID  trace this PID only
   -v, --verbose      show raw addresses
   -f, --folded       output folded format
@@ -741,4 +742,5 @@ examples:
     ./offcputime             # trace off-CPU stack time until Ctrl-C
     ./offcputime 5           # trace for 5 seconds only
     ./offcputime -f 5        # 5 seconds, and output in folded format
+    ./offcputime -u          # don't include kernel threads (user only)
     ./offcputime -p 185      # trace fo PID 185 only