Commit e778db05 authored by Teng Qin's avatar Teng Qin

Refactor offwaketime stack id error handling

parent bce2bee9
...@@ -292,7 +292,7 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value): ...@@ -292,7 +292,7 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
else: else:
line.extend([b.sym(addr, k.tgid) for addr in reversed(user_stack)]) line.extend([b.sym(addr, k.tgid) for addr in reversed(user_stack)])
if not args.user_stacks_only: if not args.user_stacks_only:
line.extend(["-"] if (need_delimiter and k.kernel_stack_id >= 0) else []) line.extend(["-"] if (need_delimiter and k.kernel_stack_id >= 0 and k.user_stack_id >= 0) else [])
if stack_id_err(k.kernel_stack_id): if stack_id_err(k.kernel_stack_id):
line.append("[Missed Kernel Stack]") line.append("[Missed Kernel Stack]")
else: else:
...@@ -307,7 +307,7 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value): ...@@ -307,7 +307,7 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
for addr in kernel_stack: for addr in kernel_stack:
print(" %s" % b.ksym(addr)) print(" %s" % b.ksym(addr))
if not args.kernel_stacks_only: if not args.kernel_stacks_only:
if need_delimiter and k.user_stack_id >= 0: if need_delimiter and k.user_stack_id >= 0 and k.kernel_stack_id >= 0:
print(" --") print(" --")
if stack_id_err(k.user_stack_id): if stack_id_err(k.user_stack_id):
print(" [Missed User Stack]") print(" [Missed User Stack]")
......
...@@ -35,6 +35,11 @@ def positive_nonzero_int(val): ...@@ -35,6 +35,11 @@ def positive_nonzero_int(val):
raise argparse.ArgumentTypeError("must be nonzero") raise argparse.ArgumentTypeError("must be nonzero")
return ival return ival
def stack_id_err(stack_id):
# -EFAULT in get_stackid normally means the stack-trace is not availible,
# Such as getting kernel stack trace in userspace code
return (stack_id < 0) and (stack_id != -errno.EFAULT)
# arguments # arguments
examples = """examples: examples = """examples:
./offwaketime # trace off-CPU + waker stack time until Ctrl-C ./offwaketime # trace off-CPU + waker stack time until Ctrl-C
...@@ -239,9 +244,8 @@ bpf_text = bpf_text.replace('MINBLOCK_US_VALUE', str(args.min_block_time)) ...@@ -239,9 +244,8 @@ bpf_text = bpf_text.replace('MINBLOCK_US_VALUE', str(args.min_block_time))
bpf_text = bpf_text.replace('MAXBLOCK_US_VALUE', str(args.max_block_time)) bpf_text = bpf_text.replace('MAXBLOCK_US_VALUE', str(args.max_block_time))
# handle stack args # handle stack args
kernel_stack_get = "stack_traces.get_stackid(ctx, BPF_F_REUSE_STACKID)" kernel_stack_get = "stack_traces.get_stackid(ctx, 0)"
user_stack_get = \ user_stack_get = "stack_traces.get_stackid(ctx, BPF_F_USER_STACK)"
"stack_traces.get_stackid(ctx, BPF_F_REUSE_STACKID | BPF_F_USER_STACK)"
stack_context = "" stack_context = ""
if args.user_stacks_only: if args.user_stacks_only:
stack_context = "user" stack_context = "user"
...@@ -288,15 +292,20 @@ missing_stacks = 0 ...@@ -288,15 +292,20 @@ missing_stacks = 0
has_enomem = False has_enomem = False
counts = b.get_table("counts") counts = b.get_table("counts")
stack_traces = b.get_table("stack_traces") stack_traces = b.get_table("stack_traces")
need_delimiter = args.delimited and not (args.kernel_stacks_only or
args.user_stacks_only)
for k, v in sorted(counts.items(), key=lambda counts: counts[1].value): for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
# handle get_stackid errors # handle get_stackid errors
# check for an ENOMEM error if not args.user_stacks_only:
if k.w_k_stack_id == -errno.ENOMEM or \ missing_stacks += int(stack_id_err(k.w_k_stack_id))
k.t_k_stack_id == -errno.ENOMEM or \ missing_stacks += int(stack_id_err(k.t_k_stack_id))
k.w_u_stack_id == -errno.ENOMEM or \ has_enomem = has_enomem or (k.w_k_stack_id == -errno.ENOMEM) or \
k.t_u_stack_id == -errno.ENOMEM: (k.t_k_stack_id == -errno.ENOMEM)
missing_stacks += 1 if not args.kernel_stacks_only:
continue missing_stacks += int(stack_id_err(k.w_u_stack_id))
missing_stacks += int(stack_id_err(k.t_u_stack_id))
has_enomem = has_enomem or (k.w_u_stack_id == -errno.ENOMEM) or \
(k.t_u_stack_id == -errno.ENOMEM)
waker_user_stack = [] if k.w_u_stack_id < 1 else \ waker_user_stack = [] if k.w_u_stack_id < 1 else \
reversed(list(stack_traces.walk(k.w_u_stack_id))[1:]) reversed(list(stack_traces.walk(k.w_u_stack_id))[1:])
...@@ -309,40 +318,69 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value): ...@@ -309,40 +318,69 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
if folded: if folded:
# print folded stack output # print folded stack output
line = \ line = [k.target.decode()]
[k.target.decode()] + \ if not args.kernel_stacks_only:
[b.sym(addr, k.t_tgid) if stack_id_err(k.t_u_stack_id):
for addr in reversed(list(target_user_stack)[1:])] + \ line.append("[Missed User Stack]")
(["-"] if args.delimited else [""]) + \ else:
[b.ksym(addr) line.extend([b.sym(addr, k.t_tgid)
for addr in reversed(list(target_kernel_stack)[1:])] + \ for addr in reversed(list(target_user_stack)[1:])])
["--"] + \ if not args.user_stacks_only:
[b.ksym(addr) line.extend(["-"] if (need_delimiter and k.t_k_stack_id > 0 and k.t_u_stack_id > 0) else [])
for addr in reversed(list(waker_kernel_stack))] + \ if stack_id_err(k.t_k_stack_id):
(["-"] if args.delimited else [""]) + \ line.append("[Missed Kernel Stack]")
[b.sym(addr, k.w_tgid) else:
for addr in reversed(list(waker_user_stack))] + \ line.extend([b.ksym(addr)
[k.waker.decode()] for addr in reversed(list(target_kernel_stack)[1:])])
line.append("--")
if not args.user_stacks_only:
if stack_id_err(k.w_k_stack_id):
line.append("[Missed Kernel Stack]")
else:
line.extend([b.ksym(addr)
for addr in reversed(list(waker_kernel_stack))])
if not args.kernel_stacks_only:
line.extend(["-"] if (need_delimiter and k.w_u_stack_id > 0 and k.w_k_stack_id > 0) else [])
if stack_id_err(k.w_u_stack_id):
line.extend("[Missed User Stack]")
else:
line.extend([b.sym(addr, k.w_tgid)
for addr in reversed(list(waker_user_stack))])
line.append(k.waker.decode())
print("%s %d" % (";".join(line), v.value)) print("%s %d" % (";".join(line), v.value))
else: else:
# print wakeup name then stack in reverse order # print wakeup name then stack in reverse order
print(" %-16s %s %s" % ("waker:", k.waker.decode(), k.t_pid)) print(" %-16s %s %s" % ("waker:", k.waker.decode(), k.t_pid))
if not args.kernel_stacks_only:
if stack_id_err(k.w_u_stack_id):
print(" [Missed User Stack]")
else:
for addr in waker_user_stack: for addr in waker_user_stack:
print(" %s" % b.sym(addr, k.w_tgid)) print(" %s" % b.sym(addr, k.w_tgid))
if args.delimited: if not args.user_stacks_only:
if need_delimiter and k.w_u_stack_id > 0 and k.w_k_stack_id > 0:
print(" -") print(" -")
if stack_id_err(k.w_k_stack_id):
print(" [Missed Kernel Stack]")
else:
for addr in waker_kernel_stack: for addr in waker_kernel_stack:
print(" %s" % b.ksym(addr)) print(" %s" % b.ksym(addr))
# print waker/wakee delimiter # print waker/wakee delimiter
print(" %-16s %s" % ("--", "--")) print(" %-16s %s" % ("--", "--"))
# print default multi-line stack output if not args.user_stacks_only:
if stack_id_err(k.t_k_stack_id):
print(" [Missed Kernel Stack]")
else:
for addr in target_kernel_stack: for addr in target_kernel_stack:
print(" %s" % b.ksym(addr)) print(" %s" % b.ksym(addr))
if args.delimited: if not args.kernel_stacks_only:
if need_delimiter and k.t_u_stack_id > 0 and k.t_k_stack_id > 0:
print(" -") print(" -")
if stack_id_err(k.t_u_stack_id):
print(" [Missed User Stack]")
else:
for addr in target_user_stack: for addr in target_user_stack:
print(" %s" % b.sym(addr, k.t_tgid)) print(" %s" % b.sym(addr, k.t_tgid))
print(" %-16s %s %s" % ("target:", k.target.decode(), k.w_pid)) print(" %-16s %s %s" % ("target:", k.target.decode(), k.w_pid))
...@@ -350,6 +388,6 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value): ...@@ -350,6 +388,6 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
if missing_stacks > 0: if missing_stacks > 0:
enomem_str = " Consider increasing --stack-storage-size." enomem_str = " Consider increasing --stack-storage-size."
print("WARNING: %d stack traces could not be displayed.%s" % print("WARNING: %d stack traces lost and could not be displayed.%s" %
(missing_stacks, enomem_str), (missing_stacks, (enomem_str if has_enomem else "")),
file=stderr) file=stderr)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment