tracing: Process constants for (un)likely() profiler

When running the likely/unlikely profiler, one of the results did not look
accurate. It noted that the unlikely() in link_path_walk() was 100%
incorrect. When I added a trace_printk() to see what was happening there, it
became 80% correct! Looking deeper into what whas happening, I found that
gcc split that if statement into two paths. One where the if statement
became a constant, the other path a variable. The other path had the if
statement always hit (making the unlikely there, always false), but since
the #define unlikely() has:

  #define unlikely() (__builtin_constant_p(x) ? !!(x) : __branch_check__(x, 0))

Where constants are ignored by the branch profiler, the "constant" path
made by the compiler was ignored, even though it was hit 80% of the time.

By just passing the constant value to the __branch_check__() function and
tracing it out of line (as always correct, as likely/unlikely isn't a factor
for constants), then we get back the accurate readings of branches that were
optimized by gcc causing part of the execution to become constant.
Signed-off-by: default avatarSteven Rostedt (VMware) <rostedt@goodmis.org>
parent 6496bb72
...@@ -107,12 +107,13 @@ struct ftrace_branch_data { ...@@ -107,12 +107,13 @@ struct ftrace_branch_data {
*/ */
#if defined(CONFIG_TRACE_BRANCH_PROFILING) \ #if defined(CONFIG_TRACE_BRANCH_PROFILING) \
&& !defined(DISABLE_BRANCH_PROFILING) && !defined(__CHECKER__) && !defined(DISABLE_BRANCH_PROFILING) && !defined(__CHECKER__)
void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect); void ftrace_likely_update(struct ftrace_branch_data *f, int val,
int expect, int is_constant);
#define likely_notrace(x) __builtin_expect(!!(x), 1) #define likely_notrace(x) __builtin_expect(!!(x), 1)
#define unlikely_notrace(x) __builtin_expect(!!(x), 0) #define unlikely_notrace(x) __builtin_expect(!!(x), 0)
#define __branch_check__(x, expect) ({ \ #define __branch_check__(x, expect, is_constant) ({ \
int ______r; \ int ______r; \
static struct ftrace_branch_data \ static struct ftrace_branch_data \
__attribute__((__aligned__(4))) \ __attribute__((__aligned__(4))) \
...@@ -122,8 +123,9 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect); ...@@ -122,8 +123,9 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect);
.file = __FILE__, \ .file = __FILE__, \
.line = __LINE__, \ .line = __LINE__, \
}; \ }; \
______r = likely_notrace(x); \ ______r = __builtin_expect(!!(x), expect); \
ftrace_likely_update(&______f, ______r, expect); \ ftrace_likely_update(&______f, ______r, \
expect, is_constant); \
______r; \ ______r; \
}) })
...@@ -133,10 +135,10 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect); ...@@ -133,10 +135,10 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect);
* written by Daniel Walker. * written by Daniel Walker.
*/ */
# ifndef likely # ifndef likely
# define likely(x) (__builtin_constant_p(x) ? !!(x) : __branch_check__(x, 1)) # define likely(x) (__branch_check__(x, 1, __builtin_constant_p(x)))
# endif # endif
# ifndef unlikely # ifndef unlikely
# define unlikely(x) (__builtin_constant_p(x) ? !!(x) : __branch_check__(x, 0)) # define unlikely(x) (__branch_check__(x, 0, __builtin_constant_p(x)))
# endif # endif
#ifdef CONFIG_PROFILE_ALL_BRANCHES #ifdef CONFIG_PROFILE_ALL_BRANCHES
......
...@@ -200,8 +200,12 @@ void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) ...@@ -200,8 +200,12 @@ void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
} }
#endif /* CONFIG_BRANCH_TRACER */ #endif /* CONFIG_BRANCH_TRACER */
void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect) void ftrace_likely_update(struct ftrace_branch_data *f, int val,
int expect, int is_constant)
{ {
/* A constant is always correct */
if (is_constant)
val = expect;
/* /*
* I would love to have a trace point here instead, but the * I would love to have a trace point here instead, but the
* trace point code is so inundated with unlikely and likely * trace point code is so inundated with unlikely and likely
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment