ftrace.c 157 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2 3 4 5 6 7 8 9 10 11 12 13
/*
 * Infrastructure for profiling code inserted by 'gcc -pg'.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
 *
 * Originally ported from the -rt patch by:
 *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
 *
 * Based on code in the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
14
 *  Copyright (C) 2004 Nadia Yvette Chambers
15 16
 */

17 18
#include <linux/stop_machine.h>
#include <linux/clocksource.h>
19
#include <linux/sched/task.h>
20
#include <linux/kallsyms.h>
21
#include <linux/security.h>
22
#include <linux/seq_file.h>
23
#include <linux/tracefs.h>
24
#include <linux/hardirq.h>
Ingo Molnar's avatar
Ingo Molnar committed
25
#include <linux/kthread.h>
26
#include <linux/uaccess.h>
27
#include <linux/bsearch.h>
28
#include <linux/module.h>
Ingo Molnar's avatar
Ingo Molnar committed
29
#include <linux/ftrace.h>
30
#include <linux/sysctl.h>
31
#include <linux/slab.h>
32
#include <linux/ctype.h>
33
#include <linux/sort.h>
34
#include <linux/list.h>
35
#include <linux/hash.h>
36
#include <linux/rcupdate.h>
37
#include <linux/kprobes.h>
38

39
#include <trace/events/sched.h>
40

41
#include <asm/sections.h>
42
#include <asm/setup.h>
43

44
#include "ftrace_internal.h"
45
#include "trace_output.h"
46
#include "trace_stat.h"
47

48
#define FTRACE_WARN_ON(cond)			\
49 50 51
	({					\
		int ___r = cond;		\
		if (WARN_ON(___r))		\
52
			ftrace_kill();		\
53 54
		___r;				\
	})
55 56

#define FTRACE_WARN_ON_ONCE(cond)		\
57 58 59
	({					\
		int ___r = cond;		\
		if (WARN_ON_ONCE(___r))		\
60
			ftrace_kill();		\
61 62
		___r;				\
	})
63

64 65 66
/* hash bits for specific function selection */
#define FTRACE_HASH_BITS 7
#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
67 68
#define FTRACE_HASH_DEFAULT_BITS 10
#define FTRACE_HASH_MAX_BITS 12
69

70
#ifdef CONFIG_DYNAMIC_FTRACE
71 72 73
#define INIT_OPS_HASH(opsname)	\
	.func_hash		= &opsname.local_hash,			\
	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
74
#else
75
#define INIT_OPS_HASH(opsname)
76 77
#endif

78 79 80 81 82
enum {
	FTRACE_MODIFY_ENABLE_FL		= (1 << 0),
	FTRACE_MODIFY_MAY_SLEEP_FL	= (1 << 1),
};

83
struct ftrace_ops ftrace_list_end __read_mostly = {
84
	.func		= ftrace_stub,
85
	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
86
	INIT_OPS_HASH(ftrace_list_end)
87 88
};

89 90
/* ftrace_enabled is a method to turn ftrace on or off */
int ftrace_enabled __read_mostly;
91
static int last_ftrace_enabled;
92

93 94
/* Current function tracing op */
struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
95 96
/* What to set function_trace_op to */
static struct ftrace_ops *set_function_trace_op;
97

98
static bool ftrace_pids_enabled(struct ftrace_ops *ops)
99
{
100 101 102 103 104 105 106 107
	struct trace_array *tr;

	if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)
		return false;

	tr = ops->private;

	return tr->function_pids != NULL;
108 109 110 111
}

static void ftrace_update_trampoline(struct ftrace_ops *ops);

112 113 114 115 116 117
/*
 * ftrace_disabled is set when an anomaly is discovered.
 * ftrace_disabled is much stronger than ftrace_enabled.
 */
static int ftrace_disabled __read_mostly;

118
DEFINE_MUTEX(ftrace_lock);
119

120
struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end;
121
ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
122
struct ftrace_ops global_ops;
123

124 125
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
126
				 struct ftrace_ops *op, struct pt_regs *regs);
127 128 129 130 131
#else
/* See comment below, where ftrace_ops_list_func is defined */
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
#endif
132

133 134 135 136
static inline void ftrace_ops_init(struct ftrace_ops *ops)
{
#ifdef CONFIG_DYNAMIC_FTRACE
	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
137 138
		mutex_init(&ops->local_hash.regex_lock);
		ops->func_hash = &ops->local_hash;
139 140 141 142 143
		ops->flags |= FTRACE_OPS_FL_INITIALIZED;
	}
#endif
}

144
static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
145
			    struct ftrace_ops *op, struct pt_regs *regs)
146
{
147 148 149
	struct trace_array *tr = op->private;

	if (tr && this_cpu_read(tr->trace_buffer.data->ftrace_ignore_pid))
150 151
		return;

152
	op->saved_func(ip, parent_ip, op, regs);
153 154
}

155 156 157 158
static void ftrace_sync(struct work_struct *work)
{
	/*
	 * This function is just a stub to implement a hard force
159
	 * of synchronize_rcu(). This requires synchronizing
160 161 162 163 164 165 166 167 168 169 170 171
	 * tasks even in userspace and idle.
	 *
	 * Yes, function tracing is rude.
	 */
}

static void ftrace_sync_ipi(void *data)
{
	/* Probably not needed, but do it anyway */
	smp_rmb();
}

172 173 174
static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
{
	/*
175
	 * If this is a dynamic, RCU, or per CPU ops, or we force list func,
176 177
	 * then it needs to call the list anyway.
	 */
178 179
	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) ||
	    FTRACE_FORCE_LIST_FUNC)
180 181 182 183 184
		return ftrace_ops_list_func;

	return ftrace_ops_get_func(ops);
}

185 186 187 188
static void update_ftrace_function(void)
{
	ftrace_func_t func;

189 190 191 192 193
	/*
	 * Prepare the ftrace_ops that the arch callback will use.
	 * If there's only one ftrace_ops registered, the ftrace_ops_list
	 * will point to the ops we want.
	 */
194 195
	set_function_trace_op = rcu_dereference_protected(ftrace_ops_list,
						lockdep_is_held(&ftrace_lock));
196 197

	/* If there's no ftrace_ops registered, just call the stub function */
198
	if (set_function_trace_op == &ftrace_list_end) {
199 200
		func = ftrace_stub;

201 202
	/*
	 * If we are at the end of the list and this ops is
203 204
	 * recursion safe and not dynamic and the arch supports passing ops,
	 * then have the mcount trampoline call the function directly.
205
	 */
206 207
	} else if (rcu_dereference_protected(ftrace_ops_list->next,
			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
208
		func = ftrace_ops_get_list_func(ftrace_ops_list);
209

210 211
	} else {
		/* Just use the default ftrace_ops */
212
		set_function_trace_op = &ftrace_list_end;
213
		func = ftrace_ops_list_func;
214
	}
215

216 217
	update_function_graph_func();

218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260
	/* If there's no change, then do nothing more here */
	if (ftrace_trace_function == func)
		return;

	/*
	 * If we are using the list function, it doesn't care
	 * about the function_trace_ops.
	 */
	if (func == ftrace_ops_list_func) {
		ftrace_trace_function = func;
		/*
		 * Don't even bother setting function_trace_ops,
		 * it would be racy to do so anyway.
		 */
		return;
	}

#ifndef CONFIG_DYNAMIC_FTRACE
	/*
	 * For static tracing, we need to be a bit more careful.
	 * The function change takes affect immediately. Thus,
	 * we need to coorditate the setting of the function_trace_ops
	 * with the setting of the ftrace_trace_function.
	 *
	 * Set the function to the list ops, which will call the
	 * function we want, albeit indirectly, but it handles the
	 * ftrace_ops and doesn't depend on function_trace_op.
	 */
	ftrace_trace_function = ftrace_ops_list_func;
	/*
	 * Make sure all CPUs see this. Yes this is slow, but static
	 * tracing is slow and nasty to have enabled.
	 */
	schedule_on_each_cpu(ftrace_sync);
	/* Now all cpus are using the list ops. */
	function_trace_op = set_function_trace_op;
	/* Make sure the function_trace_op is visible on all CPUs */
	smp_wmb();
	/* Nasty way to force a rmb on all cpus */
	smp_call_function(ftrace_sync_ipi, NULL, 1);
	/* OK, we are all set to update the ftrace_trace_function now! */
#endif /* !CONFIG_DYNAMIC_FTRACE */

261 262 263
	ftrace_trace_function = func;
}

264 265
static void add_ftrace_ops(struct ftrace_ops __rcu **list,
			   struct ftrace_ops *ops)
266
{
267 268
	rcu_assign_pointer(ops->next, *list);

269
	/*
270
	 * We are entering ops into the list but another
271 272
	 * CPU might be walking that list. We need to make sure
	 * the ops->next pointer is valid before another CPU sees
273
	 * the ops pointer included into the list.
274
	 */
275
	rcu_assign_pointer(*list, ops);
276 277
}

278 279
static int remove_ftrace_ops(struct ftrace_ops __rcu **list,
			     struct ftrace_ops *ops)
280 281 282 283
{
	struct ftrace_ops **p;

	/*
284 285
	 * If we are removing the last function, then simply point
	 * to the ftrace_stub.
286
	 */
287 288 289 290
	if (rcu_dereference_protected(*list,
			lockdep_is_held(&ftrace_lock)) == ops &&
	    rcu_dereference_protected(ops->next,
			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
291
		*list = &ftrace_list_end;
Steven Rostedt's avatar
Steven Rostedt committed
292
		return 0;
293 294
	}

295
	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
296 297 298
		if (*p == ops)
			break;

Steven Rostedt's avatar
Steven Rostedt committed
299 300
	if (*p != ops)
		return -1;
301 302

	*p = (*p)->next;
303 304
	return 0;
}
305

306 307
static void ftrace_update_trampoline(struct ftrace_ops *ops);

308
int __register_ftrace_function(struct ftrace_ops *ops)
309
{
310 311 312
	if (ops->flags & FTRACE_OPS_FL_DELETED)
		return -EINVAL;

313 314 315
	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
		return -EBUSY;

316
#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
317 318 319 320 321 322 323 324 325 326 327 328 329
	/*
	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
	 */
	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
		return -EINVAL;

	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
#endif

330 331 332
	if (!core_kernel_data((unsigned long)ops))
		ops->flags |= FTRACE_OPS_FL_DYNAMIC;

333
	add_ftrace_ops(&ftrace_ops_list, ops);
334

335 336 337
	/* Always save the function, and reset at unregistering */
	ops->saved_func = ops->func;

338
	if (ftrace_pids_enabled(ops))
339 340
		ops->func = ftrace_pid_func;

341 342
	ftrace_update_trampoline(ops);

343 344 345 346 347 348
	if (ftrace_enabled)
		update_ftrace_function();

	return 0;
}

349
int __unregister_ftrace_function(struct ftrace_ops *ops)
350 351 352
{
	int ret;

353 354 355
	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
		return -EBUSY;

356
	ret = remove_ftrace_ops(&ftrace_ops_list, ops);
357

358 359
	if (ret < 0)
		return ret;
360

361 362
	if (ftrace_enabled)
		update_ftrace_function();
363

364 365
	ops->func = ops->saved_func;

Steven Rostedt's avatar
Steven Rostedt committed
366
	return 0;
367 368
}

369 370
static void ftrace_update_pid_func(void)
{
371 372
	struct ftrace_ops *op;

373
	/* Only do something if we are tracing something */
374
	if (ftrace_trace_function == ftrace_stub)
375
		return;
376

377 378
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op->flags & FTRACE_OPS_FL_PID) {
379 380
			op->func = ftrace_pids_enabled(op) ?
				ftrace_pid_func : op->saved_func;
381 382 383 384
			ftrace_update_trampoline(op);
		}
	} while_for_each_ftrace_op(op);

385
	update_ftrace_function();
386 387
}

388 389 390 391 392
#ifdef CONFIG_FUNCTION_PROFILER
struct ftrace_profile {
	struct hlist_node		node;
	unsigned long			ip;
	unsigned long			counter;
393 394
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	unsigned long long		time;
395
	unsigned long long		time_squared;
396
#endif
397 398
};

399 400 401 402
struct ftrace_profile_page {
	struct ftrace_profile_page	*next;
	unsigned long			index;
	struct ftrace_profile		records[];
403 404
};

405 406 407 408 409 410 411 412
struct ftrace_profile_stat {
	atomic_t			disabled;
	struct hlist_head		*hash;
	struct ftrace_profile_page	*pages;
	struct ftrace_profile_page	*start;
	struct tracer_stat		stat;
};

413 414
#define PROFILE_RECORDS_SIZE						\
	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
415

416 417
#define PROFILES_PER_PAGE					\
	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
418

419 420 421
static int ftrace_profile_enabled __read_mostly;

/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
422 423
static DEFINE_MUTEX(ftrace_profile_lock);

424
static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
425

426 427
#define FTRACE_PROFILE_HASH_BITS 10
#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
428

429 430 431
static void *
function_stat_next(void *v, int idx)
{
432 433
	struct ftrace_profile *rec = v;
	struct ftrace_profile_page *pg;
434

435
	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
436 437

 again:
Li Zefan's avatar
Li Zefan committed
438 439 440
	if (idx != 0)
		rec++;

441 442 443 444 445
	if ((void *)rec >= (void *)&pg->records[pg->index]) {
		pg = pg->next;
		if (!pg)
			return NULL;
		rec = &pg->records[0];
446 447
		if (!rec->counter)
			goto again;
448 449 450 451 452 453 454
	}

	return rec;
}

static void *function_stat_start(struct tracer_stat *trace)
{
455 456 457 458 459 460 461
	struct ftrace_profile_stat *stat =
		container_of(trace, struct ftrace_profile_stat, stat);

	if (!stat || !stat->start)
		return NULL;

	return function_stat_next(&stat->start->records[0], 0);
462 463
}

464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/* function graph compares on total time */
static int function_stat_cmp(void *p1, void *p2)
{
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;

	if (a->time < b->time)
		return -1;
	if (a->time > b->time)
		return 1;
	else
		return 0;
}
#else
/* not function graph compares against hits */
480 481
static int function_stat_cmp(void *p1, void *p2)
{
482 483
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;
484 485 486 487 488 489 490 491

	if (a->counter < b->counter)
		return -1;
	if (a->counter > b->counter)
		return 1;
	else
		return 0;
}
492
#endif
493 494 495

static int function_stat_headers(struct seq_file *m)
{
496
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
497 498 499 500
	seq_puts(m, "  Function                               "
		 "Hit    Time            Avg             s^2\n"
		    "  --------                               "
		 "---    ----            ---             ---\n");
501
#else
502 503
	seq_puts(m, "  Function                               Hit\n"
		    "  --------                               ---\n");
504
#endif
505 506 507 508 509
	return 0;
}

static int function_stat_show(struct seq_file *m, void *v)
{
510
	struct ftrace_profile *rec = v;
511
	char str[KSYM_SYMBOL_LEN];
512
	int ret = 0;
513
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
514 515
	static struct trace_seq s;
	unsigned long long avg;
516
	unsigned long long stddev;
517
#endif
518 519 520 521 522 523 524
	mutex_lock(&ftrace_profile_lock);

	/* we raced with function_profile_reset() */
	if (unlikely(rec->counter == 0)) {
		ret = -EBUSY;
		goto out;
	}
525

526 527 528 529 530 531 532
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	avg = rec->time;
	do_div(avg, rec->counter);
	if (tracing_thresh && (avg < tracing_thresh))
		goto out;
#endif

533
	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
534 535 536
	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
537
	seq_puts(m, "    ");
538

539 540 541 542
	/* Sample standard deviation (s^2) */
	if (rec->counter <= 1)
		stddev = 0;
	else {
543 544 545 546 547 548 549
		/*
		 * Apply Welford's method:
		 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
		 */
		stddev = rec->counter * rec->time_squared -
			 rec->time * rec->time;

550 551 552 553
		/*
		 * Divide only 1000 for ns^2 -> us^2 conversion.
		 * trace_print_graph_duration will divide 1000 again.
		 */
554
		do_div(stddev, rec->counter * (rec->counter - 1) * 1000);
555 556
	}

557 558 559 560
	trace_seq_init(&s);
	trace_print_graph_duration(rec->time, &s);
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(avg, &s);
561 562
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(stddev, &s);
563 564 565
	trace_print_seq(m, &s);
#endif
	seq_putc(m, '\n');
566 567
out:
	mutex_unlock(&ftrace_profile_lock);
568

569
	return ret;
570 571
}

572
static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
573
{
574
	struct ftrace_profile_page *pg;
575

576
	pg = stat->pages = stat->start;
577

578 579 580 581
	while (pg) {
		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
		pg->index = 0;
		pg = pg->next;
582 583
	}

584
	memset(stat->hash, 0,
585 586
	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
}
587

588
int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
589 590
{
	struct ftrace_profile_page *pg;
591 592
	int functions;
	int pages;
593
	int i;
594

595
	/* If we already allocated, do nothing */
596
	if (stat->pages)
597
		return 0;
598

599 600
	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
	if (!stat->pages)
601
		return -ENOMEM;
602

603 604 605 606 607 608 609 610 611 612 613 614 615
#ifdef CONFIG_DYNAMIC_FTRACE
	functions = ftrace_update_tot_cnt;
#else
	/*
	 * We do not know the number of functions that exist because
	 * dynamic tracing is what counts them. With past experience
	 * we have around 20K functions. That should be more than enough.
	 * It is highly unlikely we will execute every function in
	 * the kernel.
	 */
	functions = 20000;
#endif

616
	pg = stat->start = stat->pages;
617

618 619
	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);

620
	for (i = 1; i < pages; i++) {
621 622
		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
		if (!pg->next)
623
			goto out_free;
624 625 626 627
		pg = pg->next;
	}

	return 0;
628 629 630 631 632 633 634 635 636 637 638 639 640 641

 out_free:
	pg = stat->start;
	while (pg) {
		unsigned long tmp = (unsigned long)pg;

		pg = pg->next;
		free_page(tmp);
	}

	stat->pages = NULL;
	stat->start = NULL;

	return -ENOMEM;
642 643
}

644
static int ftrace_profile_init_cpu(int cpu)
645
{
646
	struct ftrace_profile_stat *stat;
647
	int size;
648

649 650 651
	stat = &per_cpu(ftrace_profile_stats, cpu);

	if (stat->hash) {
652
		/* If the profile is already created, simply reset it */
653
		ftrace_profile_reset(stat);
654 655
		return 0;
	}
656

657 658 659 660 661
	/*
	 * We are profiling all functions, but usually only a few thousand
	 * functions are hit. We'll make a hash of 1024 items.
	 */
	size = FTRACE_PROFILE_HASH_SIZE;
662

663
	stat->hash = kcalloc(size, sizeof(struct hlist_head), GFP_KERNEL);
664

665
	if (!stat->hash)
666 667
		return -ENOMEM;

668
	/* Preallocate the function profiling pages */
669 670 671
	if (ftrace_profile_pages_init(stat) < 0) {
		kfree(stat->hash);
		stat->hash = NULL;
672 673 674 675
		return -ENOMEM;
	}

	return 0;
676 677
}

678 679 680 681 682
static int ftrace_profile_init(void)
{
	int cpu;
	int ret = 0;

683
	for_each_possible_cpu(cpu) {
684 685 686 687 688 689 690 691
		ret = ftrace_profile_init_cpu(cpu);
		if (ret)
			break;
	}

	return ret;
}

692
/* interrupts must be disabled */
693 694
static struct ftrace_profile *
ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
695
{
696
	struct ftrace_profile *rec;
697 698 699
	struct hlist_head *hhd;
	unsigned long key;

700
	key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
701
	hhd = &stat->hash[key];
702 703 704 705

	if (hlist_empty(hhd))
		return NULL;

706
	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
707
		if (rec->ip == ip)
708 709 710 711 712 713
			return rec;
	}

	return NULL;
}

714 715
static void ftrace_add_profile(struct ftrace_profile_stat *stat,
			       struct ftrace_profile *rec)
716 717 718
{
	unsigned long key;

719
	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
720
	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
721 722
}

723 724 725
/*
 * The memory is already allocated, this simply finds a new record to use.
 */
726
static struct ftrace_profile *
727
ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
728 729 730
{
	struct ftrace_profile *rec = NULL;

731
	/* prevent recursion (from NMIs) */
732
	if (atomic_inc_return(&stat->disabled) != 1)
733 734 735
		goto out;

	/*
736 737
	 * Try to find the function again since an NMI
	 * could have added it
738
	 */
739
	rec = ftrace_find_profiled_func(stat, ip);
740
	if (rec)
741
		goto out;
742

743 744 745 746
	if (stat->pages->index == PROFILES_PER_PAGE) {
		if (!stat->pages->next)
			goto out;
		stat->pages = stat->pages->next;
747
	}
748

749
	rec = &stat->pages->records[stat->pages->index++];
750
	rec->ip = ip;
751
	ftrace_add_profile(stat, rec);
752

753
 out:
754
	atomic_dec(&stat->disabled);
755 756 757 758 759

	return rec;
}

static void
760
function_profile_call(unsigned long ip, unsigned long parent_ip,
761
		      struct ftrace_ops *ops, struct pt_regs *regs)
762
{
763
	struct ftrace_profile_stat *stat;
764
	struct ftrace_profile *rec;
765 766 767 768 769 770
	unsigned long flags;

	if (!ftrace_profile_enabled)
		return;

	local_irq_save(flags);
771

772
	stat = this_cpu_ptr(&ftrace_profile_stats);
773
	if (!stat->hash || !ftrace_profile_enabled)
774 775 776
		goto out;

	rec = ftrace_find_profiled_func(stat, ip);
777
	if (!rec) {
778
		rec = ftrace_profile_alloc(stat, ip);
779 780 781
		if (!rec)
			goto out;
	}
782 783 784 785 786 787

	rec->counter++;
 out:
	local_irq_restore(flags);
}

788
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
789 790 791 792 793 794 795
static bool fgraph_graph_time = true;

void ftrace_graph_graph_time_control(bool enable)
{
	fgraph_graph_time = enable;
}

796 797
static int profile_graph_entry(struct ftrace_graph_ent *trace)
{
798
	struct ftrace_ret_stack *ret_stack;
799

800
	function_profile_call(trace->func, 0, NULL, NULL);
801

802 803 804 805
	/* If function graph is shutting down, ret_stack can be NULL */
	if (!current->ret_stack)
		return 0;

806 807 808
	ret_stack = ftrace_graph_get_ret_stack(current, 0);
	if (ret_stack)
		ret_stack->subtime = 0;
809

810 811 812 813 814
	return 1;
}

static void profile_graph_return(struct ftrace_graph_ret *trace)
{
815
	struct ftrace_ret_stack *ret_stack;
816
	struct ftrace_profile_stat *stat;
817
	unsigned long long calltime;
818
	struct ftrace_profile *rec;
819
	unsigned long flags;
820 821

	local_irq_save(flags);
822
	stat = this_cpu_ptr(&ftrace_profile_stats);
823
	if (!stat->hash || !ftrace_profile_enabled)
824 825
		goto out;

826 827 828 829
	/* If the calltime was zero'd ignore it */
	if (!trace->calltime)
		goto out;

830 831
	calltime = trace->rettime - trace->calltime;

832
	if (!fgraph_graph_time) {
833 834

		/* Append this call time to the parent time to subtract */
835 836 837
		ret_stack = ftrace_graph_get_ret_stack(current, 1);
		if (ret_stack)
			ret_stack->subtime += calltime;
838

839 840 841
		ret_stack = ftrace_graph_get_ret_stack(current, 0);
		if (ret_stack && ret_stack->subtime < calltime)
			calltime -= ret_stack->subtime;
842 843 844 845
		else
			calltime = 0;
	}

846
	rec = ftrace_find_profiled_func(stat, trace->func);
847
	if (rec) {
848
		rec->time += calltime;
849 850
		rec->time_squared += calltime * calltime;
	}
851

852
 out:
853 854 855
	local_irq_restore(flags);
}

856 857 858 859 860
static struct fgraph_ops fprofiler_ops = {
	.entryfunc = &profile_graph_entry,
	.retfunc = &profile_graph_return,
};

861 862
static int register_ftrace_profiler(void)
{
863
	return register_ftrace_graph(&fprofiler_ops);
864 865 866 867
}

static void unregister_ftrace_profiler(void)
{
868
	unregister_ftrace_graph(&fprofiler_ops);
869 870
}
#else
871
static struct ftrace_ops ftrace_profile_ops __read_mostly = {
872
	.func		= function_profile_call,
873
	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
874
	INIT_OPS_HASH(ftrace_profile_ops)
875 876
};

877 878 879 880 881 882 883 884 885 886 887
static int register_ftrace_profiler(void)
{
	return register_ftrace_function(&ftrace_profile_ops);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_function(&ftrace_profile_ops);
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

888 889 890 891 892 893 894
static ssize_t
ftrace_profile_write(struct file *filp, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	unsigned long val;
	int ret;

895 896
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
897 898 899 900 901 902 903
		return ret;

	val = !!val;

	mutex_lock(&ftrace_profile_lock);
	if (ftrace_profile_enabled ^ val) {
		if (val) {
904 905 906 907 908 909
			ret = ftrace_profile_init();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}

910 911 912 913 914
			ret = register_ftrace_profiler();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}
915 916 917
			ftrace_profile_enabled = 1;
		} else {
			ftrace_profile_enabled = 0;
918 919
			/*
			 * unregister_ftrace_profiler calls stop_machine
920
			 * so this acts like an synchronize_rcu.
921
			 */
922
			unregister_ftrace_profiler();
923 924
		}
	}
925
 out:
926 927
	mutex_unlock(&ftrace_profile_lock);

928
	*ppos += cnt;
929 930 931 932

	return cnt;
}

933 934 935 936
static ssize_t
ftrace_profile_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
937
	char buf[64];		/* big enough to hold a number */
938 939 940 941 942 943
	int r;

	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

944 945 946 947
static const struct file_operations ftrace_profile_fops = {
	.open		= tracing_open_generic,
	.read		= ftrace_profile_read,
	.write		= ftrace_profile_write,
948
	.llseek		= default_llseek,
949 950
};

951 952
/* used to initialize the real stat files */
static struct tracer_stat function_stats __initdata = {
953 954 955 956 957 958
	.name		= "functions",
	.stat_start	= function_stat_start,
	.stat_next	= function_stat_next,
	.stat_cmp	= function_stat_cmp,
	.stat_headers	= function_stat_headers,
	.stat_show	= function_stat_show
959 960
};

961
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
962
{
963
	struct ftrace_profile_stat *stat;
964
	struct dentry *entry;
965
	char *name;
966
	int ret;
967 968 969 970 971
	int cpu;

	for_each_possible_cpu(cpu) {
		stat = &per_cpu(ftrace_profile_stats, cpu);

972
		name = kasprintf(GFP_KERNEL, "function%d", cpu);
973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992
		if (!name) {
			/*
			 * The files created are permanent, if something happens
			 * we still do not free memory.
			 */
			WARN(1,
			     "Could not allocate stat file for cpu %d\n",
			     cpu);
			return;
		}
		stat->stat = function_stats;
		stat->stat.name = name;
		ret = register_stat_tracer(&stat->stat);
		if (ret) {
			WARN(1,
			     "Could not register function stat for cpu %d\n",
			     cpu);
			kfree(name);
			return;
		}
993 994
	}

995
	entry = tracefs_create_file("function_profile_enabled", 0644,
996 997
				    d_tracer, NULL, &ftrace_profile_fops);
	if (!entry)
998
		pr_warn("Could not create tracefs 'function_profile_enabled' entry\n");
999 1000 1001
}

#else /* CONFIG_FUNCTION_PROFILER */
1002
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
1003 1004 1005 1006
{
}
#endif /* CONFIG_FUNCTION_PROFILER */

1007 1008
#ifdef CONFIG_DYNAMIC_FTRACE

1009 1010
static struct ftrace_ops *removed_ops;

1011 1012 1013 1014 1015 1016
/*
 * Set when doing a global update, like enabling all recs or disabling them.
 * It is not set when just updating a single ftrace_ops.
 */
static bool update_all_ops;

1017 1018 1019 1020
#ifndef CONFIG_FTRACE_MCOUNT_RECORD
# error Dynamic ftrace depends on MCOUNT_RECORD
#endif

1021 1022 1023 1024 1025
struct ftrace_func_entry {
	struct hlist_node hlist;
	unsigned long ip;
};

1026 1027 1028 1029 1030
struct ftrace_func_probe {
	struct ftrace_probe_ops	*probe_ops;
	struct ftrace_ops	ops;
	struct trace_array	*tr;
	struct list_head	list;
1031
	void			*data;
1032 1033 1034
	int			ref;
};

1035 1036 1037 1038 1039 1040 1041 1042 1043
/*
 * We make these constant because no one should touch them,
 * but they are used as the default "empty hash", to avoid allocating
 * it all the time. These are in a read only section such that if
 * anyone does try to modify it, it will cause an exception.
 */
static const struct hlist_head empty_buckets[1];
static const struct ftrace_hash empty_hash = {
	.buckets = (struct hlist_head *)empty_buckets,
1044
};
1045
#define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
1046

1047
struct ftrace_ops global_ops = {
1048 1049 1050 1051 1052
	.func				= ftrace_stub,
	.local_hash.notrace_hash	= EMPTY_HASH,
	.local_hash.filter_hash		= EMPTY_HASH,
	INIT_OPS_HASH(global_ops)
	.flags				= FTRACE_OPS_FL_RECURSION_SAFE |
1053 1054
					  FTRACE_OPS_FL_INITIALIZED |
					  FTRACE_OPS_FL_PID,
1055 1056
};

1057
/*
1058
 * Used by the stack undwinder to know about dynamic ftrace trampolines.
1059
 */
1060
struct ftrace_ops *ftrace_ops_trampoline(unsigned long addr)
1061
{
1062
	struct ftrace_ops *op = NULL;
1063 1064 1065

	/*
	 * Some of the ops may be dynamically allocated,
1066
	 * they are freed after a synchronize_rcu().
1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078
	 */
	preempt_disable_notrace();

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/*
		 * This is to check for dynamically allocated trampolines.
		 * Trampolines that are in kernel text will have
		 * core_kernel_text() return true.
		 */
		if (op->trampoline && op->trampoline_size)
			if (addr >= op->trampoline &&
			    addr < op->trampoline + op->trampoline_size) {
1079 1080
				preempt_enable_notrace();
				return op;
1081 1082 1083 1084
			}
	} while_for_each_ftrace_op(op);
	preempt_enable_notrace();

1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096
	return NULL;
}

/*
 * This is used by __kernel_text_address() to return true if the
 * address is on a dynamically allocated trampoline that would
 * not return true for either core_kernel_text() or
 * is_module_text_address().
 */
bool is_ftrace_trampoline(unsigned long addr)
{
	return ftrace_ops_trampoline(addr) != NULL;
1097 1098
}

1099 1100
struct ftrace_page {
	struct ftrace_page	*next;
1101
	struct dyn_ftrace	*records;
1102
	int			index;
1103
	int			size;
1104 1105
};

1106 1107
#define ENTRY_SIZE sizeof(struct dyn_ftrace)
#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1108 1109 1110 1111 1112 1113 1114

/* estimate from running different kernels */
#define NR_TO_INIT		10000

static struct ftrace_page	*ftrace_pages_start;
static struct ftrace_page	*ftrace_pages;

1115 1116 1117 1118 1119 1120 1121 1122 1123
static __always_inline unsigned long
ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip)
{
	if (hash->size_bits > 0)
		return hash_long(ip, hash->size_bits);

	return 0;
}

1124 1125 1126
/* Only use this function if ftrace_hash_empty() has already been tested */
static __always_inline struct ftrace_func_entry *
__ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1127 1128 1129 1130 1131
{
	unsigned long key;
	struct ftrace_func_entry *entry;
	struct hlist_head *hhd;

1132
	key = ftrace_hash_key(hash, ip);
1133 1134
	hhd = &hash->buckets[key];

1135
	hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
1136 1137 1138 1139 1140 1141
		if (entry->ip == ip)
			return entry;
	}
	return NULL;
}

1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160
/**
 * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash
 * @hash: The hash to look at
 * @ip: The instruction pointer to test
 *
 * Search a given @hash to see if a given instruction pointer (@ip)
 * exists in it.
 *
 * Returns the entry that holds the @ip if found. NULL otherwise.
 */
struct ftrace_func_entry *
ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
{
	if (ftrace_hash_empty(hash))
		return NULL;

	return __ftrace_lookup_ip(hash, ip);
}

1161 1162
static void __add_hash_entry(struct ftrace_hash *hash,
			     struct ftrace_func_entry *entry)
1163 1164 1165 1166
{
	struct hlist_head *hhd;
	unsigned long key;

1167
	key = ftrace_hash_key(hash, entry->ip);
1168 1169 1170
	hhd = &hash->buckets[key];
	hlist_add_head(&entry->hlist, hhd);
	hash->count++;
1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
}

static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
{
	struct ftrace_func_entry *entry;

	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
	if (!entry)
		return -ENOMEM;

	entry->ip = ip;
	__add_hash_entry(hash, entry);
1183 1184 1185 1186 1187

	return 0;
}

static void
1188
free_hash_entry(struct ftrace_hash *hash,
1189 1190 1191 1192 1193 1194 1195
		  struct ftrace_func_entry *entry)
{
	hlist_del(&entry->hlist);
	kfree(entry);
	hash->count--;
}

1196 1197 1198 1199
static void
remove_hash_entry(struct ftrace_hash *hash,
		  struct ftrace_func_entry *entry)
{
1200
	hlist_del_rcu(&entry->hlist);
1201 1202 1203
	hash->count--;
}

1204 1205 1206
static void ftrace_hash_clear(struct ftrace_hash *hash)
{
	struct hlist_head *hhd;
1207
	struct hlist_node *tn;
1208 1209 1210 1211
	struct ftrace_func_entry *entry;
	int size = 1 << hash->size_bits;
	int i;

1212 1213 1214
	if (!hash->count)
		return;

1215 1216
	for (i = 0; i < size; i++) {
		hhd = &hash->buckets[i];
1217
		hlist_for_each_entry_safe(entry, tn, hhd, hlist)
1218
			free_hash_entry(hash, entry);
1219 1220 1221 1222
	}
	FTRACE_WARN_ON(hash->count);
}

1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
static void free_ftrace_mod(struct ftrace_mod_load *ftrace_mod)
{
	list_del(&ftrace_mod->list);
	kfree(ftrace_mod->module);
	kfree(ftrace_mod->func);
	kfree(ftrace_mod);
}

static void clear_ftrace_mod_list(struct list_head *head)
{
	struct ftrace_mod_load *p, *n;

	/* stack tracer isn't supported yet */
	if (!head)
		return;

	mutex_lock(&ftrace_lock);
	list_for_each_entry_safe(p, n, head, list)
		free_ftrace_mod(p);
	mutex_unlock(&ftrace_lock);
}

1245 1246 1247 1248 1249 1250 1251 1252 1253
static void free_ftrace_hash(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
	ftrace_hash_clear(hash);
	kfree(hash->buckets);
	kfree(hash);
}

1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265
static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
{
	struct ftrace_hash *hash;

	hash = container_of(rcu, struct ftrace_hash, rcu);
	free_ftrace_hash(hash);
}

static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
1266
	call_rcu(&hash->rcu, __free_ftrace_hash_rcu);
1267 1268
}

1269 1270
void ftrace_free_filter(struct ftrace_ops *ops)
{
1271
	ftrace_ops_init(ops);
1272 1273
	free_ftrace_hash(ops->func_hash->filter_hash);
	free_ftrace_hash(ops->func_hash->notrace_hash);
1274 1275
}

1276 1277 1278 1279 1280 1281 1282 1283 1284 1285
static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
{
	struct ftrace_hash *hash;
	int size;

	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
	if (!hash)
		return NULL;

	size = 1 << size_bits;
1286
	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297

	if (!hash->buckets) {
		kfree(hash);
		return NULL;
	}

	hash->size_bits = size_bits;

	return hash;
}

1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326

static int ftrace_add_mod(struct trace_array *tr,
			  const char *func, const char *module,
			  int enable)
{
	struct ftrace_mod_load *ftrace_mod;
	struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace;

	ftrace_mod = kzalloc(sizeof(*ftrace_mod), GFP_KERNEL);
	if (!ftrace_mod)
		return -ENOMEM;

	ftrace_mod->func = kstrdup(func, GFP_KERNEL);
	ftrace_mod->module = kstrdup(module, GFP_KERNEL);
	ftrace_mod->enable = enable;

	if (!ftrace_mod->func || !ftrace_mod->module)
		goto out_free;

	list_add(&ftrace_mod->list, mod_head);

	return 0;

 out_free:
	free_ftrace_mod(ftrace_mod);

	return -ENOMEM;
}

1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339
static struct ftrace_hash *
alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
{
	struct ftrace_func_entry *entry;
	struct ftrace_hash *new_hash;
	int size;
	int ret;
	int i;

	new_hash = alloc_ftrace_hash(size_bits);
	if (!new_hash)
		return NULL;

1340 1341 1342
	if (hash)
		new_hash->flags = hash->flags;

1343
	/* Empty hash? */
1344
	if (ftrace_hash_empty(hash))
1345 1346 1347 1348
		return new_hash;

	size = 1 << hash->size_bits;
	for (i = 0; i < size; i++) {
1349
		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364
			ret = add_hash_entry(new_hash, entry->ip);
			if (ret < 0)
				goto free_hash;
		}
	}

	FTRACE_WARN_ON(new_hash->count != hash->count);

	return new_hash;

 free_hash:
	free_ftrace_hash(new_hash);
	return NULL;
}

1365
static void
1366
ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
1367
static void
1368
ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
1369

1370 1371 1372
static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
				       struct ftrace_hash *new_hash);

1373 1374
static struct ftrace_hash *
__ftrace_hash_move(struct ftrace_hash *src)
1375 1376
{
	struct ftrace_func_entry *entry;
1377
	struct hlist_node *tn;
1378
	struct hlist_head *hhd;
1379
	struct ftrace_hash *new_hash;
1380 1381 1382 1383 1384
	int size = src->count;
	int bits = 0;
	int i;

	/*
1385
	 * If the new source is empty, just return the empty_hash.
1386
	 */
1387
	if (ftrace_hash_empty(src))
1388
		return EMPTY_HASH;
1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399

	/*
	 * Make the hash size about 1/2 the # found
	 */
	for (size /= 2; size; size >>= 1)
		bits++;

	/* Don't allocate too much */
	if (bits > FTRACE_HASH_MAX_BITS)
		bits = FTRACE_HASH_MAX_BITS;

1400 1401
	new_hash = alloc_ftrace_hash(bits);
	if (!new_hash)
1402
		return NULL;
1403

1404 1405
	new_hash->flags = src->flags;

1406 1407 1408
	size = 1 << src->size_bits;
	for (i = 0; i < size; i++) {
		hhd = &src->buckets[i];
1409
		hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
1410
			remove_hash_entry(src, entry);
1411
			__add_hash_entry(new_hash, entry);
1412 1413 1414
		}
	}

1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
	return new_hash;
}

static int
ftrace_hash_move(struct ftrace_ops *ops, int enable,
		 struct ftrace_hash **dst, struct ftrace_hash *src)
{
	struct ftrace_hash *new_hash;
	int ret;

	/* Reject setting notrace hash on IPMODIFY ftrace_ops */
	if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
		return -EINVAL;

	new_hash = __ftrace_hash_move(src);
	if (!new_hash)
		return -ENOMEM;

1433 1434 1435 1436 1437 1438 1439 1440 1441 1442
	/* Make sure this can be applied if it is IPMODIFY ftrace_ops */
	if (enable) {
		/* IPMODIFY should be updated only when filter_hash updating */
		ret = ftrace_hash_ipmodify_update(ops, new_hash);
		if (ret < 0) {
			free_ftrace_hash(new_hash);
			return ret;
		}
	}

1443 1444 1445 1446
	/*
	 * Remove the current set, update the hash and add
	 * them back.
	 */
1447
	ftrace_hash_rec_disable_modify(ops, enable);
1448

1449 1450
	rcu_assign_pointer(*dst, new_hash);

1451
	ftrace_hash_rec_enable_modify(ops, enable);
1452

1453
	return 0;
1454 1455
}

1456 1457 1458 1459 1460 1461 1462 1463 1464 1465
static bool hash_contains_ip(unsigned long ip,
			     struct ftrace_ops_hash *hash)
{
	/*
	 * The function record is a match if it exists in the filter
	 * hash and not in the notrace hash. Note, an emty hash is
	 * considered a match for the filter hash, but an empty
	 * notrace hash is considered not in the notrace hash.
	 */
	return (ftrace_hash_empty(hash->filter_hash) ||
1466
		__ftrace_lookup_ip(hash->filter_hash, ip)) &&
1467
		(ftrace_hash_empty(hash->notrace_hash) ||
1468
		 !__ftrace_lookup_ip(hash->notrace_hash, ip));
1469 1470
}

1471 1472 1473 1474 1475 1476 1477 1478
/*
 * Test the hashes for this ops to see if we want to call
 * the ops->func or not.
 *
 * It's a match if the ip is in the ops->filter_hash or
 * the filter_hash does not exist or is empty,
 *  AND
 * the ip is not in the ops->notrace_hash.
1479 1480
 *
 * This needs to be called with preemption disabled as
1481
 * the hashes are freed with call_rcu().
1482
 */
1483
int
1484
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
1485
{
1486
	struct ftrace_ops_hash hash;
1487 1488
	int ret;

1489 1490 1491 1492 1493 1494 1495 1496 1497 1498
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
	/*
	 * There's a small race when adding ops that the ftrace handler
	 * that wants regs, may be called without them. We can not
	 * allow that handler to be called if regs is NULL.
	 */
	if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
		return 0;
#endif

1499 1500
	rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash);
	rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash);
1501

1502
	if (hash_contains_ip(ip, &hash))
1503 1504 1505 1506 1507 1508 1509
		ret = 1;
	else
		ret = 0;

	return ret;
}

1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523
/*
 * This is a double for. Do not use 'break' to break out of the loop,
 * you must use a goto.
 */
#define do_for_each_ftrace_rec(pg, rec)					\
	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
		int _____i;						\
		for (_____i = 0; _____i < pg->index; _____i++) {	\
			rec = &pg->records[_____i];

#define while_for_each_ftrace_rec()		\
		}				\
	}

1524 1525 1526

static int ftrace_cmp_recs(const void *a, const void *b)
{
1527 1528
	const struct dyn_ftrace *key = a;
	const struct dyn_ftrace *rec = b;
1529

1530
	if (key->flags < rec->ip)
1531
		return -1;
1532 1533
	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
		return 1;
1534 1535 1536
	return 0;
}

1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549
/**
 * ftrace_location_range - return the first address of a traced location
 *	if it touches the given ip range
 * @start: start of range to search.
 * @end: end of range to search (inclusive). @end points to the last byte
 *	to check.
 *
 * Returns rec->ip if the related ftrace location is a least partly within
 * the given address range. That is, the first address of the instruction
 * that is either a NOP or call to the function tracer. It checks the ftrace
 * internal tables to determine if the address belongs or not.
 */
unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1550 1551 1552
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
1553
	struct dyn_ftrace key;
1554

1555 1556
	key.ip = start;
	key.flags = end;	/* overload flags, as it is unsigned long */
1557 1558

	for (pg = ftrace_pages_start; pg; pg = pg->next) {
1559 1560
		if (end < pg->records[0].ip ||
		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1561
			continue;
1562 1563 1564 1565
		rec = bsearch(&key, pg->records, pg->index,
			      sizeof(struct dyn_ftrace),
			      ftrace_cmp_recs);
		if (rec)
1566
			return rec->ip;
1567
	}
1568 1569 1570 1571

	return 0;
}

1572 1573 1574 1575
/**
 * ftrace_location - return true if the ip giving is a traced location
 * @ip: the instruction pointer to check
 *
1576
 * Returns rec->ip if @ip given is a pointer to a ftrace location.
1577 1578 1579 1580
 * That is, the instruction that is either a NOP or call to
 * the function tracer. It checks the ftrace internal tables to
 * determine if the address belongs or not.
 */
1581
unsigned long ftrace_location(unsigned long ip)
1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595
{
	return ftrace_location_range(ip, ip);
}

/**
 * ftrace_text_reserved - return true if range contains an ftrace location
 * @start: start of range to search
 * @end: end of range to search (inclusive). @end points to the last byte to check.
 *
 * Returns 1 if @start and @end contains a ftrace location.
 * That is, the instruction that is either a NOP or call to
 * the function tracer. It checks the ftrace internal tables to
 * determine if the address belongs or not.
 */
1596
int ftrace_text_reserved(const void *start, const void *end)
1597
{
1598 1599 1600 1601 1602 1603
	unsigned long ret;

	ret = ftrace_location_range((unsigned long)start,
				    (unsigned long)end);

	return (int)!!ret;
1604 1605
}

1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625
/* Test if ops registered to this rec needs regs */
static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
{
	struct ftrace_ops *ops;
	bool keep_regs = false;

	for (ops = ftrace_ops_list;
	     ops != &ftrace_list_end; ops = ops->next) {
		/* pass rec in as regs to have non-NULL val */
		if (ftrace_ops_test(ops, rec->ip, rec)) {
			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
				keep_regs = true;
				break;
			}
		}
	}

	return  keep_regs;
}

1626 1627 1628 1629 1630
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
static struct ftrace_ops *
ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);

1631
static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
1632 1633 1634 1635 1636 1637 1638
				     int filter_hash,
				     bool inc)
{
	struct ftrace_hash *hash;
	struct ftrace_hash *other_hash;
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
1639
	bool update = false;
1640
	int count = 0;
1641
	int all = false;
1642 1643 1644

	/* Only update if the ops has been registered */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1645
		return false;
1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658

	/*
	 * In the filter_hash case:
	 *   If the count is zero, we update all records.
	 *   Otherwise we just update the items in the hash.
	 *
	 * In the notrace_hash case:
	 *   We enable the update in the hash.
	 *   As disabling notrace means enabling the tracing,
	 *   and enabling notrace means disabling, the inc variable
	 *   gets inversed.
	 */
	if (filter_hash) {
1659 1660
		hash = ops->func_hash->filter_hash;
		other_hash = ops->func_hash->notrace_hash;
1661
		if (ftrace_hash_empty(hash))
1662
			all = true;
1663 1664
	} else {
		inc = !inc;
1665 1666
		hash = ops->func_hash->notrace_hash;
		other_hash = ops->func_hash->filter_hash;
1667 1668 1669 1670
		/*
		 * If the notrace hash has no items,
		 * then there's nothing to do.
		 */
1671
		if (ftrace_hash_empty(hash))
1672
			return false;
1673 1674 1675 1676 1677 1678 1679
	}

	do_for_each_ftrace_rec(pg, rec) {
		int in_other_hash = 0;
		int in_hash = 0;
		int match = 0;

1680 1681 1682
		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1683 1684 1685 1686 1687
		if (all) {
			/*
			 * Only the filter_hash affects all records.
			 * Update if the record is not in the notrace hash.
			 */
1688
			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1689 1690
				match = 1;
		} else {
1691 1692
			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1693 1694

			/*
1695 1696
			 * If filter_hash is set, we want to match all functions
			 * that are in the hash but not in the other hash.
1697
			 *
1698 1699 1700 1701 1702
			 * If filter_hash is not set, then we are decrementing.
			 * That means we match anything that is in the hash
			 * and also in the other_hash. That is, we need to turn
			 * off functions in the other hash because they are disabled
			 * by this hash.
1703 1704 1705 1706
			 */
			if (filter_hash && in_hash && !in_other_hash)
				match = 1;
			else if (!filter_hash && in_hash &&
1707
				 (in_other_hash || ftrace_hash_empty(other_hash)))
1708 1709 1710 1711 1712 1713 1714
				match = 1;
		}
		if (!match)
			continue;

		if (inc) {
			rec->flags++;
1715
			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
1716
				return false;
1717 1718 1719 1720 1721 1722

			/*
			 * If there's only a single callback registered to a
			 * function, and the ops has a trampoline registered
			 * for it, then we can call it directly.
			 */
1723
			if (ftrace_rec_count(rec) == 1 && ops->trampoline)
1724
				rec->flags |= FTRACE_FL_TRAMP;
1725
			else
1726 1727 1728
				/*
				 * If we are adding another function callback
				 * to this function, and the previous had a
1729 1730
				 * custom trampoline in use, then we need to go
				 * back to the default trampoline.
1731
				 */
1732
				rec->flags &= ~FTRACE_FL_TRAMP;
1733

1734 1735 1736 1737 1738 1739
			/*
			 * If any ops wants regs saved for this function
			 * then all ops will get saved regs.
			 */
			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
				rec->flags |= FTRACE_FL_REGS;
1740
		} else {
1741
			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
1742
				return false;
1743
			rec->flags--;
1744

1745 1746 1747 1748 1749 1750
			/*
			 * If the rec had REGS enabled and the ops that is
			 * being removed had REGS set, then see if there is
			 * still any ops for this record that wants regs.
			 * If not, we can stop recording them.
			 */
1751
			if (ftrace_rec_count(rec) > 0 &&
1752 1753 1754 1755 1756
			    rec->flags & FTRACE_FL_REGS &&
			    ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
				if (!test_rec_ops_needs_regs(rec))
					rec->flags &= ~FTRACE_FL_REGS;
			}
1757

1758
			/*
1759 1760 1761 1762 1763
			 * The TRAMP needs to be set only if rec count
			 * is decremented to one, and the ops that is
			 * left has a trampoline. As TRAMP can only be
			 * enabled if there is only a single ops attached
			 * to it.
1764
			 */
1765 1766 1767 1768 1769
			if (ftrace_rec_count(rec) == 1 &&
			    ftrace_find_tramp_ops_any(rec))
				rec->flags |= FTRACE_FL_TRAMP;
			else
				rec->flags &= ~FTRACE_FL_TRAMP;
1770

1771 1772 1773 1774
			/*
			 * flags will be cleared in ftrace_check_record()
			 * if rec count is zero.
			 */
1775 1776
		}
		count++;
1777 1778

		/* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */
1779
		update |= ftrace_test_record(rec, true) != FTRACE_UPDATE_IGNORE;
1780

1781 1782
		/* Shortcut, if we handled all records, we are done. */
		if (!all && count == hash->count)
1783
			return update;
1784
	} while_for_each_ftrace_rec();
1785 1786

	return update;
1787 1788
}

1789
static bool ftrace_hash_rec_disable(struct ftrace_ops *ops,
1790 1791
				    int filter_hash)
{
1792
	return __ftrace_hash_rec_update(ops, filter_hash, 0);
1793 1794
}

1795
static bool ftrace_hash_rec_enable(struct ftrace_ops *ops,
1796 1797
				   int filter_hash)
{
1798
	return __ftrace_hash_rec_update(ops, filter_hash, 1);
1799 1800
}

1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835
static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
					  int filter_hash, int inc)
{
	struct ftrace_ops *op;

	__ftrace_hash_rec_update(ops, filter_hash, inc);

	if (ops->func_hash != &global_ops.local_hash)
		return;

	/*
	 * If the ops shares the global_ops hash, then we need to update
	 * all ops that are enabled and use this hash.
	 */
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/* Already done */
		if (op == ops)
			continue;
		if (op->func_hash == &global_ops.local_hash)
			__ftrace_hash_rec_update(op, filter_hash, inc);
	} while_for_each_ftrace_op(op);
}

static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
					   int filter_hash)
{
	ftrace_hash_rec_update_modify(ops, filter_hash, 0);
}

static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
					  int filter_hash)
{
	ftrace_hash_rec_update_modify(ops, filter_hash, 1);
}

1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868
/*
 * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
 * or no-needed to update, -EBUSY if it detects a conflict of the flag
 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
 * Note that old_hash and new_hash has below meanings
 *  - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
 *  - If the hash is EMPTY_HASH, it hits nothing
 *  - Anything else hits the recs which match the hash entries.
 */
static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
					 struct ftrace_hash *old_hash,
					 struct ftrace_hash *new_hash)
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec, *end = NULL;
	int in_old, in_new;

	/* Only update if the ops has been registered */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
		return 0;

	if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
		return 0;

	/*
	 * Since the IPMODIFY is a very address sensitive action, we do not
	 * allow ftrace_ops to set all functions to new hash.
	 */
	if (!new_hash || !old_hash)
		return -EINVAL;

	/* Update rec->flags */
	do_for_each_ftrace_rec(pg, rec) {
1869 1870 1871 1872

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894
		/* We need to update only differences of filter_hash */
		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
		if (in_old == in_new)
			continue;

		if (in_new) {
			/* New entries must ensure no others are using it */
			if (rec->flags & FTRACE_FL_IPMODIFY)
				goto rollback;
			rec->flags |= FTRACE_FL_IPMODIFY;
		} else /* Removed entry */
			rec->flags &= ~FTRACE_FL_IPMODIFY;
	} while_for_each_ftrace_rec();

	return 0;

rollback:
	end = rec;

	/* Roll back what we did above */
	do_for_each_ftrace_rec(pg, rec) {
1895 1896 1897 1898

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951
		if (rec == end)
			goto err_out;

		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
		if (in_old == in_new)
			continue;

		if (in_new)
			rec->flags &= ~FTRACE_FL_IPMODIFY;
		else
			rec->flags |= FTRACE_FL_IPMODIFY;
	} while_for_each_ftrace_rec();

err_out:
	return -EBUSY;
}

static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
{
	struct ftrace_hash *hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(hash))
		hash = NULL;

	return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
}

/* Disabling always succeeds */
static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
{
	struct ftrace_hash *hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(hash))
		hash = NULL;

	__ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
}

static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
				       struct ftrace_hash *new_hash)
{
	struct ftrace_hash *old_hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(old_hash))
		old_hash = NULL;

	if (ftrace_hash_empty(new_hash))
		new_hash = NULL;

	return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
}

1952
static void print_ip_ins(const char *fmt, const unsigned char *p)
1953 1954 1955 1956 1957 1958 1959 1960 1961
{
	int i;

	printk(KERN_CONT "%s", fmt);

	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
}

1962
enum ftrace_bug_type ftrace_bug_type;
1963
const void *ftrace_expected;
1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984

static void print_bug_type(void)
{
	switch (ftrace_bug_type) {
	case FTRACE_BUG_UNKNOWN:
		break;
	case FTRACE_BUG_INIT:
		pr_info("Initializing ftrace call sites\n");
		break;
	case FTRACE_BUG_NOP:
		pr_info("Setting ftrace call site to NOP\n");
		break;
	case FTRACE_BUG_CALL:
		pr_info("Setting ftrace call site to call ftrace function\n");
		break;
	case FTRACE_BUG_UPDATE:
		pr_info("Updating ftrace call site to call a different ftrace function\n");
		break;
	}
}

1985 1986 1987
/**
 * ftrace_bug - report and shutdown function tracer
 * @failed: The failed type (EFAULT, EINVAL, EPERM)
1988
 * @rec: The record that failed
1989 1990 1991 1992 1993 1994
 *
 * The arch code that enables or disables the function tracing
 * can call ftrace_bug() when it has detected a problem in
 * modifying the code. @failed should be one of either:
 * EFAULT - if the problem happens on reading the @ip address
 * EINVAL - if what is read at @ip is not what was expected
1995
 * EPERM - if the problem happens on writing to the @ip address
1996
 */
1997
void ftrace_bug(int failed, struct dyn_ftrace *rec)
1998
{
1999 2000
	unsigned long ip = rec ? rec->ip : 0;

2001 2002 2003 2004 2005 2006 2007 2008 2009 2010
	switch (failed) {
	case -EFAULT:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on modifying ");
		print_ip_sym(ip);
		break;
	case -EINVAL:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace failed to modify ");
		print_ip_sym(ip);
2011
		print_ip_ins(" actual:   ", (unsigned char *)ip);
2012
		pr_cont("\n");
2013 2014 2015 2016
		if (ftrace_expected) {
			print_ip_ins(" expected: ", ftrace_expected);
			pr_cont("\n");
		}
2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027
		break;
	case -EPERM:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on writing ");
		print_ip_sym(ip);
		break;
	default:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on unknown error ");
		print_ip_sym(ip);
	}
2028
	print_bug_type();
2029 2030 2031 2032 2033 2034 2035 2036
	if (rec) {
		struct ftrace_ops *ops = NULL;

		pr_info("ftrace record flags: %lx\n", rec->flags);
		pr_cont(" (%ld)%s", ftrace_rec_count(rec),
			rec->flags & FTRACE_FL_REGS ? " R" : "  ");
		if (rec->flags & FTRACE_FL_TRAMP_EN) {
			ops = ftrace_find_tramp_ops_any(rec);
2037 2038 2039 2040 2041 2042 2043 2044
			if (ops) {
				do {
					pr_cont("\ttramp: %pS (%pS)",
						(void *)ops->trampoline,
						(void *)ops->func);
					ops = ftrace_find_tramp_ops_next(rec, ops);
				} while (ops);
			} else
2045 2046 2047 2048
				pr_cont("\ttramp: ERROR!");

		}
		ip = ftrace_get_addr_curr(rec);
2049
		pr_cont("\n expected tramp: %lx\n", ip);
2050
	}
2051 2052
}

2053
static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update)
2054
{
2055
	unsigned long flag = 0UL;
2056

2057 2058
	ftrace_bug_type = FTRACE_BUG_UNKNOWN;

2059 2060 2061
	if (rec->flags & FTRACE_FL_DISABLED)
		return FTRACE_UPDATE_IGNORE;

2062
	/*
2063
	 * If we are updating calls:
2064
	 *
2065 2066
	 *   If the record has a ref count, then we need to enable it
	 *   because someone is using it.
2067
	 *
2068 2069
	 *   Otherwise we make sure its disabled.
	 *
2070
	 * If we are disabling calls, then disable all records that
2071
	 * are enabled.
2072
	 */
2073
	if (enable && ftrace_rec_count(rec))
2074
		flag = FTRACE_FL_ENABLED;
2075

2076
	/*
2077 2078 2079
	 * If enabling and the REGS flag does not match the REGS_EN, or
	 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
	 * this record. Set flags to fail the compare against ENABLED.
2080
	 */
2081 2082 2083 2084 2085 2086 2087 2088 2089
	if (flag) {
		if (!(rec->flags & FTRACE_FL_REGS) != 
		    !(rec->flags & FTRACE_FL_REGS_EN))
			flag |= FTRACE_FL_REGS;

		if (!(rec->flags & FTRACE_FL_TRAMP) != 
		    !(rec->flags & FTRACE_FL_TRAMP_EN))
			flag |= FTRACE_FL_TRAMP;
	}
2090

2091 2092
	/* If the state of this record hasn't changed, then do nothing */
	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
2093
		return FTRACE_UPDATE_IGNORE;
2094

2095
	if (flag) {
2096 2097 2098 2099
		/* Save off if rec is being enabled (for return value) */
		flag ^= rec->flags & FTRACE_FL_ENABLED;

		if (update) {
2100
			rec->flags |= FTRACE_FL_ENABLED;
2101 2102 2103 2104 2105 2106
			if (flag & FTRACE_FL_REGS) {
				if (rec->flags & FTRACE_FL_REGS)
					rec->flags |= FTRACE_FL_REGS_EN;
				else
					rec->flags &= ~FTRACE_FL_REGS_EN;
			}
2107 2108 2109 2110 2111 2112
			if (flag & FTRACE_FL_TRAMP) {
				if (rec->flags & FTRACE_FL_TRAMP)
					rec->flags |= FTRACE_FL_TRAMP_EN;
				else
					rec->flags &= ~FTRACE_FL_TRAMP_EN;
			}
2113 2114 2115 2116 2117 2118 2119
		}

		/*
		 * If this record is being updated from a nop, then
		 *   return UPDATE_MAKE_CALL.
		 * Otherwise,
		 *   return UPDATE_MODIFY_CALL to tell the caller to convert
2120
		 *   from the save regs, to a non-save regs function or
2121
		 *   vice versa, or from a trampoline call.
2122
		 */
2123 2124
		if (flag & FTRACE_FL_ENABLED) {
			ftrace_bug_type = FTRACE_BUG_CALL;
2125
			return FTRACE_UPDATE_MAKE_CALL;
2126
		}
2127

2128
		ftrace_bug_type = FTRACE_BUG_UPDATE;
2129
		return FTRACE_UPDATE_MODIFY_CALL;
2130 2131
	}

2132 2133
	if (update) {
		/* If there's no more users, clear all flags */
2134
		if (!ftrace_rec_count(rec))
2135 2136
			rec->flags = 0;
		else
2137 2138 2139 2140 2141 2142
			/*
			 * Just disable the record, but keep the ops TRAMP
			 * and REGS states. The _EN flags must be disabled though.
			 */
			rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
					FTRACE_FL_REGS_EN);
2143
	}
2144

2145
	ftrace_bug_type = FTRACE_BUG_NOP;
2146 2147 2148 2149 2150 2151
	return FTRACE_UPDATE_MAKE_NOP;
}

/**
 * ftrace_update_record, set a record that now is tracing or not
 * @rec: the record to update
2152
 * @enable: set to true if the record is tracing, false to force disable
2153 2154 2155 2156
 *
 * The records that represent all functions that can be traced need
 * to be updated when tracing has been enabled.
 */
2157
int ftrace_update_record(struct dyn_ftrace *rec, bool enable)
2158
{
2159
	return ftrace_check_record(rec, enable, true);
2160 2161 2162 2163 2164
}

/**
 * ftrace_test_record, check if the record has been enabled or not
 * @rec: the record to test
2165
 * @enable: set to true to check if enabled, false if it is disabled
2166 2167 2168 2169 2170
 *
 * The arch code may need to test if a record is already set to
 * tracing to determine how to modify the function code that it
 * represents.
 */
2171
int ftrace_test_record(struct dyn_ftrace *rec, bool enable)
2172
{
2173
	return ftrace_check_record(rec, enable, false);
2174 2175
}

2176 2177 2178 2179
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2180
	unsigned long ip = rec->ip;
2181 2182 2183 2184 2185 2186

	do_for_each_ftrace_op(op, ftrace_ops_list) {

		if (!op->trampoline)
			continue;

2187
		if (hash_contains_ip(ip, op->func_hash))
2188 2189 2190 2191 2192 2193
			return op;
	} while_for_each_ftrace_op(op);

	return NULL;
}

2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211
static struct ftrace_ops *
ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
			   struct ftrace_ops *op)
{
	unsigned long ip = rec->ip;

	while_for_each_ftrace_op(op) {

		if (!op->trampoline)
			continue;

		if (hash_contains_ip(ip, op->func_hash))
			return op;
	} 

	return NULL;
}

2212 2213 2214 2215
static struct ftrace_ops *
ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2216
	unsigned long ip = rec->ip;
2217

2218 2219 2220 2221 2222 2223 2224 2225
	/*
	 * Need to check removed ops first.
	 * If they are being removed, and this rec has a tramp,
	 * and this rec is in the ops list, then it would be the
	 * one with the tramp.
	 */
	if (removed_ops) {
		if (hash_contains_ip(ip, &removed_ops->old_hash))
2226 2227 2228
			return removed_ops;
	}

2229 2230 2231 2232 2233 2234 2235
	/*
	 * Need to find the current trampoline for a rec.
	 * Now, a trampoline is only attached to a rec if there
	 * was a single 'ops' attached to it. But this can be called
	 * when we are adding another op to the rec or removing the
	 * current one. Thus, if the op is being added, we can
	 * ignore it because it hasn't attached itself to the rec
2236 2237 2238 2239 2240 2241 2242 2243 2244 2245
	 * yet.
	 *
	 * If an ops is being modified (hooking to different functions)
	 * then we don't care about the new functions that are being
	 * added, just the old ones (that are probably being removed).
	 *
	 * If we are adding an ops to a function that already is using
	 * a trampoline, it needs to be removed (trampolines are only
	 * for single ops connected), then an ops that is not being
	 * modified also needs to be checked.
2246
	 */
2247
	do_for_each_ftrace_op(op, ftrace_ops_list) {
2248 2249 2250 2251 2252 2253 2254 2255 2256

		if (!op->trampoline)
			continue;

		/*
		 * If the ops is being added, it hasn't gotten to
		 * the point to be removed from this tree yet.
		 */
		if (op->flags & FTRACE_OPS_FL_ADDING)
2257 2258
			continue;

2259

2260
		/*
2261 2262 2263
		 * If the ops is being modified and is in the old
		 * hash, then it is probably being removed from this
		 * function.
2264 2265 2266
		 */
		if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
		    hash_contains_ip(ip, &op->old_hash))
2267
			return op;
2268 2269 2270 2271 2272 2273 2274 2275
		/*
		 * If the ops is not being added or modified, and it's
		 * in its normal filter hash, then this must be the one
		 * we want!
		 */
		if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
		    hash_contains_ip(ip, op->func_hash))
			return op;
2276 2277 2278 2279 2280 2281 2282 2283 2284 2285

	} while_for_each_ftrace_op(op);

	return NULL;
}

static struct ftrace_ops *
ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2286
	unsigned long ip = rec->ip;
2287 2288 2289

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/* pass rec in as regs to have non-NULL val */
2290
		if (hash_contains_ip(ip, op->func_hash))
2291 2292 2293 2294 2295 2296
			return op;
	} while_for_each_ftrace_op(op);

	return NULL;
}

2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308
/**
 * ftrace_get_addr_new - Get the call address to set to
 * @rec:  The ftrace record descriptor
 *
 * If the record has the FTRACE_FL_REGS set, that means that it
 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
 * is not not set, then it wants to convert to the normal callback.
 *
 * Returns the address of the trampoline to set to
 */
unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
{
2309 2310 2311 2312 2313 2314
	struct ftrace_ops *ops;

	/* Trampolines take precedence over regs */
	if (rec->flags & FTRACE_FL_TRAMP) {
		ops = ftrace_find_tramp_ops_new(rec);
		if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
2315 2316
			pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
				(void *)rec->ip, (void *)rec->ip, rec->flags);
2317 2318 2319 2320 2321 2322
			/* Ftrace is shutting down, return anything */
			return (unsigned long)FTRACE_ADDR;
		}
		return ops->trampoline;
	}

2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340
	if (rec->flags & FTRACE_FL_REGS)
		return (unsigned long)FTRACE_REGS_ADDR;
	else
		return (unsigned long)FTRACE_ADDR;
}

/**
 * ftrace_get_addr_curr - Get the call address that is already there
 * @rec:  The ftrace record descriptor
 *
 * The FTRACE_FL_REGS_EN is set when the record already points to
 * a function that saves all the regs. Basically the '_EN' version
 * represents the current state of the function.
 *
 * Returns the address of the trampoline that is currently being called
 */
unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
{
2341 2342 2343 2344 2345 2346
	struct ftrace_ops *ops;

	/* Trampolines take precedence over regs */
	if (rec->flags & FTRACE_FL_TRAMP_EN) {
		ops = ftrace_find_tramp_ops_curr(rec);
		if (FTRACE_WARN_ON(!ops)) {
2347 2348
			pr_warn("Bad trampoline accounting at: %p (%pS)\n",
				(void *)rec->ip, (void *)rec->ip);
2349 2350 2351 2352 2353 2354
			/* Ftrace is shutting down, return anything */
			return (unsigned long)FTRACE_ADDR;
		}
		return ops->trampoline;
	}

2355 2356 2357 2358 2359 2360
	if (rec->flags & FTRACE_FL_REGS_EN)
		return (unsigned long)FTRACE_REGS_ADDR;
	else
		return (unsigned long)FTRACE_ADDR;
}

2361
static int
2362
__ftrace_replace_code(struct dyn_ftrace *rec, bool enable)
2363
{
2364
	unsigned long ftrace_old_addr;
2365 2366 2367
	unsigned long ftrace_addr;
	int ret;

2368
	ftrace_addr = ftrace_get_addr_new(rec);
2369

2370 2371 2372 2373
	/* This needs to be done before we call ftrace_update_record */
	ftrace_old_addr = ftrace_get_addr_curr(rec);

	ret = ftrace_update_record(rec, enable);
2374

2375 2376
	ftrace_bug_type = FTRACE_BUG_UNKNOWN;

2377 2378 2379 2380 2381
	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

	case FTRACE_UPDATE_MAKE_CALL:
2382
		ftrace_bug_type = FTRACE_BUG_CALL;
2383
		return ftrace_make_call(rec, ftrace_addr);
2384 2385

	case FTRACE_UPDATE_MAKE_NOP:
2386
		ftrace_bug_type = FTRACE_BUG_NOP;
2387
		return ftrace_make_nop(NULL, rec, ftrace_old_addr);
2388 2389

	case FTRACE_UPDATE_MODIFY_CALL:
2390
		ftrace_bug_type = FTRACE_BUG_UPDATE;
2391
		return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
2392 2393
	}

2394
	return -1; /* unknown ftrace bug */
2395 2396
}

2397
void __weak ftrace_replace_code(int mod_flags)
2398 2399 2400
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
2401
	bool enable = mod_flags & FTRACE_MODIFY_ENABLE_FL;
2402
	int schedulable = mod_flags & FTRACE_MODIFY_MAY_SLEEP_FL;
2403
	int failed;
2404

2405 2406 2407
	if (unlikely(ftrace_disabled))
		return;

2408
	do_for_each_ftrace_rec(pg, rec) {
2409 2410 2411 2412

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

2413
		failed = __ftrace_replace_code(rec, enable);
2414
		if (failed) {
2415
			ftrace_bug(failed, rec);
2416 2417
			/* Stop processing */
			return;
2418
		}
2419 2420
		if (schedulable)
			cond_resched();
2421
	} while_for_each_ftrace_rec();
2422 2423
}

2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495
struct ftrace_rec_iter {
	struct ftrace_page	*pg;
	int			index;
};

/**
 * ftrace_rec_iter_start, start up iterating over traced functions
 *
 * Returns an iterator handle that is used to iterate over all
 * the records that represent address locations where functions
 * are traced.
 *
 * May return NULL if no records are available.
 */
struct ftrace_rec_iter *ftrace_rec_iter_start(void)
{
	/*
	 * We only use a single iterator.
	 * Protected by the ftrace_lock mutex.
	 */
	static struct ftrace_rec_iter ftrace_rec_iter;
	struct ftrace_rec_iter *iter = &ftrace_rec_iter;

	iter->pg = ftrace_pages_start;
	iter->index = 0;

	/* Could have empty pages */
	while (iter->pg && !iter->pg->index)
		iter->pg = iter->pg->next;

	if (!iter->pg)
		return NULL;

	return iter;
}

/**
 * ftrace_rec_iter_next, get the next record to process.
 * @iter: The handle to the iterator.
 *
 * Returns the next iterator after the given iterator @iter.
 */
struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
{
	iter->index++;

	if (iter->index >= iter->pg->index) {
		iter->pg = iter->pg->next;
		iter->index = 0;

		/* Could have empty pages */
		while (iter->pg && !iter->pg->index)
			iter->pg = iter->pg->next;
	}

	if (!iter->pg)
		return NULL;

	return iter;
}

/**
 * ftrace_rec_iter_record, get the record at the iterator location
 * @iter: The current iterator location
 *
 * Returns the record that the current @iter is at.
 */
struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
{
	return &iter->pg->records[iter->index];
}

2496
static int
2497
ftrace_nop_initialize(struct module *mod, struct dyn_ftrace *rec)
2498
{
2499
	int ret;
2500

2501 2502 2503
	if (unlikely(ftrace_disabled))
		return 0;

2504
	ret = ftrace_init_nop(mod, rec);
2505
	if (ret) {
2506
		ftrace_bug_type = FTRACE_BUG_INIT;
2507
		ftrace_bug(ret, rec);
2508
		return 0;
2509
	}
2510
	return 1;
2511 2512
}

2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530
/*
 * archs can override this function if they must do something
 * before the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_prepare(void)
{
	return 0;
}

/*
 * archs can override this function if they must do something
 * after the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_post_process(void)
{
	return 0;
}

2531
void ftrace_modify_all_code(int command)
2532
{
2533
	int update = command & FTRACE_UPDATE_TRACE_FUNC;
2534
	int mod_flags = 0;
2535
	int err = 0;
2536

2537 2538 2539
	if (command & FTRACE_MAY_SLEEP)
		mod_flags = FTRACE_MODIFY_MAY_SLEEP_FL;

2540 2541 2542 2543 2544 2545 2546 2547 2548 2549
	/*
	 * If the ftrace_caller calls a ftrace_ops func directly,
	 * we need to make sure that it only traces functions it
	 * expects to trace. When doing the switch of functions,
	 * we need to update to the ftrace_ops_list_func first
	 * before the transition between old and new calls are set,
	 * as the ftrace_ops_list_func will check the ops hashes
	 * to make sure the ops are having the right functions
	 * traced.
	 */
2550 2551 2552 2553 2554
	if (update) {
		err = ftrace_update_ftrace_func(ftrace_ops_list_func);
		if (FTRACE_WARN_ON(err))
			return;
	}
2555

2556
	if (command & FTRACE_UPDATE_CALLS)
2557
		ftrace_replace_code(mod_flags | FTRACE_MODIFY_ENABLE_FL);
2558
	else if (command & FTRACE_DISABLE_CALLS)
2559
		ftrace_replace_code(mod_flags);
2560

2561 2562 2563 2564 2565 2566
	if (update && ftrace_trace_function != ftrace_ops_list_func) {
		function_trace_op = set_function_trace_op;
		smp_wmb();
		/* If irqs are disabled, we are in stop machine */
		if (!irqs_disabled())
			smp_call_function(ftrace_sync_ipi, NULL, 1);
2567 2568 2569
		err = ftrace_update_ftrace_func(ftrace_trace_function);
		if (FTRACE_WARN_ON(err))
			return;
2570
	}
2571

2572
	if (command & FTRACE_START_FUNC_RET)
2573
		err = ftrace_enable_ftrace_graph_caller();
2574
	else if (command & FTRACE_STOP_FUNC_RET)
2575 2576
		err = ftrace_disable_ftrace_graph_caller();
	FTRACE_WARN_ON(err);
2577 2578 2579 2580 2581 2582 2583
}

static int __ftrace_modify_code(void *data)
{
	int *command = data;

	ftrace_modify_all_code(*command);
2584

2585
	return 0;
2586 2587
}

2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611
/**
 * ftrace_run_stop_machine, go back to the stop machine method
 * @command: The command to tell ftrace what to do
 *
 * If an arch needs to fall back to the stop machine method, the
 * it can call this function.
 */
void ftrace_run_stop_machine(int command)
{
	stop_machine(__ftrace_modify_code, &command, NULL);
}

/**
 * arch_ftrace_update_code, modify the code to trace or not trace
 * @command: The command that needs to be done
 *
 * Archs can override this function if it does not need to
 * run stop_machine() to modify code.
 */
void __weak arch_ftrace_update_code(int command)
{
	ftrace_run_stop_machine(command);
}

Ingo Molnar's avatar
Ingo Molnar committed
2612
static void ftrace_run_update_code(int command)
2613
{
2614 2615 2616 2617 2618
	int ret;

	ret = ftrace_arch_code_modify_prepare();
	FTRACE_WARN_ON(ret);
	if (ret)
2619
		return;
2620

2621 2622 2623 2624 2625 2626 2627 2628
	/*
	 * By default we use stop_machine() to modify the code.
	 * But archs can do what ever they want as long as it
	 * is safe. The stop_machine() is the safest, but also
	 * produces the most overhead.
	 */
	arch_ftrace_update_code(command);

2629 2630
	ret = ftrace_arch_code_modify_post_process();
	FTRACE_WARN_ON(ret);
2631 2632
}

2633
static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
2634
				   struct ftrace_ops_hash *old_hash)
2635 2636
{
	ops->flags |= FTRACE_OPS_FL_MODIFYING;
2637 2638
	ops->old_hash.filter_hash = old_hash->filter_hash;
	ops->old_hash.notrace_hash = old_hash->notrace_hash;
2639
	ftrace_run_update_code(command);
2640
	ops->old_hash.filter_hash = NULL;
2641
	ops->old_hash.notrace_hash = NULL;
2642 2643 2644
	ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
}

2645
static ftrace_func_t saved_ftrace_func;
2646
static int ftrace_start_up;
2647

2648 2649 2650 2651
void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
{
}

2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663
static void ftrace_startup_enable(int command)
{
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}

	if (!command || !ftrace_enabled)
		return;

	ftrace_run_update_code(command);
}
2664

2665 2666 2667 2668 2669 2670 2671
static void ftrace_startup_all(int command)
{
	update_all_ops = true;
	ftrace_startup_enable(command);
	update_all_ops = false;
}

2672
int ftrace_startup(struct ftrace_ops *ops, int command)
2673
{
2674
	int ret;
2675

2676
	if (unlikely(ftrace_disabled))
2677
		return -ENODEV;
2678

2679 2680 2681 2682
	ret = __register_ftrace_function(ops);
	if (ret)
		return ret;

2683
	ftrace_start_up++;
2684

2685 2686 2687 2688 2689 2690 2691 2692 2693
	/*
	 * Note that ftrace probes uses this to start up
	 * and modify functions it will probe. But we still
	 * set the ADDING flag for modification, as probes
	 * do not have trampolines. If they add them in the
	 * future, then the probes will need to distinguish
	 * between adding and updating probes.
	 */
	ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
2694

2695 2696 2697 2698 2699 2700 2701 2702 2703
	ret = ftrace_hash_ipmodify_enable(ops);
	if (ret < 0) {
		/* Rollback registration process */
		__unregister_ftrace_function(ops);
		ftrace_start_up--;
		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
		return ret;
	}

2704 2705
	if (ftrace_hash_rec_enable(ops, 1))
		command |= FTRACE_UPDATE_CALLS;
2706

2707
	ftrace_startup_enable(command);
2708

2709 2710
	ops->flags &= ~FTRACE_OPS_FL_ADDING;

2711
	return 0;
2712 2713
}

2714
int ftrace_shutdown(struct ftrace_ops *ops, int command)
2715
{
2716
	int ret;
2717

2718
	if (unlikely(ftrace_disabled))
2719 2720 2721 2722 2723
		return -ENODEV;

	ret = __unregister_ftrace_function(ops);
	if (ret)
		return ret;
2724

2725
	ftrace_start_up--;
2726 2727 2728 2729 2730 2731 2732
	/*
	 * Just warn in case of unbalance, no need to kill ftrace, it's not
	 * critical but the ftrace_call callers may be never nopped again after
	 * further ftrace uses.
	 */
	WARN_ON_ONCE(ftrace_start_up < 0);

2733 2734
	/* Disabling ipmodify never fails */
	ftrace_hash_ipmodify_disable(ops);
2735

2736 2737
	if (ftrace_hash_rec_disable(ops, 1))
		command |= FTRACE_UPDATE_CALLS;
2738

2739
	ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2740

2741 2742 2743 2744
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}
2745

2746 2747
	if (!command || !ftrace_enabled) {
		/*
2748 2749
		 * If these are dynamic or per_cpu ops, they still
		 * need their data freed. Since, function tracing is
2750 2751 2752
		 * not currently active, we can just free them
		 * without synchronizing all CPUs.
		 */
2753
		if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
2754 2755
			goto free_ops;

2756
		return 0;
2757
	}
2758

2759 2760 2761 2762
	/*
	 * If the ops uses a trampoline, then it needs to be
	 * tested first on update.
	 */
2763
	ops->flags |= FTRACE_OPS_FL_REMOVING;
2764 2765
	removed_ops = ops;

2766 2767 2768 2769
	/* The trampoline logic checks the old hashes */
	ops->old_hash.filter_hash = ops->func_hash->filter_hash;
	ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;

2770
	ftrace_run_update_code(command);
2771

2772 2773 2774 2775
	/*
	 * If there's no more ops registered with ftrace, run a
	 * sanity check to make sure all rec flags are cleared.
	 */
2776 2777
	if (rcu_dereference_protected(ftrace_ops_list,
			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
2778 2779 2780 2781
		struct ftrace_page *pg;
		struct dyn_ftrace *rec;

		do_for_each_ftrace_rec(pg, rec) {
2782
			if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_FL_DISABLED))
2783 2784 2785 2786 2787
				pr_warn("  %pS flags:%lx\n",
					(void *)rec->ip, rec->flags);
		} while_for_each_ftrace_rec();
	}

2788 2789 2790 2791
	ops->old_hash.filter_hash = NULL;
	ops->old_hash.notrace_hash = NULL;

	removed_ops = NULL;
2792
	ops->flags &= ~FTRACE_OPS_FL_REMOVING;
2793

2794 2795 2796
	/*
	 * Dynamic ops may be freed, we must make sure that all
	 * callers are done before leaving this function.
2797
	 * The same goes for freeing the per_cpu data of the per_cpu
2798 2799
	 * ops.
	 */
2800
	if (ops->flags & FTRACE_OPS_FL_DYNAMIC) {
2801 2802 2803 2804 2805 2806 2807 2808
		/*
		 * We need to do a hard force of sched synchronization.
		 * This is because we use preempt_disable() to do RCU, but
		 * the function tracers can be called where RCU is not watching
		 * (like before user_exit()). We can not rely on the RCU
		 * infrastructure to do the synchronization, thus we must do it
		 * ourselves.
		 */
2809 2810
		schedule_on_each_cpu(ftrace_sync);

2811 2812 2813 2814 2815 2816 2817
		/*
		 * When the kernel is preeptive, tasks can be preempted
		 * while on a ftrace trampoline. Just scheduling a task on
		 * a CPU is not good enough to flush them. Calling
		 * synchornize_rcu_tasks() will wait for those tasks to
		 * execute and either schedule voluntarily or enter user space.
		 */
2818
		if (IS_ENABLED(CONFIG_PREEMPTION))
2819 2820
			synchronize_rcu_tasks();

2821
 free_ops:
2822
		arch_ftrace_trampoline_free(ops);
2823 2824
	}

2825
	return 0;
2826 2827
}

Ingo Molnar's avatar
Ingo Molnar committed
2828
static void ftrace_startup_sysctl(void)
2829
{
2830 2831
	int command;

2832 2833 2834
	if (unlikely(ftrace_disabled))
		return;

2835 2836
	/* Force update next time */
	saved_ftrace_func = NULL;
2837
	/* ftrace_start_up is true if we want ftrace running */
2838 2839 2840 2841
	if (ftrace_start_up) {
		command = FTRACE_UPDATE_CALLS;
		if (ftrace_graph_active)
			command |= FTRACE_START_FUNC_RET;
2842
		ftrace_startup_enable(command);
2843
	}
2844 2845
}

Ingo Molnar's avatar
Ingo Molnar committed
2846
static void ftrace_shutdown_sysctl(void)
2847
{
2848 2849
	int command;

2850 2851 2852
	if (unlikely(ftrace_disabled))
		return;

2853
	/* ftrace_start_up is true if ftrace is running */
2854 2855 2856 2857 2858 2859
	if (ftrace_start_up) {
		command = FTRACE_DISABLE_CALLS;
		if (ftrace_graph_active)
			command |= FTRACE_STOP_FUNC_RET;
		ftrace_run_update_code(command);
	}
2860 2861
}

2862
static u64		ftrace_update_time;
2863 2864
unsigned long		ftrace_update_tot_cnt;

2865
static inline int ops_traces_mod(struct ftrace_ops *ops)
2866
{
2867 2868 2869 2870
	/*
	 * Filter_hash being empty will default to trace module.
	 * But notrace hash requires a test of individual module functions.
	 */
2871 2872
	return ftrace_hash_empty(ops->func_hash->filter_hash) &&
		ftrace_hash_empty(ops->func_hash->notrace_hash);
2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886
}

/*
 * Check if the current ops references the record.
 *
 * If the ops traces all functions, then it was already accounted for.
 * If the ops does not trace the current record function, skip it.
 * If the ops ignores the function via notrace filter, skip it.
 */
static inline bool
ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	/* If ops isn't enabled, ignore it */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
2887
		return false;
2888

2889
	/* If ops traces all then it includes this function */
2890
	if (ops_traces_mod(ops))
2891
		return true;
2892 2893

	/* The function must be in the filter */
2894
	if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
2895
	    !__ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))
2896
		return false;
2897

2898
	/* If in notrace hash, we ignore it too */
2899
	if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip))
2900
		return false;
2901

2902
	return true;
2903 2904
}

2905
static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
2906
{
2907
	struct ftrace_page *pg;
2908
	struct dyn_ftrace *p;
2909
	u64 start, stop;
2910
	unsigned long update_cnt = 0;
2911
	unsigned long rec_flags = 0;
2912
	int i;
2913

2914 2915
	start = ftrace_now(raw_smp_processor_id());

2916
	/*
2917 2918 2919 2920 2921 2922 2923 2924 2925
	 * When a module is loaded, this function is called to convert
	 * the calls to mcount in its text to nops, and also to create
	 * an entry in the ftrace data. Now, if ftrace is activated
	 * after this call, but before the module sets its text to
	 * read-only, the modification of enabling ftrace can fail if
	 * the read-only is done while ftrace is converting the calls.
	 * To prevent this, the module's records are set as disabled
	 * and will be enabled after the call to set the module's text
	 * to read-only.
2926
	 */
2927 2928
	if (mod)
		rec_flags |= FTRACE_FL_DISABLED;
2929

2930
	for (pg = new_pgs; pg; pg = pg->next) {
2931

2932
		for (i = 0; i < pg->index; i++) {
2933

2934 2935 2936
			/* If something went wrong, bail without enabling anything */
			if (unlikely(ftrace_disabled))
				return -1;
2937

2938
			p = &pg->records[i];
2939
			p->flags = rec_flags;
2940

2941 2942 2943 2944
			/*
			 * Do the initial record conversion from mcount jump
			 * to the NOP instructions.
			 */
2945
			if (!__is_defined(CC_USING_NOP_MCOUNT) &&
2946
			    !ftrace_nop_initialize(mod, p))
2947
				break;
2948

2949
			update_cnt++;
2950
		}
2951 2952
	}

2953
	stop = ftrace_now(raw_smp_processor_id());
2954
	ftrace_update_time = stop - start;
2955
	ftrace_update_tot_cnt += update_cnt;
2956

2957 2958 2959
	return 0;
}

2960
static int ftrace_allocate_records(struct ftrace_page *pg, int count)
2961
{
2962
	int order;
2963 2964
	int cnt;

2965 2966 2967 2968
	if (WARN_ON(!count))
		return -EINVAL;

	order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
2969 2970

	/*
2971 2972
	 * We want to fill as much as possible. No more than a page
	 * may be empty.
2973
	 */
2974 2975
	while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
		order--;
2976

2977 2978
 again:
	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
2979

2980 2981 2982 2983 2984 2985 2986
	if (!pg->records) {
		/* if we can't allocate this size, try something smaller */
		if (!order)
			return -ENOMEM;
		order >>= 1;
		goto again;
	}
2987

2988 2989
	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
	pg->size = cnt;
2990

2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005
	if (cnt > count)
		cnt = count;

	return cnt;
}

static struct ftrace_page *
ftrace_allocate_pages(unsigned long num_to_init)
{
	struct ftrace_page *start_pg;
	struct ftrace_page *pg;
	int order;
	int cnt;

	if (!num_to_init)
3006
		return NULL;
3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023

	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
	if (!pg)
		return NULL;

	/*
	 * Try to allocate as much as possible in one continues
	 * location that fills in all of the space. We want to
	 * waste as little space as possible.
	 */
	for (;;) {
		cnt = ftrace_allocate_records(pg, num_to_init);
		if (cnt < 0)
			goto free_pages;

		num_to_init -= cnt;
		if (!num_to_init)
3024 3025
			break;

3026 3027 3028 3029
		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
		if (!pg->next)
			goto free_pages;

3030 3031 3032
		pg = pg->next;
	}

3033 3034 3035
	return start_pg;

 free_pages:
3036 3037
	pg = start_pg;
	while (pg) {
3038 3039 3040 3041 3042 3043 3044 3045 3046 3047
		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
		free_pages((unsigned long)pg->records, order);
		start_pg = pg->next;
		kfree(pg);
		pg = start_pg;
	}
	pr_info("ftrace: FAILED to allocate memory for functions\n");
	return NULL;
}

3048 3049 3050
#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */

struct ftrace_iterator {
3051
	loff_t				pos;
3052
	loff_t				func_pos;
3053
	loff_t				mod_pos;
3054 3055 3056
	struct ftrace_page		*pg;
	struct dyn_ftrace		*func;
	struct ftrace_func_probe	*probe;
3057
	struct ftrace_func_entry	*probe_entry;
3058
	struct trace_parser		parser;
3059
	struct ftrace_hash		*hash;
3060
	struct ftrace_ops		*ops;
3061 3062
	struct trace_array		*tr;
	struct list_head		*mod_list;
3063
	int				pidx;
3064 3065
	int				idx;
	unsigned			flags;
3066 3067
};

3068
static void *
3069
t_probe_next(struct seq_file *m, loff_t *pos)
3070 3071
{
	struct ftrace_iterator *iter = m->private;
3072
	struct trace_array *tr = iter->ops->private;
3073
	struct list_head *func_probes;
3074 3075
	struct ftrace_hash *hash;
	struct list_head *next;
3076
	struct hlist_node *hnd = NULL;
3077
	struct hlist_head *hhd;
3078
	int size;
3079 3080

	(*pos)++;
3081
	iter->pos = *pos;
3082

3083
	if (!tr)
3084 3085
		return NULL;

3086 3087
	func_probes = &tr->func_probes;
	if (list_empty(func_probes))
3088 3089
		return NULL;

3090
	if (!iter->probe) {
3091
		next = func_probes->next;
3092
		iter->probe = list_entry(next, struct ftrace_func_probe, list);
3093 3094 3095 3096 3097 3098
	}

	if (iter->probe_entry)
		hnd = &iter->probe_entry->hlist;

	hash = iter->probe->ops.func_hash->filter_hash;
3099

3100 3101 3102 3103 3104
	/*
	 * A probe being registered may temporarily have an empty hash
	 * and it's at the end of the func_probes list.
	 */
	if (!hash || hash == EMPTY_HASH)
3105 3106
		return NULL;

3107 3108 3109 3110
	size = 1 << hash->size_bits;

 retry:
	if (iter->pidx >= size) {
3111
		if (iter->probe->list.next == func_probes)
3112 3113
			return NULL;
		next = iter->probe->list.next;
3114
		iter->probe = list_entry(next, struct ftrace_func_probe, list);
3115 3116 3117 3118 3119 3120
		hash = iter->probe->ops.func_hash->filter_hash;
		size = 1 << hash->size_bits;
		iter->pidx = 0;
	}

	hhd = &hash->buckets[iter->pidx];
3121 3122

	if (hlist_empty(hhd)) {
3123
		iter->pidx++;
3124 3125 3126 3127 3128 3129 3130 3131 3132
		hnd = NULL;
		goto retry;
	}

	if (!hnd)
		hnd = hhd->first;
	else {
		hnd = hnd->next;
		if (!hnd) {
3133
			iter->pidx++;
3134 3135 3136 3137
			goto retry;
		}
	}

3138 3139 3140
	if (WARN_ON_ONCE(!hnd))
		return NULL;

3141
	iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist);
3142 3143

	return iter;
3144 3145
}

3146
static void *t_probe_start(struct seq_file *m, loff_t *pos)
3147 3148 3149
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
Li Zefan's avatar
Li Zefan committed
3150 3151
	loff_t l;

3152
	if (!(iter->flags & FTRACE_ITER_DO_PROBES))
3153 3154
		return NULL;

3155
	if (iter->mod_pos > *pos)
3156
		return NULL;
3157

3158 3159 3160
	iter->probe = NULL;
	iter->probe_entry = NULL;
	iter->pidx = 0;
3161
	for (l = 0; l <= (*pos - iter->mod_pos); ) {
3162
		p = t_probe_next(m, &l);
Li Zefan's avatar
Li Zefan committed
3163 3164 3165
		if (!p)
			break;
	}
3166 3167 3168
	if (!p)
		return NULL;

3169
	/* Only set this if we have an item */
3170
	iter->flags |= FTRACE_ITER_PROBE;
3171

3172
	return iter;
3173 3174
}

3175
static int
3176
t_probe_show(struct seq_file *m, struct ftrace_iterator *iter)
3177
{
3178
	struct ftrace_func_entry *probe_entry;
3179 3180
	struct ftrace_probe_ops *probe_ops;
	struct ftrace_func_probe *probe;
3181

3182 3183
	probe = iter->probe;
	probe_entry = iter->probe_entry;
3184

3185
	if (WARN_ON_ONCE(!probe || !probe_entry))
3186
		return -EIO;
3187

3188
	probe_ops = probe->probe_ops;
3189

3190
	if (probe_ops->print)
3191
		return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data);
3192

3193 3194
	seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip,
		   (void *)probe_ops->func);
3195 3196 3197 3198

	return 0;
}

3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274
static void *
t_mod_next(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	struct trace_array *tr = iter->tr;

	(*pos)++;
	iter->pos = *pos;

	iter->mod_list = iter->mod_list->next;

	if (iter->mod_list == &tr->mod_trace ||
	    iter->mod_list == &tr->mod_notrace) {
		iter->flags &= ~FTRACE_ITER_MOD;
		return NULL;
	}

	iter->mod_pos = *pos;

	return iter;
}

static void *t_mod_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
	loff_t l;

	if (iter->func_pos > *pos)
		return NULL;

	iter->mod_pos = iter->func_pos;

	/* probes are only available if tr is set */
	if (!iter->tr)
		return NULL;

	for (l = 0; l <= (*pos - iter->func_pos); ) {
		p = t_mod_next(m, &l);
		if (!p)
			break;
	}
	if (!p) {
		iter->flags &= ~FTRACE_ITER_MOD;
		return t_probe_start(m, pos);
	}

	/* Only set this if we have an item */
	iter->flags |= FTRACE_ITER_MOD;

	return iter;
}

static int
t_mod_show(struct seq_file *m, struct ftrace_iterator *iter)
{
	struct ftrace_mod_load *ftrace_mod;
	struct trace_array *tr = iter->tr;

	if (WARN_ON_ONCE(!iter->mod_list) ||
			 iter->mod_list == &tr->mod_trace ||
			 iter->mod_list == &tr->mod_notrace)
		return -EIO;

	ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list);

	if (ftrace_mod->func)
		seq_printf(m, "%s", ftrace_mod->func);
	else
		seq_putc(m, '*');

	seq_printf(m, ":mod:%s\n", ftrace_mod->module);

	return 0;
}

Ingo Molnar's avatar
Ingo Molnar committed
3275
static void *
3276
t_func_next(struct seq_file *m, loff_t *pos)
3277 3278 3279 3280 3281
{
	struct ftrace_iterator *iter = m->private;
	struct dyn_ftrace *rec = NULL;

	(*pos)++;
3282

3283 3284 3285 3286 3287 3288 3289 3290 3291
 retry:
	if (iter->idx >= iter->pg->index) {
		if (iter->pg->next) {
			iter->pg = iter->pg->next;
			iter->idx = 0;
			goto retry;
		}
	} else {
		rec = &iter->pg->records[iter->idx++];
3292 3293
		if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
		     !ftrace_lookup_ip(iter->hash, rec->ip)) ||
3294 3295

		    ((iter->flags & FTRACE_ITER_ENABLED) &&
3296
		     !(rec->flags & FTRACE_FL_ENABLED))) {
3297

3298 3299 3300 3301 3302
			rec = NULL;
			goto retry;
		}
	}

3303
	if (!rec)
3304
		return NULL;
3305

3306
	iter->pos = iter->func_pos = *pos;
3307 3308 3309
	iter->func = rec;

	return iter;
3310 3311
}

3312 3313 3314 3315
static void *
t_next(struct seq_file *m, void *v, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
3316
	loff_t l = *pos; /* t_probe_start() must use original pos */
3317 3318 3319 3320 3321
	void *ret;

	if (unlikely(ftrace_disabled))
		return NULL;

3322 3323
	if (iter->flags & FTRACE_ITER_PROBE)
		return t_probe_next(m, pos);
3324

3325 3326 3327
	if (iter->flags & FTRACE_ITER_MOD)
		return t_mod_next(m, pos);

3328
	if (iter->flags & FTRACE_ITER_PRINTALL) {
3329
		/* next must increment pos, and t_probe_start does not */
3330
		(*pos)++;
3331
		return t_mod_start(m, &l);
3332 3333 3334 3335 3336
	}

	ret = t_func_next(m, pos);

	if (!ret)
3337
		return t_mod_start(m, &l);
3338 3339 3340 3341

	return ret;
}

3342 3343 3344 3345
static void reset_iter_read(struct ftrace_iterator *iter)
{
	iter->pos = 0;
	iter->func_pos = 0;
3346
	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD);
3347 3348 3349 3350 3351 3352
}

static void *t_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
3353
	loff_t l;
3354

3355
	mutex_lock(&ftrace_lock);
3356 3357 3358 3359

	if (unlikely(ftrace_disabled))
		return NULL;

3360 3361 3362 3363 3364 3365
	/*
	 * If an lseek was done, then reset and start from beginning.
	 */
	if (*pos < iter->pos)
		reset_iter_read(iter);

3366 3367 3368 3369 3370
	/*
	 * For set_ftrace_filter reading, if we have the filter
	 * off, we can short cut and just print out that all
	 * functions are enabled.
	 */
3371 3372
	if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
	    ftrace_hash_empty(iter->hash)) {
3373
		iter->func_pos = 1; /* Account for the message */
3374
		if (*pos > 0)
3375
			return t_mod_start(m, pos);
3376
		iter->flags |= FTRACE_ITER_PRINTALL;
3377
		/* reset in case of seek/pread */
3378
		iter->flags &= ~FTRACE_ITER_PROBE;
3379 3380 3381
		return iter;
	}

3382 3383
	if (iter->flags & FTRACE_ITER_MOD)
		return t_mod_start(m, pos);
3384

3385 3386 3387 3388 3389
	/*
	 * Unfortunately, we need to restart at ftrace_pages_start
	 * every time we let go of the ftrace_mutex. This is because
	 * those pointers can change without the lock.
	 */
3390 3391 3392
	iter->pg = ftrace_pages_start;
	iter->idx = 0;
	for (l = 0; l <= *pos; ) {
3393
		p = t_func_next(m, &l);
3394 3395
		if (!p)
			break;
3396
	}
3397

3398
	if (!p)
3399
		return t_mod_start(m, pos);
3400 3401

	return iter;
3402 3403 3404 3405
}

static void t_stop(struct seq_file *m, void *p)
{
3406
	mutex_unlock(&ftrace_lock);
3407 3408
}

3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424
void * __weak
arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	return NULL;
}

static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
				struct dyn_ftrace *rec)
{
	void *ptr;

	ptr = arch_ftrace_trampoline_func(ops, rec);
	if (ptr)
		seq_printf(m, " ->%pS", ptr);
}

3425 3426
static int t_show(struct seq_file *m, void *v)
{
3427
	struct ftrace_iterator *iter = m->private;
3428
	struct dyn_ftrace *rec;
3429

3430 3431
	if (iter->flags & FTRACE_ITER_PROBE)
		return t_probe_show(m, iter);
3432

3433 3434 3435
	if (iter->flags & FTRACE_ITER_MOD)
		return t_mod_show(m, iter);

3436
	if (iter->flags & FTRACE_ITER_PRINTALL) {
3437
		if (iter->flags & FTRACE_ITER_NOTRACE)
3438
			seq_puts(m, "#### no functions disabled ####\n");
3439
		else
3440
			seq_puts(m, "#### all functions enabled ####\n");
3441 3442 3443
		return 0;
	}

3444 3445
	rec = iter->func;

3446 3447 3448
	if (!rec)
		return 0;

3449
	seq_printf(m, "%ps", (void *)rec->ip);
3450
	if (iter->flags & FTRACE_ITER_ENABLED) {
3451
		struct ftrace_ops *ops;
3452

3453
		seq_printf(m, " (%ld)%s%s",
3454
			   ftrace_rec_count(rec),
3455 3456
			   rec->flags & FTRACE_FL_REGS ? " R" : "  ",
			   rec->flags & FTRACE_FL_IPMODIFY ? " I" : "  ");
3457
		if (rec->flags & FTRACE_FL_TRAMP_EN) {
3458
			ops = ftrace_find_tramp_ops_any(rec);
3459 3460 3461 3462 3463
			if (ops) {
				do {
					seq_printf(m, "\ttramp: %pS (%pS)",
						   (void *)ops->trampoline,
						   (void *)ops->func);
3464
					add_trampoline_func(m, ops, rec);
3465 3466 3467
					ops = ftrace_find_tramp_ops_next(rec, ops);
				} while (ops);
			} else
3468
				seq_puts(m, "\ttramp: ERROR!");
3469 3470
		} else {
			add_trampoline_func(m, NULL, rec);
3471 3472 3473
		}
	}	

3474
	seq_putc(m, '\n');
3475 3476 3477 3478

	return 0;
}

3479
static const struct seq_operations show_ftrace_seq_ops = {
3480 3481 3482 3483 3484 3485
	.start = t_start,
	.next = t_next,
	.stop = t_stop,
	.show = t_show,
};

Ingo Molnar's avatar
Ingo Molnar committed
3486
static int
3487 3488 3489
ftrace_avail_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;
3490 3491 3492 3493 3494
	int ret;

	ret = security_locked_down(LOCKDOWN_TRACEFS);
	if (ret)
		return ret;
3495

3496 3497 3498
	if (unlikely(ftrace_disabled))
		return -ENODEV;

3499
	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3500 3501
	if (!iter)
		return -ENOMEM;
3502

3503 3504 3505 3506
	iter->pg = ftrace_pages_start;
	iter->ops = &global_ops;

	return 0;
3507 3508
}

3509 3510 3511 3512 3513
static int
ftrace_enabled_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;

3514 3515 3516 3517 3518 3519 3520 3521 3522
	/*
	 * This shows us what functions are currently being
	 * traced and by what. Not sure if we want lockdown
	 * to hide such critical information for an admin.
	 * Although, perhaps it can show information we don't
	 * want people to see, but if something is tracing
	 * something, we probably want to know about it.
	 */

3523
	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3524 3525
	if (!iter)
		return -ENOMEM;
3526

3527 3528 3529 3530 3531
	iter->pg = ftrace_pages_start;
	iter->flags = FTRACE_ITER_ENABLED;
	iter->ops = &global_ops;

	return 0;
3532 3533
}

3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546
/**
 * ftrace_regex_open - initialize function tracer filter files
 * @ops: The ftrace_ops that hold the hash filters
 * @flag: The type of filter to process
 * @inode: The inode, usually passed in to your open routine
 * @file: The file, usually passed in to your open routine
 *
 * ftrace_regex_open() initializes the filter files for the
 * @ops. Depending on @flag it may process the filter hash or
 * the notrace hash of @ops. With this called from the open
 * routine, you can use ftrace_filter_write() for the write
 * routine if @flag has FTRACE_ITER_FILTER set, or
 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
3547
 * tracing_lseek() should be used as the lseek routine, and
3548 3549 3550
 * release must call ftrace_regex_release().
 */
int
3551
ftrace_regex_open(struct ftrace_ops *ops, int flag,
3552
		  struct inode *inode, struct file *file)
3553 3554
{
	struct ftrace_iterator *iter;
3555
	struct ftrace_hash *hash;
3556 3557
	struct list_head *mod_head;
	struct trace_array *tr = ops->private;
3558
	int ret = -ENOMEM;
3559

3560 3561
	ftrace_ops_init(ops);

3562 3563 3564
	if (unlikely(ftrace_disabled))
		return -ENODEV;

3565
	if (tracing_check_open_get_tr(tr))
3566 3567
		return -ENODEV;

3568 3569
	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
	if (!iter)
3570
		goto out;
3571

3572 3573
	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX))
		goto out;
3574

3575 3576
	iter->ops = ops;
	iter->flags = flag;
3577
	iter->tr = tr;
3578

3579
	mutex_lock(&ops->func_hash->regex_lock);
3580

3581
	if (flag & FTRACE_ITER_NOTRACE) {
3582
		hash = ops->func_hash->notrace_hash;
3583
		mod_head = tr ? &tr->mod_notrace : NULL;
3584
	} else {
3585
		hash = ops->func_hash->filter_hash;
3586
		mod_head = tr ? &tr->mod_trace : NULL;
3587
	}
3588

3589 3590
	iter->mod_list = mod_head;

3591
	if (file->f_mode & FMODE_WRITE) {
3592 3593
		const int size_bits = FTRACE_HASH_DEFAULT_BITS;

3594
		if (file->f_flags & O_TRUNC) {
3595
			iter->hash = alloc_ftrace_hash(size_bits);
3596 3597
			clear_ftrace_mod_list(mod_head);
	        } else {
3598
			iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);
3599
		}
3600

3601 3602
		if (!iter->hash) {
			trace_parser_put(&iter->parser);
3603
			goto out_unlock;
3604
		}
3605 3606
	} else
		iter->hash = hash;
3607

3608 3609
	ret = 0;

3610 3611 3612 3613 3614 3615 3616
	if (file->f_mode & FMODE_READ) {
		iter->pg = ftrace_pages_start;

		ret = seq_open(file, &show_ftrace_seq_ops);
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = iter;
3617
		} else {
3618 3619
			/* Failed */
			free_ftrace_hash(iter->hash);
3620 3621
			trace_parser_put(&iter->parser);
		}
3622 3623
	} else
		file->private_data = iter;
3624 3625

 out_unlock:
3626
	mutex_unlock(&ops->func_hash->regex_lock);
3627

3628 3629 3630 3631 3632 3633 3634
 out:
	if (ret) {
		kfree(iter);
		if (tr)
			trace_array_put(tr);
	}

3635 3636 3637
	return ret;
}

3638 3639 3640
static int
ftrace_filter_open(struct inode *inode, struct file *file)
{
3641 3642
	struct ftrace_ops *ops = inode->i_private;

3643
	/* Checks for tracefs lockdown */
3644
	return ftrace_regex_open(ops,
3645
			FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES,
3646
			inode, file);
3647 3648 3649 3650 3651
}

static int
ftrace_notrace_open(struct inode *inode, struct file *file)
{
3652 3653
	struct ftrace_ops *ops = inode->i_private;

3654
	/* Checks for tracefs lockdown */
3655
	return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
3656
				 inode, file);
3657 3658
}

3659 3660 3661 3662 3663 3664 3665
/* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
struct ftrace_glob {
	char *search;
	unsigned len;
	int type;
};

3666 3667 3668 3669 3670 3671 3672 3673 3674 3675
/*
 * If symbols in an architecture don't correspond exactly to the user-visible
 * name of what they represent, it is possible to define this function to
 * perform the necessary adjustments.
*/
char * __weak arch_ftrace_match_adjust(char *str, const char *search)
{
	return str;
}

3676
static int ftrace_match(char *str, struct ftrace_glob *g)
3677 3678
{
	int matched = 0;
3679
	int slen;
3680

3681 3682
	str = arch_ftrace_match_adjust(str, g->search);

3683
	switch (g->type) {
3684
	case MATCH_FULL:
3685
		if (strcmp(str, g->search) == 0)
3686 3687 3688
			matched = 1;
		break;
	case MATCH_FRONT_ONLY:
3689
		if (strncmp(str, g->search, g->len) == 0)
3690 3691 3692
			matched = 1;
		break;
	case MATCH_MIDDLE_ONLY:
3693
		if (strstr(str, g->search))
3694 3695 3696
			matched = 1;
		break;
	case MATCH_END_ONLY:
3697
		slen = strlen(str);
3698 3699
		if (slen >= g->len &&
		    memcmp(str + slen - g->len, g->search, g->len) == 0)
3700 3701
			matched = 1;
		break;
3702 3703 3704 3705
	case MATCH_GLOB:
		if (glob_match(g->search, str))
			matched = 1;
		break;
3706 3707 3708 3709 3710
	}

	return matched;
}

3711
static int
3712
enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
3713
{
3714 3715 3716
	struct ftrace_func_entry *entry;
	int ret = 0;

3717
	entry = ftrace_lookup_ip(hash, rec->ip);
3718
	if (clear_filter) {
3719 3720 3721
		/* Do nothing if it doesn't exist */
		if (!entry)
			return 0;
3722

3723
		free_hash_entry(hash, entry);
3724 3725 3726 3727
	} else {
		/* Do nothing if it exists */
		if (entry)
			return 0;
3728

3729
		ret = add_hash_entry(hash, rec->ip);
3730 3731
	}
	return ret;
3732 3733
}

3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758
static int
add_rec_by_index(struct ftrace_hash *hash, struct ftrace_glob *func_g,
		 int clear_filter)
{
	long index = simple_strtoul(func_g->search, NULL, 0);
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;

	/* The index starts at 1 */
	if (--index < 0)
		return 0;

	do_for_each_ftrace_rec(pg, rec) {
		if (pg->index <= index) {
			index -= pg->index;
			/* this is a double loop, break goes to the next page */
			break;
		}
		rec = &pg->records[index];
		enter_record(hash, rec, clear_filter);
		return 1;
	} while_for_each_ftrace_rec();
	return 0;
}

3759
static int
3760 3761
ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
		struct ftrace_glob *mod_g, int exclude_mod)
3762 3763
{
	char str[KSYM_SYMBOL_LEN];
3764 3765 3766 3767
	char *modname;

	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);

3768 3769 3770 3771 3772 3773
	if (mod_g) {
		int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0;

		/* blank module name to match all modules */
		if (!mod_g->len) {
			/* blank module globbing: modname xor exclude_mod */
3774
			if (!exclude_mod != !modname)
3775 3776 3777 3778
				goto func_match;
			return 0;
		}

3779 3780 3781 3782 3783 3784 3785 3786
		/*
		 * exclude_mod is set to trace everything but the given
		 * module. If it is set and the module matches, then
		 * return 0. If it is not set, and the module doesn't match
		 * also return 0. Otherwise, check the function to see if
		 * that matches.
		 */
		if (!mod_matches == !exclude_mod)
3787
			return 0;
3788
func_match:
3789
		/* blank search means to match all funcs in the mod */
3790
		if (!func_g->len)
3791 3792
			return 1;
	}
3793

3794
	return ftrace_match(str, func_g);
3795 3796
}

3797
static int
3798
match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
3799 3800 3801
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
3802
	struct ftrace_glob func_g = { .type = MATCH_FULL };
3803 3804 3805
	struct ftrace_glob mod_g = { .type = MATCH_FULL };
	struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL;
	int exclude_mod = 0;
3806
	int found = 0;
3807
	int ret;
3808
	int clear_filter = 0;
3809

3810
	if (func) {
3811 3812 3813
		func_g.type = filter_parse_regex(func, len, &func_g.search,
						 &clear_filter);
		func_g.len = strlen(func_g.search);
3814
	}
3815

3816 3817 3818 3819
	if (mod) {
		mod_g.type = filter_parse_regex(mod, strlen(mod),
				&mod_g.search, &exclude_mod);
		mod_g.len = strlen(mod_g.search);
3820
	}
3821

3822
	mutex_lock(&ftrace_lock);
3823

3824 3825
	if (unlikely(ftrace_disabled))
		goto out_unlock;
3826

3827 3828 3829 3830 3831
	if (func_g.type == MATCH_INDEX) {
		found = add_rec_by_index(hash, &func_g, clear_filter);
		goto out_unlock;
	}

3832
	do_for_each_ftrace_rec(pg, rec) {
3833 3834 3835 3836

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

3837
		if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
3838
			ret = enter_record(hash, rec, clear_filter);
3839 3840 3841 3842
			if (ret < 0) {
				found = ret;
				goto out_unlock;
			}
3843
			found = 1;
3844 3845
		}
	} while_for_each_ftrace_rec();
3846
 out_unlock:
3847
	mutex_unlock(&ftrace_lock);
3848 3849

	return found;
3850 3851
}

3852
static int
3853
ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
3854
{
3855
	return match_records(hash, buff, len, NULL);
3856 3857
}

3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907
static void ftrace_ops_update_code(struct ftrace_ops *ops,
				   struct ftrace_ops_hash *old_hash)
{
	struct ftrace_ops *op;

	if (!ftrace_enabled)
		return;

	if (ops->flags & FTRACE_OPS_FL_ENABLED) {
		ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
		return;
	}

	/*
	 * If this is the shared global_ops filter, then we need to
	 * check if there is another ops that shares it, is enabled.
	 * If so, we still need to run the modify code.
	 */
	if (ops->func_hash != &global_ops.local_hash)
		return;

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op->func_hash == &global_ops.local_hash &&
		    op->flags & FTRACE_OPS_FL_ENABLED) {
			ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
			/* Only need to do this once */
			return;
		}
	} while_for_each_ftrace_op(op);
}

static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops,
					   struct ftrace_hash **orig_hash,
					   struct ftrace_hash *hash,
					   int enable)
{
	struct ftrace_ops_hash old_hash_ops;
	struct ftrace_hash *old_hash;
	int ret;

	old_hash = *orig_hash;
	old_hash_ops.filter_hash = ops->func_hash->filter_hash;
	old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
	if (!ret) {
		ftrace_ops_update_code(ops, &old_hash_ops);
		free_ftrace_hash_rcu(old_hash);
	}
	return ret;
}
3908

3909 3910 3911
static bool module_exists(const char *module)
{
	/* All modules have the symbol __this_module */
3912
	static const char this_mod[] = "__this_module";
3913
	char modname[MAX_PARAM_PREFIX_LEN + sizeof(this_mod) + 2];
3914 3915 3916
	unsigned long val;
	int n;

3917
	n = snprintf(modname, sizeof(modname), "%s:%s", module, this_mod);
3918

3919
	if (n > sizeof(modname) - 1)
3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945
		return false;

	val = module_kallsyms_lookup_name(modname);
	return val != 0;
}

static int cache_mod(struct trace_array *tr,
		     const char *func, char *module, int enable)
{
	struct ftrace_mod_load *ftrace_mod, *n;
	struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace;
	int ret;

	mutex_lock(&ftrace_lock);

	/* We do not cache inverse filters */
	if (func[0] == '!') {
		func++;
		ret = -EINVAL;

		/* Look to remove this hash */
		list_for_each_entry_safe(ftrace_mod, n, head, list) {
			if (strcmp(ftrace_mod->module, module) != 0)
				continue;

			/* no func matches all */
3946
			if (strcmp(func, "*") == 0 ||
3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969
			    (ftrace_mod->func &&
			     strcmp(ftrace_mod->func, func) == 0)) {
				ret = 0;
				free_ftrace_mod(ftrace_mod);
				continue;
			}
		}
		goto out;
	}

	ret = -EINVAL;
	/* We only care about modules that have not been loaded yet */
	if (module_exists(module))
		goto out;

	/* Save this string off, and execute it when the module is loaded */
	ret = ftrace_add_mod(tr, func, module, enable);
 out:
	mutex_unlock(&ftrace_lock);

	return ret;
}

3970 3971 3972 3973
static int
ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
		 int reset, int enable);

3974
#ifdef CONFIG_MODULES
3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993
static void process_mod_list(struct list_head *head, struct ftrace_ops *ops,
			     char *mod, bool enable)
{
	struct ftrace_mod_load *ftrace_mod, *n;
	struct ftrace_hash **orig_hash, *new_hash;
	LIST_HEAD(process_mods);
	char *func;
	int ret;

	mutex_lock(&ops->func_hash->regex_lock);

	if (enable)
		orig_hash = &ops->func_hash->filter_hash;
	else
		orig_hash = &ops->func_hash->notrace_hash;

	new_hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS,
					      *orig_hash);
	if (!new_hash)
3994
		goto out; /* warn? */
3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029

	mutex_lock(&ftrace_lock);

	list_for_each_entry_safe(ftrace_mod, n, head, list) {

		if (strcmp(ftrace_mod->module, mod) != 0)
			continue;

		if (ftrace_mod->func)
			func = kstrdup(ftrace_mod->func, GFP_KERNEL);
		else
			func = kstrdup("*", GFP_KERNEL);

		if (!func) /* warn? */
			continue;

		list_del(&ftrace_mod->list);
		list_add(&ftrace_mod->list, &process_mods);

		/* Use the newly allocated func, as it may be "*" */
		kfree(ftrace_mod->func);
		ftrace_mod->func = func;
	}

	mutex_unlock(&ftrace_lock);

	list_for_each_entry_safe(ftrace_mod, n, &process_mods, list) {

		func = ftrace_mod->func;

		/* Grabs ftrace_lock, which is why we have this extra step */
		match_records(new_hash, func, strlen(func), mod);
		free_ftrace_mod(ftrace_mod);
	}

4030 4031 4032
	if (enable && list_empty(head))
		new_hash->flags &= ~FTRACE_HASH_FL_MOD;

4033 4034 4035 4036 4037 4038
	mutex_lock(&ftrace_lock);

	ret = ftrace_hash_move_and_update_ops(ops, orig_hash,
					      new_hash, enable);
	mutex_unlock(&ftrace_lock);

4039
 out:
4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064
	mutex_unlock(&ops->func_hash->regex_lock);

	free_ftrace_hash(new_hash);
}

static void process_cached_mods(const char *mod_name)
{
	struct trace_array *tr;
	char *mod;

	mod = kstrdup(mod_name, GFP_KERNEL);
	if (!mod)
		return;

	mutex_lock(&trace_types_lock);
	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
		if (!list_empty(&tr->mod_trace))
			process_mod_list(&tr->mod_trace, tr->ops, mod, true);
		if (!list_empty(&tr->mod_notrace))
			process_mod_list(&tr->mod_notrace, tr->ops, mod, false);
	}
	mutex_unlock(&trace_types_lock);

	kfree(mod);
}
4065
#endif
4066

4067 4068 4069 4070 4071 4072
/*
 * We register the module command as a template to show others how
 * to register the a command as well.
 */

static int
4073
ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash,
4074
		    char *func_orig, char *cmd, char *module, int enable)
4075
{
4076
	char *func;
4077
	int ret;
4078

4079 4080 4081 4082 4083
	/* match_records() modifies func, and we need the original */
	func = kstrdup(func_orig, GFP_KERNEL);
	if (!func)
		return -ENOMEM;

4084 4085 4086 4087 4088 4089 4090
	/*
	 * cmd == 'mod' because we only registered this func
	 * for the 'mod' ftrace_func_command.
	 * But if you register one func with multiple commands,
	 * you can tell which command was used by the cmd
	 * parameter.
	 */
4091
	ret = match_records(hash, func, strlen(func), module);
4092 4093
	kfree(func);

4094
	if (!ret)
4095
		return cache_mod(tr, func_orig, module, enable);
4096 4097 4098
	if (ret < 0)
		return ret;
	return 0;
4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109
}

static struct ftrace_func_command ftrace_mod_cmd = {
	.name			= "mod",
	.func			= ftrace_mod_callback,
};

static int __init ftrace_mod_cmd_init(void)
{
	return register_ftrace_command(&ftrace_mod_cmd);
}
4110
core_initcall(ftrace_mod_cmd_init);
4111

4112
static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
4113
				      struct ftrace_ops *op, struct pt_regs *pt_regs)
4114
{
4115
	struct ftrace_probe_ops *probe_ops;
4116
	struct ftrace_func_probe *probe;
4117

4118 4119
	probe = container_of(op, struct ftrace_func_probe, ops);
	probe_ops = probe->probe_ops;
4120 4121 4122 4123 4124 4125

	/*
	 * Disable preemption for these calls to prevent a RCU grace
	 * period. This syncs the hash iteration and freeing of items
	 * on the hash. rcu_read_lock is too dangerous here.
	 */
4126
	preempt_disable_notrace();
4127
	probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data);
4128
	preempt_enable_notrace();
4129 4130
}

4131 4132 4133
struct ftrace_func_map {
	struct ftrace_func_entry	entry;
	void				*data;
4134 4135
};

4136 4137 4138
struct ftrace_func_mapper {
	struct ftrace_hash		hash;
};
4139

4140 4141 4142 4143 4144 4145
/**
 * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper
 *
 * Returns a ftrace_func_mapper descriptor that can be used to map ips to data.
 */
struct ftrace_func_mapper *allocate_ftrace_func_mapper(void)
4146
{
4147
	struct ftrace_hash *hash;
4148

4149 4150 4151 4152 4153 4154 4155 4156
	/*
	 * The mapper is simply a ftrace_hash, but since the entries
	 * in the hash are not ftrace_func_entry type, we define it
	 * as a separate structure.
	 */
	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
	return (struct ftrace_func_mapper *)hash;
}
4157

4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173
/**
 * ftrace_func_mapper_find_ip - Find some data mapped to an ip
 * @mapper: The mapper that has the ip maps
 * @ip: the instruction pointer to find the data for
 *
 * Returns the data mapped to @ip if found otherwise NULL. The return
 * is actually the address of the mapper data pointer. The address is
 * returned for use cases where the data is no bigger than a long, and
 * the user can use the data pointer as its data instead of having to
 * allocate more memory for the reference.
 */
void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
				  unsigned long ip)
{
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;
4174

4175 4176 4177
	entry = ftrace_lookup_ip(&mapper->hash, ip);
	if (!entry)
		return NULL;
4178

4179 4180
	map = (struct ftrace_func_map *)entry;
	return &map->data;
4181 4182
}

4183 4184 4185 4186 4187 4188 4189 4190 4191 4192
/**
 * ftrace_func_mapper_add_ip - Map some data to an ip
 * @mapper: The mapper that has the ip maps
 * @ip: The instruction pointer address to map @data to
 * @data: The data to map to @ip
 *
 * Returns 0 on succes otherwise an error.
 */
int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
			      unsigned long ip, void *data)
4193
{
4194 4195
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;
4196

4197 4198 4199
	entry = ftrace_lookup_ip(&mapper->hash, ip);
	if (entry)
		return -EBUSY;
4200

4201 4202 4203
	map = kmalloc(sizeof(*map), GFP_KERNEL);
	if (!map)
		return -ENOMEM;
4204

4205 4206
	map->entry.ip = ip;
	map->data = data;
4207

4208
	__add_hash_entry(&mapper->hash, &map->entry);
4209

4210 4211
	return 0;
}
4212

4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224
/**
 * ftrace_func_mapper_remove_ip - Remove an ip from the mapping
 * @mapper: The mapper that has the ip maps
 * @ip: The instruction pointer address to remove the data from
 *
 * Returns the data if it is found, otherwise NULL.
 * Note, if the data pointer is used as the data itself, (see 
 * ftrace_func_mapper_find_ip(), then the return value may be meaningless,
 * if the data pointer was set to zero.
 */
void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
				   unsigned long ip)
4225
{
4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;
	void *data;

	entry = ftrace_lookup_ip(&mapper->hash, ip);
	if (!entry)
		return NULL;

	map = (struct ftrace_func_map *)entry;
	data = map->data;

	remove_hash_entry(&mapper->hash, entry);
4238
	kfree(entry);
4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256

	return data;
}

/**
 * free_ftrace_func_mapper - free a mapping of ips and data
 * @mapper: The mapper that has the ip maps
 * @free_func: A function to be called on each data item.
 *
 * This is used to free the function mapper. The @free_func is optional
 * and can be used if the data needs to be freed as well.
 */
void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
			     ftrace_mapper_func free_func)
{
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;
	struct hlist_head *hhd;
4257 4258 4259 4260
	int size, i;

	if (!mapper)
		return;
4261 4262

	if (free_func && mapper->hash.count) {
4263
		size = 1 << mapper->hash.size_bits;
4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274
		for (i = 0; i < size; i++) {
			hhd = &mapper->hash.buckets[i];
			hlist_for_each_entry(entry, hhd, hlist) {
				map = (struct ftrace_func_map *)entry;
				free_func(map);
			}
		}
	}
	free_ftrace_hash(&mapper->hash);
}

4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287
static void release_probe(struct ftrace_func_probe *probe)
{
	struct ftrace_probe_ops *probe_ops;

	mutex_lock(&ftrace_lock);

	WARN_ON(probe->ref <= 0);

	/* Subtract the ref that was used to protect this instance */
	probe->ref--;

	if (!probe->ref) {
		probe_ops = probe->probe_ops;
4288 4289 4290 4291 4292 4293
		/*
		 * Sending zero as ip tells probe_ops to free
		 * the probe->data itself
		 */
		if (probe_ops->free)
			probe_ops->free(probe_ops, probe->tr, 0, probe->data);
4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306
		list_del(&probe->list);
		kfree(probe);
	}
	mutex_unlock(&ftrace_lock);
}

static void acquire_probe_locked(struct ftrace_func_probe *probe)
{
	/*
	 * Add one ref to keep it from being freed when releasing the
	 * ftrace_lock mutex.
	 */
	probe->ref++;
4307 4308 4309
}

int
4310
register_ftrace_function_probe(char *glob, struct trace_array *tr,
4311 4312
			       struct ftrace_probe_ops *probe_ops,
			       void *data)
4313
{
4314
	struct ftrace_func_entry *entry;
4315
	struct ftrace_func_probe *probe;
4316 4317
	struct ftrace_hash **orig_hash;
	struct ftrace_hash *old_hash;
4318
	struct ftrace_hash *hash;
4319
	int count = 0;
4320
	int size;
4321
	int ret;
4322
	int i;
4323

4324
	if (WARN_ON(!tr))
4325 4326
		return -EINVAL;

4327 4328
	/* We do not support '!' for function probes */
	if (WARN_ON(glob[0] == '!'))
4329 4330
		return -EINVAL;

4331

4332 4333 4334 4335 4336
	mutex_lock(&ftrace_lock);
	/* Check if the probe_ops is already registered */
	list_for_each_entry(probe, &tr->func_probes, list) {
		if (probe->probe_ops == probe_ops)
			break;
4337
	}
4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348
	if (&probe->list == &tr->func_probes) {
		probe = kzalloc(sizeof(*probe), GFP_KERNEL);
		if (!probe) {
			mutex_unlock(&ftrace_lock);
			return -ENOMEM;
		}
		probe->probe_ops = probe_ops;
		probe->ops.func = function_trace_probe_call;
		probe->tr = tr;
		ftrace_ops_init(&probe->ops);
		list_add(&probe->list, &tr->func_probes);
4349
	}
4350

4351
	acquire_probe_locked(probe);
4352

4353
	mutex_unlock(&ftrace_lock);
4354

4355 4356 4357 4358
	/*
	 * Note, there's a small window here that the func_hash->filter_hash
	 * may be NULL or empty. Need to be carefule when reading the loop.
	 */
4359
	mutex_lock(&probe->ops.func_hash->regex_lock);
4360

4361
	orig_hash = &probe->ops.func_hash->filter_hash;
4362 4363
	old_hash = *orig_hash;
	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
4364

4365 4366 4367 4368 4369
	if (!hash) {
		ret = -ENOMEM;
		goto out;
	}

4370
	ret = ftrace_match_records(hash, glob, strlen(glob));
4371

4372 4373 4374
	/* Nothing found? */
	if (!ret)
		ret = -EINVAL;
4375

4376 4377
	if (ret < 0)
		goto out;
4378

4379 4380 4381 4382
	size = 1 << hash->size_bits;
	for (i = 0; i < size; i++) {
		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
			if (ftrace_lookup_ip(old_hash, entry->ip))
4383
				continue;
4384 4385 4386 4387 4388
			/*
			 * The caller might want to do something special
			 * for each function we find. We call the callback
			 * to give the caller an opportunity to do so.
			 */
4389 4390
			if (probe_ops->init) {
				ret = probe_ops->init(probe_ops, tr,
4391 4392 4393 4394 4395 4396 4397
						      entry->ip, data,
						      &probe->data);
				if (ret < 0) {
					if (probe_ops->free && count)
						probe_ops->free(probe_ops, tr,
								0, probe->data);
					probe->data = NULL;
4398
					goto out;
4399
				}
4400
			}
4401
			count++;
4402
		}
4403
	}
4404

4405
	mutex_lock(&ftrace_lock);
4406

4407 4408 4409 4410 4411
	if (!count) {
		/* Nothing was added? */
		ret = -EINVAL;
		goto out_unlock;
	}
4412

4413 4414
	ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
					      hash, 1);
4415
	if (ret < 0)
4416
		goto err_unlock;
4417

4418 4419
	/* One ref for each new function traced */
	probe->ref += count;
4420

4421 4422
	if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED))
		ret = ftrace_startup(&probe->ops, 0);
4423

4424
 out_unlock:
4425
	mutex_unlock(&ftrace_lock);
4426

4427
	if (!ret)
4428
		ret = count;
4429
 out:
4430
	mutex_unlock(&probe->ops.func_hash->regex_lock);
4431
	free_ftrace_hash(hash);
4432

4433
	release_probe(probe);
4434

4435
	return ret;
4436

4437
 err_unlock:
4438
	if (!probe_ops->free || !count)
4439 4440 4441 4442 4443 4444 4445
		goto out_unlock;

	/* Failed to do the move, need to call the free functions */
	for (i = 0; i < size; i++) {
		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
			if (ftrace_lookup_ip(old_hash, entry->ip))
				continue;
4446
			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
4447 4448 4449
		}
	}
	goto out_unlock;
4450 4451
}

4452
int
4453 4454
unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
				      struct ftrace_probe_ops *probe_ops)
4455
{
4456
	struct ftrace_ops_hash old_hash_ops;
4457
	struct ftrace_func_entry *entry;
4458
	struct ftrace_func_probe *probe;
4459
	struct ftrace_glob func_g;
4460 4461 4462
	struct ftrace_hash **orig_hash;
	struct ftrace_hash *old_hash;
	struct ftrace_hash *hash = NULL;
4463
	struct hlist_node *tmp;
4464
	struct hlist_head hhd;
4465
	char str[KSYM_SYMBOL_LEN];
4466 4467
	int count = 0;
	int i, ret = -ENODEV;
4468
	int size;
4469

4470
	if (!glob || !strlen(glob) || !strcmp(glob, "*"))
4471
		func_g.search = NULL;
4472
	else {
4473 4474
		int not;

4475 4476 4477
		func_g.type = filter_parse_regex(glob, strlen(glob),
						 &func_g.search, &not);
		func_g.len = strlen(func_g.search);
4478

4479
		/* we do not support '!' for function probes */
4480
		if (WARN_ON(not))
4481
			return -EINVAL;
4482 4483
	}

4484 4485 4486 4487 4488
	mutex_lock(&ftrace_lock);
	/* Check if the probe_ops is already registered */
	list_for_each_entry(probe, &tr->func_probes, list) {
		if (probe->probe_ops == probe_ops)
			break;
4489
	}
4490 4491 4492 4493 4494 4495 4496 4497 4498 4499
	if (&probe->list == &tr->func_probes)
		goto err_unlock_ftrace;

	ret = -EINVAL;
	if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED))
		goto err_unlock_ftrace;

	acquire_probe_locked(probe);

	mutex_unlock(&ftrace_lock);
4500

4501
	mutex_lock(&probe->ops.func_hash->regex_lock);
4502

4503
	orig_hash = &probe->ops.func_hash->filter_hash;
4504 4505 4506 4507
	old_hash = *orig_hash;

	if (ftrace_hash_empty(old_hash))
		goto out_unlock;
4508

4509 4510 4511 4512
	old_hash_ops.filter_hash = old_hash;
	/* Probes only have filters */
	old_hash_ops.notrace_hash = NULL;

4513
	ret = -ENOMEM;
4514
	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
4515 4516 4517
	if (!hash)
		goto out_unlock;

4518
	INIT_HLIST_HEAD(&hhd);
4519

4520 4521 4522
	size = 1 << hash->size_bits;
	for (i = 0; i < size; i++) {
		hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) {
4523

4524
			if (func_g.search) {
4525 4526
				kallsyms_lookup(entry->ip, NULL, NULL,
						NULL, str);
4527
				if (!ftrace_match(str, &func_g))
4528 4529
					continue;
			}
4530
			count++;
4531 4532
			remove_hash_entry(hash, entry);
			hlist_add_head(&entry->hlist, &hhd);
4533 4534
		}
	}
4535 4536

	/* Nothing found? */
4537
	if (!count) {
4538 4539 4540 4541
		ret = -EINVAL;
		goto out_unlock;
	}

4542
	mutex_lock(&ftrace_lock);
4543

4544
	WARN_ON(probe->ref < count);
4545

4546
	probe->ref -= count;
4547

4548 4549 4550 4551
	if (ftrace_hash_empty(hash))
		ftrace_shutdown(&probe->ops, 0);

	ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
4552
					      hash, 1);
4553 4554

	/* still need to update the function call sites */
4555
	if (ftrace_enabled && !ftrace_hash_empty(hash))
4556
		ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS,
4557
				       &old_hash_ops);
4558
	synchronize_rcu();
4559

4560 4561
	hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) {
		hlist_del(&entry->hlist);
4562
		if (probe_ops->free)
4563
			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
4564
		kfree(entry);
4565
	}
4566
	mutex_unlock(&ftrace_lock);
4567

4568
 out_unlock:
4569
	mutex_unlock(&probe->ops.func_hash->regex_lock);
4570
	free_ftrace_hash(hash);
4571

4572
	release_probe(probe);
4573

4574
	return ret;
4575

4576 4577
 err_unlock_ftrace:
	mutex_unlock(&ftrace_lock);
4578
	return ret;
4579 4580
}

4581 4582 4583 4584 4585 4586 4587 4588
void clear_ftrace_function_probes(struct trace_array *tr)
{
	struct ftrace_func_probe *probe, *n;

	list_for_each_entry_safe(probe, n, &tr->func_probes, list)
		unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops);
}

4589 4590 4591
static LIST_HEAD(ftrace_commands);
static DEFINE_MUTEX(ftrace_cmd_mutex);

4592 4593 4594 4595 4596
/*
 * Currently we only register ftrace commands from __init, so mark this
 * __init too.
 */
__init int register_ftrace_command(struct ftrace_func_command *cmd)
4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614
{
	struct ftrace_func_command *p;
	int ret = 0;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = -EBUSY;
			goto out_unlock;
		}
	}
	list_add(&cmd->list, &ftrace_commands);
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

4615 4616 4617 4618 4619
/*
 * Currently we only unregister ftrace commands from __init, so mark
 * this __init too.
 */
__init int unregister_ftrace_command(struct ftrace_func_command *cmd)
4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637
{
	struct ftrace_func_command *p, *n;
	int ret = -ENODEV;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = 0;
			list_del_init(&p->list);
			goto out_unlock;
		}
	}
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

4638
static int ftrace_process_regex(struct ftrace_iterator *iter,
4639
				char *buff, int len, int enable)
4640
{
4641
	struct ftrace_hash *hash = iter->hash;
4642
	struct trace_array *tr = iter->ops->private;
4643
	char *func, *command, *next = buff;
4644
	struct ftrace_func_command *p;
4645
	int ret = -EINVAL;
4646 4647 4648 4649

	func = strsep(&next, ":");

	if (!next) {
4650
		ret = ftrace_match_records(hash, func, len);
4651 4652 4653 4654 4655
		if (!ret)
			ret = -EINVAL;
		if (ret < 0)
			return ret;
		return 0;
4656 4657
	}

4658
	/* command found */
4659 4660 4661

	command = strsep(&next, ":");

4662 4663 4664
	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(p->name, command) == 0) {
4665
			ret = p->func(tr, hash, func, command, next, enable);
4666 4667
			goto out_unlock;
		}
4668
	}
4669 4670
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);
4671

4672
	return ret;
4673 4674
}

Ingo Molnar's avatar
Ingo Molnar committed
4675
static ssize_t
4676 4677
ftrace_regex_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos, int enable)
4678 4679
{
	struct ftrace_iterator *iter;
4680 4681
	struct trace_parser *parser;
	ssize_t ret, read;
4682

4683
	if (!cnt)
4684 4685 4686 4687 4688 4689 4690 4691
		return 0;

	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;
		iter = m->private;
	} else
		iter = file->private_data;

4692
	if (unlikely(ftrace_disabled))
4693 4694 4695
		return -ENODEV;

	/* iter->hash is a local copy, so we don't need regex_lock */
4696

4697 4698
	parser = &iter->parser;
	read = trace_get_user(parser, ubuf, cnt, ppos);
4699

4700
	if (read >= 0 && trace_parser_loaded(parser) &&
4701
	    !trace_parser_cont(parser)) {
4702
		ret = ftrace_process_regex(iter, parser->buffer,
4703
					   parser->idx, enable);
4704
		trace_parser_clear(parser);
4705
		if (ret < 0)
4706
			goto out;
4707
	}
4708 4709

	ret = read;
4710
 out:
4711 4712 4713
	return ret;
}

4714
ssize_t
4715 4716 4717 4718 4719 4720
ftrace_filter_write(struct file *file, const char __user *ubuf,
		    size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
}

4721
ssize_t
4722 4723 4724 4725 4726 4727
ftrace_notrace_write(struct file *file, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
}

4728
static int
4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749
ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
{
	struct ftrace_func_entry *entry;

	if (!ftrace_location(ip))
		return -EINVAL;

	if (remove) {
		entry = ftrace_lookup_ip(hash, ip);
		if (!entry)
			return -ENOENT;
		free_hash_entry(hash, entry);
		return 0;
	}

	return add_hash_entry(hash, ip);
}

static int
ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
		unsigned long ip, int remove, int reset, int enable)
4750
{
4751
	struct ftrace_hash **orig_hash;
4752
	struct ftrace_hash *hash;
4753
	int ret;
4754

4755
	if (unlikely(ftrace_disabled))
4756
		return -ENODEV;
4757

4758
	mutex_lock(&ops->func_hash->regex_lock);
4759

4760
	if (enable)
4761
		orig_hash = &ops->func_hash->filter_hash;
4762
	else
4763
		orig_hash = &ops->func_hash->notrace_hash;
4764

4765 4766 4767 4768 4769
	if (reset)
		hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
	else
		hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);

4770 4771 4772 4773
	if (!hash) {
		ret = -ENOMEM;
		goto out_regex_unlock;
	}
4774

4775 4776 4777 4778
	if (buf && !ftrace_match_records(hash, buf, len)) {
		ret = -EINVAL;
		goto out_regex_unlock;
	}
4779 4780 4781 4782 4783
	if (ip) {
		ret = ftrace_match_addr(hash, ip, remove);
		if (ret < 0)
			goto out_regex_unlock;
	}
4784 4785

	mutex_lock(&ftrace_lock);
4786
	ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable);
4787 4788
	mutex_unlock(&ftrace_lock);

4789
 out_regex_unlock:
4790
	mutex_unlock(&ops->func_hash->regex_lock);
4791 4792 4793

	free_ftrace_hash(hash);
	return ret;
4794 4795
}

4796 4797 4798 4799
static int
ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
		int reset, int enable)
{
4800
	return ftrace_set_hash(ops, NULL, 0, ip, remove, reset, enable);
4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815
}

/**
 * ftrace_set_filter_ip - set a function to filter on in ftrace by address
 * @ops - the ops to set the filter with
 * @ip - the address to add to or remove from the filter.
 * @remove - non zero to remove the ip from the filter
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled
 * If @ip is NULL, it failes to update filter.
 */
int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
			 int remove, int reset)
{
4816
	ftrace_ops_init(ops);
4817 4818 4819 4820
	return ftrace_set_addr(ops, ip, remove, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);

4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837
/**
 * ftrace_ops_set_global_filter - setup ops to use global filters
 * @ops - the ops which will use the global filters
 *
 * ftrace users who need global function trace filtering should call this.
 * It can set the global filter only if ops were not initialized before.
 */
void ftrace_ops_set_global_filter(struct ftrace_ops *ops)
{
	if (ops->flags & FTRACE_OPS_FL_INITIALIZED)
		return;

	ftrace_ops_init(ops);
	ops->func_hash = &global_ops.local_hash;
}
EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter);

4838 4839 4840 4841 4842 4843 4844
static int
ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
		 int reset, int enable)
{
	return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
}

4845 4846
/**
 * ftrace_set_filter - set a function to filter on in ftrace
4847 4848 4849 4850 4851 4852 4853 4854
 * @ops - the ops to set the filter with
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
4855
int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
4856 4857
		       int len, int reset)
{
4858
	ftrace_ops_init(ops);
4859
	return ftrace_set_regex(ops, buf, len, reset, 1);
4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873
}
EXPORT_SYMBOL_GPL(ftrace_set_filter);

/**
 * ftrace_set_notrace - set a function to not trace in ftrace
 * @ops - the ops to set the notrace filter with
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
4874
int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
4875 4876
			int len, int reset)
{
4877
	ftrace_ops_init(ops);
4878
	return ftrace_set_regex(ops, buf, len, reset, 0);
4879 4880 4881
}
EXPORT_SYMBOL_GPL(ftrace_set_notrace);
/**
4882
 * ftrace_set_global_filter - set a function to filter on with global tracers
4883 4884 4885 4886 4887 4888 4889
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
4890
void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
4891
{
4892
	ftrace_set_regex(&global_ops, buf, len, reset, 1);
4893
}
4894
EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
4895

4896
/**
4897
 * ftrace_set_global_notrace - set a function to not trace with global tracers
4898 4899 4900 4901 4902 4903 4904 4905
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
4906
void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
4907
{
4908
	ftrace_set_regex(&global_ops, buf, len, reset, 0);
4909
}
4910
EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
4911

4912 4913 4914 4915 4916 4917 4918
/*
 * command line interface to allow users to set filters on boot up.
 */
#define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;

4919 4920 4921
/* Used by function selftest to not test if filter is set */
bool ftrace_filter_param __initdata;

4922 4923
static int __init set_ftrace_notrace(char *str)
{
4924
	ftrace_filter_param = true;
4925
	strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
4926 4927 4928 4929 4930 4931
	return 1;
}
__setup("ftrace_notrace=", set_ftrace_notrace);

static int __init set_ftrace_filter(char *str)
{
4932
	ftrace_filter_param = true;
4933
	strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
4934 4935 4936 4937
	return 1;
}
__setup("ftrace_filter=", set_ftrace_filter);

4938
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4939
static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
4940
static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
4941
static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer);
4942

4943 4944
static int __init set_graph_function(char *str)
{
4945
	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
4946 4947 4948 4949
	return 1;
}
__setup("ftrace_graph_filter=", set_graph_function);

4950 4951 4952 4953 4954 4955 4956
static int __init set_graph_notrace_function(char *str)
{
	strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE);
	return 1;
}
__setup("ftrace_graph_notrace=", set_graph_notrace_function);

4957 4958 4959 4960 4961 4962 4963 4964
static int __init set_graph_max_depth_function(char *str)
{
	if (!str)
		return 0;
	fgraph_max_depth = simple_strtoul(str, NULL, 0);
	return 1;
}
__setup("ftrace_graph_max_depth=", set_graph_max_depth_function);
4965 4966

static void __init set_ftrace_early_graph(char *buf, int enable)
4967 4968 4969
{
	int ret;
	char *func;
4970
	struct ftrace_hash *hash;
4971

4972 4973 4974
	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
	if (WARN_ON(!hash))
		return;
4975 4976 4977 4978

	while (buf) {
		func = strsep(&buf, ",");
		/* we allow only one expression at a time */
4979
		ret = ftrace_graph_set_hash(hash, func);
4980 4981 4982 4983
		if (ret)
			printk(KERN_DEBUG "ftrace: function %s not "
					  "traceable\n", func);
	}
4984 4985 4986 4987 4988

	if (enable)
		ftrace_graph_hash = hash;
	else
		ftrace_graph_notrace_hash = hash;
4989 4990 4991
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

4992 4993
void __init
ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
4994 4995 4996
{
	char *func;

4997 4998
	ftrace_ops_init(ops);

4999 5000
	while (buf) {
		func = strsep(&buf, ",");
5001
		ftrace_set_regex(ops, func, strlen(func), 0, enable);
5002 5003 5004 5005 5006 5007
	}
}

static void __init set_ftrace_early_filters(void)
{
	if (ftrace_filter_buf[0])
5008
		ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
5009
	if (ftrace_notrace_buf[0])
5010
		ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
5011 5012
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	if (ftrace_graph_buf[0])
5013 5014 5015
		set_ftrace_early_graph(ftrace_graph_buf, 1);
	if (ftrace_graph_notrace_buf[0])
		set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
5016
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
5017 5018
}

5019
int ftrace_regex_release(struct inode *inode, struct file *file)
5020 5021 5022
{
	struct seq_file *m = (struct seq_file *)file->private_data;
	struct ftrace_iterator *iter;
5023
	struct ftrace_hash **orig_hash;
5024
	struct trace_parser *parser;
5025
	int filter_hash;
5026
	int ret;
5027 5028 5029 5030 5031 5032 5033

	if (file->f_mode & FMODE_READ) {
		iter = m->private;
		seq_release(inode, file);
	} else
		iter = file->private_data;

5034 5035
	parser = &iter->parser;
	if (trace_parser_loaded(parser)) {
5036
		ftrace_match_records(iter->hash, parser->buffer, parser->idx);
5037 5038
	}

5039 5040
	trace_parser_put(parser);

5041
	mutex_lock(&iter->ops->func_hash->regex_lock);
5042

5043
	if (file->f_mode & FMODE_WRITE) {
5044 5045
		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);

5046
		if (filter_hash) {
5047
			orig_hash = &iter->ops->func_hash->filter_hash;
5048
			if (iter->tr && !list_empty(&iter->tr->mod_trace))
5049 5050
				iter->hash->flags |= FTRACE_HASH_FL_MOD;
		} else
5051
			orig_hash = &iter->ops->func_hash->notrace_hash;
5052

5053
		mutex_lock(&ftrace_lock);
5054 5055
		ret = ftrace_hash_move_and_update_ops(iter->ops, orig_hash,
						      iter->hash, filter_hash);
5056
		mutex_unlock(&ftrace_lock);
5057 5058 5059
	} else {
		/* For read only, the hash is the ops hash */
		iter->hash = NULL;
5060
	}
5061

5062
	mutex_unlock(&iter->ops->func_hash->regex_lock);
5063
	free_ftrace_hash(iter->hash);
5064 5065
	if (iter->tr)
		trace_array_put(iter->tr);
5066
	kfree(iter);
5067

5068 5069 5070
	return 0;
}

5071
static const struct file_operations ftrace_avail_fops = {
5072 5073 5074
	.open = ftrace_avail_open,
	.read = seq_read,
	.llseek = seq_lseek,
Li Zefan's avatar
Li Zefan committed
5075
	.release = seq_release_private,
5076 5077
};

5078 5079 5080 5081 5082 5083 5084
static const struct file_operations ftrace_enabled_fops = {
	.open = ftrace_enabled_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = seq_release_private,
};

5085
static const struct file_operations ftrace_filter_fops = {
5086
	.open = ftrace_filter_open,
Lai Jiangshan's avatar
Lai Jiangshan committed
5087
	.read = seq_read,
5088
	.write = ftrace_filter_write,
5089
	.llseek = tracing_lseek,
5090
	.release = ftrace_regex_release,
5091 5092
};

5093
static const struct file_operations ftrace_notrace_fops = {
5094
	.open = ftrace_notrace_open,
Lai Jiangshan's avatar
Lai Jiangshan committed
5095
	.read = seq_read,
5096
	.write = ftrace_notrace_write,
5097
	.llseek = tracing_lseek,
5098
	.release = ftrace_regex_release,
5099 5100
};

5101 5102 5103 5104
#ifdef CONFIG_FUNCTION_GRAPH_TRACER

static DEFINE_MUTEX(graph_lock);

5105 5106 5107 5108 5109 5110 5111
struct ftrace_hash *ftrace_graph_hash = EMPTY_HASH;
struct ftrace_hash *ftrace_graph_notrace_hash = EMPTY_HASH;

enum graph_filter_type {
	GRAPH_FILTER_NOTRACE	= 0,
	GRAPH_FILTER_FUNCTION,
};
5112

5113 5114
#define FTRACE_GRAPH_EMPTY	((void *)1)

5115
struct ftrace_graph_data {
5116 5117 5118 5119 5120 5121 5122
	struct ftrace_hash		*hash;
	struct ftrace_func_entry	*entry;
	int				idx;   /* for hash table iteration */
	enum graph_filter_type		type;
	struct ftrace_hash		*new_hash;
	const struct seq_operations	*seq_ops;
	struct trace_parser		parser;
5123 5124
};

5125
static void *
5126
__g_next(struct seq_file *m, loff_t *pos)
5127
{
5128
	struct ftrace_graph_data *fgd = m->private;
5129 5130 5131
	struct ftrace_func_entry *entry = fgd->entry;
	struct hlist_head *head;
	int i, idx = fgd->idx;
5132

5133
	if (*pos >= fgd->hash->count)
5134
		return NULL;
5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153

	if (entry) {
		hlist_for_each_entry_continue(entry, hlist) {
			fgd->entry = entry;
			return entry;
		}

		idx++;
	}

	for (i = idx; i < 1 << fgd->hash->size_bits; i++) {
		head = &fgd->hash->buckets[i];
		hlist_for_each_entry(entry, head, hlist) {
			fgd->entry = entry;
			fgd->idx = i;
			return entry;
		}
	}
	return NULL;
5154
}
5155

5156 5157 5158 5159 5160
static void *
g_next(struct seq_file *m, void *v, loff_t *pos)
{
	(*pos)++;
	return __g_next(m, pos);
5161 5162 5163 5164
}

static void *g_start(struct seq_file *m, loff_t *pos)
{
5165 5166
	struct ftrace_graph_data *fgd = m->private;

5167 5168
	mutex_lock(&graph_lock);

5169 5170 5171 5172 5173 5174 5175
	if (fgd->type == GRAPH_FILTER_FUNCTION)
		fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
					lockdep_is_held(&graph_lock));
	else
		fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
					lockdep_is_held(&graph_lock));

5176
	/* Nothing, tell g_show to print all functions are enabled */
5177
	if (ftrace_hash_empty(fgd->hash) && !*pos)
5178
		return FTRACE_GRAPH_EMPTY;
5179

5180 5181
	fgd->idx = 0;
	fgd->entry = NULL;
5182
	return __g_next(m, pos);
5183 5184 5185 5186 5187 5188 5189 5190 5191
}

static void g_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&graph_lock);
}

static int g_show(struct seq_file *m, void *v)
{
5192
	struct ftrace_func_entry *entry = v;
5193

5194
	if (!entry)
5195 5196
		return 0;

5197
	if (entry == FTRACE_GRAPH_EMPTY) {
5198 5199
		struct ftrace_graph_data *fgd = m->private;

5200
		if (fgd->type == GRAPH_FILTER_FUNCTION)
5201
			seq_puts(m, "#### all functions enabled ####\n");
5202
		else
5203
			seq_puts(m, "#### no functions disabled ####\n");
5204 5205 5206
		return 0;
	}

5207
	seq_printf(m, "%ps\n", (void *)entry->ip);
5208 5209 5210 5211

	return 0;
}

5212
static const struct seq_operations ftrace_graph_seq_ops = {
5213 5214 5215 5216 5217 5218 5219
	.start = g_start,
	.next = g_next,
	.stop = g_stop,
	.show = g_show,
};

static int
5220 5221
__ftrace_graph_open(struct inode *inode, struct file *file,
		    struct ftrace_graph_data *fgd)
5222
{
5223
	int ret;
5224
	struct ftrace_hash *new_hash = NULL;
5225

5226 5227 5228 5229
	ret = security_locked_down(LOCKDOWN_TRACEFS);
	if (ret)
		return ret;

5230 5231 5232
	if (file->f_mode & FMODE_WRITE) {
		const int size_bits = FTRACE_HASH_DEFAULT_BITS;

5233 5234 5235
		if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX))
			return -ENOMEM;

5236 5237 5238 5239 5240 5241 5242 5243 5244
		if (file->f_flags & O_TRUNC)
			new_hash = alloc_ftrace_hash(size_bits);
		else
			new_hash = alloc_and_copy_ftrace_hash(size_bits,
							      fgd->hash);
		if (!new_hash) {
			ret = -ENOMEM;
			goto out;
		}
5245 5246
	}

5247
	if (file->f_mode & FMODE_READ) {
5248
		ret = seq_open(file, &ftrace_graph_seq_ops);
5249 5250 5251
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = fgd;
5252 5253 5254 5255
		} else {
			/* Failed */
			free_ftrace_hash(new_hash);
			new_hash = NULL;
5256 5257 5258
		}
	} else
		file->private_data = fgd;
5259

5260
out:
5261 5262 5263
	if (ret < 0 && file->f_mode & FMODE_WRITE)
		trace_parser_put(&fgd->parser);

5264
	fgd->new_hash = new_hash;
5265 5266 5267 5268 5269 5270 5271 5272

	/*
	 * All uses of fgd->hash must be taken with the graph_lock
	 * held. The graph_lock is going to be released, so force
	 * fgd->hash to be reinitialized when it is taken again.
	 */
	fgd->hash = NULL;

5273 5274 5275
	return ret;
}

5276 5277 5278 5279
static int
ftrace_graph_open(struct inode *inode, struct file *file)
{
	struct ftrace_graph_data *fgd;
5280
	int ret;
5281 5282 5283 5284 5285 5286 5287 5288

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
	if (fgd == NULL)
		return -ENOMEM;

5289 5290
	mutex_lock(&graph_lock);

5291 5292
	fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
					lockdep_is_held(&graph_lock));
5293
	fgd->type = GRAPH_FILTER_FUNCTION;
5294 5295
	fgd->seq_ops = &ftrace_graph_seq_ops;

5296 5297 5298 5299 5300 5301
	ret = __ftrace_graph_open(inode, file, fgd);
	if (ret < 0)
		kfree(fgd);

	mutex_unlock(&graph_lock);
	return ret;
5302 5303
}

5304 5305 5306 5307
static int
ftrace_graph_notrace_open(struct inode *inode, struct file *file)
{
	struct ftrace_graph_data *fgd;
5308
	int ret;
5309 5310 5311 5312 5313 5314 5315 5316

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
	if (fgd == NULL)
		return -ENOMEM;

5317 5318
	mutex_lock(&graph_lock);

5319 5320
	fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
					lockdep_is_held(&graph_lock));
5321
	fgd->type = GRAPH_FILTER_NOTRACE;
5322 5323
	fgd->seq_ops = &ftrace_graph_seq_ops;

5324 5325 5326 5327 5328 5329
	ret = __ftrace_graph_open(inode, file, fgd);
	if (ret < 0)
		kfree(fgd);

	mutex_unlock(&graph_lock);
	return ret;
5330 5331
}

5332 5333 5334
static int
ftrace_graph_release(struct inode *inode, struct file *file)
{
5335
	struct ftrace_graph_data *fgd;
5336 5337 5338
	struct ftrace_hash *old_hash, *new_hash;
	struct trace_parser *parser;
	int ret = 0;
5339

5340 5341 5342
	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;

5343
		fgd = m->private;
5344
		seq_release(inode, file);
5345
	} else {
5346
		fgd = file->private_data;
5347 5348
	}

5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381

	if (file->f_mode & FMODE_WRITE) {

		parser = &fgd->parser;

		if (trace_parser_loaded((parser))) {
			ret = ftrace_graph_set_hash(fgd->new_hash,
						    parser->buffer);
		}

		trace_parser_put(parser);

		new_hash = __ftrace_hash_move(fgd->new_hash);
		if (!new_hash) {
			ret = -ENOMEM;
			goto out;
		}

		mutex_lock(&graph_lock);

		if (fgd->type == GRAPH_FILTER_FUNCTION) {
			old_hash = rcu_dereference_protected(ftrace_graph_hash,
					lockdep_is_held(&graph_lock));
			rcu_assign_pointer(ftrace_graph_hash, new_hash);
		} else {
			old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
					lockdep_is_held(&graph_lock));
			rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash);
		}

		mutex_unlock(&graph_lock);

		/* Wait till all users are no longer using the old hash */
5382
		synchronize_rcu();
5383 5384 5385 5386 5387

		free_ftrace_hash(old_hash);
	}

 out:
5388
	free_ftrace_hash(fgd->new_hash);
5389 5390
	kfree(fgd);

5391
	return ret;
5392 5393
}

5394
static int
5395
ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer)
5396
{
5397
	struct ftrace_glob func_g;
5398 5399
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
5400
	struct ftrace_func_entry *entry;
5401
	int fail = 1;
5402
	int not;
5403

5404
	/* decode regex */
5405 5406
	func_g.type = filter_parse_regex(buffer, strlen(buffer),
					 &func_g.search, &not);
5407

5408
	func_g.len = strlen(func_g.search);
5409

5410
	mutex_lock(&ftrace_lock);
5411 5412 5413 5414 5415 5416

	if (unlikely(ftrace_disabled)) {
		mutex_unlock(&ftrace_lock);
		return -ENODEV;
	}

5417 5418
	do_for_each_ftrace_rec(pg, rec) {

5419 5420 5421
		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

5422
		if (ftrace_match_record(rec, &func_g, NULL, 0)) {
5423
			entry = ftrace_lookup_ip(hash, rec->ip);
5424 5425 5426

			if (!not) {
				fail = 0;
5427 5428 5429 5430 5431

				if (entry)
					continue;
				if (add_hash_entry(hash, rec->ip) < 0)
					goto out;
5432
			} else {
5433 5434
				if (entry) {
					free_hash_entry(hash, entry);
5435 5436 5437
					fail = 0;
				}
			}
5438
		}
5439
	} while_for_each_ftrace_rec();
5440
out:
5441
	mutex_unlock(&ftrace_lock);
5442

5443 5444 5445 5446
	if (fail)
		return -EINVAL;

	return 0;
5447 5448 5449 5450 5451 5452
}

static ssize_t
ftrace_graph_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
5453
	ssize_t read, ret = 0;
5454
	struct ftrace_graph_data *fgd = file->private_data;
5455
	struct trace_parser *parser;
5456

5457
	if (!cnt)
5458 5459
		return 0;

5460 5461 5462 5463 5464 5465
	/* Read mode uses seq functions */
	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;
		fgd = m->private;
	}

5466
	parser = &fgd->parser;
5467

5468
	read = trace_get_user(parser, ubuf, cnt, ppos);
5469

5470 5471
	if (read >= 0 && trace_parser_loaded(parser) &&
	    !trace_parser_cont(parser)) {
5472

5473
		ret = ftrace_graph_set_hash(fgd->new_hash,
5474 5475
					    parser->buffer);
		trace_parser_clear(parser);
5476 5477
	}

5478 5479
	if (!ret)
		ret = read;
5480

5481 5482 5483 5484
	return ret;
}

static const struct file_operations ftrace_graph_fops = {
5485 5486 5487
	.open		= ftrace_graph_open,
	.read		= seq_read,
	.write		= ftrace_graph_write,
5488
	.llseek		= tracing_lseek,
5489
	.release	= ftrace_graph_release,
5490
};
5491 5492 5493 5494 5495

static const struct file_operations ftrace_graph_notrace_fops = {
	.open		= ftrace_graph_notrace_open,
	.read		= seq_read,
	.write		= ftrace_graph_write,
5496
	.llseek		= tracing_lseek,
5497 5498
	.release	= ftrace_graph_release,
};
5499 5500
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513
void ftrace_create_filter_files(struct ftrace_ops *ops,
				struct dentry *parent)
{

	trace_create_file("set_ftrace_filter", 0644, parent,
			  ops, &ftrace_filter_fops);

	trace_create_file("set_ftrace_notrace", 0644, parent,
			  ops, &ftrace_notrace_fops);
}

/*
 * The name "destroy_filter_files" is really a misnomer. Although
5514
 * in the future, it may actually delete the files, but this is
5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527
 * really intended to make sure the ops passed in are disabled
 * and that when this function returns, the caller is free to
 * free the ops.
 *
 * The "destroy" name is only to match the "create" name that this
 * should be paired with.
 */
void ftrace_destroy_filter_files(struct ftrace_ops *ops)
{
	mutex_lock(&ftrace_lock);
	if (ops->flags & FTRACE_OPS_FL_ENABLED)
		ftrace_shutdown(ops, 0);
	ops->flags |= FTRACE_OPS_FL_DELETED;
5528
	ftrace_free_filter(ops);
5529 5530 5531
	mutex_unlock(&ftrace_lock);
}

5532
static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
5533 5534
{

5535 5536
	trace_create_file("available_filter_functions", 0444,
			d_tracer, NULL, &ftrace_avail_fops);
5537

5538 5539 5540
	trace_create_file("enabled_functions", 0444,
			d_tracer, NULL, &ftrace_enabled_fops);

5541
	ftrace_create_filter_files(&global_ops, d_tracer);
5542

5543
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
5544
	trace_create_file("set_graph_function", 0644, d_tracer,
5545 5546
				    NULL,
				    &ftrace_graph_fops);
5547
	trace_create_file("set_graph_notrace", 0644, d_tracer,
5548 5549
				    NULL,
				    &ftrace_graph_notrace_fops);
5550 5551
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

5552 5553 5554
	return 0;
}

5555
static int ftrace_cmp_ips(const void *a, const void *b)
5556
{
5557 5558
	const unsigned long *ipa = a;
	const unsigned long *ipb = b;
5559

5560 5561 5562 5563 5564 5565 5566
	if (*ipa > *ipb)
		return 1;
	if (*ipa < *ipb)
		return -1;
	return 0;
}

5567
static int ftrace_process_locs(struct module *mod,
5568
			       unsigned long *start,
5569 5570
			       unsigned long *end)
{
5571
	struct ftrace_page *start_pg;
5572
	struct ftrace_page *pg;
5573
	struct dyn_ftrace *rec;
5574
	unsigned long count;
5575 5576
	unsigned long *p;
	unsigned long addr;
5577
	unsigned long flags = 0; /* Shut up gcc */
5578 5579 5580 5581 5582 5583 5584
	int ret = -ENOMEM;

	count = end - start;

	if (!count)
		return 0;

5585
	sort(start, count, sizeof(*start),
5586
	     ftrace_cmp_ips, NULL);
5587

5588 5589
	start_pg = ftrace_allocate_pages(count);
	if (!start_pg)
5590
		return -ENOMEM;
5591

Steven Rostedt's avatar
Steven Rostedt committed
5592
	mutex_lock(&ftrace_lock);
5593

5594 5595 5596 5597 5598
	/*
	 * Core and each module needs their own pages, as
	 * modules will free them when they are removed.
	 * Force a new page to be allocated for modules.
	 */
5599 5600 5601
	if (!mod) {
		WARN_ON(ftrace_pages || ftrace_pages_start);
		/* First initialization */
5602
		ftrace_pages = ftrace_pages_start = start_pg;
5603
	} else {
5604
		if (!ftrace_pages)
5605
			goto out;
5606

5607 5608 5609 5610
		if (WARN_ON(ftrace_pages->next)) {
			/* Hmm, we have free pages? */
			while (ftrace_pages->next)
				ftrace_pages = ftrace_pages->next;
5611
		}
5612

5613
		ftrace_pages->next = start_pg;
5614 5615
	}

5616
	p = start;
5617
	pg = start_pg;
5618 5619
	while (p < end) {
		addr = ftrace_call_adjust(*p++);
5620 5621 5622 5623 5624 5625 5626 5627
		/*
		 * Some architecture linkers will pad between
		 * the different mcount_loc sections of different
		 * object files to satisfy alignments.
		 * Skip any NULL pointers.
		 */
		if (!addr)
			continue;
5628 5629 5630 5631 5632 5633 5634 5635 5636 5637

		if (pg->index == pg->size) {
			/* We should have allocated enough */
			if (WARN_ON(!pg->next))
				break;
			pg = pg->next;
		}

		rec = &pg->records[pg->index++];
		rec->ip = addr;
5638 5639
	}

5640 5641 5642 5643 5644 5645
	/* We should have used all pages */
	WARN_ON(pg->next);

	/* Assign the last page to ftrace_pages */
	ftrace_pages = pg;

5646
	/*
5647 5648 5649 5650 5651 5652
	 * We only need to disable interrupts on start up
	 * because we are modifying code that an interrupt
	 * may execute, and the modification is not atomic.
	 * But for modules, nothing runs the code we modify
	 * until we are finished with it, and there's no
	 * reason to cause large interrupt latencies while we do it.
5653
	 */
5654 5655
	if (!mod)
		local_irq_save(flags);
5656
	ftrace_update_code(mod, start_pg);
5657 5658
	if (!mod)
		local_irq_restore(flags);
5659 5660
	ret = 0;
 out:
Steven Rostedt's avatar
Steven Rostedt committed
5661
	mutex_unlock(&ftrace_lock);
5662

5663
	return ret;
5664 5665
}

5666 5667 5668 5669 5670 5671 5672 5673
struct ftrace_mod_func {
	struct list_head	list;
	char			*name;
	unsigned long		ip;
	unsigned int		size;
};

struct ftrace_mod_map {
5674
	struct rcu_head		rcu;
5675 5676 5677 5678 5679
	struct list_head	list;
	struct module		*mod;
	unsigned long		start_addr;
	unsigned long		end_addr;
	struct list_head	funcs;
5680
	unsigned int		num_funcs;
5681 5682
};

5683
#ifdef CONFIG_MODULES
5684 5685 5686

#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)

5687 5688
static LIST_HEAD(ftrace_mod_maps);

5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701
static int referenced_filters(struct dyn_ftrace *rec)
{
	struct ftrace_ops *ops;
	int cnt = 0;

	for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
		if (ops_references_rec(ops, rec))
		    cnt++;
	}

	return cnt;
}

5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741
static void
clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash)
{
	struct ftrace_func_entry *entry;
	struct dyn_ftrace *rec;
	int i;

	if (ftrace_hash_empty(hash))
		return;

	for (i = 0; i < pg->index; i++) {
		rec = &pg->records[i];
		entry = __ftrace_lookup_ip(hash, rec->ip);
		/*
		 * Do not allow this rec to match again.
		 * Yeah, it may waste some memory, but will be removed
		 * if/when the hash is modified again.
		 */
		if (entry)
			entry->ip = 0;
	}
}

/* Clear any records from hashs */
static void clear_mod_from_hashes(struct ftrace_page *pg)
{
	struct trace_array *tr;

	mutex_lock(&trace_types_lock);
	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
		if (!tr->ops || !tr->ops->func_hash)
			continue;
		mutex_lock(&tr->ops->func_hash->regex_lock);
		clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash);
		clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash);
		mutex_unlock(&tr->ops->func_hash->regex_lock);
	}
	mutex_unlock(&trace_types_lock);
}

5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757
static void ftrace_free_mod_map(struct rcu_head *rcu)
{
	struct ftrace_mod_map *mod_map = container_of(rcu, struct ftrace_mod_map, rcu);
	struct ftrace_mod_func *mod_func;
	struct ftrace_mod_func *n;

	/* All the contents of mod_map are now not visible to readers */
	list_for_each_entry_safe(mod_func, n, &mod_map->funcs, list) {
		kfree(mod_func->name);
		list_del(&mod_func->list);
		kfree(mod_func);
	}

	kfree(mod_map);
}

5758
void ftrace_release_mod(struct module *mod)
5759
{
5760 5761
	struct ftrace_mod_map *mod_map;
	struct ftrace_mod_map *n;
5762
	struct dyn_ftrace *rec;
5763
	struct ftrace_page **last_pg;
5764
	struct ftrace_page *tmp_page = NULL;
5765
	struct ftrace_page *pg;
5766
	int order;
5767

5768 5769
	mutex_lock(&ftrace_lock);

5770
	if (ftrace_disabled)
5771
		goto out_unlock;
5772

5773 5774 5775
	list_for_each_entry_safe(mod_map, n, &ftrace_mod_maps, list) {
		if (mod_map->mod == mod) {
			list_del_rcu(&mod_map->list);
5776
			call_rcu(&mod_map->rcu, ftrace_free_mod_map);
5777 5778 5779 5780
			break;
		}
	}

5781 5782 5783 5784 5785 5786 5787
	/*
	 * Each module has its own ftrace_pages, remove
	 * them from the list.
	 */
	last_pg = &ftrace_pages_start;
	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
		rec = &pg->records[0];
5788 5789
		if (within_module_core(rec->ip, mod) ||
		    within_module_init(rec->ip, mod)) {
5790
			/*
5791 5792
			 * As core pages are first, the first
			 * page should never be a module page.
5793
			 */
5794 5795 5796 5797 5798 5799 5800
			if (WARN_ON(pg == ftrace_pages_start))
				goto out_unlock;

			/* Check if we are deleting the last page */
			if (pg == ftrace_pages)
				ftrace_pages = next_to_ftrace_page(last_pg);

5801
			ftrace_update_tot_cnt -= pg->index;
5802
			*last_pg = pg->next;
5803 5804 5805

			pg->next = tmp_page;
			tmp_page = pg;
5806 5807 5808
		} else
			last_pg = &pg->next;
	}
5809
 out_unlock:
5810
	mutex_unlock(&ftrace_lock);
5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821

	for (pg = tmp_page; pg; pg = tmp_page) {

		/* Needs to be called outside of ftrace_lock */
		clear_mod_from_hashes(pg);

		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
		free_pages((unsigned long)pg->records, order);
		tmp_page = pg->next;
		kfree(pg);
	}
5822 5823
}

5824
void ftrace_module_enable(struct module *mod)
5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;

	mutex_lock(&ftrace_lock);

	if (ftrace_disabled)
		goto out_unlock;

	/*
	 * If the tracing is enabled, go ahead and enable the record.
	 *
5837
	 * The reason not to enable the record immediately is the
5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857
	 * inherent check of ftrace_make_nop/ftrace_make_call for
	 * correct previous instructions.  Making first the NOP
	 * conversion puts the module to the correct state, thus
	 * passing the ftrace_make_call check.
	 *
	 * We also delay this to after the module code already set the
	 * text to read-only, as we now need to set it back to read-write
	 * so that we can modify the text.
	 */
	if (ftrace_start_up)
		ftrace_arch_code_modify_prepare();

	do_for_each_ftrace_rec(pg, rec) {
		int cnt;
		/*
		 * do_for_each_ftrace_rec() is a double loop.
		 * module text shares the pg. If a record is
		 * not part of this module, then skip this pg,
		 * which the "break" will do.
		 */
5858 5859
		if (!within_module_core(rec->ip, mod) &&
		    !within_module_init(rec->ip, mod))
5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884 5885 5886 5887 5888 5889 5890 5891
			break;

		cnt = 0;

		/*
		 * When adding a module, we need to check if tracers are
		 * currently enabled and if they are, and can trace this record,
		 * we need to enable the module functions as well as update the
		 * reference counts for those function records.
		 */
		if (ftrace_start_up)
			cnt += referenced_filters(rec);

		/* This clears FTRACE_FL_DISABLED */
		rec->flags = cnt;

		if (ftrace_start_up && cnt) {
			int failed = __ftrace_replace_code(rec, 1);
			if (failed) {
				ftrace_bug(failed, rec);
				goto out_loop;
			}
		}

	} while_for_each_ftrace_rec();

 out_loop:
	if (ftrace_start_up)
		ftrace_arch_code_modify_post_process();

 out_unlock:
	mutex_unlock(&ftrace_lock);
5892 5893

	process_cached_mods(mod->name);
5894 5895
}

5896
void ftrace_module_init(struct module *mod)
5897
{
5898
	if (ftrace_disabled || !mod->num_ftrace_callsites)
5899
		return;
5900

5901 5902
	ftrace_process_locs(mod, mod->ftrace_callsites,
			    mod->ftrace_callsites + mod->num_ftrace_callsites);
5903
}
5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931

static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
				struct dyn_ftrace *rec)
{
	struct ftrace_mod_func *mod_func;
	unsigned long symsize;
	unsigned long offset;
	char str[KSYM_SYMBOL_LEN];
	char *modname;
	const char *ret;

	ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str);
	if (!ret)
		return;

	mod_func = kmalloc(sizeof(*mod_func), GFP_KERNEL);
	if (!mod_func)
		return;

	mod_func->name = kstrdup(str, GFP_KERNEL);
	if (!mod_func->name) {
		kfree(mod_func);
		return;
	}

	mod_func->ip = rec->ip - offset;
	mod_func->size = symsize;

5932 5933
	mod_map->num_funcs++;

5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949
	list_add_rcu(&mod_func->list, &mod_map->funcs);
}

static struct ftrace_mod_map *
allocate_ftrace_mod_map(struct module *mod,
			unsigned long start, unsigned long end)
{
	struct ftrace_mod_map *mod_map;

	mod_map = kmalloc(sizeof(*mod_map), GFP_KERNEL);
	if (!mod_map)
		return NULL;

	mod_map->mod = mod;
	mod_map->start_addr = start;
	mod_map->end_addr = end;
5950
	mod_map->num_funcs = 0;
5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995

	INIT_LIST_HEAD_RCU(&mod_map->funcs);

	list_add_rcu(&mod_map->list, &ftrace_mod_maps);

	return mod_map;
}

static const char *
ftrace_func_address_lookup(struct ftrace_mod_map *mod_map,
			   unsigned long addr, unsigned long *size,
			   unsigned long *off, char *sym)
{
	struct ftrace_mod_func *found_func =  NULL;
	struct ftrace_mod_func *mod_func;

	list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
		if (addr >= mod_func->ip &&
		    addr < mod_func->ip + mod_func->size) {
			found_func = mod_func;
			break;
		}
	}

	if (found_func) {
		if (size)
			*size = found_func->size;
		if (off)
			*off = addr - found_func->ip;
		if (sym)
			strlcpy(sym, found_func->name, KSYM_NAME_LEN);

		return found_func->name;
	}

	return NULL;
}

const char *
ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
		   unsigned long *off, char **modname, char *sym)
{
	struct ftrace_mod_map *mod_map;
	const char *ret = NULL;

5996
	/* mod_map is freed via call_rcu() */
5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010
	preempt_disable();
	list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
		ret = ftrace_func_address_lookup(mod_map, addr, size, off, sym);
		if (ret) {
			if (modname)
				*modname = mod_map->mod->name;
			break;
		}
	}
	preempt_enable();

	return ret;
}

6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046
int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
			   char *type, char *name,
			   char *module_name, int *exported)
{
	struct ftrace_mod_map *mod_map;
	struct ftrace_mod_func *mod_func;

	preempt_disable();
	list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {

		if (symnum >= mod_map->num_funcs) {
			symnum -= mod_map->num_funcs;
			continue;
		}

		list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
			if (symnum > 1) {
				symnum--;
				continue;
			}

			*value = mod_func->ip;
			*type = 'T';
			strlcpy(name, mod_func->name, KSYM_NAME_LEN);
			strlcpy(module_name, mod_map->mod->name, MODULE_NAME_LEN);
			*exported = 1;
			preempt_enable();
			return 0;
		}
		WARN_ON(1);
		break;
	}
	preempt_enable();
	return -ERANGE;
}

6047 6048 6049 6050 6051 6052 6053 6054 6055
#else
static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
				struct dyn_ftrace *rec) { }
static inline struct ftrace_mod_map *
allocate_ftrace_mod_map(struct module *mod,
			unsigned long start, unsigned long end)
{
	return NULL;
}
6056 6057
#endif /* CONFIG_MODULES */

6058 6059 6060 6061 6062 6063 6064 6065
struct ftrace_init_func {
	struct list_head list;
	unsigned long ip;
};

/* Clear any init ips from hashes */
static void
clear_func_from_hash(struct ftrace_init_func *func, struct ftrace_hash *hash)
6066
{
6067 6068
	struct ftrace_func_entry *entry;

6069
	entry = ftrace_lookup_ip(hash, func->ip);
6070 6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110
	/*
	 * Do not allow this rec to match again.
	 * Yeah, it may waste some memory, but will be removed
	 * if/when the hash is modified again.
	 */
	if (entry)
		entry->ip = 0;
}

static void
clear_func_from_hashes(struct ftrace_init_func *func)
{
	struct trace_array *tr;

	mutex_lock(&trace_types_lock);
	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
		if (!tr->ops || !tr->ops->func_hash)
			continue;
		mutex_lock(&tr->ops->func_hash->regex_lock);
		clear_func_from_hash(func, tr->ops->func_hash->filter_hash);
		clear_func_from_hash(func, tr->ops->func_hash->notrace_hash);
		mutex_unlock(&tr->ops->func_hash->regex_lock);
	}
	mutex_unlock(&trace_types_lock);
}

static void add_to_clear_hash_list(struct list_head *clear_list,
				   struct dyn_ftrace *rec)
{
	struct ftrace_init_func *func;

	func = kmalloc(sizeof(*func), GFP_KERNEL);
	if (!func) {
		WARN_ONCE(1, "alloc failure, ftrace filter could be stale\n");
		return;
	}

	func->ip = rec->ip;
	list_add(&func->list, clear_list);
}

6111
void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr)
6112
{
6113 6114
	unsigned long start = (unsigned long)(start_ptr);
	unsigned long end = (unsigned long)(end_ptr);
6115 6116 6117 6118
	struct ftrace_page **last_pg = &ftrace_pages_start;
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
	struct dyn_ftrace key;
6119
	struct ftrace_mod_map *mod_map = NULL;
6120 6121
	struct ftrace_init_func *func, *func_next;
	struct list_head clear_hash;
6122 6123
	int order;

6124 6125
	INIT_LIST_HEAD(&clear_hash);

6126 6127 6128 6129 6130
	key.ip = start;
	key.flags = end;	/* overload flags, as it is unsigned long */

	mutex_lock(&ftrace_lock);

6131 6132 6133 6134 6135 6136 6137 6138
	/*
	 * If we are freeing module init memory, then check if
	 * any tracer is active. If so, we need to save a mapping of
	 * the module functions being freed with the address.
	 */
	if (mod && ftrace_ops_list != &ftrace_list_end)
		mod_map = allocate_ftrace_mod_map(mod, start, end);

6139 6140 6141 6142 6143 6144 6145 6146 6147 6148
	for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) {
		if (end < pg->records[0].ip ||
		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
			continue;
 again:
		rec = bsearch(&key, pg->records, pg->index,
			      sizeof(struct dyn_ftrace),
			      ftrace_cmp_recs);
		if (!rec)
			continue;
6149

6150 6151 6152
		/* rec will be cleared from hashes after ftrace_lock unlock */
		add_to_clear_hash_list(&clear_hash, rec);

6153 6154 6155
		if (mod_map)
			save_ftrace_mod_rec(mod_map, rec);

6156
		pg->index--;
6157
		ftrace_update_tot_cnt--;
6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173
		if (!pg->index) {
			*last_pg = pg->next;
			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
			free_pages((unsigned long)pg->records, order);
			kfree(pg);
			pg = container_of(last_pg, struct ftrace_page, next);
			if (!(*last_pg))
				ftrace_pages = pg;
			continue;
		}
		memmove(rec, rec + 1,
			(pg->index - (rec - pg->records)) * sizeof(*rec));
		/* More than one function may be in this block */
		goto again;
	}
	mutex_unlock(&ftrace_lock);
6174 6175 6176 6177 6178

	list_for_each_entry_safe(func, func_next, &clear_hash, list) {
		clear_func_from_hashes(func);
		kfree(func);
	}
6179 6180
}

6181 6182 6183 6184 6185
void __init ftrace_free_init_mem(void)
{
	void *start = (void *)(&__init_begin);
	void *end = (void *)(&__init_end);

6186
	ftrace_free_mem(NULL, start, end);
6187 6188
}

6189 6190
void __init ftrace_init(void)
{
6191 6192
	extern unsigned long __start_mcount_loc[];
	extern unsigned long __stop_mcount_loc[];
6193
	unsigned long count, flags;
6194 6195 6196
	int ret;

	local_irq_save(flags);
6197
	ret = ftrace_dyn_arch_init();
6198
	local_irq_restore(flags);
6199
	if (ret)
6200 6201 6202
		goto failed;

	count = __stop_mcount_loc - __start_mcount_loc;
6203 6204
	if (!count) {
		pr_info("ftrace: No functions to be traced?\n");
6205
		goto failed;
6206 6207 6208 6209
	}

	pr_info("ftrace: allocating %ld entries in %ld pages\n",
		count, count / ENTRIES_PER_PAGE + 1);
6210 6211 6212

	last_ftrace_enabled = ftrace_enabled = 1;

6213
	ret = ftrace_process_locs(NULL,
6214
				  __start_mcount_loc,
6215 6216
				  __stop_mcount_loc);

6217 6218
	set_ftrace_early_filters();

6219 6220 6221 6222 6223
	return;
 failed:
	ftrace_disabled = 1;
}

6224 6225 6226 6227 6228 6229 6230 6231 6232 6233
/* Do nothing if arch does not support this */
void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
{
}

static void ftrace_update_trampoline(struct ftrace_ops *ops)
{
	arch_ftrace_update_trampoline(ops);
}

6234 6235 6236
void ftrace_init_trace_array(struct trace_array *tr)
{
	INIT_LIST_HEAD(&tr->func_probes);
6237 6238
	INIT_LIST_HEAD(&tr->mod_trace);
	INIT_LIST_HEAD(&tr->mod_notrace);
6239
}
6240
#else
6241

6242
struct ftrace_ops global_ops = {
6243
	.func			= ftrace_stub,
6244 6245 6246
	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
				  FTRACE_OPS_FL_INITIALIZED |
				  FTRACE_OPS_FL_PID,
6247 6248
};

6249 6250 6251 6252 6253
static int __init ftrace_nodyn_init(void)
{
	ftrace_enabled = 1;
	return 0;
}
6254
core_initcall(ftrace_nodyn_init);
6255

6256
static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
6257
static inline void ftrace_startup_enable(int command) { }
6258
static inline void ftrace_startup_all(int command) { }
6259

Ingo Molnar's avatar
Ingo Molnar committed
6260 6261
# define ftrace_startup_sysctl()	do { } while (0)
# define ftrace_shutdown_sysctl()	do { } while (0)
6262

6263 6264 6265 6266
static void ftrace_update_trampoline(struct ftrace_ops *ops)
{
}

6267 6268
#endif /* CONFIG_DYNAMIC_FTRACE */

6269 6270 6271 6272
__init void ftrace_init_global_array_ops(struct trace_array *tr)
{
	tr->ops = &global_ops;
	tr->ops->private = tr;
6273
	ftrace_init_trace_array(tr);
6274 6275 6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292
}

void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
{
	/* If we filter on pids, update to use the pid function */
	if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
		if (WARN_ON(tr->ops->func != ftrace_stub))
			printk("ftrace ops had %pS for function\n",
			       tr->ops->func);
	}
	tr->ops->func = func;
	tr->ops->private = tr;
}

void ftrace_reset_array_ops(struct trace_array *tr)
{
	tr->ops->func = ftrace_stub;
}

6293
static nokprobe_inline void
6294
__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
6295
		       struct ftrace_ops *ignored, struct pt_regs *regs)
6296
{
6297
	struct ftrace_ops *op;
6298
	int bit;
6299

6300 6301 6302
	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
	if (bit < 0)
		return;
6303

6304 6305
	/*
	 * Some of the ops may be dynamically allocated,
6306
	 * they must be freed after a synchronize_rcu().
6307 6308
	 */
	preempt_disable_notrace();
6309

6310
	do_for_each_ftrace_op(op, ftrace_ops_list) {
6311 6312 6313
		/* Stub functions don't need to be called nor tested */
		if (op->flags & FTRACE_OPS_FL_STUB)
			continue;
6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324
		/*
		 * Check the following for each ops before calling their func:
		 *  if RCU flag is set, then rcu_is_watching() must be true
		 *  if PER_CPU is set, then ftrace_function_local_disable()
		 *                          must be false
		 *  Otherwise test if the ip matches the ops filter
		 *
		 * If any of the above fails then the op->func() is not executed.
		 */
		if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
		    ftrace_ops_test(op, ip, regs)) {
6325 6326
			if (FTRACE_WARN_ON(!op->func)) {
				pr_warn("op=%p %pS\n", op, op);
6327 6328
				goto out;
			}
6329
			op->func(ip, parent_ip, op, regs);
6330
		}
6331
	} while_for_each_ftrace_op(op);
6332
out:
6333
	preempt_enable_notrace();
6334
	trace_clear_recursion(bit);
6335 6336
}

6337 6338 6339 6340 6341
/*
 * Some archs only support passing ip and parent_ip. Even though
 * the list function ignores the op parameter, we do not want any
 * C side effects, where a function is called without the caller
 * sending a third parameter.
6342 6343 6344
 * Archs are to support both the regs and ftrace_ops at the same time.
 * If they support ftrace_ops, it is assumed they support regs.
 * If call backs want to use regs, they must either check for regs
6345 6346
 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
6347
 * An architecture can pass partial regs with ftrace_ops and still
Li Bin's avatar
Li Bin committed
6348
 * set the ARCH_SUPPORTS_FTRACE_OPS.
6349 6350 6351
 */
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
6352
				 struct ftrace_ops *op, struct pt_regs *regs)
6353
{
6354
	__ftrace_ops_list_func(ip, parent_ip, NULL, regs);
6355
}
6356
NOKPROBE_SYMBOL(ftrace_ops_list_func);
6357 6358 6359
#else
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
{
6360
	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
6361
}
6362
NOKPROBE_SYMBOL(ftrace_ops_no_ops);
6363 6364
#endif

6365 6366
/*
 * If there's only one function registered but it does not support
6367 6368
 * recursion, needs RCU protection and/or requires per cpu handling, then
 * this function will be called by the mcount trampoline.
6369
 */
6370
static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
6371 6372 6373 6374
				   struct ftrace_ops *op, struct pt_regs *regs)
{
	int bit;

6375 6376 6377
	if ((op->flags & FTRACE_OPS_FL_RCU) && !rcu_is_watching())
		return;

6378 6379 6380 6381
	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
	if (bit < 0)
		return;

6382
	preempt_disable_notrace();
6383

6384
	op->func(ip, parent_ip, op, regs);
6385 6386

	preempt_enable_notrace();
6387 6388
	trace_clear_recursion(bit);
}
6389
NOKPROBE_SYMBOL(ftrace_ops_assist_func);
6390

6391 6392 6393 6394 6395 6396 6397
/**
 * ftrace_ops_get_func - get the function a trampoline should call
 * @ops: the ops to get the function for
 *
 * Normally the mcount trampoline will call the ops->func, but there
 * are times that it should not. For example, if the ops does not
 * have its own recursion protection, then it should call the
6398
 * ftrace_ops_assist_func() instead.
6399 6400 6401 6402 6403 6404
 *
 * Returns the function that the trampoline should call for @ops.
 */
ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
{
	/*
6405 6406
	 * If the function does not handle recursion, needs to be RCU safe,
	 * or does per cpu logic, then we need to call the assist handler.
6407
	 */
6408
	if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE) ||
6409
	    ops->flags & FTRACE_OPS_FL_RCU)
6410
		return ftrace_ops_assist_func;
6411 6412 6413 6414

	return ops->func;
}

6415 6416 6417
static void
ftrace_filter_pid_sched_switch_probe(void *data, bool preempt,
		    struct task_struct *prev, struct task_struct *next)
Steven Rostedt's avatar
Steven Rostedt committed
6418
{
6419 6420
	struct trace_array *tr = data;
	struct trace_pid_list *pid_list;
Steven Rostedt's avatar
Steven Rostedt committed
6421

6422
	pid_list = rcu_dereference_sched(tr->function_pids);
6423

6424 6425
	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
		       trace_ignore_this_task(pid_list, next));
Steven Rostedt's avatar
Steven Rostedt committed
6426 6427
}

6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 6462 6463 6464
static void
ftrace_pid_follow_sched_process_fork(void *data,
				     struct task_struct *self,
				     struct task_struct *task)
{
	struct trace_pid_list *pid_list;
	struct trace_array *tr = data;

	pid_list = rcu_dereference_sched(tr->function_pids);
	trace_filter_add_remove_task(pid_list, self, task);
}

static void
ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task)
{
	struct trace_pid_list *pid_list;
	struct trace_array *tr = data;

	pid_list = rcu_dereference_sched(tr->function_pids);
	trace_filter_add_remove_task(pid_list, NULL, task);
}

void ftrace_pid_follow_fork(struct trace_array *tr, bool enable)
{
	if (enable) {
		register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
						  tr);
		register_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit,
						  tr);
	} else {
		unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
						    tr);
		unregister_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit,
						    tr);
	}
}

6465
static void clear_ftrace_pids(struct trace_array *tr)
6466
{
6467 6468
	struct trace_pid_list *pid_list;
	int cpu;
6469

6470 6471 6472 6473
	pid_list = rcu_dereference_protected(tr->function_pids,
					     lockdep_is_held(&ftrace_lock));
	if (!pid_list)
		return;
6474

6475
	unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
6476

6477 6478
	for_each_possible_cpu(cpu)
		per_cpu_ptr(tr->trace_buffer.data, cpu)->ftrace_ignore_pid = false;
Steven Rostedt's avatar
Steven Rostedt committed
6479

6480
	rcu_assign_pointer(tr->function_pids, NULL);
Steven Rostedt's avatar
Steven Rostedt committed
6481

6482
	/* Wait till all users are no longer using pid filtering */
6483
	synchronize_rcu();
6484

6485
	trace_free_pid_list(pid_list);
6486 6487
}

6488 6489 6490 6491 6492 6493 6494 6495 6496
void ftrace_clear_pids(struct trace_array *tr)
{
	mutex_lock(&ftrace_lock);

	clear_ftrace_pids(tr);

	mutex_unlock(&ftrace_lock);
}

6497
static void ftrace_pid_reset(struct trace_array *tr)
6498
{
6499
	mutex_lock(&ftrace_lock);
6500
	clear_ftrace_pids(tr);
Steven Rostedt's avatar
Steven Rostedt committed
6501

6502
	ftrace_update_pid_func();
6503
	ftrace_startup_all(0);
6504 6505 6506 6507

	mutex_unlock(&ftrace_lock);
}

6508 6509
/* Greater than any max PID */
#define FTRACE_NO_PIDS		(void *)(PID_MAX_LIMIT + 1)
6510

6511
static void *fpid_start(struct seq_file *m, loff_t *pos)
6512
	__acquires(RCU)
6513
{
6514 6515 6516
	struct trace_pid_list *pid_list;
	struct trace_array *tr = m->private;

6517
	mutex_lock(&ftrace_lock);
6518 6519 6520
	rcu_read_lock_sched();

	pid_list = rcu_dereference_sched(tr->function_pids);
6521

6522 6523
	if (!pid_list)
		return !(*pos) ? FTRACE_NO_PIDS : NULL;
6524

6525
	return trace_pid_start(pid_list, pos);
6526 6527 6528 6529
}

static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
{
6530 6531 6532 6533
	struct trace_array *tr = m->private;
	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);

	if (v == FTRACE_NO_PIDS)
6534 6535
		return NULL;

6536
	return trace_pid_next(pid_list, v, pos);
6537 6538 6539
}

static void fpid_stop(struct seq_file *m, void *p)
6540
	__releases(RCU)
6541
{
6542
	rcu_read_unlock_sched();
6543 6544 6545 6546 6547
	mutex_unlock(&ftrace_lock);
}

static int fpid_show(struct seq_file *m, void *v)
{
6548
	if (v == FTRACE_NO_PIDS) {
6549
		seq_puts(m, "no pid\n");
6550 6551 6552
		return 0;
	}

6553
	return trace_pid_show(m, v);
6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565
}

static const struct seq_operations ftrace_pid_sops = {
	.start = fpid_start,
	.next = fpid_next,
	.stop = fpid_stop,
	.show = fpid_show,
};

static int
ftrace_pid_open(struct inode *inode, struct file *file)
{
6566 6567
	struct trace_array *tr = inode->i_private;
	struct seq_file *m;
6568 6569
	int ret = 0;

6570 6571 6572
	ret = tracing_check_open_get_tr(tr);
	if (ret)
		return ret;
6573

6574 6575
	if ((file->f_mode & FMODE_WRITE) &&
	    (file->f_flags & O_TRUNC))
6576
		ftrace_pid_reset(tr);
6577

6578 6579 6580 6581 6582 6583 6584 6585
	ret = seq_open(file, &ftrace_pid_sops);
	if (ret < 0) {
		trace_array_put(tr);
	} else {
		m = file->private_data;
		/* copy tr over to seq ops */
		m->private = tr;
	}
6586 6587 6588 6589

	return ret;
}

6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604 6605
static void ignore_task_cpu(void *data)
{
	struct trace_array *tr = data;
	struct trace_pid_list *pid_list;

	/*
	 * This function is called by on_each_cpu() while the
	 * event_mutex is held.
	 */
	pid_list = rcu_dereference_protected(tr->function_pids,
					     mutex_is_locked(&ftrace_lock));

	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
		       trace_ignore_this_task(pid_list, current));
}

6606 6607 6608 6609
static ssize_t
ftrace_pid_write(struct file *filp, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
6610 6611 6612 6613 6614
	struct seq_file *m = filp->private_data;
	struct trace_array *tr = m->private;
	struct trace_pid_list *filtered_pids = NULL;
	struct trace_pid_list *pid_list;
	ssize_t ret;
6615

6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626
	if (!cnt)
		return 0;

	mutex_lock(&ftrace_lock);

	filtered_pids = rcu_dereference_protected(tr->function_pids,
					     lockdep_is_held(&ftrace_lock));

	ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
	if (ret < 0)
		goto out;
6627

6628
	rcu_assign_pointer(tr->function_pids, pid_list);
6629

6630
	if (filtered_pids) {
6631
		synchronize_rcu();
6632 6633 6634 6635 6636
		trace_free_pid_list(filtered_pids);
	} else if (pid_list) {
		/* Register a probe to set whether to ignore the tracing of a task */
		register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
	}
6637

6638
	/*
6639 6640 6641
	 * Ignoring of pids is done at task switch. But we have to
	 * check for those tasks that are currently running.
	 * Always do this in case a pid was appended or removed.
6642
	 */
6643
	on_each_cpu(ignore_task_cpu, tr, 1);
6644

6645 6646 6647 6648
	ftrace_update_pid_func();
	ftrace_startup_all(0);
 out:
	mutex_unlock(&ftrace_lock);
6649

6650 6651
	if (ret > 0)
		*ppos += ret;
6652

6653
	return ret;
6654
}
6655

6656 6657 6658
static int
ftrace_pid_release(struct inode *inode, struct file *file)
{
6659
	struct trace_array *tr = inode->i_private;
6660

6661 6662 6663
	trace_array_put(tr);

	return seq_release(inode, file);
6664 6665
}

6666
static const struct file_operations ftrace_pid_fops = {
6667 6668 6669
	.open		= ftrace_pid_open,
	.write		= ftrace_pid_write,
	.read		= seq_read,
6670
	.llseek		= tracing_lseek,
6671
	.release	= ftrace_pid_release,
6672 6673
};

6674
void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
6675
{
6676
	trace_create_file("set_ftrace_pid", 0644, d_tracer,
6677
			    tr, &ftrace_pid_fops);
6678 6679
}

6680 6681 6682 6683 6684 6685 6686 6687 6688 6689
void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,
					 struct dentry *d_tracer)
{
	/* Only the top level directory has the dyn_tracefs and profile */
	WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));

	ftrace_init_dyn_tracefs(d_tracer);
	ftrace_profile_tracefs(d_tracer);
}

6690
/**
6691
 * ftrace_kill - kill ftrace
6692 6693 6694 6695 6696
 *
 * This function should be used by panic code. It stops ftrace
 * but in a not so nice way. If you need to simply kill ftrace
 * from a non-atomic section, use ftrace_kill.
 */
6697
void ftrace_kill(void)
6698 6699 6700
{
	ftrace_disabled = 1;
	ftrace_enabled = 0;
6701
	ftrace_trace_function = ftrace_stub;
6702 6703
}

6704 6705 6706 6707 6708 6709 6710 6711
/**
 * Test if ftrace is dead or not.
 */
int ftrace_is_dead(void)
{
	return ftrace_disabled;
}

6712
/**
6713 6714
 * register_ftrace_function - register a function for profiling
 * @ops - ops structure that holds the function for profiling.
6715
 *
6716 6717 6718 6719 6720 6721
 * Register a function to be called by all functions in the
 * kernel.
 *
 * Note: @ops->func and all the functions it calls must be labeled
 *       with "notrace", otherwise it will go into a
 *       recursive loop.
6722
 */
6723
int register_ftrace_function(struct ftrace_ops *ops)
6724
{
6725
	int ret = -1;
6726

6727 6728
	ftrace_ops_init(ops);

Steven Rostedt's avatar
Steven Rostedt committed
6729
	mutex_lock(&ftrace_lock);
6730

6731
	ret = ftrace_startup(ops, 0);
6732

Steven Rostedt's avatar
Steven Rostedt committed
6733
	mutex_unlock(&ftrace_lock);
6734

6735
	return ret;
6736
}
6737
EXPORT_SYMBOL_GPL(register_ftrace_function);
6738 6739

/**
6740
 * unregister_ftrace_function - unregister a function for profiling.
6741 6742 6743 6744 6745 6746 6747 6748
 * @ops - ops structure that holds the function to unregister
 *
 * Unregister a function that was added to be called by ftrace profiling.
 */
int unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

Steven Rostedt's avatar
Steven Rostedt committed
6749
	mutex_lock(&ftrace_lock);
6750
	ret = ftrace_shutdown(ops, 0);
Steven Rostedt's avatar
Steven Rostedt committed
6751
	mutex_unlock(&ftrace_lock);
6752 6753 6754

	return ret;
}
6755
EXPORT_SYMBOL_GPL(unregister_ftrace_function);
6756

Ingo Molnar's avatar
Ingo Molnar committed
6757
int
6758
ftrace_enable_sysctl(struct ctl_table *table, int write,
6759
		     void __user *buffer, size_t *lenp,
6760 6761
		     loff_t *ppos)
{
6762
	int ret = -ENODEV;
6763

Steven Rostedt's avatar
Steven Rostedt committed
6764
	mutex_lock(&ftrace_lock);
6765

6766 6767 6768 6769
	if (unlikely(ftrace_disabled))
		goto out;

	ret = proc_dointvec(table, write, buffer, lenp, ppos);
6770

6771
	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
6772 6773
		goto out;

6774
	last_ftrace_enabled = !!ftrace_enabled;
6775 6776 6777 6778

	if (ftrace_enabled) {

		/* we are starting ftrace again */
6779 6780
		if (rcu_dereference_protected(ftrace_ops_list,
			lockdep_is_held(&ftrace_lock)) != &ftrace_list_end)
6781
			update_ftrace_function();
6782

6783 6784
		ftrace_startup_sysctl();

6785 6786 6787 6788 6789 6790 6791 6792
	} else {
		/* stopping ftrace calls (just send to ftrace_stub) */
		ftrace_trace_function = ftrace_stub;

		ftrace_shutdown_sysctl();
	}

 out:
Steven Rostedt's avatar
Steven Rostedt committed
6793
	mutex_unlock(&ftrace_lock);
6794
	return ret;
6795
}