perf_event.h 22.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 * Performance events x86 architecture header
 *
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2009 Jaswinder Singh Rajput
 *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
 *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
 *  Copyright (C) 2009 Google, Inc., Stephane Eranian
 *
 *  For licencing details see kernel-base/COPYING
 */

#include <linux/perf_event.h>

17 18 19 20 21 22 23 24 25 26 27 28
#if 0
#undef wrmsrl
#define wrmsrl(msr, val) 						\
do {									\
	unsigned int _msr = (msr);					\
	u64 _val = (val);						\
	trace_printk("wrmsrl(%x, %Lx)\n", (unsigned int)(_msr),		\
			(unsigned long long)(_val));			\
	native_write_msr((_msr), (u32)(_val), (u32)(_val >> 32));	\
} while (0)
#endif

29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
/*
 *          |   NHM/WSM    |      SNB     |
 * register -------------------------------
 *          |  HT  | no HT |  HT  | no HT |
 *-----------------------------------------
 * offcore  | core | core  | cpu  | core  |
 * lbr_sel  | core | core  | cpu  | core  |
 * ld_lat   | cpu  | core  | cpu  | core  |
 *-----------------------------------------
 *
 * Given that there is a small number of shared regs,
 * we can pre-allocate their slot in the per-cpu
 * per-core reg tables.
 */
enum extra_reg_type {
	EXTRA_REG_NONE  = -1,	/* not used */

	EXTRA_REG_RSP_0 = 0,	/* offcore_response_0 */
	EXTRA_REG_RSP_1 = 1,	/* offcore_response_1 */
48
	EXTRA_REG_LBR   = 2,	/* lbr_select */
49
	EXTRA_REG_LDLAT = 3,	/* ld_lat_threshold */
50 51 52 53 54 55 56 57 58 59 60 61

	EXTRA_REG_MAX		/* number of entries needed */
};

struct event_constraint {
	union {
		unsigned long	idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
		u64		idxmsk64;
	};
	u64	code;
	u64	cmask;
	int	weight;
62
	int	overlap;
63
	int	flags;
64
};
65
/*
66
 * struct hw_perf_event.flags flags
67 68
 */
#define PERF_X86_EVENT_PEBS_LDLAT	0x1 /* ld+ldlat data address sampling */
69
#define PERF_X86_EVENT_PEBS_ST		0x2 /* st data address sampling */
70
#define PERF_X86_EVENT_PEBS_ST_HSW	0x4 /* haswell style datala, store */
71
#define PERF_X86_EVENT_COMMITTED	0x8 /* event passed commit_txn */
72 73
#define PERF_X86_EVENT_PEBS_LD_HSW	0x10 /* haswell style datala, load */
#define PERF_X86_EVENT_PEBS_NA_HSW	0x20 /* haswell style datala, unknown */
74 75
#define PERF_X86_EVENT_RDPMC_ALLOWED	0x40 /* grant rdpmc permission */

76 77 78 79 80 81 82 83 84

struct amd_nb {
	int nb_id;  /* NorthBridge id */
	int refcnt; /* reference count */
	struct perf_event *owners[X86_PMC_IDX_MAX];
	struct event_constraint event_constraints[X86_PMC_IDX_MAX];
};

/* The maximal number of PEBS events: */
85
#define MAX_PEBS_EVENTS		8
86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136

/*
 * A debug store configuration.
 *
 * We only support architectures that use 64bit fields.
 */
struct debug_store {
	u64	bts_buffer_base;
	u64	bts_index;
	u64	bts_absolute_maximum;
	u64	bts_interrupt_threshold;
	u64	pebs_buffer_base;
	u64	pebs_index;
	u64	pebs_absolute_maximum;
	u64	pebs_interrupt_threshold;
	u64	pebs_event_reset[MAX_PEBS_EVENTS];
};

/*
 * Per register state.
 */
struct er_account {
	raw_spinlock_t		lock;	/* per-core: protect structure */
	u64                 config;	/* extra MSR config */
	u64                 reg;	/* extra MSR number */
	atomic_t            ref;	/* reference count */
};

/*
 * Per core/cpu state
 *
 * Used to coordinate shared registers between HT threads or
 * among events on a single PMU.
 */
struct intel_shared_regs {
	struct er_account       regs[EXTRA_REG_MAX];
	int                     refcnt;		/* per-core: #HT threads */
	unsigned                core_id;	/* per-core: core id */
};

#define MAX_LBR_ENTRIES		16

struct cpu_hw_events {
	/*
	 * Generic x86 PMC bits
	 */
	struct perf_event	*events[X86_PMC_IDX_MAX]; /* in counter order */
	unsigned long		active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
	unsigned long		running[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
	int			enabled;

137 138 139 140 141
	int			n_events; /* the # of events in the below arrays */
	int			n_added;  /* the # last events in the below arrays;
					     they've never been enabled yet */
	int			n_txn;    /* the # last events in the below arrays;
					     added in the current transaction */
142 143 144 145 146
	int			assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
	u64			tags[X86_PMC_IDX_MAX];
	struct perf_event	*event_list[X86_PMC_IDX_MAX]; /* in enabled order */

	unsigned int		group_flag;
147
	int			is_fake;
148 149 150 151 152 153 154 155 156 157 158 159 160 161

	/*
	 * Intel DebugStore bits
	 */
	struct debug_store	*ds;
	u64			pebs_enabled;

	/*
	 * Intel LBR bits
	 */
	int				lbr_users;
	void				*lbr_context;
	struct perf_branch_stack	lbr_stack;
	struct perf_branch_entry	lbr_entries[MAX_LBR_ENTRIES];
162
	struct er_account		*lbr_sel;
163
	u64				br_sel;
164

165 166 167 168 169 170 171
	/*
	 * Intel host/guest exclude bits
	 */
	u64				intel_ctrl_guest_mask;
	u64				intel_ctrl_host_mask;
	struct perf_guest_switch_msr	guest_switch_msrs[X86_PMC_IDX_MAX];

172 173 174 175 176
	/*
	 * Intel checkpoint mask
	 */
	u64				intel_cp_status;

177 178 179 180 181 182 183 184 185
	/*
	 * manage shared (per-core, per-cpu) registers
	 * used on Intel NHM/WSM/SNB
	 */
	struct intel_shared_regs	*shared_regs;

	/*
	 * AMD specific bits
	 */
186 187 188
	struct amd_nb			*amd_nb;
	/* Inverted mask of bits to clear in the perf_ctr ctrl registers */
	u64				perf_ctr_virt_mask;
189 190 191 192

	void				*kfree_on_online;
};

193
#define __EVENT_CONSTRAINT(c, n, m, w, o, f) {\
194 195 196 197
	{ .idxmsk64 = (n) },		\
	.code = (c),			\
	.cmask = (m),			\
	.weight = (w),			\
198
	.overlap = (o),			\
199
	.flags = f,			\
200 201 202
}

#define EVENT_CONSTRAINT(c, n, m)	\
203
	__EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0, 0)
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226

/*
 * The overlap flag marks event constraints with overlapping counter
 * masks. This is the case if the counter mask of such an event is not
 * a subset of any other counter mask of a constraint with an equal or
 * higher weight, e.g.:
 *
 *  c_overlaps = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
 *  c_another1 = EVENT_CONSTRAINT(0, 0x07, 0);
 *  c_another2 = EVENT_CONSTRAINT(0, 0x38, 0);
 *
 * The event scheduler may not select the correct counter in the first
 * cycle because it needs to know which subsequent events will be
 * scheduled. It may fail to schedule the events then. So we set the
 * overlap flag for such constraints to give the scheduler a hint which
 * events to select for counter rescheduling.
 *
 * Care must be taken as the rescheduling algorithm is O(n!) which
 * will increase scheduling cycles for an over-commited system
 * dramatically.  The number of such EVENT_CONSTRAINT_OVERLAP() macros
 * and its counter masks must be kept at a minimum.
 */
#define EVENT_CONSTRAINT_OVERLAP(c, n, m)	\
227
	__EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1, 0)
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242

/*
 * Constraint on the Event code.
 */
#define INTEL_EVENT_CONSTRAINT(c, n)	\
	EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)

/*
 * Constraint on the Event code + UMask + fixed-mask
 *
 * filter mask to validate fixed counter events.
 * the following filters disqualify for fixed counters:
 *  - inv
 *  - edge
 *  - cnt-mask
243 244
 *  - in_tx
 *  - in_tx_checkpointed
245 246 247
 *  The other filters are supported by fixed counters.
 *  The any-thread option is supported starting with v3.
 */
248
#define FIXED_EVENT_FLAGS (X86_RAW_EVENT_MASK|HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)
249
#define FIXED_EVENT_CONSTRAINT(c, n)	\
250
	EVENT_CONSTRAINT(c, (1ULL << (32+n)), FIXED_EVENT_FLAGS)
251 252 253 254 255 256 257

/*
 * Constraint on the Event code + UMask
 */
#define INTEL_UEVENT_CONSTRAINT(c, n)	\
	EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)

258 259 260 261
/* Like UEVENT_CONSTRAINT, but match flags too */
#define INTEL_FLAGS_UEVENT_CONSTRAINT(c, n)	\
	EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS)

262
#define INTEL_PLD_CONSTRAINT(c, n)	\
263
	__EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
264 265
			   HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LDLAT)

266
#define INTEL_PST_CONSTRAINT(c, n)	\
267
	__EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
268 269
			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST)

270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293
/* Event constraint, but match on all event flags too. */
#define INTEL_FLAGS_EVENT_CONSTRAINT(c, n) \
	EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS)

/* Check only flags, but allow all event/umask */
#define INTEL_ALL_EVENT_CONSTRAINT(code, n)	\
	EVENT_CONSTRAINT(code, n, X86_ALL_EVENT_FLAGS)

/* Check flags and event code, and set the HSW store flag */
#define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_ST(code, n) \
	__EVENT_CONSTRAINT(code, n, 			\
			  ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)

/* Check flags and event code, and set the HSW load flag */
#define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(code, n) \
	__EVENT_CONSTRAINT(code, n, 			\
			  ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)

/* Check flags and event code/umask, and set the HSW store flag */
#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(code, n) \
	__EVENT_CONSTRAINT(code, n, 			\
			  INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
294 295
			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)

296 297 298 299 300 301 302 303 304 305 306 307 308
/* Check flags and event code/umask, and set the HSW load flag */
#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(code, n) \
	__EVENT_CONSTRAINT(code, n, 			\
			  INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)

/* Check flags and event code/umask, and set the HSW N/A flag */
#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(code, n) \
	__EVENT_CONSTRAINT(code, n, 			\
			  INTEL_ARCH_EVENT_MASK|INTEL_ARCH_EVENT_MASK, \
			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_NA_HSW)


309 310 311 312 313 314 315 316
/*
 * We define the end marker as having a weight of -1
 * to enable blacklisting of events using a counter bitmask
 * of zero and thus a weight of zero.
 * The end marker has a weight that cannot possibly be
 * obtained from counting the bits in the bitmask.
 */
#define EVENT_CONSTRAINT_END { .weight = -1 }
317

318 319 320
/*
 * Check for end marker with weight == -1
 */
321
#define for_each_event_constraint(e, c)	\
322
	for ((e) = (c); (e)->weight != -1; (e)++)
323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339

/*
 * Extra registers for specific events.
 *
 * Some events need large masks and require external MSRs.
 * Those extra MSRs end up being shared for all events on
 * a PMU and sometimes between PMU of sibling HT threads.
 * In either case, the kernel needs to handle conflicting
 * accesses to those extra, shared, regs. The data structure
 * to manage those registers is stored in cpu_hw_event.
 */
struct extra_reg {
	unsigned int		event;
	unsigned int		msr;
	u64			config_mask;
	u64			valid_mask;
	int			idx;  /* per_xxx->regs[] reg index */
340
	bool			extra_msr_access;
341 342 343
};

#define EVENT_EXTRA_REG(e, ms, m, vm, i) {	\
344 345 346 347 348 349
	.event = (e),			\
	.msr = (ms),			\
	.config_mask = (m),		\
	.valid_mask = (vm),		\
	.idx = EXTRA_REG_##i,		\
	.extra_msr_access = true,	\
350 351 352 353 354
	}

#define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx)	\
	EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx)

355 356 357 358 359 360 361 362 363 364
#define INTEL_UEVENT_EXTRA_REG(event, msr, vm, idx) \
	EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT | \
			ARCH_PERFMON_EVENTSEL_UMASK, vm, idx)

#define INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(c) \
	INTEL_UEVENT_EXTRA_REG(c, \
			       MSR_PEBS_LD_LAT_THRESHOLD, \
			       0xffff, \
			       LDLAT)

365 366 367 368 369 370 371 372 373
#define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0)

union perf_capabilities {
	struct {
		u64	lbr_format:6;
		u64	pebs_trap:1;
		u64	pebs_arch_reg:1;
		u64	pebs_format:4;
		u64	smm_freeze:1;
374 375 376 377 378
		/*
		 * PMU supports separate counter range for writing
		 * values > 32bit.
		 */
		u64	full_width_write:1;
379 380 381 382
	};
	u64	capabilities;
};

383 384 385 386 387
struct x86_pmu_quirk {
	struct x86_pmu_quirk *next;
	void (*func)(void);
};

388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410
union x86_pmu_config {
	struct {
		u64 event:8,
		    umask:8,
		    usr:1,
		    os:1,
		    edge:1,
		    pc:1,
		    interrupt:1,
		    __reserved1:1,
		    en:1,
		    inv:1,
		    cmask:8,
		    event2:4,
		    __reserved2:4,
		    go:1,
		    ho:1;
	} bits;
	u64 value;
};

#define X86_CONFIG(args...) ((union x86_pmu_config){.bits = {args}}).value

411 412
enum {
	x86_lbr_exclusive_lbr,
413
	x86_lbr_exclusive_bts,
414 415 416 417
	x86_lbr_exclusive_pt,
	x86_lbr_exclusive_max,
};

418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
/*
 * struct x86_pmu - generic x86 pmu
 */
struct x86_pmu {
	/*
	 * Generic x86 PMC bits
	 */
	const char	*name;
	int		version;
	int		(*handle_irq)(struct pt_regs *);
	void		(*disable_all)(void);
	void		(*enable_all)(int added);
	void		(*enable)(struct perf_event *);
	void		(*disable)(struct perf_event *);
	int		(*hw_config)(struct perf_event *event);
	int		(*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
	unsigned	eventsel;
	unsigned	perfctr;
436
	int		(*addr_offset)(int index, bool eventsel);
437
	int		(*rdpmc_index)(int index);
438 439 440 441 442 443
	u64		(*event_map)(int);
	int		max_events;
	int		num_counters;
	int		num_counters_fixed;
	int		cntval_bits;
	u64		cntval_mask;
444 445 446 447 448
	union {
			unsigned long events_maskl;
			unsigned long events_mask[BITS_TO_LONGS(ARCH_PERFMON_EVENTS_COUNT)];
	};
	int		events_mask_len;
449 450 451 452 453 454 455 456 457
	int		apic;
	u64		max_period;
	struct event_constraint *
			(*get_event_constraints)(struct cpu_hw_events *cpuc,
						 struct perf_event *event);

	void		(*put_event_constraints)(struct cpu_hw_events *cpuc,
						 struct perf_event *event);
	struct event_constraint *event_constraints;
458
	struct x86_pmu_quirk *quirks;
459
	int		perfctr_second_write;
460
	bool		late_ack;
461
	unsigned	(*limit_period)(struct perf_event *event, unsigned l);
462

463 464 465
	/*
	 * sysfs attrs
	 */
466
	int		attr_rdpmc_broken;
467
	int		attr_rdpmc;
468
	struct attribute **format_attrs;
469
	struct attribute **event_attrs;
470

471
	ssize_t		(*events_sysfs_show)(char *page, u64 config);
472
	struct attribute **cpu_events;
473

474 475 476
	/*
	 * CPU Hotplug hooks
	 */
477 478 479 480
	int		(*cpu_prepare)(int cpu);
	void		(*cpu_starting)(int cpu);
	void		(*cpu_dying)(int cpu);
	void		(*cpu_dead)(int cpu);
481 482

	void		(*check_microcode)(void);
483 484
	void		(*sched_task)(struct perf_event_context *ctx,
				      bool sched_in);
485 486 487 488 489 490 491 492 493 494

	/*
	 * Intel Arch Perfmon v2+
	 */
	u64			intel_ctrl;
	union perf_capabilities intel_cap;

	/*
	 * Intel DebugStore bits
	 */
495
	unsigned int	bts		:1,
496 497 498 499
			bts_active	:1,
			pebs		:1,
			pebs_active	:1,
			pebs_broken	:1;
500 501 502
	int		pebs_record_size;
	void		(*drain_pebs)(struct pt_regs *regs);
	struct event_constraint *pebs_constraints;
503
	void		(*pebs_aliases)(struct perf_event *event);
504
	int 		max_pebs_events;
505 506 507 508 509 510

	/*
	 * Intel LBR
	 */
	unsigned long	lbr_tos, lbr_from, lbr_to; /* MSR base regs       */
	int		lbr_nr;			   /* hardware stack size */
511 512
	u64		lbr_sel_mask;		   /* LBR_SELECT valid bits */
	const int	*lbr_sel_map;		   /* lbr_select mappings */
513
	bool		lbr_double_abort;	   /* duplicated lbr aborts */
514

515 516 517 518 519
	/*
	 * Intel PT/LBR/BTS are exclusive
	 */
	atomic_t	lbr_exclusive[x86_lbr_exclusive_max];

520 521 522 523
	/*
	 * Extra registers for events
	 */
	struct extra_reg *extra_regs;
524
	unsigned int flags;
525 526 527 528 529

	/*
	 * Intel host/guest support (KVM)
	 */
	struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr);
530 531
};

532 533 534 535 536 537 538
struct x86_perf_task_context {
	u64 lbr_from[MAX_LBR_ENTRIES];
	u64 lbr_to[MAX_LBR_ENTRIES];
	int lbr_callstack_users;
	int lbr_stack_state;
};

539 540 541 542 543 544 545 546 547
#define x86_add_quirk(func_)						\
do {									\
	static struct x86_pmu_quirk __quirk __initdata = {		\
		.func = func_,						\
	};								\
	__quirk.next = x86_pmu.quirks;					\
	x86_pmu.quirks = &__quirk;					\
} while (0)

548 549 550 551 552
/*
 * x86_pmu flags
 */
#define PMU_FL_NO_HT_SHARING	0x1 /* no hyper-threading resource sharing */
#define PMU_FL_HAS_RSP_1	0x2 /* has 2 equivalent offcore_rsp regs   */
553

554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570
#define EVENT_VAR(_id)  event_attr_##_id
#define EVENT_PTR(_id) &event_attr_##_id.attr.attr

#define EVENT_ATTR(_name, _id)						\
static struct perf_pmu_events_attr EVENT_VAR(_id) = {			\
	.attr		= __ATTR(_name, 0444, events_sysfs_show, NULL),	\
	.id		= PERF_COUNT_HW_##_id,				\
	.event_str	= NULL,						\
};

#define EVENT_ATTR_STR(_name, v, str)					\
static struct perf_pmu_events_attr event_attr_##v = {			\
	.attr		= __ATTR(_name, 0444, events_sysfs_show, NULL),	\
	.id		= 0,						\
	.event_str	= str,						\
};

571 572
extern struct x86_pmu x86_pmu __read_mostly;

573 574 575 576 577 578
static inline bool x86_pmu_has_lbr_callstack(void)
{
	return  x86_pmu.lbr_sel_map &&
		x86_pmu.lbr_sel_map[PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] > 0;
}

579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605
DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events);

int x86_perf_event_set_period(struct perf_event *event);

/*
 * Generalized hw caching related hw_event table, filled
 * in on a per model basis. A value of 0 means
 * 'not supported', -1 means 'hw_event makes no sense on
 * this CPU', any other value means the raw hw_event
 * ID.
 */

#define C(x) PERF_COUNT_HW_CACHE_##x

extern u64 __read_mostly hw_cache_event_ids
				[PERF_COUNT_HW_CACHE_MAX]
				[PERF_COUNT_HW_CACHE_OP_MAX]
				[PERF_COUNT_HW_CACHE_RESULT_MAX];
extern u64 __read_mostly hw_cache_extra_regs
				[PERF_COUNT_HW_CACHE_MAX]
				[PERF_COUNT_HW_CACHE_OP_MAX]
				[PERF_COUNT_HW_CACHE_RESULT_MAX];

u64 x86_perf_event_update(struct perf_event *event);

static inline unsigned int x86_pmu_config_addr(int index)
{
606 607
	return x86_pmu.eventsel + (x86_pmu.addr_offset ?
				   x86_pmu.addr_offset(index, true) : index);
608 609 610 611
}

static inline unsigned int x86_pmu_event_addr(int index)
{
612 613
	return x86_pmu.perfctr + (x86_pmu.addr_offset ?
				  x86_pmu.addr_offset(index, false) : index);
614 615
}

616 617 618 619 620
static inline int x86_pmu_rdpmc_index(int index)
{
	return x86_pmu.rdpmc_index ? x86_pmu.rdpmc_index(index) : index;
}

621 622 623 624 625 626
int x86_add_exclusive(unsigned int what);

void x86_del_exclusive(unsigned int what);

void hw_perf_lbr_event_destroy(struct perf_event *event);

627 628 629 630 631 632 633 634 635
int x86_setup_perfctr(struct perf_event *event);

int x86_pmu_hw_config(struct perf_event *event);

void x86_pmu_disable_all(void);

static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
					  u64 enable_mask)
{
636 637
	u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);

638 639
	if (hwc->extra_reg.reg)
		wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config);
640
	wrmsrl(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
641 642 643 644
}

void x86_pmu_enable_all(int added);

645
int perf_assign_events(struct perf_event **events, int n,
646
			int wmin, int wmax, int *assign);
647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665
int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign);

void x86_pmu_stop(struct perf_event *event, int flags);

static inline void x86_pmu_disable_event(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	wrmsrl(hwc->config_base, hwc->config);
}

void x86_pmu_enable_event(struct perf_event *event);

int x86_pmu_handle_irq(struct pt_regs *regs);

extern struct event_constraint emptyconstraint;

extern struct event_constraint unconstrained;

666 667 668 669 670 671 672 673 674
static inline bool kernel_ip(unsigned long ip)
{
#ifdef CONFIG_X86_32
	return ip > PAGE_OFFSET;
#else
	return (long)ip < 0;
#endif
}

675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694
/*
 * Not all PMUs provide the right context information to place the reported IP
 * into full context. Specifically segment registers are typically not
 * supplied.
 *
 * Assuming the address is a linear address (it is for IBS), we fake the CS and
 * vm86 mode using the known zero-based code segment and 'fix up' the registers
 * to reflect this.
 *
 * Intel PEBS/LBR appear to typically provide the effective address, nothing
 * much we can do about that but pray and treat it like a linear address.
 */
static inline void set_linear_ip(struct pt_regs *regs, unsigned long ip)
{
	regs->cs = kernel_ip(ip) ? __KERNEL_CS : __USER_CS;
	if (regs->flags & X86_VM_MASK)
		regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK);
	regs->ip = ip;
}

695
ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event);
696
ssize_t intel_event_sysfs_show(char *page, u64 config);
697

698 699 700 701 702 703 704 705 706 707 708 709 710 711 712
#ifdef CONFIG_CPU_SUP_AMD

int amd_pmu_init(void);

#else /* CONFIG_CPU_SUP_AMD */

static inline int amd_pmu_init(void)
{
	return 0;
}

#endif /* CONFIG_CPU_SUP_AMD */

#ifdef CONFIG_CPU_SUP_INTEL

713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735
static inline bool intel_pmu_needs_lbr_smpl(struct perf_event *event)
{
	/* user explicitly requested branch sampling */
	if (has_branch_stack(event))
		return true;

	/* implicit branch sampling to correct PEBS skid */
	if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1 &&
	    x86_pmu.intel_cap.pebs_format < 2)
		return true;

	return false;
}

static inline bool intel_pmu_has_bts(struct perf_event *event)
{
	if (event->attr.config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS &&
	    !event->attr.freq && event->hw.sample_period == 1)
		return true;

	return false;
}

736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764
int intel_pmu_save_and_restart(struct perf_event *event);

struct event_constraint *
x86_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event);

struct intel_shared_regs *allocate_shared_regs(int cpu);

int intel_pmu_init(void);

void init_debug_store_on_cpu(int cpu);

void fini_debug_store_on_cpu(int cpu);

void release_ds_buffers(void);

void reserve_ds_buffers(void);

extern struct event_constraint bts_constraint;

void intel_pmu_enable_bts(u64 config);

void intel_pmu_disable_bts(void);

int intel_pmu_drain_bts_buffer(void);

extern struct event_constraint intel_core2_pebs_event_constraints[];

extern struct event_constraint intel_atom_pebs_event_constraints[];

765 766
extern struct event_constraint intel_slm_pebs_event_constraints[];

767 768 769 770 771 772
extern struct event_constraint intel_nehalem_pebs_event_constraints[];

extern struct event_constraint intel_westmere_pebs_event_constraints[];

extern struct event_constraint intel_snb_pebs_event_constraints[];

773 774
extern struct event_constraint intel_ivb_pebs_event_constraints[];

775 776
extern struct event_constraint intel_hsw_pebs_event_constraints[];

777 778 779 780 781 782 783 784 785 786 787 788
struct event_constraint *intel_pebs_constraints(struct perf_event *event);

void intel_pmu_pebs_enable(struct perf_event *event);

void intel_pmu_pebs_disable(struct perf_event *event);

void intel_pmu_pebs_enable_all(void);

void intel_pmu_pebs_disable_all(void);

void intel_ds_init(void);

789 790
void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in);

791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808
void intel_pmu_lbr_reset(void);

void intel_pmu_lbr_enable(struct perf_event *event);

void intel_pmu_lbr_disable(struct perf_event *event);

void intel_pmu_lbr_enable_all(void);

void intel_pmu_lbr_disable_all(void);

void intel_pmu_lbr_read(void);

void intel_pmu_lbr_init_core(void);

void intel_pmu_lbr_init_nhm(void);

void intel_pmu_lbr_init_atom(void);

809 810
void intel_pmu_lbr_init_snb(void);

811 812
void intel_pmu_lbr_init_hsw(void);

813 814
int intel_pmu_setup_lbr_filter(struct perf_event *event);

815 816
void intel_pt_interrupt(void);

817 818 819 820 821 822
int intel_bts_interrupt(void);

void intel_bts_enable_local(void);

void intel_bts_disable_local(void);

823 824 825 826
int p4_pmu_init(void);

int p6_pmu_init(void);

827 828
int knc_pmu_init(void);

829 830 831
ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr,
			  char *page);

832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852
#else /* CONFIG_CPU_SUP_INTEL */

static inline void reserve_ds_buffers(void)
{
}

static inline void release_ds_buffers(void)
{
}

static inline int intel_pmu_init(void)
{
	return 0;
}

static inline struct intel_shared_regs *allocate_shared_regs(int cpu)
{
	return NULL;
}

#endif /* CONFIG_CPU_SUP_INTEL */