coresight-etm4x-core.c 61.8 KB
Newer Older
1 2 3
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2014, The Linux Foundation. All rights reserved.
4 5
 */

6
#include <linux/bitops.h>
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
#include <linux/kernel.h>
#include <linux/moduleparam.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/device.h>
#include <linux/io.h>
#include <linux/err.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/delay.h>
#include <linux/smp.h>
#include <linux/sysfs.h>
#include <linux/stat.h>
#include <linux/clk.h>
#include <linux/cpu.h>
22
#include <linux/cpu_pm.h>
23
#include <linux/coresight.h>
24
#include <linux/coresight-pmu.h>
25 26 27 28
#include <linux/pm_wakeup.h>
#include <linux/amba/bus.h>
#include <linux/seq_file.h>
#include <linux/uaccess.h>
29
#include <linux/perf_event.h>
30
#include <linux/platform_device.h>
31
#include <linux/pm_runtime.h>
32
#include <linux/property.h>
33

34
#include <asm/barrier.h>
35
#include <asm/sections.h>
36
#include <asm/sysreg.h>
37
#include <asm/local.h>
38
#include <asm/virt.h>
39 40

#include "coresight-etm4x.h"
41
#include "coresight-etm-perf.h"
42
#include "coresight-etm4x-cfg.h"
43
#include "coresight-self-hosted-trace.h"
44
#include "coresight-syscfg.h"
45 46

static int boot_enable;
47 48
module_param(boot_enable, int, 0444);
MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
49

50 51 52 53 54 55 56 57 58
#define PARAM_PM_SAVE_FIRMWARE	  0 /* save self-hosted state as per firmware */
#define PARAM_PM_SAVE_NEVER	  1 /* never save any state */
#define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */

static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
module_param(pm_save_enable, int, 0444);
MODULE_PARM_DESC(pm_save_enable,
	"Save/restore state on power down: 1 = never, 2 = self-hosted");

59
static struct etmv4_drvdata *etmdrvdata[NR_CPUS];
60 61 62
static void etm4_set_default_config(struct etmv4_config *config);
static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
				  struct perf_event *event);
63
static u64 etm4_get_access_type(struct etmv4_config *config);
64

65 66
static enum cpuhp_state hp_online;

67
struct etm4_init_arg {
68
	unsigned int		pid;
69 70 71 72
	struct etmv4_drvdata	*drvdata;
	struct csdev_access	*csa;
};

73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
/*
 * Check if TRCSSPCICRn(i) is implemented for a given instance.
 *
 * TRCSSPCICRn is implemented only if :
 *	TRCSSPCICR<n> is present only if all of the following are true:
 *		TRCIDR4.NUMSSCC > n.
 *		TRCIDR4.NUMPC > 0b0000 .
 *		TRCSSCSR<n>.PC == 0b1
 */
static inline bool etm4x_sspcicrn_present(struct etmv4_drvdata *drvdata, int n)
{
	return (n < drvdata->nr_ss_cmp) &&
	       drvdata->nr_pe &&
	       (drvdata->config.ss_status[n] & TRCSSCSRn_PC);
}

89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
u64 etm4x_sysreg_read(u32 offset, bool _relaxed, bool _64bit)
{
	u64 res = 0;

	switch (offset) {
	ETM4x_READ_SYSREG_CASES(res)
	default :
		pr_warn_ratelimited("etm4x: trying to read unsupported register @%x\n",
			 offset);
	}

	if (!_relaxed)
		__iormb(res);	/* Imitate the !relaxed I/O helpers */

	return res;
}

void etm4x_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit)
{
	if (!_relaxed)
		__iowmb();	/* Imitate the !relaxed I/O helpers */
	if (!_64bit)
		val &= GENMASK(31, 0);

	switch (offset) {
	ETM4x_WRITE_SYSREG_CASES(val)
	default :
		pr_warn_ratelimited("etm4x: trying to write to unsupported register @%x\n",
			offset);
	}
}

121
static u64 ete_sysreg_read(u32 offset, bool _relaxed, bool _64bit)
122
{
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
	u64 res = 0;

	switch (offset) {
	ETE_READ_CASES(res)
	default :
		pr_warn_ratelimited("ete: trying to read unsupported register @%x\n",
				    offset);
	}

	if (!_relaxed)
		__iormb(res);	/* Imitate the !relaxed I/O helpers */

	return res;
}

static void ete_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit)
{
	if (!_relaxed)
		__iowmb();	/* Imitate the !relaxed I/O helpers */
	if (!_64bit)
		val &= GENMASK(31, 0);

	switch (offset) {
	ETE_WRITE_CASES(val)
	default :
		pr_warn_ratelimited("ete: trying to write to unsupported register @%x\n",
				    offset);
	}
}

153 154
static void etm_detect_os_lock(struct etmv4_drvdata *drvdata,
			       struct csdev_access *csa)
155
{
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
	u32 oslsr = etm4x_relaxed_read32(csa, TRCOSLSR);

	drvdata->os_lock_model = ETM_OSLSR_OSLM(oslsr);
}

static void etm_write_os_lock(struct etmv4_drvdata *drvdata,
			      struct csdev_access *csa, u32 val)
{
	val = !!val;

	switch (drvdata->os_lock_model) {
	case ETM_OSLOCK_PRESENT:
		etm4x_relaxed_write32(csa, val, TRCOSLAR);
		break;
	case ETM_OSLOCK_PE:
		write_sysreg_s(val, SYS_OSLAR_EL1);
		break;
	default:
		pr_warn_once("CPU%d: Unsupported Trace OSLock model: %x\n",
			     smp_processor_id(), drvdata->os_lock_model);
		fallthrough;
	case ETM_OSLOCK_NI:
		return;
	}
180 181 182
	isb();
}

183 184 185 186 187 188 189 190 191 192
static inline void etm4_os_unlock_csa(struct etmv4_drvdata *drvdata,
				      struct csdev_access *csa)
{
	WARN_ON(drvdata->cpu != smp_processor_id());

	/* Writing 0 to OS Lock unlocks the trace unit registers */
	etm_write_os_lock(drvdata, csa, 0x0);
	drvdata->os_unlock = true;
}

193 194 195 196 197 198
static void etm4_os_unlock(struct etmv4_drvdata *drvdata)
{
	if (!WARN_ON(!drvdata->csdev))
		etm4_os_unlock_csa(drvdata, &drvdata->csdev->access);
}

199 200
static void etm4_os_lock(struct etmv4_drvdata *drvdata)
{
201 202
	if (WARN_ON(!drvdata->csdev))
		return;
203 204
	/* Writing 0x1 to OS Lock locks the trace registers */
	etm_write_os_lock(drvdata, &drvdata->csdev->access, 0x1);
205 206 207
	drvdata->os_unlock = false;
}

208 209 210 211 212 213 214 215 216 217 218 219 220 221 222
static void etm4_cs_lock(struct etmv4_drvdata *drvdata,
			 struct csdev_access *csa)
{
	/* Software Lock is only accessible via memory mapped interface */
	if (csa->io_mem)
		CS_LOCK(csa->base);
}

static void etm4_cs_unlock(struct etmv4_drvdata *drvdata,
			   struct csdev_access *csa)
{
	if (csa->io_mem)
		CS_UNLOCK(csa->base);
}

223 224 225 226 227 228 229
static int etm4_cpu_id(struct coresight_device *csdev)
{
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

	return drvdata->cpu;
}

230 231 232 233
static int etm4_trace_id(struct coresight_device *csdev)
{
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

234
	return drvdata->trcid;
235 236
}

237 238 239 240 241
struct etm4_enable_arg {
	struct etmv4_drvdata *drvdata;
	int rc;
};

242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
/*
 * etm4x_prohibit_trace - Prohibit the CPU from tracing at all ELs.
 * When the CPU supports FEAT_TRF, we could move the ETM to a trace
 * prohibited state by filtering the Exception levels via TRFCR_EL1.
 */
static void etm4x_prohibit_trace(struct etmv4_drvdata *drvdata)
{
	/* If the CPU doesn't support FEAT_TRF, nothing to do */
	if (!drvdata->trfcr)
		return;
	cpu_prohibit_trace();
}

/*
 * etm4x_allow_trace - Allow CPU tracing in the respective ELs,
 * as configured by the drvdata->config.mode for the current
 * session. Even though we have TRCVICTLR bits to filter the
 * trace in the ELs, it doesn't prevent the ETM from generating
 * a packet (e.g, TraceInfo) that might contain the addresses from
 * the excluded levels. Thus we use the additional controls provided
 * via the Trace Filtering controls (FEAT_TRF) to make sure no trace
 * is generated for the excluded ELs.
 */
static void etm4x_allow_trace(struct etmv4_drvdata *drvdata)
{
	u64 trfcr = drvdata->trfcr;

	/* If the CPU doesn't support FEAT_TRF, nothing to do */
	if (!trfcr)
		return;

	if (drvdata->config.mode & ETM_MODE_EXCL_KERN)
		trfcr &= ~TRFCR_ELx_ExTRE;
	if (drvdata->config.mode & ETM_MODE_EXCL_USER)
		trfcr &= ~TRFCR_ELx_E0TRE;

	write_trfcr(trfcr);
}

281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
#ifdef CONFIG_ETM4X_IMPDEF_FEATURE

#define HISI_HIP08_AMBA_ID		0x000b6d01
#define ETM4_AMBA_MASK			0xfffff
#define HISI_HIP08_CORE_COMMIT_MASK	0x3000
#define HISI_HIP08_CORE_COMMIT_SHIFT	12
#define HISI_HIP08_CORE_COMMIT_FULL	0b00
#define HISI_HIP08_CORE_COMMIT_LVL_1	0b01
#define HISI_HIP08_CORE_COMMIT_REG	sys_reg(3, 1, 15, 2, 5)

struct etm4_arch_features {
	void (*arch_callback)(bool enable);
};

static bool etm4_hisi_match_pid(unsigned int id)
{
	return (id & ETM4_AMBA_MASK) == HISI_HIP08_AMBA_ID;
}

static void etm4_hisi_config_core_commit(bool enable)
{
	u8 commit = enable ? HISI_HIP08_CORE_COMMIT_LVL_1 :
		    HISI_HIP08_CORE_COMMIT_FULL;
	u64 val;

	/*
	 * bit 12 and 13 of HISI_HIP08_CORE_COMMIT_REG are used together
	 * to set core-commit, 2'b00 means cpu is at full speed, 2'b01,
	 * 2'b10, 2'b11 mean reduce pipeline speed, and 2'b01 means level-1
	 * speed(minimun value). So bit 12 and 13 should be cleared together.
	 */
	val = read_sysreg_s(HISI_HIP08_CORE_COMMIT_REG);
	val &= ~HISI_HIP08_CORE_COMMIT_MASK;
	val |= commit << HISI_HIP08_CORE_COMMIT_SHIFT;
	write_sysreg_s(val, HISI_HIP08_CORE_COMMIT_REG);
}

static struct etm4_arch_features etm4_features[] = {
	[ETM4_IMPDEF_HISI_CORE_COMMIT] = {
		.arch_callback = etm4_hisi_config_core_commit,
	},
	{},
};

static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)
{
	struct etm4_arch_features *ftr;
	int bit;

	for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) {
		ftr = &etm4_features[bit];

		if (ftr->arch_callback)
			ftr->arch_callback(true);
	}
}

static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata)
{
	struct etm4_arch_features *ftr;
	int bit;

	for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) {
		ftr = &etm4_features[bit];

		if (ftr->arch_callback)
			ftr->arch_callback(false);
	}
}

static void etm4_check_arch_features(struct etmv4_drvdata *drvdata,
				      unsigned int id)
{
	if (etm4_hisi_match_pid(id))
		set_bit(ETM4_IMPDEF_HISI_CORE_COMMIT, drvdata->arch_features);
}
#else
static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)
{
}

static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata)
{
}

static void etm4_check_arch_features(struct etmv4_drvdata *drvdata,
				     unsigned int id)
{
}
#endif /* CONFIG_ETM4X_IMPDEF_FEATURE */

372
static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
373
{
374
	int i, rc;
375
	struct etmv4_config *config = &drvdata->config;
376 377 378
	struct coresight_device *csdev = drvdata->csdev;
	struct device *etm_dev = &csdev->dev;
	struct csdev_access *csa = &csdev->access;
379

380 381

	etm4_cs_unlock(drvdata, csa);
382
	etm4_enable_arch_specific(drvdata);
383 384 385

	etm4_os_unlock(drvdata);

386
	rc = coresight_claim_device_unlocked(csdev);
387 388 389
	if (rc)
		goto done;

390
	/* Disable the trace unit before programming trace registers */
391
	etm4x_relaxed_write32(csa, 0, TRCPRGCTLR);
392

393 394 395 396 397 398 399 400 401
	/*
	 * If we use system instructions, we need to synchronize the
	 * write to the TRCPRGCTLR, before accessing the TRCSTATR.
	 * See ARM IHI0064F, section
	 * "4.3.7 Synchronization of register updates"
	 */
	if (!csa->io_mem)
		isb();

402
	/* wait for TRCSTATR.IDLE to go up */
403
	if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1))
404
		dev_err(etm_dev,
405
			"timeout while waiting for Idle Trace Status\n");
406
	if (drvdata->nr_pe)
407 408
		etm4x_relaxed_write32(csa, config->pe_sel, TRCPROCSELR);
	etm4x_relaxed_write32(csa, config->cfg, TRCCONFIGR);
409
	/* nothing specific implemented */
410 411 412
	etm4x_relaxed_write32(csa, 0x0, TRCAUXCTLR);
	etm4x_relaxed_write32(csa, config->eventctrl0, TRCEVENTCTL0R);
	etm4x_relaxed_write32(csa, config->eventctrl1, TRCEVENTCTL1R);
413 414
	if (drvdata->stallctl)
		etm4x_relaxed_write32(csa, config->stall_ctrl, TRCSTALLCTLR);
415 416 417 418 419 420 421 422
	etm4x_relaxed_write32(csa, config->ts_ctrl, TRCTSCTLR);
	etm4x_relaxed_write32(csa, config->syncfreq, TRCSYNCPR);
	etm4x_relaxed_write32(csa, config->ccctlr, TRCCCCTLR);
	etm4x_relaxed_write32(csa, config->bb_ctrl, TRCBBCTLR);
	etm4x_relaxed_write32(csa, drvdata->trcid, TRCTRACEIDR);
	etm4x_relaxed_write32(csa, config->vinst_ctrl, TRCVICTLR);
	etm4x_relaxed_write32(csa, config->viiectlr, TRCVIIECTLR);
	etm4x_relaxed_write32(csa, config->vissctlr, TRCVISSCTLR);
423
	if (drvdata->nr_pe_cmp)
424
		etm4x_relaxed_write32(csa, config->vipcssctlr, TRCVIPCSSCTLR);
425
	for (i = 0; i < drvdata->nrseqstate - 1; i++)
426 427 428 429
		etm4x_relaxed_write32(csa, config->seq_ctrl[i], TRCSEQEVRn(i));
	etm4x_relaxed_write32(csa, config->seq_rst, TRCSEQRSTEVR);
	etm4x_relaxed_write32(csa, config->seq_state, TRCSEQSTR);
	etm4x_relaxed_write32(csa, config->ext_inp, TRCEXTINSELR);
430
	for (i = 0; i < drvdata->nr_cntr; i++) {
431 432 433
		etm4x_relaxed_write32(csa, config->cntrldvr[i], TRCCNTRLDVRn(i));
		etm4x_relaxed_write32(csa, config->cntr_ctrl[i], TRCCNTCTLRn(i));
		etm4x_relaxed_write32(csa, config->cntr_val[i], TRCCNTVRn(i));
434
	}
435

436 437 438 439 440
	/*
	 * Resource selector pair 0 is always implemented and reserved.  As
	 * such start at 2.
	 */
	for (i = 2; i < drvdata->nr_resource * 2; i++)
441
		etm4x_relaxed_write32(csa, config->res_ctrl[i], TRCRSCTLRn(i));
442 443

	for (i = 0; i < drvdata->nr_ss_cmp; i++) {
444 445
		/* always clear status bit on restart if using single-shot */
		if (config->ss_ctrl[i] || config->ss_pe_cmp[i])
446
			config->ss_status[i] &= ~TRCSSCSRn_STATUS;
447 448
		etm4x_relaxed_write32(csa, config->ss_ctrl[i], TRCSSCCRn(i));
		etm4x_relaxed_write32(csa, config->ss_status[i], TRCSSCSRn(i));
449
		if (etm4x_sspcicrn_present(drvdata, i))
450
			etm4x_relaxed_write32(csa, config->ss_pe_cmp[i], TRCSSPCICRn(i));
451 452
	}
	for (i = 0; i < drvdata->nr_addr_cmp; i++) {
453 454
		etm4x_relaxed_write64(csa, config->addr_val[i], TRCACVRn(i));
		etm4x_relaxed_write64(csa, config->addr_acc[i], TRCACATRn(i));
455 456
	}
	for (i = 0; i < drvdata->numcidc; i++)
457 458
		etm4x_relaxed_write64(csa, config->ctxid_pid[i], TRCCIDCVRn(i));
	etm4x_relaxed_write32(csa, config->ctxid_mask0, TRCCIDCCTLR0);
459
	if (drvdata->numcidc > 4)
460
		etm4x_relaxed_write32(csa, config->ctxid_mask1, TRCCIDCCTLR1);
461 462

	for (i = 0; i < drvdata->numvmidc; i++)
463 464
		etm4x_relaxed_write64(csa, config->vmid_val[i], TRCVMIDCVRn(i));
	etm4x_relaxed_write32(csa, config->vmid_mask0, TRCVMIDCCTLR0);
465
	if (drvdata->numvmidc > 4)
466
		etm4x_relaxed_write32(csa, config->vmid_mask1, TRCVMIDCCTLR1);
467

468
	if (!drvdata->skip_power_up) {
469 470
		u32 trcpdcr = etm4x_relaxed_read32(csa, TRCPDCR);

471 472 473 474
		/*
		 * Request to keep the trace unit powered and also
		 * emulation of powerdown
		 */
475
		etm4x_relaxed_write32(csa, trcpdcr | TRCPDCR_PU, TRCPDCR);
476
	}
477

478 479 480 481 482 483 484
	/*
	 * ETE mandates that the TRCRSR is written to before
	 * enabling it.
	 */
	if (etm4x_is_ete(drvdata))
		etm4x_relaxed_write32(csa, TRCRSR_TA, TRCRSR);

485
	etm4x_allow_trace(drvdata);
486
	/* Enable the trace unit */
487
	etm4x_relaxed_write32(csa, 1, TRCPRGCTLR);
488

489 490 491 492
	/* Synchronize the register updates for sysreg access */
	if (!csa->io_mem)
		isb();

493
	/* wait for TRCSTATR.IDLE to go back down to '0' */
494
	if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 0))
495
		dev_err(etm_dev,
496
			"timeout while waiting for Idle Trace Status\n");
497

498 499 500 501 502 503 504
	/*
	 * As recommended by section 4.3.7 ("Synchronization when using the
	 * memory-mapped interface") of ARM IHI 0064D
	 */
	dsb(sy);
	isb();

505
done:
506
	etm4_cs_lock(drvdata, csa);
507

508
	dev_dbg(etm_dev, "cpu: %d enable smp call done: %d\n",
509 510
		drvdata->cpu, rc);
	return rc;
511 512 513 514 515 516 517 518 519
}

static void etm4_enable_hw_smp_call(void *info)
{
	struct etm4_enable_arg *arg = info;

	if (WARN_ON(!arg))
		return;
	arg->rc = etm4_enable_hw(arg->drvdata);
520 521
}

522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606
/*
 * The goal of function etm4_config_timestamp_event() is to configure a
 * counter that will tell the tracer to emit a timestamp packet when it
 * reaches zero.  This is done in order to get a more fine grained idea
 * of when instructions are executed so that they can be correlated
 * with execution on other CPUs.
 *
 * To do this the counter itself is configured to self reload and
 * TRCRSCTLR1 (always true) used to get the counter to decrement.  From
 * there a resource selector is configured with the counter and the
 * timestamp control register to use the resource selector to trigger the
 * event that will insert a timestamp packet in the stream.
 */
static int etm4_config_timestamp_event(struct etmv4_drvdata *drvdata)
{
	int ctridx, ret = -EINVAL;
	int counter, rselector;
	u32 val = 0;
	struct etmv4_config *config = &drvdata->config;

	/* No point in trying if we don't have at least one counter */
	if (!drvdata->nr_cntr)
		goto out;

	/* Find a counter that hasn't been initialised */
	for (ctridx = 0; ctridx < drvdata->nr_cntr; ctridx++)
		if (config->cntr_val[ctridx] == 0)
			break;

	/* All the counters have been configured already, bail out */
	if (ctridx == drvdata->nr_cntr) {
		pr_debug("%s: no available counter found\n", __func__);
		ret = -ENOSPC;
		goto out;
	}

	/*
	 * Searching for an available resource selector to use, starting at
	 * '2' since every implementation has at least 2 resource selector.
	 * ETMIDR4 gives the number of resource selector _pairs_,
	 * hence multiply by 2.
	 */
	for (rselector = 2; rselector < drvdata->nr_resource * 2; rselector++)
		if (!config->res_ctrl[rselector])
			break;

	if (rselector == drvdata->nr_resource * 2) {
		pr_debug("%s: no available resource selector found\n",
			 __func__);
		ret = -ENOSPC;
		goto out;
	}

	/* Remember what counter we used */
	counter = 1 << ctridx;

	/*
	 * Initialise original and reload counter value to the smallest
	 * possible value in order to get as much precision as we can.
	 */
	config->cntr_val[ctridx] = 1;
	config->cntrldvr[ctridx] = 1;

	/* Set the trace counter control register */
	val =  0x1 << 16	|  /* Bit 16, reload counter automatically */
	       0x0 << 7		|  /* Select single resource selector */
	       0x1;		   /* Resource selector 1, i.e always true */

	config->cntr_ctrl[ctridx] = val;

	val = 0x2 << 16		| /* Group 0b0010 - Counter and sequencers */
	      counter << 0;	  /* Counter to use */

	config->res_ctrl[rselector] = val;

	val = 0x0 << 7		| /* Select single resource selector */
	      rselector;	  /* Resource selector */

	config->ts_ctrl = val;

	ret = 0;
out:
	return ret;
}

607
static int etm4_parse_event_config(struct coresight_device *csdev,
608
				   struct perf_event *event)
609
{
610
	int ret = 0;
611
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
612
	struct etmv4_config *config = &drvdata->config;
613
	struct perf_event_attr *attr = &event->attr;
614 615
	unsigned long cfg_hash;
	int preset;
616 617 618 619 620 621 622 623 624 625 626

	/* Clear configuration from previous run */
	memset(config, 0, sizeof(struct etmv4_config));

	if (attr->exclude_kernel)
		config->mode = ETM_MODE_EXCL_KERN;

	if (attr->exclude_user)
		config->mode = ETM_MODE_EXCL_USER;

	/* Always start from the default config */
627 628 629 630 631 632
	etm4_set_default_config(config);

	/* Configure filters specified on the perf cmd line, if any. */
	ret = etm4_set_event_filters(drvdata, event);
	if (ret)
		goto out;
633 634

	/* Go from generic option to ETMv4 specifics */
635
	if (attr->config & BIT(ETM_OPT_CYCACC)) {
636
		config->cfg |= TRCCONFIGR_CCI;
637 638 639
		/* TRM: Must program this for cycacc to work */
		config->ccctlr = ETM_CYC_THRESHOLD_DEFAULT;
	}
640 641 642 643 644 645 646 647 648 649 650 651 652 653 654
	if (attr->config & BIT(ETM_OPT_TS)) {
		/*
		 * Configure timestamps to be emitted at regular intervals in
		 * order to correlate instructions executed on different CPUs
		 * (CPU-wide trace scenarios).
		 */
		ret = etm4_config_timestamp_event(drvdata);

		/*
		 * No need to go further if timestamp intervals can't
		 * be configured.
		 */
		if (ret)
			goto out;

655
		/* bit[11], Global timestamp tracing bit */
656
		config->cfg |= TRCCONFIGR_TS;
657
	}
658

659 660 661
	/* Only trace contextID when runs in root PID namespace */
	if ((attr->config & BIT(ETM_OPT_CTXTID)) &&
	    task_is_in_init_pid_ns(current))
662
		/* bit[6], Context ID tracing bit */
663
		config->cfg |= TRCCONFIGR_CID;
664

665 666 667 668 669 670 671 672 673 674
	/*
	 * If set bit ETM_OPT_CTXTID2 in perf config, this asks to trace VMID
	 * for recording CONTEXTIDR_EL2.  Do not enable VMID tracing if the
	 * kernel is not running in EL2.
	 */
	if (attr->config & BIT(ETM_OPT_CTXTID2)) {
		if (!is_kernel_in_hyp_mode()) {
			ret = -EINVAL;
			goto out;
		}
675 676
		/* Only trace virtual contextID when runs in root PID namespace */
		if (task_is_in_init_pid_ns(current))
677
			config->cfg |= TRCCONFIGR_VMID | TRCCONFIGR_VMIDOPT;
678 679
	}

680 681 682
	/* return stack - enable if selected and supported */
	if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack)
		/* bit[12], Return stack enable bit */
683
		config->cfg |= TRCCONFIGR_RS;
684

685 686 687 688 689 690 691 692 693 694 695 696 697 698
	/*
	 * Set any selected configuration and preset.
	 *
	 * This extracts the values of PMU_FORMAT_ATTR(configid) and PMU_FORMAT_ATTR(preset)
	 * in the perf attributes defined in coresight-etm-perf.c.
	 * configid uses bits 63:32 of attr->config2, preset uses bits 3:0 of attr->config.
	 * A zero configid means no configuration active, preset = 0 means no preset selected.
	 */
	if (attr->config2 & GENMASK_ULL(63, 32)) {
		cfg_hash = (u32)(attr->config2 >> 32);
		preset = attr->config & 0xF;
		ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset);
	}

699 700 701 702 703 704 705 706 707 708 709 710 711 712
	/* branch broadcast - enable if selected and supported */
	if (attr->config & BIT(ETM_OPT_BRANCH_BROADCAST)) {
		if (!drvdata->trcbb) {
			/*
			 * Missing BB support could cause silent decode errors
			 * so fail to open if it's not supported.
			 */
			ret = -EINVAL;
			goto out;
		} else {
			config->cfg |= BIT(ETM4_CFG_BIT_BB);
		}
	}

713 714
out:
	return ret;
715 716 717
}

static int etm4_enable_perf(struct coresight_device *csdev,
718
			    struct perf_event *event)
719
{
720
	int ret = 0;
721 722
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

723 724 725 726
	if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) {
		ret = -EINVAL;
		goto out;
	}
727 728

	/* Configure the tracer based on the session's specifics */
729
	ret = etm4_parse_event_config(csdev, event);
730 731
	if (ret)
		goto out;
732
	/* And enable it */
733
	ret = etm4_enable_hw(drvdata);
734

735 736
out:
	return ret;
737 738
}

739
static int etm4_enable_sysfs(struct coresight_device *csdev)
740 741
{
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
742
	struct etm4_enable_arg arg = { };
743 744 745 746 747 748 749 750 751 752
	unsigned long cfg_hash;
	int ret, preset;

	/* enable any config activated by configfs */
	cscfg_config_sysfs_get_active_cfg(&cfg_hash, &preset);
	if (cfg_hash) {
		ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset);
		if (ret)
			return ret;
	}
753 754 755 756 757 758 759

	spin_lock(&drvdata->spinlock);

	/*
	 * Executing etm4_enable_hw on the cpu whose ETM is being enabled
	 * ensures that register writes occur when cpu is powered.
	 */
760
	arg.drvdata = drvdata;
761
	ret = smp_call_function_single(drvdata->cpu,
762 763 764 765 766
				       etm4_enable_hw_smp_call, &arg, 1);
	if (!ret)
		ret = arg.rc;
	if (!ret)
		drvdata->sticky_enable = true;
767 768
	spin_unlock(&drvdata->spinlock);

769
	if (!ret)
770
		dev_dbg(&csdev->dev, "ETM tracing enabled\n");
771 772 773
	return ret;
}

774
static int etm4_enable(struct coresight_device *csdev,
775
		       struct perf_event *event, u32 mode)
776 777 778 779 780 781 782 783 784 785 786 787 788 789 790
{
	int ret;
	u32 val;
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

	val = local_cmpxchg(&drvdata->mode, CS_MODE_DISABLED, mode);

	/* Someone is already using the tracer */
	if (val)
		return -EBUSY;

	switch (mode) {
	case CS_MODE_SYSFS:
		ret = etm4_enable_sysfs(csdev);
		break;
791
	case CS_MODE_PERF:
792
		ret = etm4_enable_perf(csdev, event);
793
		break;
794 795 796 797 798 799 800 801 802 803 804
	default:
		ret = -EINVAL;
	}

	/* The tracer didn't start */
	if (ret)
		local_set(&drvdata->mode, CS_MODE_DISABLED);

	return ret;
}

805 806 807 808
static void etm4_disable_hw(void *info)
{
	u32 control;
	struct etmv4_drvdata *drvdata = info;
809
	struct etmv4_config *config = &drvdata->config;
810 811 812
	struct coresight_device *csdev = drvdata->csdev;
	struct device *etm_dev = &csdev->dev;
	struct csdev_access *csa = &csdev->access;
813
	int i;
814

815
	etm4_cs_unlock(drvdata, csa);
816
	etm4_disable_arch_specific(drvdata);
817

818 819
	if (!drvdata->skip_power_up) {
		/* power can be removed from the trace unit now */
820
		control = etm4x_relaxed_read32(csa, TRCPDCR);
821
		control &= ~TRCPDCR_PU;
822
		etm4x_relaxed_write32(csa, control, TRCPDCR);
823
	}
824

825
	control = etm4x_relaxed_read32(csa, TRCPRGCTLR);
826 827 828 829

	/* EN, bit[0] Trace unit enable bit */
	control &= ~0x1;

830 831 832 833
	/*
	 * If the CPU supports v8.4 Trace filter Control,
	 * set the ETM to trace prohibited region.
	 */
834
	etm4x_prohibit_trace(drvdata);
835 836 837 838 839 840
	/*
	 * Make sure everything completes before disabling, as recommended
	 * by section 7.3.77 ("TRCVICTLR, ViewInst Main Control Register,
	 * SSTATUS") of ARM IHI 0064D
	 */
	dsb(sy);
841
	isb();
842 843
	/* Trace synchronization barrier, is a nop if not supported */
	tsb_csync();
844
	etm4x_relaxed_write32(csa, control, TRCPRGCTLR);
845

846
	/* wait for TRCSTATR.PMSTABLE to go to '1' */
847
	if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1))
848 849 850 851 852
		dev_err(etm_dev,
			"timeout while waiting for PM stable Trace Status\n");
	/* read the status of the single shot comparators */
	for (i = 0; i < drvdata->nr_ss_cmp; i++) {
		config->ss_status[i] =
853
			etm4x_relaxed_read32(csa, TRCSSCSRn(i));
854 855
	}

856 857 858
	/* read back the current counter values */
	for (i = 0; i < drvdata->nr_cntr; i++) {
		config->cntr_val[i] =
859
			etm4x_relaxed_read32(csa, TRCCNTVRn(i));
860 861
	}

862
	coresight_disclaim_device_unlocked(csdev);
863
	etm4_cs_lock(drvdata, csa);
864

865 866
	dev_dbg(&drvdata->csdev->dev,
		"cpu: %d disable smp call done\n", drvdata->cpu);
867 868
}

869 870
static int etm4_disable_perf(struct coresight_device *csdev,
			     struct perf_event *event)
871
{
872 873
	u32 control;
	struct etm_filters *filters = event->hw.addr_filters;
874
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
875
	struct perf_event_attr *attr = &event->attr;
876 877 878 879 880

	if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id()))
		return -EINVAL;

	etm4_disable_hw(drvdata);
881 882 883 884 885 886
	/*
	 * The config_id occupies bits 63:32 of the config2 perf event attr
	 * field. If this is non-zero then we will have enabled a config.
	 */
	if (attr->config2 & GENMASK_ULL(63, 32))
		cscfg_csdev_disable_active_config(csdev);
887 888 889 890 891 892 893

	/*
	 * Check if the start/stop logic was active when the unit was stopped.
	 * That way we can re-enable the start/stop logic when the process is
	 * scheduled again.  Configuration of the start/stop logic happens in
	 * function etm4_set_event_filters().
	 */
894
	control = etm4x_relaxed_read32(&csdev->access, TRCVICTLR);
895 896 897
	/* TRCVICTLR::SSSTATUS, bit[9] */
	filters->ssstatus = (control & BIT(9));

898 899 900
	return 0;
}

901
static void etm4_disable_sysfs(struct coresight_device *csdev)
902 903 904 905 906 907 908 909 910
{
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

	/*
	 * Taking hotplug lock here protects from clocks getting disabled
	 * with tracing being left on (crash scenario) if user disable occurs
	 * after cpu online mask indicates the cpu is offline but before the
	 * DYING hotplug callback is serviced by the ETM driver.
	 */
911
	cpus_read_lock();
912 913 914 915 916 917 918 919 920
	spin_lock(&drvdata->spinlock);

	/*
	 * Executing etm4_disable_hw on the cpu whose ETM is being disabled
	 * ensures that register writes occur when cpu is powered.
	 */
	smp_call_function_single(drvdata->cpu, etm4_disable_hw, drvdata, 1);

	spin_unlock(&drvdata->spinlock);
921
	cpus_read_unlock();
922

923
	dev_dbg(&csdev->dev, "ETM tracing disabled\n");
924 925
}

926 927
static void etm4_disable(struct coresight_device *csdev,
			 struct perf_event *event)
928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944
{
	u32 mode;
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

	/*
	 * For as long as the tracer isn't disabled another entity can't
	 * change its status.  As such we can read the status here without
	 * fearing it will change under us.
	 */
	mode = local_read(&drvdata->mode);

	switch (mode) {
	case CS_MODE_DISABLED:
		break;
	case CS_MODE_SYSFS:
		etm4_disable_sysfs(csdev);
		break;
945
	case CS_MODE_PERF:
946
		etm4_disable_perf(csdev, event);
947
		break;
948 949 950 951 952 953
	}

	if (mode)
		local_set(&drvdata->mode, CS_MODE_DISABLED);
}

954
static const struct coresight_ops_source etm4_source_ops = {
955
	.cpu_id		= etm4_cpu_id,
956 957 958 959 960 961 962 963 964
	.trace_id	= etm4_trace_id,
	.enable		= etm4_enable,
	.disable	= etm4_disable,
};

static const struct coresight_ops etm4_cs_ops = {
	.source_ops	= &etm4_source_ops,
};

965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983
static inline bool cpu_supports_sysreg_trace(void)
{
	u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1);

	return ((dfr0 >> ID_AA64DFR0_TRACEVER_SHIFT) & 0xfUL) > 0;
}

static bool etm4_init_sysreg_access(struct etmv4_drvdata *drvdata,
				    struct csdev_access *csa)
{
	u32 devarch;

	if (!cpu_supports_sysreg_trace())
		return false;

	/*
	 * ETMs implementing sysreg access must implement TRCDEVARCH.
	 */
	devarch = read_etm4x_sysreg_const_offset(TRCDEVARCH);
984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999
	switch (devarch & ETM_DEVARCH_ID_MASK) {
	case ETM_DEVARCH_ETMv4x_ARCH:
		*csa = (struct csdev_access) {
			.io_mem	= false,
			.read	= etm4x_sysreg_read,
			.write	= etm4x_sysreg_write,
		};
		break;
	case ETM_DEVARCH_ETE_ARCH:
		*csa = (struct csdev_access) {
			.io_mem	= false,
			.read	= ete_sysreg_read,
			.write	= ete_sysreg_write,
		};
		break;
	default:
1000
		return false;
1001
	}
1002 1003 1004 1005 1006

	drvdata->arch = etm_devarch_to_arch(devarch);
	return true;
}

1007 1008 1009
static bool etm4_init_iomem_access(struct etmv4_drvdata *drvdata,
				   struct csdev_access *csa)
{
1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
	u32 devarch = readl_relaxed(drvdata->base + TRCDEVARCH);
	u32 idr1 = readl_relaxed(drvdata->base + TRCIDR1);

	/*
	 * All ETMs must implement TRCDEVARCH to indicate that
	 * the component is an ETMv4. To support any broken
	 * implementations we fall back to TRCIDR1 check, which
	 * is not really reliable.
	 */
	if ((devarch & ETM_DEVARCH_ID_MASK) == ETM_DEVARCH_ETMv4x_ARCH) {
		drvdata->arch = etm_devarch_to_arch(devarch);
	} else {
		pr_warn("CPU%d: ETM4x incompatible TRCDEVARCH: %x, falling back to TRCIDR1\n",
			smp_processor_id(), devarch);

		if (ETM_TRCIDR1_ARCH_MAJOR(idr1) != ETM_TRCIDR1_ARCH_ETMv4)
			return false;
		drvdata->arch = etm_trcidr_to_arch(idr1);
	}

1030 1031 1032 1033 1034 1035 1036
	*csa = CSDEV_ACCESS_IOMEM(drvdata->base);
	return true;
}

static bool etm4_init_csdev_access(struct etmv4_drvdata *drvdata,
				   struct csdev_access *csa)
{
1037 1038 1039 1040 1041
	/*
	 * Always choose the memory mapped io, if there is
	 * a memory map to prevent sysreg access on broken
	 * systems.
	 */
1042 1043 1044
	if (drvdata->base)
		return etm4_init_iomem_access(drvdata, csa);

1045 1046 1047
	if (etm4_init_sysreg_access(drvdata, csa))
		return true;

1048 1049 1050
	return false;
}

1051
static void cpu_detect_trace_filtering(struct etmv4_drvdata *drvdata)
1052 1053 1054 1055
{
	u64 dfr0 = read_sysreg(id_aa64dfr0_el1);
	u64 trfcr;

1056
	drvdata->trfcr = 0;
1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072
	if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_TRACE_FILT_SHIFT))
		return;

	/*
	 * If the CPU supports v8.4 SelfHosted Tracing, enable
	 * tracing at the kernel EL and EL0, forcing to use the
	 * virtual time as the timestamp.
	 */
	trfcr = (TRFCR_ELx_TS_VIRTUAL |
		 TRFCR_ELx_ExTRE |
		 TRFCR_ELx_E0TRE);

	/* If we are running at EL2, allow tracing the CONTEXTIDR_EL2. */
	if (is_kernel_in_hyp_mode())
		trfcr |= TRFCR_EL2_CX;

1073
	drvdata->trfcr = trfcr;
1074 1075
}

1076 1077 1078 1079 1080 1081 1082
static void etm4_init_arch_data(void *info)
{
	u32 etmidr0;
	u32 etmidr2;
	u32 etmidr3;
	u32 etmidr4;
	u32 etmidr5;
1083 1084 1085
	struct etm4_init_arg *init_arg = info;
	struct etmv4_drvdata *drvdata;
	struct csdev_access *csa;
1086
	int i;
1087

1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098
	drvdata = init_arg->drvdata;
	csa = init_arg->csa;

	/*
	 * If we are unable to detect the access mechanism,
	 * or unable to detect the trace unit type, fail
	 * early.
	 */
	if (!etm4_init_csdev_access(drvdata, csa))
		return;

1099 1100 1101
	/* Detect the support for OS Lock before we actually use it */
	etm_detect_os_lock(drvdata, csa);

1102
	/* Make sure all registers are accessible */
1103
	etm4_os_unlock_csa(drvdata, csa);
1104
	etm4_cs_unlock(drvdata, csa);
1105

1106 1107
	etm4_check_arch_features(drvdata, init_arg->pid);

1108
	/* find all capabilities of the tracing unit */
1109
	etmidr0 = etm4x_relaxed_read32(csa, TRCIDR0);
1110 1111

	/* INSTP0, bits[2:1] P0 tracing support field */
1112
	drvdata->instrp0 = !!(FIELD_GET(TRCIDR0_INSTP0_MASK, etmidr0) == 0b11);
1113
	/* TRCBB, bit[5] Branch broadcast tracing support bit */
1114
	drvdata->trcbb = !!(etmidr0 & TRCIDR0_TRCBB);
1115
	/* TRCCOND, bit[6] Conditional instruction tracing support bit */
1116
	drvdata->trccond = !!(etmidr0 & TRCIDR0_TRCCOND);
1117
	/* TRCCCI, bit[7] Cycle counting instruction bit */
1118
	drvdata->trccci = !!(etmidr0 & TRCIDR0_TRCCCI);
1119
	/* RETSTACK, bit[9] Return stack bit */
1120
	drvdata->retstack = !!(etmidr0 & TRCIDR0_RETSTACK);
1121
	/* NUMEVENT, bits[11:10] Number of events field */
1122
	drvdata->nr_event = FIELD_GET(TRCIDR0_NUMEVENT_MASK, etmidr0);
1123
	/* QSUPP, bits[16:15] Q element support field */
1124
	drvdata->q_support = FIELD_GET(TRCIDR0_QSUPP_MASK, etmidr0);
1125
	/* TSSIZE, bits[28:24] Global timestamp size field */
1126
	drvdata->ts_size = FIELD_GET(TRCIDR0_TSSIZE_MASK, etmidr0);
1127 1128

	/* maximum size of resources */
1129
	etmidr2 = etm4x_relaxed_read32(csa, TRCIDR2);
1130
	/* CIDSIZE, bits[9:5] Indicates the Context ID size */
1131
	drvdata->ctxid_size = FIELD_GET(TRCIDR2_CIDSIZE_MASK, etmidr2);
1132
	/* VMIDSIZE, bits[14:10] Indicates the VMID size */
1133
	drvdata->vmid_size = FIELD_GET(TRCIDR2_VMIDSIZE_MASK, etmidr2);
1134
	/* CCSIZE, bits[28:25] size of the cycle counter in bits minus 12 */
1135
	drvdata->ccsize = FIELD_GET(TRCIDR2_CCSIZE_MASK, etmidr2);
1136

1137
	etmidr3 = etm4x_relaxed_read32(csa, TRCIDR3);
1138
	/* CCITMIN, bits[11:0] minimum threshold value that can be programmed */
1139
	drvdata->ccitmin = FIELD_GET(TRCIDR3_CCITMIN_MASK, etmidr3);
1140
	/* EXLEVEL_S, bits[19:16] Secure state instruction tracing */
1141
	drvdata->s_ex_level = FIELD_GET(TRCIDR3_EXLEVEL_S_MASK, etmidr3);
1142
	drvdata->config.s_ex_level = drvdata->s_ex_level;
1143
	/* EXLEVEL_NS, bits[23:20] Non-secure state instruction tracing */
1144
	drvdata->ns_ex_level = FIELD_GET(TRCIDR3_EXLEVEL_NS_MASK, etmidr3);
1145 1146 1147 1148
	/*
	 * TRCERR, bit[24] whether a trace unit can trace a
	 * system error exception.
	 */
1149
	drvdata->trc_error = !!(etmidr3 & TRCIDR3_TRCERR);
1150
	/* SYNCPR, bit[25] implementation has a fixed synchronization period? */
1151
	drvdata->syncpr = !!(etmidr3 & TRCIDR3_SYNCPR);
1152
	/* STALLCTL, bit[26] is stall control implemented? */
1153
	drvdata->stallctl = !!(etmidr3 & TRCIDR3_STALLCTL);
1154
	/* SYSSTALL, bit[27] implementation can support stall control? */
1155
	drvdata->sysstall = !!(etmidr3 & TRCIDR3_SYSSTALL);
1156 1157 1158 1159 1160 1161
	/*
	 * NUMPROC - the number of PEs available for tracing, 5bits
	 *         = TRCIDR3.bits[13:12]bits[30:28]
	 *  bits[4:3] = TRCIDR3.bits[13:12] (since etm-v4.2, otherwise RES0)
	 *  bits[3:0] = TRCIDR3.bits[30:28]
	 */
1162 1163
	drvdata->nr_pe =  (FIELD_GET(TRCIDR3_NUMPROC_HI_MASK, etmidr3) << 3) |
			   FIELD_GET(TRCIDR3_NUMPROC_LO_MASK, etmidr3);
1164
	/* NOOVERFLOW, bit[31] is trace overflow prevention supported */
1165
	drvdata->nooverflow = !!(etmidr3 & TRCIDR3_NOOVERFLOW);
1166 1167

	/* number of resources trace unit supports */
1168
	etmidr4 = etm4x_relaxed_read32(csa, TRCIDR4);
1169
	/* NUMACPAIRS, bits[0:3] number of addr comparator pairs for tracing */
1170
	drvdata->nr_addr_cmp = FIELD_GET(TRCIDR4_NUMACPAIRS_MASK, etmidr4);
1171
	/* NUMPC, bits[15:12] number of PE comparator inputs for tracing */
1172
	drvdata->nr_pe_cmp = FIELD_GET(TRCIDR4_NUMPC_MASK, etmidr4);
1173 1174 1175 1176 1177
	/*
	 * NUMRSPAIR, bits[19:16]
	 * The number of resource pairs conveyed by the HW starts at 0, i.e a
	 * value of 0x0 indicate 1 resource pair, 0x1 indicate two and so on.
	 * As such add 1 to the value of NUMRSPAIR for a better representation.
1178 1179 1180 1181
	 *
	 * For ETM v4.3 and later, 0x0 means 0, and no pairs are available -
	 * the default TRUE and FALSE resource selectors are omitted.
	 * Otherwise for values 0x1 and above the number is N + 1 as per v4.2.
1182
	 */
1183
	drvdata->nr_resource = FIELD_GET(TRCIDR4_NUMRSPAIR_MASK, etmidr4);
1184
	if ((drvdata->arch < ETM_ARCH_V4_3) || (drvdata->nr_resource > 0))
1185
		drvdata->nr_resource += 1;
1186 1187
	/*
	 * NUMSSCC, bits[23:20] the number of single-shot
1188 1189
	 * comparator control for tracing. Read any status regs as these
	 * also contain RO capability data.
1190
	 */
1191
	drvdata->nr_ss_cmp = FIELD_GET(TRCIDR4_NUMSSCC_MASK, etmidr4);
1192 1193
	for (i = 0; i < drvdata->nr_ss_cmp; i++) {
		drvdata->config.ss_status[i] =
1194
			etm4x_relaxed_read32(csa, TRCSSCSRn(i));
1195
	}
1196
	/* NUMCIDC, bits[27:24] number of Context ID comparators for tracing */
1197
	drvdata->numcidc = FIELD_GET(TRCIDR4_NUMCIDC_MASK, etmidr4);
1198
	/* NUMVMIDC, bits[31:28] number of VMID comparators for tracing */
1199
	drvdata->numvmidc = FIELD_GET(TRCIDR4_NUMVMIDC_MASK, etmidr4);
1200

1201
	etmidr5 = etm4x_relaxed_read32(csa, TRCIDR5);
1202
	/* NUMEXTIN, bits[8:0] number of external inputs implemented */
1203
	drvdata->nr_ext_inp = FIELD_GET(TRCIDR5_NUMEXTIN_MASK, etmidr5);
1204
	/* TRACEIDSIZE, bits[21:16] indicates the trace ID width */
1205
	drvdata->trcid_size = FIELD_GET(TRCIDR5_TRACEIDSIZE_MASK, etmidr5);
1206
	/* ATBTRIG, bit[22] implementation can support ATB triggers? */
1207
	drvdata->atbtrig = !!(etmidr5 & TRCIDR5_ATBTRIG);
1208 1209 1210 1211
	/*
	 * LPOVERRIDE, bit[23] implementation supports
	 * low-power state override
	 */
1212
	drvdata->lpoverride = (etmidr5 & TRCIDR5_LPOVERRIDE) && (!drvdata->skip_power_up);
1213
	/* NUMSEQSTATE, bits[27:25] number of sequencer states implemented */
1214
	drvdata->nrseqstate = FIELD_GET(TRCIDR5_NUMSEQSTATE_MASK, etmidr5);
1215
	/* NUMCNTR, bits[30:28] number of counters available for tracing */
1216
	drvdata->nr_cntr = FIELD_GET(TRCIDR5_NUMCNTR_MASK, etmidr5);
1217
	etm4_cs_lock(drvdata, csa);
1218
	cpu_detect_trace_filtering(drvdata);
1219 1220
}

1221 1222
static inline u32 etm4_get_victlr_access_type(struct etmv4_config *config)
{
1223
	return etm4_get_access_type(config) << __bf_shf(TRCVICTLR_EXLEVEL_MASK);
1224 1225
}

1226 1227 1228
/* Set ELx trace filter access in the TRCVICTLR register */
static void etm4_set_victlr_access(struct etmv4_config *config)
{
1229 1230
	config->vinst_ctrl &= ~TRCVICTLR_EXLEVEL_MASK;
	config->vinst_ctrl |= etm4_get_victlr_access_type(config);
1231 1232
}

1233
static void etm4_set_default_config(struct etmv4_config *config)
1234 1235
{
	/* disable all events tracing */
1236 1237
	config->eventctrl0 = 0x0;
	config->eventctrl1 = 0x0;
1238 1239

	/* disable stalling */
1240
	config->stall_ctrl = 0x0;
1241

1242 1243 1244
	/* enable trace synchronization every 4096 bytes, if available */
	config->syncfreq = 0xC;

1245
	/* disable timestamp event */
1246
	config->ts_ctrl = 0x0;
1247

1248
	/* TRCVICTLR::EVENT = 0x01, select the always on logic */
1249
	config->vinst_ctrl = FIELD_PREP(TRCVICTLR_EVENT_MASK, 0x01);
1250 1251 1252

	/* TRCVICTLR::EXLEVEL_NS:EXLEVELS: Set kernel / user filtering */
	etm4_set_victlr_access(config);
1253
}
1254

1255
static u64 etm4_get_ns_access_type(struct etmv4_config *config)
1256
{
1257
	u64 access_type = 0;
1258

1259
	/*
1260 1261 1262
	 * EXLEVEL_NS, for NonSecure Exception levels.
	 * The mask here is a generic value and must be
	 * shifted to the corresponding field for the registers
1263
	 */
1264 1265 1266 1267 1268 1269 1270 1271
	if (!is_kernel_in_hyp_mode()) {
		/* Stay away from hypervisor mode for non-VHE */
		access_type =  ETM_EXLEVEL_NS_HYP;
		if (config->mode & ETM_MODE_EXCL_KERN)
			access_type |= ETM_EXLEVEL_NS_OS;
	} else if (config->mode & ETM_MODE_EXCL_KERN) {
		access_type = ETM_EXLEVEL_NS_HYP;
	}
1272 1273 1274 1275

	if (config->mode & ETM_MODE_EXCL_USER)
		access_type |= ETM_EXLEVEL_NS_APP;

1276 1277 1278
	return access_type;
}

1279 1280 1281 1282 1283
/*
 * Construct the exception level masks for a given config.
 * This must be shifted to the corresponding register field
 * for usage.
 */
1284 1285
static u64 etm4_get_access_type(struct etmv4_config *config)
{
1286 1287 1288
	/* All Secure exception levels are excluded from the trace */
	return etm4_get_ns_access_type(config) | (u64)config->s_ex_level;
}
1289

1290 1291 1292
static u64 etm4_get_comparator_access_type(struct etmv4_config *config)
{
	return etm4_get_access_type(config) << TRCACATR_EXLEVEL_SHIFT;
1293 1294 1295 1296 1297
}

static void etm4_set_comparator_filter(struct etmv4_config *config,
				       u64 start, u64 stop, int comparator)
{
1298
	u64 access_type = etm4_get_comparator_access_type(config);
1299

1300
	/* First half of default address comparator */
1301 1302 1303
	config->addr_val[comparator] = start;
	config->addr_acc[comparator] = access_type;
	config->addr_type[comparator] = ETM_ADDR_TYPE_RANGE;
1304 1305

	/* Second half of default address comparator */
1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327
	config->addr_val[comparator + 1] = stop;
	config->addr_acc[comparator + 1] = access_type;
	config->addr_type[comparator + 1] = ETM_ADDR_TYPE_RANGE;

	/*
	 * Configure the ViewInst function to include this address range
	 * comparator.
	 *
	 * @comparator is divided by two since it is the index in the
	 * etmv4_config::addr_val array but register TRCVIIECTLR deals with
	 * address range comparator _pairs_.
	 *
	 * Therefore:
	 *	index 0 -> compatator pair 0
	 *	index 2 -> comparator pair 1
	 *	index 4 -> comparator pair 2
	 *	...
	 *	index 14 -> comparator pair 7
	 */
	config->viiectlr |= BIT(comparator / 2);
}

1328 1329 1330 1331 1332
static void etm4_set_start_stop_filter(struct etmv4_config *config,
				       u64 address, int comparator,
				       enum etm_addr_type type)
{
	int shift;
1333
	u64 access_type = etm4_get_comparator_access_type(config);
1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348

	/* Configure the comparator */
	config->addr_val[comparator] = address;
	config->addr_acc[comparator] = access_type;
	config->addr_type[comparator] = type;

	/*
	 * Configure ViewInst Start-Stop control register.
	 * Addresses configured to start tracing go from bit 0 to n-1,
	 * while those configured to stop tracing from 16 to 16 + n-1.
	 */
	shift = (type == ETM_ADDR_TYPE_START ? 0 : 16);
	config->vissctlr |= BIT(shift + comparator);
}

1349 1350
static void etm4_set_default_filter(struct etmv4_config *config)
{
1351 1352
	/* Trace everything 'default' filter achieved by no filtering */
	config->viiectlr = 0x0;
1353

1354 1355 1356 1357
	/*
	 * TRCVICTLR::SSSTATUS == 1, the start-stop logic is
	 * in the started state
	 */
1358
	config->vinst_ctrl |= TRCVICTLR_SSSTATUS;
1359
	config->mode |= ETM_MODE_VIEWINST_STARTSTOP;
1360 1361

	/* No start-stop filtering for ViewInst */
1362
	config->vissctlr = 0x0;
1363 1364
}

1365 1366 1367 1368 1369 1370 1371 1372
static void etm4_set_default(struct etmv4_config *config)
{
	if (WARN_ON_ONCE(!config))
		return;

	/*
	 * Make default initialisation trace everything
	 *
1373 1374 1375
	 * This is done by a minimum default config sufficient to enable
	 * full instruction trace - with a default filter for trace all
	 * achieved by having no filtering.
1376 1377 1378 1379 1380
	 */
	etm4_set_default_config(config);
	etm4_set_default_filter(config);
}

1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402
static int etm4_get_next_comparator(struct etmv4_drvdata *drvdata, u32 type)
{
	int nr_comparator, index = 0;
	struct etmv4_config *config = &drvdata->config;

	/*
	 * nr_addr_cmp holds the number of comparator _pair_, so time 2
	 * for the total number of comparators.
	 */
	nr_comparator = drvdata->nr_addr_cmp * 2;

	/* Go through the tally of comparators looking for a free one. */
	while (index < nr_comparator) {
		switch (type) {
		case ETM_ADDR_TYPE_RANGE:
			if (config->addr_type[index] == ETM_ADDR_TYPE_NONE &&
			    config->addr_type[index + 1] == ETM_ADDR_TYPE_NONE)
				return index;

			/* Address range comparators go in pairs */
			index += 2;
			break;
1403 1404 1405 1406 1407 1408 1409 1410
		case ETM_ADDR_TYPE_START:
		case ETM_ADDR_TYPE_STOP:
			if (config->addr_type[index] == ETM_ADDR_TYPE_NONE)
				return index;

			/* Start/stop address can have odd indexes */
			index += 1;
			break;
1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423
		default:
			return -EINVAL;
		}
	}

	/* If we are here all the comparators have been used. */
	return -ENOSPC;
}

static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
				  struct perf_event *event)
{
	int i, comparator, ret = 0;
1424
	u64 address;
1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461
	struct etmv4_config *config = &drvdata->config;
	struct etm_filters *filters = event->hw.addr_filters;

	if (!filters)
		goto default_filter;

	/* Sync events with what Perf got */
	perf_event_addr_filters_sync(event);

	/*
	 * If there are no filters to deal with simply go ahead with
	 * the default filter, i.e the entire address range.
	 */
	if (!filters->nr_filters)
		goto default_filter;

	for (i = 0; i < filters->nr_filters; i++) {
		struct etm_filter *filter = &filters->etm_filter[i];
		enum etm_addr_type type = filter->type;

		/* See if a comparator is free. */
		comparator = etm4_get_next_comparator(drvdata, type);
		if (comparator < 0) {
			ret = comparator;
			goto out;
		}

		switch (type) {
		case ETM_ADDR_TYPE_RANGE:
			etm4_set_comparator_filter(config,
						   filter->start_addr,
						   filter->stop_addr,
						   comparator);
			/*
			 * TRCVICTLR::SSSTATUS == 1, the start-stop logic is
			 * in the started state
			 */
1462
			config->vinst_ctrl |= TRCVICTLR_SSSTATUS;
1463 1464 1465 1466

			/* No start-stop filtering for ViewInst */
			config->vissctlr = 0x0;
			break;
1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489
		case ETM_ADDR_TYPE_START:
		case ETM_ADDR_TYPE_STOP:
			/* Get the right start or stop address */
			address = (type == ETM_ADDR_TYPE_START ?
				   filter->start_addr :
				   filter->stop_addr);

			/* Configure comparator */
			etm4_set_start_stop_filter(config, address,
						   comparator, type);

			/*
			 * If filters::ssstatus == 1, trace acquisition was
			 * started but the process was yanked away before the
			 * the stop address was hit.  As such the start/stop
			 * logic needs to be re-started so that tracing can
			 * resume where it left.
			 *
			 * The start/stop logic status when a process is
			 * scheduled out is checked in function
			 * etm4_disable_perf().
			 */
			if (filters->ssstatus)
1490
				config->vinst_ctrl |= TRCVICTLR_SSSTATUS;
1491 1492 1493 1494

			/* No include/exclude filtering for ViewInst */
			config->viiectlr = 0x0;
			break;
1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510
		default:
			ret = -EINVAL;
			goto out;
		}
	}

	goto out;


default_filter:
	etm4_set_default_filter(config);

out:
	return ret;
}

1511 1512
void etm4_config_trace_mode(struct etmv4_config *config)
{
1513
	u32 mode;
1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524

	mode = config->mode;
	mode &= (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER);

	/* excluding kernel AND user space doesn't make sense */
	WARN_ON_ONCE(mode == (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER));

	/* nothing to do if neither flags are set */
	if (!(mode & ETM_MODE_EXCL_KERN) && !(mode & ETM_MODE_EXCL_USER))
		return;

1525
	etm4_set_victlr_access(config);
1526 1527
}

1528
static int etm4_online_cpu(unsigned int cpu)
1529 1530
{
	if (!etmdrvdata[cpu])
1531
		return 0;
1532

1533 1534 1535 1536
	if (etmdrvdata[cpu]->boot_enable && !etmdrvdata[cpu]->sticky_enable)
		coresight_enable(etmdrvdata[cpu]->csdev);
	return 0;
}
1537

1538 1539 1540 1541 1542 1543
static int etm4_starting_cpu(unsigned int cpu)
{
	if (!etmdrvdata[cpu])
		return 0;

	spin_lock(&etmdrvdata[cpu]->spinlock);
1544
	if (!etmdrvdata[cpu]->os_unlock)
1545 1546 1547 1548 1549 1550
		etm4_os_unlock(etmdrvdata[cpu]);

	if (local_read(&etmdrvdata[cpu]->mode))
		etm4_enable_hw(etmdrvdata[cpu]);
	spin_unlock(&etmdrvdata[cpu]->spinlock);
	return 0;
1551 1552
}

1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563
static int etm4_dying_cpu(unsigned int cpu)
{
	if (!etmdrvdata[cpu])
		return 0;

	spin_lock(&etmdrvdata[cpu]->spinlock);
	if (local_read(&etmdrvdata[cpu]->mode))
		etm4_disable_hw(etmdrvdata[cpu]);
	spin_unlock(&etmdrvdata[cpu]->spinlock);
	return 0;
}
1564

1565 1566 1567 1568 1569
static void etm4_init_trace_id(struct etmv4_drvdata *drvdata)
{
	drvdata->trcid = coresight_get_trace_id(drvdata->cpu);
}

1570
static int __etm4_cpu_save(struct etmv4_drvdata *drvdata)
1571 1572 1573
{
	int i, ret = 0;
	struct etmv4_save_state *state;
1574 1575 1576 1577 1578 1579 1580 1581 1582
	struct coresight_device *csdev = drvdata->csdev;
	struct csdev_access *csa;
	struct device *etm_dev;

	if (WARN_ON(!csdev))
		return -ENODEV;

	etm_dev = &csdev->dev;
	csa = &csdev->access;
1583 1584 1585 1586 1587 1588 1589 1590

	/*
	 * As recommended by 3.4.1 ("The procedure when powering down the PE")
	 * of ARM IHI 0064D
	 */
	dsb(sy);
	isb();

1591
	etm4_cs_unlock(drvdata, csa);
1592 1593 1594 1595
	/* Lock the OS lock to disable trace and external debugger access */
	etm4_os_lock(drvdata);

	/* wait for TRCSTATR.PMSTABLE to go up */
1596
	if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) {
1597 1598 1599 1600 1601 1602 1603 1604 1605
		dev_err(etm_dev,
			"timeout while waiting for PM Stable Status\n");
		etm4_os_unlock(drvdata);
		ret = -EBUSY;
		goto out;
	}

	state = drvdata->save_state;

1606
	state->trcprgctlr = etm4x_read32(csa, TRCPRGCTLR);
1607
	if (drvdata->nr_pe)
1608 1609 1610 1611 1612
		state->trcprocselr = etm4x_read32(csa, TRCPROCSELR);
	state->trcconfigr = etm4x_read32(csa, TRCCONFIGR);
	state->trcauxctlr = etm4x_read32(csa, TRCAUXCTLR);
	state->trceventctl0r = etm4x_read32(csa, TRCEVENTCTL0R);
	state->trceventctl1r = etm4x_read32(csa, TRCEVENTCTL1R);
1613 1614
	if (drvdata->stallctl)
		state->trcstallctlr = etm4x_read32(csa, TRCSTALLCTLR);
1615 1616 1617 1618 1619 1620 1621 1622 1623 1624
	state->trctsctlr = etm4x_read32(csa, TRCTSCTLR);
	state->trcsyncpr = etm4x_read32(csa, TRCSYNCPR);
	state->trcccctlr = etm4x_read32(csa, TRCCCCTLR);
	state->trcbbctlr = etm4x_read32(csa, TRCBBCTLR);
	state->trctraceidr = etm4x_read32(csa, TRCTRACEIDR);
	state->trcqctlr = etm4x_read32(csa, TRCQCTLR);

	state->trcvictlr = etm4x_read32(csa, TRCVICTLR);
	state->trcviiectlr = etm4x_read32(csa, TRCVIIECTLR);
	state->trcvissctlr = etm4x_read32(csa, TRCVISSCTLR);
1625
	if (drvdata->nr_pe_cmp)
1626 1627 1628 1629
		state->trcvipcssctlr = etm4x_read32(csa, TRCVIPCSSCTLR);
	state->trcvdctlr = etm4x_read32(csa, TRCVDCTLR);
	state->trcvdsacctlr = etm4x_read32(csa, TRCVDSACCTLR);
	state->trcvdarcctlr = etm4x_read32(csa, TRCVDARCCTLR);
1630

1631
	for (i = 0; i < drvdata->nrseqstate - 1; i++)
1632
		state->trcseqevr[i] = etm4x_read32(csa, TRCSEQEVRn(i));
1633

1634 1635 1636
	state->trcseqrstevr = etm4x_read32(csa, TRCSEQRSTEVR);
	state->trcseqstr = etm4x_read32(csa, TRCSEQSTR);
	state->trcextinselr = etm4x_read32(csa, TRCEXTINSELR);
1637 1638

	for (i = 0; i < drvdata->nr_cntr; i++) {
1639 1640 1641
		state->trccntrldvr[i] = etm4x_read32(csa, TRCCNTRLDVRn(i));
		state->trccntctlr[i] = etm4x_read32(csa, TRCCNTCTLRn(i));
		state->trccntvr[i] = etm4x_read32(csa, TRCCNTVRn(i));
1642 1643 1644
	}

	for (i = 0; i < drvdata->nr_resource * 2; i++)
1645
		state->trcrsctlr[i] = etm4x_read32(csa, TRCRSCTLRn(i));
1646 1647

	for (i = 0; i < drvdata->nr_ss_cmp; i++) {
1648 1649
		state->trcssccr[i] = etm4x_read32(csa, TRCSSCCRn(i));
		state->trcsscsr[i] = etm4x_read32(csa, TRCSSCSRn(i));
1650
		if (etm4x_sspcicrn_present(drvdata, i))
1651
			state->trcsspcicr[i] = etm4x_read32(csa, TRCSSPCICRn(i));
1652 1653 1654
	}

	for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) {
1655 1656
		state->trcacvr[i] = etm4x_read64(csa, TRCACVRn(i));
		state->trcacatr[i] = etm4x_read64(csa, TRCACATRn(i));
1657 1658 1659 1660 1661 1662 1663 1664 1665 1666
	}

	/*
	 * Data trace stream is architecturally prohibited for A profile cores
	 * so we don't save (or later restore) trcdvcvr and trcdvcmr - As per
	 * section 1.3.4 ("Possible functional configurations of an ETMv4 trace
	 * unit") of ARM IHI 0064D.
	 */

	for (i = 0; i < drvdata->numcidc; i++)
1667
		state->trccidcvr[i] = etm4x_read64(csa, TRCCIDCVRn(i));
1668 1669

	for (i = 0; i < drvdata->numvmidc; i++)
1670
		state->trcvmidcvr[i] = etm4x_read64(csa, TRCVMIDCVRn(i));
1671

1672
	state->trccidcctlr0 = etm4x_read32(csa, TRCCIDCCTLR0);
1673
	if (drvdata->numcidc > 4)
1674
		state->trccidcctlr1 = etm4x_read32(csa, TRCCIDCCTLR1);
1675

1676
	state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR0);
1677
	if (drvdata->numvmidc > 4)
1678
		state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR1);
1679

1680
	state->trcclaimset = etm4x_read32(csa, TRCCLAIMCLR);
1681

1682
	if (!drvdata->skip_power_up)
1683
		state->trcpdcr = etm4x_read32(csa, TRCPDCR);
1684 1685

	/* wait for TRCSTATR.IDLE to go up */
1686
	if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) {
1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700
		dev_err(etm_dev,
			"timeout while waiting for Idle Trace Status\n");
		etm4_os_unlock(drvdata);
		ret = -EBUSY;
		goto out;
	}

	drvdata->state_needs_restore = true;

	/*
	 * Power can be removed from the trace unit now. We do this to
	 * potentially save power on systems that respect the TRCPDCR_PU
	 * despite requesting software to save/restore state.
	 */
1701
	if (!drvdata->skip_power_up)
1702 1703
		etm4x_relaxed_write32(csa, (state->trcpdcr & ~TRCPDCR_PU),
				      TRCPDCR);
1704
out:
1705
	etm4_cs_lock(drvdata, csa);
1706 1707 1708
	return ret;
}

1709 1710 1711 1712 1713
static int etm4_cpu_save(struct etmv4_drvdata *drvdata)
{
	int ret = 0;

	/* Save the TRFCR irrespective of whether the ETM is ON */
1714
	if (drvdata->trfcr)
1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725
		drvdata->save_trfcr = read_trfcr();
	/*
	 * Save and restore the ETM Trace registers only if
	 * the ETM is active.
	 */
	if (local_read(&drvdata->mode) && drvdata->save_state)
		ret = __etm4_cpu_save(drvdata);
	return ret;
}

static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata)
1726 1727 1728
{
	int i;
	struct etmv4_save_state *state = drvdata->save_state;
1729 1730
	struct csdev_access tmp_csa = CSDEV_ACCESS_IOMEM(drvdata->base);
	struct csdev_access *csa = &tmp_csa;
1731

1732
	etm4_cs_unlock(drvdata, csa);
1733
	etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET);
1734

1735
	etm4x_relaxed_write32(csa, state->trcprgctlr, TRCPRGCTLR);
1736
	if (drvdata->nr_pe)
1737 1738 1739 1740 1741
		etm4x_relaxed_write32(csa, state->trcprocselr, TRCPROCSELR);
	etm4x_relaxed_write32(csa, state->trcconfigr, TRCCONFIGR);
	etm4x_relaxed_write32(csa, state->trcauxctlr, TRCAUXCTLR);
	etm4x_relaxed_write32(csa, state->trceventctl0r, TRCEVENTCTL0R);
	etm4x_relaxed_write32(csa, state->trceventctl1r, TRCEVENTCTL1R);
1742 1743
	if (drvdata->stallctl)
		etm4x_relaxed_write32(csa, state->trcstallctlr, TRCSTALLCTLR);
1744 1745 1746 1747 1748 1749 1750 1751 1752 1753
	etm4x_relaxed_write32(csa, state->trctsctlr, TRCTSCTLR);
	etm4x_relaxed_write32(csa, state->trcsyncpr, TRCSYNCPR);
	etm4x_relaxed_write32(csa, state->trcccctlr, TRCCCCTLR);
	etm4x_relaxed_write32(csa, state->trcbbctlr, TRCBBCTLR);
	etm4x_relaxed_write32(csa, state->trctraceidr, TRCTRACEIDR);
	etm4x_relaxed_write32(csa, state->trcqctlr, TRCQCTLR);

	etm4x_relaxed_write32(csa, state->trcvictlr, TRCVICTLR);
	etm4x_relaxed_write32(csa, state->trcviiectlr, TRCVIIECTLR);
	etm4x_relaxed_write32(csa, state->trcvissctlr, TRCVISSCTLR);
1754
	if (drvdata->nr_pe_cmp)
1755 1756 1757 1758
		etm4x_relaxed_write32(csa, state->trcvipcssctlr, TRCVIPCSSCTLR);
	etm4x_relaxed_write32(csa, state->trcvdctlr, TRCVDCTLR);
	etm4x_relaxed_write32(csa, state->trcvdsacctlr, TRCVDSACCTLR);
	etm4x_relaxed_write32(csa, state->trcvdarcctlr, TRCVDARCCTLR);
1759

1760
	for (i = 0; i < drvdata->nrseqstate - 1; i++)
1761
		etm4x_relaxed_write32(csa, state->trcseqevr[i], TRCSEQEVRn(i));
1762

1763 1764 1765
	etm4x_relaxed_write32(csa, state->trcseqrstevr, TRCSEQRSTEVR);
	etm4x_relaxed_write32(csa, state->trcseqstr, TRCSEQSTR);
	etm4x_relaxed_write32(csa, state->trcextinselr, TRCEXTINSELR);
1766 1767

	for (i = 0; i < drvdata->nr_cntr; i++) {
1768 1769 1770
		etm4x_relaxed_write32(csa, state->trccntrldvr[i], TRCCNTRLDVRn(i));
		etm4x_relaxed_write32(csa, state->trccntctlr[i], TRCCNTCTLRn(i));
		etm4x_relaxed_write32(csa, state->trccntvr[i], TRCCNTVRn(i));
1771 1772 1773
	}

	for (i = 0; i < drvdata->nr_resource * 2; i++)
1774
		etm4x_relaxed_write32(csa, state->trcrsctlr[i], TRCRSCTLRn(i));
1775 1776

	for (i = 0; i < drvdata->nr_ss_cmp; i++) {
1777 1778
		etm4x_relaxed_write32(csa, state->trcssccr[i], TRCSSCCRn(i));
		etm4x_relaxed_write32(csa, state->trcsscsr[i], TRCSSCSRn(i));
1779
		if (etm4x_sspcicrn_present(drvdata, i))
1780
			etm4x_relaxed_write32(csa, state->trcsspcicr[i], TRCSSPCICRn(i));
1781 1782 1783
	}

	for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) {
1784 1785
		etm4x_relaxed_write64(csa, state->trcacvr[i], TRCACVRn(i));
		etm4x_relaxed_write64(csa, state->trcacatr[i], TRCACATRn(i));
1786 1787 1788
	}

	for (i = 0; i < drvdata->numcidc; i++)
1789
		etm4x_relaxed_write64(csa, state->trccidcvr[i], TRCCIDCVRn(i));
1790 1791

	for (i = 0; i < drvdata->numvmidc; i++)
1792
		etm4x_relaxed_write64(csa, state->trcvmidcvr[i], TRCVMIDCVRn(i));
1793

1794
	etm4x_relaxed_write32(csa, state->trccidcctlr0, TRCCIDCCTLR0);
1795
	if (drvdata->numcidc > 4)
1796
		etm4x_relaxed_write32(csa, state->trccidcctlr1, TRCCIDCCTLR1);
1797

1798
	etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR0);
1799
	if (drvdata->numvmidc > 4)
1800
		etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR1);
1801

1802
	etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET);
1803

1804
	if (!drvdata->skip_power_up)
1805
		etm4x_relaxed_write32(csa, state->trcpdcr, TRCPDCR);
1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817

	drvdata->state_needs_restore = false;

	/*
	 * As recommended by section 4.3.7 ("Synchronization when using the
	 * memory-mapped interface") of ARM IHI 0064D
	 */
	dsb(sy);
	isb();

	/* Unlock the OS lock to re-enable trace and external debug access */
	etm4_os_unlock(drvdata);
1818
	etm4_cs_lock(drvdata, csa);
1819 1820
}

1821 1822
static void etm4_cpu_restore(struct etmv4_drvdata *drvdata)
{
1823
	if (drvdata->trfcr)
1824 1825 1826 1827 1828
		write_trfcr(drvdata->save_trfcr);
	if (drvdata->state_needs_restore)
		__etm4_cpu_restore(drvdata);
}

1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844
static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd,
			      void *v)
{
	struct etmv4_drvdata *drvdata;
	unsigned int cpu = smp_processor_id();

	if (!etmdrvdata[cpu])
		return NOTIFY_OK;

	drvdata = etmdrvdata[cpu];

	if (WARN_ON_ONCE(drvdata->cpu != cpu))
		return NOTIFY_BAD;

	switch (cmd) {
	case CPU_PM_ENTER:
1845 1846
		if (etm4_cpu_save(drvdata))
			return NOTIFY_BAD;
1847 1848 1849
		break;
	case CPU_PM_EXIT:
	case CPU_PM_ENTER_FAILED:
1850
		etm4_cpu_restore(drvdata);
1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862
		break;
	default:
		return NOTIFY_DONE;
	}

	return NOTIFY_OK;
}

static struct notifier_block etm4_cpu_pm_nb = {
	.notifier_call = etm4_cpu_pm_notify,
};

1863 1864
/* Setup PM. Deals with error conditions and counts */
static int __init etm4_pm_setup(void)
1865
{
1866
	int ret;
1867

1868 1869
	ret = cpu_pm_register_notifier(&etm4_cpu_pm_nb);
	if (ret)
1870
		return ret;
1871

1872 1873 1874
	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING,
					"arm/coresight4:starting",
					etm4_starting_cpu, etm4_dying_cpu);
1875 1876 1877 1878

	if (ret)
		goto unregister_notifier;

1879 1880 1881
	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
					"arm/coresight4:online",
					etm4_online_cpu, NULL);
1882 1883 1884 1885 1886 1887 1888 1889

	/* HP dyn state ID returned in ret on success */
	if (ret > 0) {
		hp_online = ret;
		return 0;
	}

	/* failed dyn state - remove others */
1890
	cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
1891 1892 1893 1894

unregister_notifier:
	cpu_pm_unregister_notifier(&etm4_cpu_pm_nb);
	return ret;
1895 1896
}

1897
static void etm4_pm_clear(void)
1898
{
1899 1900 1901 1902 1903 1904
	cpu_pm_unregister_notifier(&etm4_cpu_pm_nb);
	cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
	if (hp_online) {
		cpuhp_remove_state_nocalls(hp_online);
		hp_online = 0;
	}
1905 1906
}

1907
static int etm4_probe(struct device *dev, void __iomem *base, u32 etm_pid)
1908 1909 1910 1911
{
	int ret;
	struct coresight_platform_data *pdata = NULL;
	struct etmv4_drvdata *drvdata;
1912
	struct coresight_desc desc = { 0 };
1913
	struct etm4_init_arg init_arg = { 0 };
1914 1915
	u8 major, minor;
	char *type_name;
1916 1917 1918 1919 1920 1921 1922

	drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
	if (!drvdata)
		return -ENOMEM;

	dev_set_drvdata(dev, drvdata);

1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933
	if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE)
		pm_save_enable = coresight_loses_context_with_cpu(dev) ?
			       PARAM_PM_SAVE_SELF_HOSTED : PARAM_PM_SAVE_NEVER;

	if (pm_save_enable != PARAM_PM_SAVE_NEVER) {
		drvdata->save_state = devm_kmalloc(dev,
				sizeof(struct etmv4_save_state), GFP_KERNEL);
		if (!drvdata->save_state)
			return -ENOMEM;
	}

1934 1935 1936 1937
	drvdata->base = base;

	spin_lock_init(&drvdata->spinlock);

1938
	drvdata->cpu = coresight_get_cpu(dev);
1939 1940 1941
	if (drvdata->cpu < 0)
		return drvdata->cpu;

1942 1943
	init_arg.drvdata = drvdata;
	init_arg.csa = &desc.access;
1944
	init_arg.pid = etm_pid;
1945

1946
	if (smp_call_function_single(drvdata->cpu,
1947
				etm4_init_arch_data,  &init_arg, 1))
1948 1949
		dev_err(dev, "ETM arch init failed\n");

1950
	if (!drvdata->arch)
1951
		return -EINVAL;
1952

1953 1954 1955 1956 1957
	/* TRCPDCR is not accessible with system instructions. */
	if (!desc.access.io_mem ||
	    fwnode_property_present(dev_fwnode(dev), "qcom,skip-power-up"))
		drvdata->skip_power_up = true;

1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973
	major = ETM_ARCH_MAJOR_VERSION(drvdata->arch);
	minor = ETM_ARCH_MINOR_VERSION(drvdata->arch);

	if (etm4x_is_ete(drvdata)) {
		type_name = "ete";
		/* ETE v1 has major version == 0b101. Adjust this for logging.*/
		major -= 4;
	} else {
		type_name = "etm";
	}

	desc.name = devm_kasprintf(dev, GFP_KERNEL,
				   "%s%d", type_name, drvdata->cpu);
	if (!desc.name)
		return -ENOMEM;

1974 1975
	etm4_init_trace_id(drvdata);
	etm4_set_default(&drvdata->config);
1976

1977
	pdata = coresight_get_platform_data(dev);
1978 1979 1980
	if (IS_ERR(pdata))
		return PTR_ERR(pdata);

1981
	dev->platform_data = pdata;
1982

1983 1984 1985 1986 1987 1988 1989
	desc.type = CORESIGHT_DEV_TYPE_SOURCE;
	desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
	desc.ops = &etm4_cs_ops;
	desc.pdata = pdata;
	desc.dev = dev;
	desc.groups = coresight_etmv4_groups;
	drvdata->csdev = coresight_register(&desc);
1990 1991
	if (IS_ERR(drvdata->csdev))
		return PTR_ERR(drvdata->csdev);
1992

1993 1994 1995
	ret = etm_perf_symlink(drvdata->csdev, true);
	if (ret) {
		coresight_unregister(drvdata->csdev);
1996
		return ret;
1997 1998
	}

1999 2000 2001 2002 2003 2004 2005
	/* register with config infrastructure & load any current features */
	ret = etm4_cscfg_register(drvdata->csdev);
	if (ret) {
		coresight_unregister(drvdata->csdev);
		return ret;
	}

2006 2007
	etmdrvdata[drvdata->cpu] = drvdata;

2008 2009
	dev_info(&drvdata->csdev->dev, "CPU%d: %s v%d.%d initialized\n",
		 drvdata->cpu, type_name, major, minor);
2010 2011 2012 2013 2014 2015 2016 2017 2018

	if (boot_enable) {
		coresight_enable(drvdata->csdev);
		drvdata->boot_enable = true;
	}

	return 0;
}

2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037
static int etm4_probe_amba(struct amba_device *adev, const struct amba_id *id)
{
	void __iomem *base;
	struct device *dev = &adev->dev;
	struct resource *res = &adev->res;
	int ret;

	/* Validity for the resource is already checked by the AMBA core */
	base = devm_ioremap_resource(dev, res);
	if (IS_ERR(base))
		return PTR_ERR(base);

	ret = etm4_probe(dev, base, id->id);
	if (!ret)
		pm_runtime_put(&adev->dev);

	return ret;
}

2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056
static int etm4_probe_platform_dev(struct platform_device *pdev)
{
	int ret;

	pm_runtime_get_noresume(&pdev->dev);
	pm_runtime_set_active(&pdev->dev);
	pm_runtime_enable(&pdev->dev);

	/*
	 * System register based devices could match the
	 * HW by reading appropriate registers on the HW
	 * and thus we could skip the PID.
	 */
	ret = etm4_probe(&pdev->dev, NULL, 0);

	pm_runtime_put(&pdev->dev);
	return ret;
}

2057 2058 2059
static struct amba_cs_uci_id uci_id_etm4[] = {
	{
		/*  ETMv4 UCI data */
2060 2061
		.devarch	= ETM_DEVARCH_ETMv4x_ARCH,
		.devarch_mask	= ETM_DEVARCH_ID_MASK,
2062
		.devtype	= 0x00000013,
2063
	}
2064
};
2065

2066
static void clear_etmdrvdata(void *info)
2067 2068 2069 2070 2071 2072
{
	int cpu = *(int *)info;

	etmdrvdata[cpu] = NULL;
}

2073
static int __exit etm4_remove_dev(struct etmv4_drvdata *drvdata)
2074 2075 2076
{
	etm_perf_symlink(drvdata->csdev, false);
	/*
2077 2078
	 * Taking hotplug lock here to avoid racing between etm4_remove_dev()
	 * and CPU hotplug call backs.
2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091
	 */
	cpus_read_lock();
	/*
	 * The readers for etmdrvdata[] are CPU hotplug call backs
	 * and PM notification call backs. Change etmdrvdata[i] on
	 * CPU i ensures these call backs has consistent view
	 * inside one call back function.
	 */
	if (smp_call_function_single(drvdata->cpu, clear_etmdrvdata, &drvdata->cpu, 1))
		etmdrvdata[drvdata->cpu] = NULL;

	cpus_read_unlock();

2092
	cscfg_unregister_csdev(drvdata->csdev);
2093
	coresight_unregister(drvdata->csdev);
2094 2095

	return 0;
2096 2097
}

2098
static void __exit etm4_remove_amba(struct amba_device *adev)
2099 2100 2101 2102
{
	struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev);

	if (drvdata)
2103
		etm4_remove_dev(drvdata);
2104 2105
}

2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116
static int __exit etm4_remove_platform_dev(struct platform_device *pdev)
{
	int ret = 0;
	struct etmv4_drvdata *drvdata = dev_get_drvdata(&pdev->dev);

	if (drvdata)
		ret = etm4_remove_dev(drvdata);
	pm_runtime_disable(&pdev->dev);
	return ret;
}

2117
static const struct amba_id etm4_ids[] = {
2118 2119 2120 2121 2122
	CS_AMBA_ID(0x000bb95d),			/* Cortex-A53 */
	CS_AMBA_ID(0x000bb95e),			/* Cortex-A57 */
	CS_AMBA_ID(0x000bb95a),			/* Cortex-A72 */
	CS_AMBA_ID(0x000bb959),			/* Cortex-A73 */
	CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4),/* Cortex-A35 */
2123 2124
	CS_AMBA_UCI_ID(0x000bbd05, uci_id_etm4),/* Cortex-A55 */
	CS_AMBA_UCI_ID(0x000bbd0a, uci_id_etm4),/* Cortex-A75 */
2125
	CS_AMBA_UCI_ID(0x000bbd0c, uci_id_etm4),/* Neoverse N1 */
2126
	CS_AMBA_UCI_ID(0x000bbd41, uci_id_etm4),/* Cortex-A78 */
2127 2128
	CS_AMBA_UCI_ID(0x000f0205, uci_id_etm4),/* Qualcomm Kryo */
	CS_AMBA_UCI_ID(0x000f0211, uci_id_etm4),/* Qualcomm Kryo */
2129 2130
	CS_AMBA_UCI_ID(0x000bb802, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A55 */
	CS_AMBA_UCI_ID(0x000bb803, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A75 */
2131 2132
	CS_AMBA_UCI_ID(0x000bb805, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A55 */
	CS_AMBA_UCI_ID(0x000bb804, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A76 */
2133
	CS_AMBA_UCI_ID(0x000bbd0d, uci_id_etm4),/* Qualcomm Kryo 5XX Cortex-A77 */
2134
	CS_AMBA_UCI_ID(0x000cc0af, uci_id_etm4),/* Marvell ThunderX2 */
2135 2136
	CS_AMBA_UCI_ID(0x000b6d01, uci_id_etm4),/* HiSilicon-Hip08 */
	CS_AMBA_UCI_ID(0x000b6d02, uci_id_etm4),/* HiSilicon-Hip09 */
2137
	{},
2138 2139
};

2140 2141
MODULE_DEVICE_TABLE(amba, etm4_ids);

2142
static struct amba_driver etm4x_amba_driver = {
2143 2144
	.drv = {
		.name   = "coresight-etm4x",
2145
		.owner  = THIS_MODULE,
2146
		.suppress_bind_attrs = true,
2147
	},
2148 2149
	.probe		= etm4_probe_amba,
	.remove         = etm4_remove_amba,
2150 2151
	.id_table	= etm4_ids,
};
2152

2153 2154
static const struct of_device_id etm4_sysreg_match[] = {
	{ .compatible	= "arm,coresight-etm4x-sysreg" },
2155
	{ .compatible	= "arm,embedded-trace-extension" },
2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168
	{}
};

static struct platform_driver etm4_platform_driver = {
	.probe		= etm4_probe_platform_dev,
	.remove		= etm4_remove_platform_dev,
	.driver			= {
		.name			= "coresight-etm4x",
		.of_match_table		= etm4_sysreg_match,
		.suppress_bind_attrs	= true,
	},
};

2169 2170 2171 2172 2173 2174 2175 2176 2177 2178
static int __init etm4x_init(void)
{
	int ret;

	ret = etm4_pm_setup();

	/* etm4_pm_setup() does its own cleanup - exit on error */
	if (ret)
		return ret;

2179
	ret = amba_driver_register(&etm4x_amba_driver);
2180
	if (ret) {
2181 2182
		pr_err("Error registering etm4x AMBA driver\n");
		goto clear_pm;
2183 2184
	}

2185 2186 2187 2188 2189 2190 2191 2192 2193
	ret = platform_driver_register(&etm4_platform_driver);
	if (!ret)
		return 0;

	pr_err("Error registering etm4x platform driver\n");
	amba_driver_unregister(&etm4x_amba_driver);

clear_pm:
	etm4_pm_clear();
2194 2195
	return ret;
}
2196 2197 2198

static void __exit etm4x_exit(void)
{
2199
	amba_driver_unregister(&etm4x_amba_driver);
2200
	platform_driver_unregister(&etm4_platform_driver);
2201 2202 2203 2204 2205 2206 2207 2208 2209 2210
	etm4_pm_clear();
}

module_init(etm4x_init);
module_exit(etm4x_exit);

MODULE_AUTHOR("Pratik Patel <pratikp@codeaurora.org>");
MODULE_AUTHOR("Mathieu Poirier <mathieu.poirier@linaro.org>");
MODULE_DESCRIPTION("Arm CoreSight Program Flow Trace v4.x driver");
MODULE_LICENSE("GPL v2");