radix_tlb.c 33.8 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-or-later
2 3 4 5 6 7 8 9 10
/*
 * TLB flush routines for radix kernels.
 *
 * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
 */

#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/memblock.h>
11 12
#include <linux/mmu_context.h>
#include <linux/sched/mm.h>
13

14
#include <asm/ppc-opcode.h>
15 16
#include <asm/tlb.h>
#include <asm/tlbflush.h>
17
#include <asm/trace.h>
18
#include <asm/cputhreads.h>
19
#include <asm/plpar_wrappers.h>
20

21 22 23 24
#define RIC_FLUSH_TLB 0
#define RIC_FLUSH_PWC 1
#define RIC_FLUSH_ALL 2

25 26 27 28
/*
 * tlbiel instruction for radix, set invalidation
 * i.e., r=1 and is=01 or is=10 or is=11
 */
29
static __always_inline void tlbiel_radix_set_isa300(unsigned int set, unsigned int is,
30 31 32 33 34 35 36 37 38
					unsigned int pid,
					unsigned int ric, unsigned int prs)
{
	unsigned long rb;
	unsigned long rs;

	rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53));
	rs = ((unsigned long)pid << PPC_BITLSHIFT(31));

39 40
	asm volatile(PPC_TLBIEL(%0, %1, %2, %3, 1)
		     : : "r"(rb), "r"(rs), "i"(ric), "i"(prs)
41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
		     : "memory");
}

static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is)
{
	unsigned int set;

	asm volatile("ptesync": : :"memory");

	/*
	 * Flush the first set of the TLB, and the entire Page Walk Cache
	 * and partition table entries. Then flush the remaining sets of the
	 * TLB.
	 */

56 57 58
	if (early_cpu_has_feature(CPU_FTR_HVMODE)) {
		/* MSR[HV] should flush partition scope translations first. */
		tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0);
59 60 61 62 63 64

		if (!early_cpu_has_feature(CPU_FTR_ARCH_31)) {
			for (set = 1; set < num_sets; set++)
				tlbiel_radix_set_isa300(set, is, 0,
							RIC_FLUSH_TLB, 0);
		}
65 66 67
	}

	/* Flush process scoped entries. */
68
	tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1);
69 70 71 72 73

	if (!early_cpu_has_feature(CPU_FTR_ARCH_31)) {
		for (set = 1; set < num_sets; set++)
			tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1);
	}
74

75
	ppc_after_tlbiel_barrier();
76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97
}

void radix__tlbiel_all(unsigned int action)
{
	unsigned int is;

	switch (action) {
	case TLB_INVAL_SCOPE_GLOBAL:
		is = 3;
		break;
	case TLB_INVAL_SCOPE_LPID:
		is = 2;
		break;
	default:
		BUG();
	}

	if (early_cpu_has_feature(CPU_FTR_ARCH_300))
		tlbiel_all_isa300(POWER9_TLB_SETS_RADIX, is);
	else
		WARN(1, "%s called on pre-POWER9 CPU\n", __func__);

98
	asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory");
99 100
}

101
static __always_inline void __tlbiel_pid(unsigned long pid, int set,
102
				unsigned long ric)
103
{
104
	unsigned long rb,rs,prs,r;
105 106 107 108 109

	rb = PPC_BIT(53); /* IS = 1 */
	rb |= set << PPC_BITLSHIFT(51);
	rs = ((unsigned long)pid) << PPC_BITLSHIFT(31);
	prs = 1; /* process scoped */
110
	r = 1;   /* radix format */
111

112
	asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
113
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
114
	trace_tlbie(0, 1, rb, rs, ric, prs, r);
115 116
}

117
static __always_inline void __tlbie_pid(unsigned long pid, unsigned long ric)
118 119 120 121 122 123
{
	unsigned long rb,rs,prs,r;

	rb = PPC_BIT(53); /* IS = 1 */
	rs = pid << PPC_BITLSHIFT(31);
	prs = 1; /* process scoped */
124
	r = 1;   /* radix format */
125 126 127 128 129 130

	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	trace_tlbie(0, 0, rb, rs, ric, prs, r);
}

131
static __always_inline void __tlbie_lpid(unsigned long lpid, unsigned long ric)
132 133 134 135 136 137 138 139 140 141 142 143 144
{
	unsigned long rb,rs,prs,r;

	rb = PPC_BIT(52); /* IS = 2 */
	rs = lpid;
	prs = 0; /* partition scoped */
	r = 1;   /* radix format */

	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
}

145
static __always_inline void __tlbie_lpid_guest(unsigned long lpid, unsigned long ric)
146 147 148 149
{
	unsigned long rb,rs,prs,r;

	rb = PPC_BIT(52); /* IS = 2 */
150
	rs = lpid;
151 152 153
	prs = 1; /* process scoped */
	r = 1;   /* radix format */

154
	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
155
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
156
	trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
157 158
}

159 160
static __always_inline void __tlbiel_va(unsigned long va, unsigned long pid,
					unsigned long ap, unsigned long ric)
161 162 163 164 165 166 167
{
	unsigned long rb,rs,prs,r;

	rb = va & ~(PPC_BITMASK(52, 63));
	rb |= ap << PPC_BITLSHIFT(58);
	rs = pid << PPC_BITLSHIFT(31);
	prs = 1; /* process scoped */
168
	r = 1;   /* radix format */
169 170 171 172 173 174

	asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	trace_tlbie(0, 1, rb, rs, ric, prs, r);
}

175 176
static __always_inline void __tlbie_va(unsigned long va, unsigned long pid,
				       unsigned long ap, unsigned long ric)
177 178 179 180 181 182 183
{
	unsigned long rb,rs,prs,r;

	rb = va & ~(PPC_BITMASK(52, 63));
	rb |= ap << PPC_BITLSHIFT(58);
	rs = pid << PPC_BITLSHIFT(31);
	prs = 1; /* process scoped */
184
	r = 1;   /* radix format */
185 186 187 188 189 190

	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	trace_tlbie(0, 0, rb, rs, ric, prs, r);
}

191 192
static __always_inline void __tlbie_lpid_va(unsigned long va, unsigned long lpid,
					    unsigned long ap, unsigned long ric)
193 194 195 196 197 198 199 200 201 202 203 204 205 206
{
	unsigned long rb,rs,prs,r;

	rb = va & ~(PPC_BITMASK(52, 63));
	rb |= ap << PPC_BITLSHIFT(58);
	rs = lpid;
	prs = 0; /* partition scoped */
	r = 1;   /* radix format */

	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
	trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
}

207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236

static inline void fixup_tlbie_va(unsigned long va, unsigned long pid,
				  unsigned long ap)
{
	if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
		asm volatile("ptesync": : :"memory");
		__tlbie_va(va, 0, ap, RIC_FLUSH_TLB);
	}

	if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
		asm volatile("ptesync": : :"memory");
		__tlbie_va(va, pid, ap, RIC_FLUSH_TLB);
	}
}

static inline void fixup_tlbie_va_range(unsigned long va, unsigned long pid,
					unsigned long ap)
{
	if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
		asm volatile("ptesync": : :"memory");
		__tlbie_pid(0, RIC_FLUSH_TLB);
	}

	if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
		asm volatile("ptesync": : :"memory");
		__tlbie_va(va, pid, ap, RIC_FLUSH_TLB);
	}
}

static inline void fixup_tlbie_pid(unsigned long pid)
237
{
238 239 240 241
	/*
	 * We can use any address for the invalidation, pick one which is
	 * probably unused as an optimisation.
	 */
242 243
	unsigned long va = ((1UL << 52) - 1);

244 245 246 247 248
	if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
		asm volatile("ptesync": : :"memory");
		__tlbie_pid(0, RIC_FLUSH_TLB);
	}

249
	if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
250 251 252 253 254
		asm volatile("ptesync": : :"memory");
		__tlbie_va(va, pid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB);
	}
}

255 256 257 258 259 260 261 262 263 264 265 266 267 268 269

static inline void fixup_tlbie_lpid_va(unsigned long va, unsigned long lpid,
				       unsigned long ap)
{
	if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
		asm volatile("ptesync": : :"memory");
		__tlbie_lpid_va(va, 0, ap, RIC_FLUSH_TLB);
	}

	if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
		asm volatile("ptesync": : :"memory");
		__tlbie_lpid_va(va, lpid, ap, RIC_FLUSH_TLB);
	}
}

270 271
static inline void fixup_tlbie_lpid(unsigned long lpid)
{
272 273 274 275
	/*
	 * We can use any address for the invalidation, pick one which is
	 * probably unused as an optimisation.
	 */
276 277
	unsigned long va = ((1UL << 52) - 1);

278 279 280 281 282
	if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
		asm volatile("ptesync": : :"memory");
		__tlbie_lpid(0, RIC_FLUSH_TLB);
	}

283
	if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
284 285 286 287 288
		asm volatile("ptesync": : :"memory");
		__tlbie_lpid_va(va, lpid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB);
	}
}

289 290 291
/*
 * We use 128 set in radix mode and 256 set in hpt mode.
 */
292
static __always_inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
293 294 295
{
	int set;

296
	asm volatile("ptesync": : :"memory");
297 298 299 300 301 302 303

	/*
	 * Flush the first set of the TLB, and if we're doing a RIC_FLUSH_ALL,
	 * also flush the entire Page Walk Cache.
	 */
	__tlbiel_pid(pid, 0, ric);

304 305
	/* For PWC, only one flush is needed */
	if (ric == RIC_FLUSH_PWC) {
306
		ppc_after_tlbiel_barrier();
307 308
		return;
	}
309

310 311 312 313 314
	if (!cpu_has_feature(CPU_FTR_ARCH_31)) {
		/* For the remaining sets, just flush the TLB */
		for (set = 1; set < POWER9_TLB_SETS_RADIX ; set++)
			__tlbiel_pid(pid, set, RIC_FLUSH_TLB);
	}
315

316
	ppc_after_tlbiel_barrier();
317
	asm volatile(PPC_RADIX_INVALIDATE_ERAT_USER "; isync" : : :"memory");
318 319
}

320
static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
321 322 323
{
	asm volatile("ptesync": : :"memory");

324 325 326 327 328 329 330 331
	/*
	 * Workaround the fact that the "ric" argument to __tlbie_pid
	 * must be a compile-time contraint to match the "i" constraint
	 * in the asm statement.
	 */
	switch (ric) {
	case RIC_FLUSH_TLB:
		__tlbie_pid(pid, RIC_FLUSH_TLB);
332
		fixup_tlbie_pid(pid);
333 334 335 336 337 338 339
		break;
	case RIC_FLUSH_PWC:
		__tlbie_pid(pid, RIC_FLUSH_PWC);
		break;
	case RIC_FLUSH_ALL:
	default:
		__tlbie_pid(pid, RIC_FLUSH_ALL);
340
		fixup_tlbie_pid(pid);
341
	}
342 343 344
	asm volatile("eieio; tlbsync; ptesync": : :"memory");
}

345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377
struct tlbiel_pid {
	unsigned long pid;
	unsigned long ric;
};

static void do_tlbiel_pid(void *info)
{
	struct tlbiel_pid *t = info;

	if (t->ric == RIC_FLUSH_TLB)
		_tlbiel_pid(t->pid, RIC_FLUSH_TLB);
	else if (t->ric == RIC_FLUSH_PWC)
		_tlbiel_pid(t->pid, RIC_FLUSH_PWC);
	else
		_tlbiel_pid(t->pid, RIC_FLUSH_ALL);
}

static inline void _tlbiel_pid_multicast(struct mm_struct *mm,
				unsigned long pid, unsigned long ric)
{
	struct cpumask *cpus = mm_cpumask(mm);
	struct tlbiel_pid t = { .pid = pid, .ric = ric };

	on_each_cpu_mask(cpus, do_tlbiel_pid, &t, 1);
	/*
	 * Always want the CPU translations to be invalidated with tlbiel in
	 * these paths, so while coprocessors must use tlbie, we can not
	 * optimise away the tlbiel component.
	 */
	if (atomic_read(&mm->context.copros) > 0)
		_tlbie_pid(pid, RIC_FLUSH_ALL);
}

378 379 380 381 382 383 384 385 386 387 388 389
static inline void _tlbie_lpid(unsigned long lpid, unsigned long ric)
{
	asm volatile("ptesync": : :"memory");

	/*
	 * Workaround the fact that the "ric" argument to __tlbie_pid
	 * must be a compile-time contraint to match the "i" constraint
	 * in the asm statement.
	 */
	switch (ric) {
	case RIC_FLUSH_TLB:
		__tlbie_lpid(lpid, RIC_FLUSH_TLB);
390
		fixup_tlbie_lpid(lpid);
391 392 393 394 395 396 397
		break;
	case RIC_FLUSH_PWC:
		__tlbie_lpid(lpid, RIC_FLUSH_PWC);
		break;
	case RIC_FLUSH_ALL:
	default:
		__tlbie_lpid(lpid, RIC_FLUSH_ALL);
398
		fixup_tlbie_lpid(lpid);
399 400 401 402
	}
	asm volatile("eieio; tlbsync; ptesync": : :"memory");
}

403
static __always_inline void _tlbie_lpid_guest(unsigned long lpid, unsigned long ric)
404 405
{
	/*
406 407 408
	 * Workaround the fact that the "ric" argument to __tlbie_pid
	 * must be a compile-time contraint to match the "i" constraint
	 * in the asm statement.
409
	 */
410 411 412 413 414 415 416 417 418 419
	switch (ric) {
	case RIC_FLUSH_TLB:
		__tlbie_lpid_guest(lpid, RIC_FLUSH_TLB);
		break;
	case RIC_FLUSH_PWC:
		__tlbie_lpid_guest(lpid, RIC_FLUSH_PWC);
		break;
	case RIC_FLUSH_ALL:
	default:
		__tlbie_lpid_guest(lpid, RIC_FLUSH_ALL);
420
	}
421 422
	fixup_tlbie_lpid(lpid);
	asm volatile("eieio; tlbsync; ptesync": : :"memory");
423 424
}

425 426 427 428 429 430 431 432 433 434 435
static inline void __tlbiel_va_range(unsigned long start, unsigned long end,
				    unsigned long pid, unsigned long page_size,
				    unsigned long psize)
{
	unsigned long addr;
	unsigned long ap = mmu_get_ap(psize);

	for (addr = start; addr < end; addr += page_size)
		__tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
}

436 437
static __always_inline void _tlbiel_va(unsigned long va, unsigned long pid,
				       unsigned long psize, unsigned long ric)
438
{
439 440
	unsigned long ap = mmu_get_ap(psize);

441 442
	asm volatile("ptesync": : :"memory");
	__tlbiel_va(va, pid, ap, ric);
443
	ppc_after_tlbiel_barrier();
444 445
}

446 447
static inline void _tlbiel_va_range(unsigned long start, unsigned long end,
				    unsigned long pid, unsigned long page_size,
448
				    unsigned long psize, bool also_pwc)
449 450
{
	asm volatile("ptesync": : :"memory");
451 452
	if (also_pwc)
		__tlbiel_pid(pid, 0, RIC_FLUSH_PWC);
453
	__tlbiel_va_range(start, end, pid, page_size, psize);
454
	ppc_after_tlbiel_barrier();
455 456
}

457 458 459 460 461 462 463 464 465
static inline void __tlbie_va_range(unsigned long start, unsigned long end,
				    unsigned long pid, unsigned long page_size,
				    unsigned long psize)
{
	unsigned long addr;
	unsigned long ap = mmu_get_ap(psize);

	for (addr = start; addr < end; addr += page_size)
		__tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
466 467

	fixup_tlbie_va_range(addr - page_size, pid, ap);
468 469
}

470 471
static __always_inline void _tlbie_va(unsigned long va, unsigned long pid,
				      unsigned long psize, unsigned long ric)
472
{
473 474
	unsigned long ap = mmu_get_ap(psize);

475 476
	asm volatile("ptesync": : :"memory");
	__tlbie_va(va, pid, ap, ric);
477
	fixup_tlbie_va(va, pid, ap);
478 479 480
	asm volatile("eieio; tlbsync; ptesync": : :"memory");
}

481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527
struct tlbiel_va {
	unsigned long pid;
	unsigned long va;
	unsigned long psize;
	unsigned long ric;
};

static void do_tlbiel_va(void *info)
{
	struct tlbiel_va *t = info;

	if (t->ric == RIC_FLUSH_TLB)
		_tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_TLB);
	else if (t->ric == RIC_FLUSH_PWC)
		_tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_PWC);
	else
		_tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_ALL);
}

static inline void _tlbiel_va_multicast(struct mm_struct *mm,
				unsigned long va, unsigned long pid,
				unsigned long psize, unsigned long ric)
{
	struct cpumask *cpus = mm_cpumask(mm);
	struct tlbiel_va t = { .va = va, .pid = pid, .psize = psize, .ric = ric };
	on_each_cpu_mask(cpus, do_tlbiel_va, &t, 1);
	if (atomic_read(&mm->context.copros) > 0)
		_tlbie_va(va, pid, psize, RIC_FLUSH_TLB);
}

struct tlbiel_va_range {
	unsigned long pid;
	unsigned long start;
	unsigned long end;
	unsigned long page_size;
	unsigned long psize;
	bool also_pwc;
};

static void do_tlbiel_va_range(void *info)
{
	struct tlbiel_va_range *t = info;

	_tlbiel_va_range(t->start, t->end, t->pid, t->page_size,
				    t->psize, t->also_pwc);
}

528
static __always_inline void _tlbie_lpid_va(unsigned long va, unsigned long lpid,
529 530 531 532 533 534
			      unsigned long psize, unsigned long ric)
{
	unsigned long ap = mmu_get_ap(psize);

	asm volatile("ptesync": : :"memory");
	__tlbie_lpid_va(va, lpid, ap, ric);
535
	fixup_tlbie_lpid_va(va, lpid, ap);
536 537 538
	asm volatile("eieio; tlbsync; ptesync": : :"memory");
}

539 540
static inline void _tlbie_va_range(unsigned long start, unsigned long end,
				    unsigned long pid, unsigned long page_size,
541
				    unsigned long psize, bool also_pwc)
542 543
{
	asm volatile("ptesync": : :"memory");
544 545
	if (also_pwc)
		__tlbie_pid(pid, RIC_FLUSH_PWC);
546
	__tlbie_va_range(start, end, pid, page_size, psize);
547 548
	asm volatile("eieio; tlbsync; ptesync": : :"memory");
}
549

550 551 552 553 554 555 556 557 558 559 560 561 562 563 564
static inline void _tlbiel_va_range_multicast(struct mm_struct *mm,
				unsigned long start, unsigned long end,
				unsigned long pid, unsigned long page_size,
				unsigned long psize, bool also_pwc)
{
	struct cpumask *cpus = mm_cpumask(mm);
	struct tlbiel_va_range t = { .start = start, .end = end,
				.pid = pid, .page_size = page_size,
				.psize = psize, .also_pwc = also_pwc };

	on_each_cpu_mask(cpus, do_tlbiel_va_range, &t, 1);
	if (atomic_read(&mm->context.copros) > 0)
		_tlbie_va_range(start, end, pid, page_size, psize, also_pwc);
}

565 566 567 568 569 570 571 572 573 574 575 576 577
/*
 * Base TLB flushing operations:
 *
 *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
 *  - flush_tlb_page(vma, vmaddr) flushes one page
 *  - flush_tlb_range(vma, start, end) flushes a range of pages
 *  - flush_tlb_kernel_range(start, end) flushes kernel pages
 *
 *  - local_* variants of page and mm only apply to the current
 *    processor
 */
void radix__local_flush_tlb_mm(struct mm_struct *mm)
{
578
	unsigned long pid;
579 580 581 582

	preempt_disable();
	pid = mm->context.id;
	if (pid != MMU_NO_CONTEXT)
583
		_tlbiel_pid(pid, RIC_FLUSH_TLB);
584 585 586 587
	preempt_enable();
}
EXPORT_SYMBOL(radix__local_flush_tlb_mm);

588
#ifndef CONFIG_SMP
589
void radix__local_flush_all_mm(struct mm_struct *mm)
590 591 592 593 594 595
{
	unsigned long pid;

	preempt_disable();
	pid = mm->context.id;
	if (pid != MMU_NO_CONTEXT)
596
		_tlbiel_pid(pid, RIC_FLUSH_ALL);
597 598
	preempt_enable();
}
599
EXPORT_SYMBOL(radix__local_flush_all_mm);
600 601 602 603 604

static void __flush_all_mm(struct mm_struct *mm, bool fullmm)
{
	radix__local_flush_all_mm(mm);
}
605
#endif /* CONFIG_SMP */
606

607
void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
608
				       int psize)
609
{
610
	unsigned long pid;
611 612

	preempt_disable();
613
	pid = mm->context.id;
614
	if (pid != MMU_NO_CONTEXT)
615
		_tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
616 617 618 619 620
	preempt_enable();
}

void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
621 622
#ifdef CONFIG_HUGETLB_PAGE
	/* need the return fix for nohash.c */
623 624
	if (is_vm_hugetlb_page(vma))
		return radix__local_flush_hugetlb_page(vma, vmaddr);
625
#endif
626
	radix__local_flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize);
627 628 629
}
EXPORT_SYMBOL(radix__local_flush_tlb_page);

630 631 632 633 634 635 636 637 638
static bool mm_is_singlethreaded(struct mm_struct *mm)
{
	if (atomic_read(&mm->context.copros) > 0)
		return false;
	if (atomic_read(&mm->mm_users) <= 1 && current->mm == mm)
		return true;
	return false;
}

639 640 641 642 643 644 645
static bool mm_needs_flush_escalation(struct mm_struct *mm)
{
	/*
	 * P9 nest MMU has issues with the page walk cache
	 * caching PTEs and not flushing them properly when
	 * RIC = 0 for a PID/LPID invalidate
	 */
646 647 648
	if (atomic_read(&mm->context.copros) > 0)
		return true;
	return false;
649 650
}

651
#ifdef CONFIG_SMP
652 653 654 655 656
static void do_exit_flush_lazy_tlb(void *arg)
{
	struct mm_struct *mm = arg;
	unsigned long pid = mm->context.id;

657 658 659 660 661 662 663
	/*
	 * A kthread could have done a mmget_not_zero() after the flushing CPU
	 * checked mm_is_singlethreaded, and be in the process of
	 * kthread_use_mm when interrupted here. In that case, current->mm will
	 * be set to mm, because kthread_use_mm() setting ->mm and switching to
	 * the mm is done with interrupts off.
	 */
664
	if (current->mm == mm)
665
		goto out_flush;
666 667

	if (current->active_mm == mm) {
668 669
		WARN_ON_ONCE(current->mm != NULL);
		/* Is a kernel thread and is using mm as the lazy tlb */
670 671
		mmgrab(&init_mm);
		current->active_mm = &init_mm;
672
		switch_mm_irqs_off(mm, &init_mm, current);
673 674
		mmdrop(mm);
	}
675 676 677 678 679

	atomic_dec(&mm->context.active_cpus);
	cpumask_clear_cpu(smp_processor_id(), mm_cpumask(mm));

out_flush:
680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695
	_tlbiel_pid(pid, RIC_FLUSH_ALL);
}

static void exit_flush_lazy_tlbs(struct mm_struct *mm)
{
	/*
	 * Would be nice if this was async so it could be run in
	 * parallel with our local flush, but generic code does not
	 * give a good API for it. Could extend the generic code or
	 * make a special powerpc IPI for flushing TLBs.
	 * For now it's not too performance critical.
	 */
	smp_call_function_many(mm_cpumask(mm), do_exit_flush_lazy_tlb,
				(void *)mm, 1);
}

696 697
void radix__flush_tlb_mm(struct mm_struct *mm)
{
698
	unsigned long pid;
699 700 701

	pid = mm->context.id;
	if (unlikely(pid == MMU_NO_CONTEXT))
702
		return;
703

704
	preempt_disable();
705 706 707 708 709
	/*
	 * Order loads of mm_cpumask vs previous stores to clear ptes before
	 * the invalidate. See barrier in switch_mm_irqs_off
	 */
	smp_mb();
710
	if (!mm_is_thread_local(mm)) {
711 712 713 714 715
		if (unlikely(mm_is_singlethreaded(mm))) {
			exit_flush_lazy_tlbs(mm);
			goto local;
		}

716 717 718 719 720 721 722 723
		if (!mmu_has_feature(MMU_FTR_GTSE)) {
			unsigned long tgt = H_RPTI_TARGET_CMMU;

			if (atomic_read(&mm->context.copros) > 0)
				tgt |= H_RPTI_TARGET_NMMU;
			pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB,
					       H_RPTI_PAGE_ALL, 0, -1UL);
		} else if (cputlb_use_tlbie()) {
724 725 726 727 728 729 730
			if (mm_needs_flush_escalation(mm))
				_tlbie_pid(pid, RIC_FLUSH_ALL);
			else
				_tlbie_pid(pid, RIC_FLUSH_TLB);
		} else {
			_tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB);
		}
731 732
	} else {
local:
733
		_tlbiel_pid(pid, RIC_FLUSH_TLB);
734
	}
735 736 737 738
	preempt_enable();
}
EXPORT_SYMBOL(radix__flush_tlb_mm);

739
static void __flush_all_mm(struct mm_struct *mm, bool fullmm)
740 741 742 743 744
{
	unsigned long pid;

	pid = mm->context.id;
	if (unlikely(pid == MMU_NO_CONTEXT))
745
		return;
746

747
	preempt_disable();
748
	smp_mb(); /* see radix__flush_tlb_mm */
749 750 751 752 753 754 755
	if (!mm_is_thread_local(mm)) {
		if (unlikely(mm_is_singlethreaded(mm))) {
			if (!fullmm) {
				exit_flush_lazy_tlbs(mm);
				goto local;
			}
		}
756 757 758 759 760 761 762 763 764 765
		if (!mmu_has_feature(MMU_FTR_GTSE)) {
			unsigned long tgt = H_RPTI_TARGET_CMMU;
			unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
					     H_RPTI_TYPE_PRT;

			if (atomic_read(&mm->context.copros) > 0)
				tgt |= H_RPTI_TARGET_NMMU;
			pseries_rpt_invalidate(pid, tgt, type,
					       H_RPTI_PAGE_ALL, 0, -1UL);
		} else if (cputlb_use_tlbie())
766 767 768
			_tlbie_pid(pid, RIC_FLUSH_ALL);
		else
			_tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL);
769 770
	} else {
local:
771
		_tlbiel_pid(pid, RIC_FLUSH_ALL);
772
	}
773 774
	preempt_enable();
}
775

776 777 778 779
void radix__flush_all_mm(struct mm_struct *mm)
{
	__flush_all_mm(mm, false);
}
780
EXPORT_SYMBOL(radix__flush_all_mm);
781

782
void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
783
				 int psize)
784
{
785
	unsigned long pid;
786

787
	pid = mm->context.id;
788
	if (unlikely(pid == MMU_NO_CONTEXT))
789 790 791
		return;

	preempt_disable();
792
	smp_mb(); /* see radix__flush_tlb_mm */
793 794 795 796 797
	if (!mm_is_thread_local(mm)) {
		if (unlikely(mm_is_singlethreaded(mm))) {
			exit_flush_lazy_tlbs(mm);
			goto local;
		}
798 799 800 801 802 803 804 805 806 807 808 809 810
		if (!mmu_has_feature(MMU_FTR_GTSE)) {
			unsigned long tgt, pg_sizes, size;

			tgt = H_RPTI_TARGET_CMMU;
			pg_sizes = psize_to_rpti_pgsize(psize);
			size = 1UL << mmu_psize_to_shift(psize);

			if (atomic_read(&mm->context.copros) > 0)
				tgt |= H_RPTI_TARGET_NMMU;
			pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB,
					       pg_sizes, vmaddr,
					       vmaddr + size);
		} else if (cputlb_use_tlbie())
811 812 813
			_tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
		else
			_tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB);
814 815
	} else {
local:
816
		_tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
817
	}
818 819 820 821 822
	preempt_enable();
}

void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
823
#ifdef CONFIG_HUGETLB_PAGE
824 825
	if (is_vm_hugetlb_page(vma))
		return radix__flush_hugetlb_page(vma, vmaddr);
826
#endif
827
	radix__flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize);
828 829 830
}
EXPORT_SYMBOL(radix__flush_tlb_page);

831
#else /* CONFIG_SMP */
832
static inline void exit_flush_lazy_tlbs(struct mm_struct *mm) { }
833 834
#endif /* CONFIG_SMP */

835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852
static void do_tlbiel_kernel(void *info)
{
	_tlbiel_pid(0, RIC_FLUSH_ALL);
}

static inline void _tlbiel_kernel_broadcast(void)
{
	on_each_cpu(do_tlbiel_kernel, NULL, 1);
	if (tlbie_capable) {
		/*
		 * Coherent accelerators don't refcount kernel memory mappings,
		 * so have to always issue a tlbie for them. This is quite a
		 * slow path anyway.
		 */
		_tlbie_pid(0, RIC_FLUSH_ALL);
	}
}

853 854 855 856 857
/*
 * If kernel TLBIs ever become local rather than global, then
 * drivers/misc/ocxl/link.c:ocxl_link_add_pe will need some work, as it
 * assumes kernel TLBIs are global.
 */
858 859
void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
860 861 862 863 864 865 866 867
	if (!mmu_has_feature(MMU_FTR_GTSE)) {
		unsigned long tgt = H_RPTI_TARGET_CMMU | H_RPTI_TARGET_NMMU;
		unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
				     H_RPTI_TYPE_PRT;

		pseries_rpt_invalidate(0, tgt, type, H_RPTI_PAGE_ALL,
				       start, end);
	} else if (cputlb_use_tlbie())
868 869 870
		_tlbie_pid(0, RIC_FLUSH_ALL);
	else
		_tlbiel_kernel_broadcast();
871 872 873
}
EXPORT_SYMBOL(radix__flush_tlb_kernel_range);

874 875
#define TLB_FLUSH_ALL -1UL

876
/*
877 878 879 880 881 882 883
 * Number of pages above which we invalidate the entire PID rather than
 * flush individual pages, for local and global flushes respectively.
 *
 * tlbie goes out to the interconnect and individual ops are more costly.
 * It also does not iterate over sets like the local tlbiel variant when
 * invalidating a full PID, so it has a far lower threshold to change from
 * individual page flushes to full-pid flushes.
884
 */
885
static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;
886
static unsigned long tlb_local_single_page_flush_ceiling __read_mostly = POWER9_TLB_SETS_RADIX * 2;
887

888
static inline void __radix__flush_tlb_range(struct mm_struct *mm,
889
					    unsigned long start, unsigned long end)
890 891

{
892 893 894 895 896
	unsigned long pid;
	unsigned int page_shift = mmu_psize_defs[mmu_virtual_psize].shift;
	unsigned long page_size = 1UL << page_shift;
	unsigned long nr_pages = (end - start) >> page_shift;
	bool local, full;
897

898 899 900 901 902
	pid = mm->context.id;
	if (unlikely(pid == MMU_NO_CONTEXT))
		return;

	preempt_disable();
903
	smp_mb(); /* see radix__flush_tlb_mm */
904 905 906 907 908 909 910
	if (!mm_is_thread_local(mm)) {
		if (unlikely(mm_is_singlethreaded(mm))) {
			if (end != TLB_FLUSH_ALL) {
				exit_flush_lazy_tlbs(mm);
				goto is_local;
			}
		}
911 912 913
		local = false;
		full = (end == TLB_FLUSH_ALL ||
				nr_pages > tlb_single_page_flush_ceiling);
914 915 916 917 918
	} else {
is_local:
		local = true;
		full = (end == TLB_FLUSH_ALL ||
				nr_pages > tlb_local_single_page_flush_ceiling);
919
	}
920

921 922 923 924 925 926 927 928 929 930 931
	if (!mmu_has_feature(MMU_FTR_GTSE) && !local) {
		unsigned long tgt = H_RPTI_TARGET_CMMU;
		unsigned long pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize);

		if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
			pg_sizes |= psize_to_rpti_pgsize(MMU_PAGE_2M);
		if (atomic_read(&mm->context.copros) > 0)
			tgt |= H_RPTI_TARGET_NMMU;
		pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, pg_sizes,
				       start, end);
	} else if (full) {
932
		if (local) {
933
			_tlbiel_pid(pid, RIC_FLUSH_TLB);
934
		} else {
935 936 937 938 939 940 941 942
			if (cputlb_use_tlbie()) {
				if (mm_needs_flush_escalation(mm))
					_tlbie_pid(pid, RIC_FLUSH_ALL);
				else
					_tlbie_pid(pid, RIC_FLUSH_TLB);
			} else {
				_tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB);
			}
943
		}
944
	} else {
945
		bool hflush = false;
946 947
		unsigned long hstart, hend;

948
		if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
949 950
			hstart = (start + PMD_SIZE - 1) & PMD_MASK;
			hend = end & PMD_MASK;
951
			if (hstart < hend)
952
				hflush = true;
953 954 955
		}

		if (local) {
956
			asm volatile("ptesync": : :"memory");
957 958 959
			__tlbiel_va_range(start, end, pid, page_size, mmu_virtual_psize);
			if (hflush)
				__tlbiel_va_range(hstart, hend, pid,
960
						PMD_SIZE, MMU_PAGE_2M);
961
			ppc_after_tlbiel_barrier();
962 963
		} else if (cputlb_use_tlbie()) {
			asm volatile("ptesync": : :"memory");
964 965 966
			__tlbie_va_range(start, end, pid, page_size, mmu_virtual_psize);
			if (hflush)
				__tlbie_va_range(hstart, hend, pid,
967
						PMD_SIZE, MMU_PAGE_2M);
968
			asm volatile("eieio; tlbsync; ptesync": : :"memory");
969 970 971 972 973 974
		} else {
			_tlbiel_va_range_multicast(mm,
					start, end, pid, page_size, mmu_virtual_psize, false);
			if (hflush)
				_tlbiel_va_range_multicast(mm,
					hstart, hend, pid, PMD_SIZE, MMU_PAGE_2M, false);
975 976 977
		}
	}
	preempt_enable();
978
}
979 980 981 982 983 984 985 986 987 988

void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
		     unsigned long end)

{
#ifdef CONFIG_HUGETLB_PAGE
	if (is_vm_hugetlb_page(vma))
		return radix__flush_hugetlb_tlb_range(vma, start, end);
#endif

989
	__radix__flush_tlb_range(vma->vm_mm, start, end);
990
}
991 992
EXPORT_SYMBOL(radix__flush_tlb_range);

993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006
static int radix_get_mmu_psize(int page_size)
{
	int psize;

	if (page_size == (1UL << mmu_psize_defs[mmu_virtual_psize].shift))
		psize = mmu_virtual_psize;
	else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_2M].shift))
		psize = MMU_PAGE_2M;
	else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_1G].shift))
		psize = MMU_PAGE_1G;
	else
		return -1;
	return psize;
}
1007

1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
/*
 * Flush partition scoped LPID address translation for all CPUs.
 */
void radix__flush_tlb_lpid_page(unsigned int lpid,
					unsigned long addr,
					unsigned long page_size)
{
	int psize = radix_get_mmu_psize(page_size);

	_tlbie_lpid_va(addr, lpid, psize, RIC_FLUSH_TLB);
}
EXPORT_SYMBOL_GPL(radix__flush_tlb_lpid_page);

/*
 * Flush partition scoped PWC from LPID for all CPUs.
 */
void radix__flush_pwc_lpid(unsigned int lpid)
{
	_tlbie_lpid(lpid, RIC_FLUSH_PWC);
}
EXPORT_SYMBOL_GPL(radix__flush_pwc_lpid);

1030 1031 1032
/*
 * Flush partition scoped translations from LPID (=LPIDR)
 */
1033
void radix__flush_all_lpid(unsigned int lpid)
1034 1035 1036
{
	_tlbie_lpid(lpid, RIC_FLUSH_ALL);
}
1037
EXPORT_SYMBOL_GPL(radix__flush_all_lpid);
1038

1039
/*
1040
 * Flush process scoped translations from LPID (=LPIDR)
1041
 */
1042
void radix__flush_all_lpid_guest(unsigned int lpid)
1043
{
1044
	_tlbie_lpid_guest(lpid, RIC_FLUSH_ALL);
1045 1046
}

1047 1048 1049
static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
				  unsigned long end, int psize);

1050 1051
void radix__tlb_flush(struct mmu_gather *tlb)
{
1052
	int psize = 0;
1053
	struct mm_struct *mm = tlb->mm;
1054
	int page_size = tlb->page_size;
1055 1056
	unsigned long start = tlb->start;
	unsigned long end = tlb->end;
1057 1058 1059

	/*
	 * if page size is not something we understand, do a full mm flush
1060 1061 1062 1063
	 *
	 * A "fullmm" flush must always do a flush_all_mm (RIC=2) flush
	 * that flushes the process table entry cache upon process teardown.
	 * See the comment for radix in arch_exit_mmap().
1064
	 */
1065
	if (tlb->fullmm || tlb->need_flush_all) {
1066
		__flush_all_mm(mm, true);
1067
	} else if ( (psize = radix_get_mmu_psize(page_size)) == -1) {
1068
		if (!tlb->freed_tables)
1069 1070 1071 1072
			radix__flush_tlb_mm(mm);
		else
			radix__flush_all_mm(mm);
	} else {
1073
		if (!tlb->freed_tables)
1074 1075 1076 1077
			radix__flush_tlb_range_psize(mm, start, end, psize);
		else
			radix__flush_tlb_pwc_range_psize(mm, start, end, psize);
	}
1078 1079
}

1080
static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm,
1081 1082
				unsigned long start, unsigned long end,
				int psize, bool also_pwc)
1083 1084
{
	unsigned long pid;
1085 1086 1087 1088
	unsigned int page_shift = mmu_psize_defs[psize].shift;
	unsigned long page_size = 1UL << page_shift;
	unsigned long nr_pages = (end - start) >> page_shift;
	bool local, full;
1089

1090
	pid = mm->context.id;
1091
	if (unlikely(pid == MMU_NO_CONTEXT))
1092
		return;
1093

1094
	preempt_disable();
1095
	smp_mb(); /* see radix__flush_tlb_mm */
1096 1097 1098 1099 1100 1101 1102
	if (!mm_is_thread_local(mm)) {
		if (unlikely(mm_is_singlethreaded(mm))) {
			if (end != TLB_FLUSH_ALL) {
				exit_flush_lazy_tlbs(mm);
				goto is_local;
			}
		}
1103 1104 1105
		local = false;
		full = (end == TLB_FLUSH_ALL ||
				nr_pages > tlb_single_page_flush_ceiling);
1106 1107 1108 1109 1110
	} else {
is_local:
		local = true;
		full = (end == TLB_FLUSH_ALL ||
				nr_pages > tlb_local_single_page_flush_ceiling);
1111
	}
1112

1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123
	if (!mmu_has_feature(MMU_FTR_GTSE) && !local) {
		unsigned long tgt = H_RPTI_TARGET_CMMU;
		unsigned long type = H_RPTI_TYPE_TLB;
		unsigned long pg_sizes = psize_to_rpti_pgsize(psize);

		if (also_pwc)
			type |= H_RPTI_TYPE_PWC;
		if (atomic_read(&mm->context.copros) > 0)
			tgt |= H_RPTI_TARGET_NMMU;
		pseries_rpt_invalidate(pid, tgt, type, pg_sizes, start, end);
	} else if (full) {
1124
		if (local) {
1125
			_tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
1126
		} else {
1127 1128 1129 1130 1131 1132 1133 1134 1135 1136
			if (cputlb_use_tlbie()) {
				if (mm_needs_flush_escalation(mm))
					also_pwc = true;

				_tlbie_pid(pid,
					also_pwc ?  RIC_FLUSH_ALL : RIC_FLUSH_TLB);
			} else {
				_tlbiel_pid_multicast(mm, pid,
					also_pwc ?  RIC_FLUSH_ALL : RIC_FLUSH_TLB);
			}
1137 1138

		}
1139
	} else {
1140
		if (local)
1141
			_tlbiel_va_range(start, end, pid, page_size, psize, also_pwc);
1142
		else if (cputlb_use_tlbie())
1143
			_tlbie_va_range(start, end, pid, page_size, psize, also_pwc);
1144 1145 1146
		else
			_tlbiel_va_range_multicast(mm,
					start, end, pid, page_size, psize, also_pwc);
1147 1148
	}
	preempt_enable();
1149
}
1150

1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162
void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
				  unsigned long end, int psize)
{
	return __radix__flush_tlb_range_psize(mm, start, end, psize, false);
}

static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
				  unsigned long end, int psize)
{
	__radix__flush_tlb_range_psize(mm, start, end, psize, true);
}

1163 1164 1165 1166 1167
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
{
	unsigned long pid, end;

1168
	pid = mm->context.id;
1169
	if (unlikely(pid == MMU_NO_CONTEXT))
1170
		return;
1171 1172 1173 1174 1175 1176 1177

	/* 4k page size, just blow the world */
	if (PAGE_SIZE == 0x1000) {
		radix__flush_all_mm(mm);
		return;
	}

1178 1179 1180
	end = addr + HPAGE_PMD_SIZE;

	/* Otherwise first do the PWC, then iterate the pages. */
1181
	preempt_disable();
1182
	smp_mb(); /* see radix__flush_tlb_mm */
1183 1184 1185 1186 1187
	if (!mm_is_thread_local(mm)) {
		if (unlikely(mm_is_singlethreaded(mm))) {
			exit_flush_lazy_tlbs(mm);
			goto local;
		}
1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200
		if (!mmu_has_feature(MMU_FTR_GTSE)) {
			unsigned long tgt, type, pg_sizes;

			tgt = H_RPTI_TARGET_CMMU;
			type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
			       H_RPTI_TYPE_PRT;
			pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize);

			if (atomic_read(&mm->context.copros) > 0)
				tgt |= H_RPTI_TARGET_NMMU;
			pseries_rpt_invalidate(pid, tgt, type, pg_sizes,
					       addr, end);
		} else if (cputlb_use_tlbie())
1201 1202 1203 1204
			_tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
		else
			_tlbiel_va_range_multicast(mm,
					addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
1205 1206 1207
	} else {
local:
		_tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
1208
	}
1209

1210 1211 1212 1213
	preempt_enable();
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

1214 1215 1216 1217 1218 1219
void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
				unsigned long start, unsigned long end)
{
	radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M);
}
EXPORT_SYMBOL(radix__flush_pmd_tlb_range);
1220 1221 1222 1223 1224 1225 1226 1227

void radix__flush_tlb_all(void)
{
	unsigned long rb,prs,r,rs;
	unsigned long ric = RIC_FLUSH_ALL;

	rb = 0x3 << PPC_BITLSHIFT(53); /* IS = 3 */
	prs = 0; /* partition scoped */
1228
	r = 1;   /* radix format */
1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
	rs = 1 & ((1UL << 32) - 1); /* any LPID value to flush guest mappings */

	asm volatile("ptesync": : :"memory");
	/*
	 * now flush guest entries by passing PRS = 1 and LPID != 0
	 */
	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
		     : : "r"(rb), "i"(r), "i"(1), "i"(ric), "r"(rs) : "memory");
	/*
	 * now flush host entires by passing PRS = 0 and LPID == 0
	 */
	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(0) : "memory");
	asm volatile("eieio; tlbsync; ptesync": : :"memory");
}
1244

1245 1246 1247
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
extern void radix_kvm_prefetch_workaround(struct mm_struct *mm)
{
1248
	unsigned long pid = mm->context.id;
1249 1250 1251 1252

	if (unlikely(pid == MMU_NO_CONTEXT))
		return;

1253 1254 1255
	if (!cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG))
		return;

1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
	/*
	 * If this context hasn't run on that CPU before and KVM is
	 * around, there's a slim chance that the guest on another
	 * CPU just brought in obsolete translation into the TLB of
	 * this CPU due to a bad prefetch using the guest PID on
	 * the way into the hypervisor.
	 *
	 * We work around this here. If KVM is possible, we check if
	 * any sibling thread is in KVM. If it is, the window may exist
	 * and thus we flush that PID from the core.
	 *
	 * A potential future improvement would be to mark which PIDs
	 * have never been used on the system and avoid it if the PID
	 * is new and the process has no other cpumask bit set.
	 */
	if (cpu_has_feature(CPU_FTR_HVMODE) && radix_enabled()) {
		int cpu = smp_processor_id();
		int sib = cpu_first_thread_sibling(cpu);
		bool flush = false;

		for (; sib <= cpu_last_thread_sibling(cpu) && !flush; sib++) {
			if (sib == cpu)
				continue;
1279 1280
			if (!cpu_possible(sib))
				continue;
1281
			if (paca_ptrs[sib]->kvm_hstate.kvm_vcpu)
1282 1283 1284 1285 1286 1287 1288 1289
				flush = true;
		}
		if (flush)
			_tlbiel_pid(pid, RIC_FLUSH_ALL);
	}
}
EXPORT_SYMBOL_GPL(radix_kvm_prefetch_workaround);
#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */