Commit 81a91edb authored by David S. Miller's avatar David S. Miller Committed by Greg Kroah-Hartman

sparc64: Handle extremely large kernel TLB range flushes more gracefully.

[ Upstream commit a74ad5e6 ]

When the vmalloc area gets fragmented, and because the firmware
mapping area sits between where modules live and the vmalloc area, we
can sometimes receive requests for enormous kernel TLB range flushes.

When this happens the cpu just spins flushing billions of pages and
this triggers the NMI watchdog and other problems.

We took care of this on the TSB side by doing a linear scan of the
table once we pass a certain threshold.

Do something similar for the TLB flush, however we are limited by
the TLB flush facilities provided by the different chip variants.

First of all we use an (mostly arbitrary) cut-off of 256K which is
about 32 pages.  This can be tuned in the future.

The huge range code path for each chip works as follows:

1) On spitfire we flush all non-locked TLB entries using diagnostic
   acceses.

2) On cheetah we use the "flush all" TLB flush.

3) On sun4v/hypervisor we do a TLB context flush on context 0, which
   unlike previous chips does not remove "permanent" or locked
   entries.

We could probably do something better on spitfire, such as limiting
the flush to kernel TLB entries or even doing range comparisons.
However that probably isn't worth it since those chips are old and
the TLB only had 64 entries.
Reported-by: default avatarJames Clarke <jrtc27@jrtc27.com>
Tested-by: default avatarJames Clarke <jrtc27@jrtc27.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent 7f8a50eb
...@@ -113,12 +113,14 @@ __flush_tlb_pending: /* 27 insns */ ...@@ -113,12 +113,14 @@ __flush_tlb_pending: /* 27 insns */
.align 32 .align 32
.globl __flush_tlb_kernel_range .globl __flush_tlb_kernel_range
__flush_tlb_kernel_range: /* 19 insns */ __flush_tlb_kernel_range: /* 31 insns */
/* %o0=start, %o1=end */ /* %o0=start, %o1=end */
cmp %o0, %o1 cmp %o0, %o1
be,pn %xcc, 2f be,pn %xcc, 2f
sethi %hi(PAGE_SIZE), %o4
sub %o1, %o0, %o3 sub %o1, %o0, %o3
srlx %o3, 18, %o4
brnz,pn %o4, __spitfire_flush_tlb_kernel_range_slow
sethi %hi(PAGE_SIZE), %o4
sub %o3, %o4, %o3 sub %o3, %o4, %o3
or %o0, 0x20, %o0 ! Nucleus or %o0, 0x20, %o0 ! Nucleus
1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
...@@ -134,6 +136,38 @@ __flush_tlb_kernel_range: /* 19 insns */ ...@@ -134,6 +136,38 @@ __flush_tlb_kernel_range: /* 19 insns */
nop nop
nop nop
nop nop
nop
nop
nop
nop
nop
nop
nop
nop
nop
nop
__spitfire_flush_tlb_kernel_range_slow:
mov 63 * 8, %o4
1: ldxa [%o4] ASI_ITLB_DATA_ACCESS, %o3
andcc %o3, 0x40, %g0 /* _PAGE_L_4U */
bne,pn %xcc, 2f
mov TLB_TAG_ACCESS, %o3
stxa %g0, [%o3] ASI_IMMU
stxa %g0, [%o4] ASI_ITLB_DATA_ACCESS
membar #Sync
2: ldxa [%o4] ASI_DTLB_DATA_ACCESS, %o3
andcc %o3, 0x40, %g0
bne,pn %xcc, 2f
mov TLB_TAG_ACCESS, %o3
stxa %g0, [%o3] ASI_DMMU
stxa %g0, [%o4] ASI_DTLB_DATA_ACCESS
membar #Sync
2: sub %o4, 8, %o4
brgez,pt %o4, 1b
nop
retl
nop
__spitfire_flush_tlb_mm_slow: __spitfire_flush_tlb_mm_slow:
rdpr %pstate, %g1 rdpr %pstate, %g1
...@@ -288,6 +322,40 @@ __cheetah_flush_tlb_pending: /* 27 insns */ ...@@ -288,6 +322,40 @@ __cheetah_flush_tlb_pending: /* 27 insns */
retl retl
wrpr %g7, 0x0, %pstate wrpr %g7, 0x0, %pstate
__cheetah_flush_tlb_kernel_range: /* 31 insns */
/* %o0=start, %o1=end */
cmp %o0, %o1
be,pn %xcc, 2f
sub %o1, %o0, %o3
srlx %o3, 18, %o4
brnz,pn %o4, 3f
sethi %hi(PAGE_SIZE), %o4
sub %o3, %o4, %o3
or %o0, 0x20, %o0 ! Nucleus
1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP
membar #Sync
brnz,pt %o3, 1b
sub %o3, %o4, %o3
2: sethi %hi(KERNBASE), %o3
flush %o3
retl
nop
3: mov 0x80, %o4
stxa %g0, [%o4] ASI_DMMU_DEMAP
membar #Sync
stxa %g0, [%o4] ASI_IMMU_DEMAP
membar #Sync
retl
nop
nop
nop
nop
nop
nop
nop
nop
#ifdef DCACHE_ALIASING_POSSIBLE #ifdef DCACHE_ALIASING_POSSIBLE
__cheetah_flush_dcache_page: /* 11 insns */ __cheetah_flush_dcache_page: /* 11 insns */
sethi %hi(PAGE_OFFSET), %g1 sethi %hi(PAGE_OFFSET), %g1
...@@ -388,13 +456,15 @@ __hypervisor_flush_tlb_pending: /* 27 insns */ ...@@ -388,13 +456,15 @@ __hypervisor_flush_tlb_pending: /* 27 insns */
nop nop
nop nop
__hypervisor_flush_tlb_kernel_range: /* 19 insns */ __hypervisor_flush_tlb_kernel_range: /* 31 insns */
/* %o0=start, %o1=end */ /* %o0=start, %o1=end */
cmp %o0, %o1 cmp %o0, %o1
be,pn %xcc, 2f be,pn %xcc, 2f
sethi %hi(PAGE_SIZE), %g3 sub %o1, %o0, %g2
srlx %g2, 18, %g3
brnz,pn %g3, 4f
mov %o0, %g1 mov %o0, %g1
sub %o1, %g1, %g2 sethi %hi(PAGE_SIZE), %g3
sub %g2, %g3, %g2 sub %g2, %g3, %g2
1: add %g1, %g2, %o0 /* ARG0: virtual address */ 1: add %g1, %g2, %o0 /* ARG0: virtual address */
mov 0, %o1 /* ARG1: mmu context */ mov 0, %o1 /* ARG1: mmu context */
...@@ -409,6 +479,16 @@ __hypervisor_flush_tlb_kernel_range: /* 19 insns */ ...@@ -409,6 +479,16 @@ __hypervisor_flush_tlb_kernel_range: /* 19 insns */
3: sethi %hi(__hypervisor_tlb_tl0_error), %o2 3: sethi %hi(__hypervisor_tlb_tl0_error), %o2
jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
nop nop
4: mov 0, %o0 /* ARG0: CPU lists unimplemented */
mov 0, %o1 /* ARG1: CPU lists unimplemented */
mov 0, %o2 /* ARG2: mmu context == nucleus */
mov HV_MMU_ALL, %o3 /* ARG3: flags */
mov HV_FAST_MMU_DEMAP_CTX, %o5
ta HV_FAST_TRAP
brnz,pn %o0, 3b
mov HV_FAST_MMU_DEMAP_CTX, %o1
retl
nop
#ifdef DCACHE_ALIASING_POSSIBLE #ifdef DCACHE_ALIASING_POSSIBLE
/* XXX Niagara and friends have an 8K cache, so no aliasing is /* XXX Niagara and friends have an 8K cache, so no aliasing is
...@@ -431,43 +511,6 @@ tlb_patch_one: ...@@ -431,43 +511,6 @@ tlb_patch_one:
retl retl
nop nop
.globl cheetah_patch_cachetlbops
cheetah_patch_cachetlbops:
save %sp, -128, %sp
sethi %hi(__flush_tlb_mm), %o0
or %o0, %lo(__flush_tlb_mm), %o0
sethi %hi(__cheetah_flush_tlb_mm), %o1
or %o1, %lo(__cheetah_flush_tlb_mm), %o1
call tlb_patch_one
mov 19, %o2
sethi %hi(__flush_tlb_page), %o0
or %o0, %lo(__flush_tlb_page), %o0
sethi %hi(__cheetah_flush_tlb_page), %o1
or %o1, %lo(__cheetah_flush_tlb_page), %o1
call tlb_patch_one
mov 22, %o2
sethi %hi(__flush_tlb_pending), %o0
or %o0, %lo(__flush_tlb_pending), %o0
sethi %hi(__cheetah_flush_tlb_pending), %o1
or %o1, %lo(__cheetah_flush_tlb_pending), %o1
call tlb_patch_one
mov 27, %o2
#ifdef DCACHE_ALIASING_POSSIBLE
sethi %hi(__flush_dcache_page), %o0
or %o0, %lo(__flush_dcache_page), %o0
sethi %hi(__cheetah_flush_dcache_page), %o1
or %o1, %lo(__cheetah_flush_dcache_page), %o1
call tlb_patch_one
mov 11, %o2
#endif /* DCACHE_ALIASING_POSSIBLE */
ret
restore
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
/* These are all called by the slaves of a cross call, at /* These are all called by the slaves of a cross call, at
* trap level 1, with interrupts fully disabled. * trap level 1, with interrupts fully disabled.
...@@ -535,12 +578,14 @@ xcall_flush_tlb_page: /* 20 insns */ ...@@ -535,12 +578,14 @@ xcall_flush_tlb_page: /* 20 insns */
nop nop
.globl xcall_flush_tlb_kernel_range .globl xcall_flush_tlb_kernel_range
xcall_flush_tlb_kernel_range: /* 28 insns */ xcall_flush_tlb_kernel_range: /* 44 insns */
sethi %hi(PAGE_SIZE - 1), %g2 sethi %hi(PAGE_SIZE - 1), %g2
or %g2, %lo(PAGE_SIZE - 1), %g2 or %g2, %lo(PAGE_SIZE - 1), %g2
andn %g1, %g2, %g1 andn %g1, %g2, %g1
andn %g7, %g2, %g7 andn %g7, %g2, %g7
sub %g7, %g1, %g3 sub %g7, %g1, %g3
srlx %g3, 18, %g2
brnz,pn %g2, 2f
add %g2, 1, %g2 add %g2, 1, %g2
sub %g3, %g2, %g3 sub %g3, %g2, %g3
or %g1, 0x20, %g1 ! Nucleus or %g1, 0x20, %g1 ! Nucleus
...@@ -550,11 +595,25 @@ xcall_flush_tlb_kernel_range: /* 28 insns */ ...@@ -550,11 +595,25 @@ xcall_flush_tlb_kernel_range: /* 28 insns */
brnz,pt %g3, 1b brnz,pt %g3, 1b
sub %g3, %g2, %g3 sub %g3, %g2, %g3
retry retry
2: mov 63 * 8, %g1
1: ldxa [%g1] ASI_ITLB_DATA_ACCESS, %g2
andcc %g2, 0x40, %g0 /* _PAGE_L_4U */
bne,pn %xcc, 2f
mov TLB_TAG_ACCESS, %g2
stxa %g0, [%g2] ASI_IMMU
stxa %g0, [%g1] ASI_ITLB_DATA_ACCESS
membar #Sync
2: ldxa [%g1] ASI_DTLB_DATA_ACCESS, %g2
andcc %g2, 0x40, %g0
bne,pn %xcc, 2f
mov TLB_TAG_ACCESS, %g2
stxa %g0, [%g2] ASI_DMMU
stxa %g0, [%g1] ASI_DTLB_DATA_ACCESS
membar #Sync
2: sub %g1, 8, %g1
brgez,pt %g1, 1b
nop nop
nop retry
nop
nop
nop
nop nop
nop nop
nop nop
...@@ -683,6 +742,52 @@ xcall_fetch_glob_pmu_n4: ...@@ -683,6 +742,52 @@ xcall_fetch_glob_pmu_n4:
retry retry
__cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */
sethi %hi(PAGE_SIZE - 1), %g2
or %g2, %lo(PAGE_SIZE - 1), %g2
andn %g1, %g2, %g1
andn %g7, %g2, %g7
sub %g7, %g1, %g3
srlx %g3, 18, %g2
brnz,pn %g2, 2f
add %g2, 1, %g2
sub %g3, %g2, %g3
or %g1, 0x20, %g1 ! Nucleus
1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP
membar #Sync
brnz,pt %g3, 1b
sub %g3, %g2, %g3
retry
2: mov 0x80, %g2
stxa %g0, [%g2] ASI_DMMU_DEMAP
membar #Sync
stxa %g0, [%g2] ASI_IMMU_DEMAP
membar #Sync
retry
nop
nop
nop
nop
nop
nop
nop
nop
nop
nop
nop
nop
nop
nop
nop
nop
nop
nop
nop
nop
nop
nop
#ifdef DCACHE_ALIASING_POSSIBLE #ifdef DCACHE_ALIASING_POSSIBLE
.align 32 .align 32
.globl xcall_flush_dcache_page_cheetah .globl xcall_flush_dcache_page_cheetah
...@@ -798,17 +903,19 @@ __hypervisor_xcall_flush_tlb_page: /* 20 insns */ ...@@ -798,17 +903,19 @@ __hypervisor_xcall_flush_tlb_page: /* 20 insns */
nop nop
.globl __hypervisor_xcall_flush_tlb_kernel_range .globl __hypervisor_xcall_flush_tlb_kernel_range
__hypervisor_xcall_flush_tlb_kernel_range: /* 28 insns */ __hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */
/* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
sethi %hi(PAGE_SIZE - 1), %g2 sethi %hi(PAGE_SIZE - 1), %g2
or %g2, %lo(PAGE_SIZE - 1), %g2 or %g2, %lo(PAGE_SIZE - 1), %g2
andn %g1, %g2, %g1 andn %g1, %g2, %g1
andn %g7, %g2, %g7 andn %g7, %g2, %g7
sub %g7, %g1, %g3 sub %g7, %g1, %g3
srlx %g3, 18, %g7
add %g2, 1, %g2 add %g2, 1, %g2
sub %g3, %g2, %g3 sub %g3, %g2, %g3
mov %o0, %g2 mov %o0, %g2
mov %o1, %g4 mov %o1, %g4
brnz,pn %g7, 2f
mov %o2, %g7 mov %o2, %g7
1: add %g1, %g3, %o0 /* ARG0: virtual address */ 1: add %g1, %g3, %o0 /* ARG0: virtual address */
mov 0, %o1 /* ARG1: mmu context */ mov 0, %o1 /* ARG1: mmu context */
...@@ -820,7 +927,7 @@ __hypervisor_xcall_flush_tlb_kernel_range: /* 28 insns */ ...@@ -820,7 +927,7 @@ __hypervisor_xcall_flush_tlb_kernel_range: /* 28 insns */
sethi %hi(PAGE_SIZE), %o2 sethi %hi(PAGE_SIZE), %o2
brnz,pt %g3, 1b brnz,pt %g3, 1b
sub %g3, %o2, %g3 sub %g3, %o2, %g3
mov %g2, %o0 5: mov %g2, %o0
mov %g4, %o1 mov %g4, %o1
mov %g7, %o2 mov %g7, %o2
membar #Sync membar #Sync
...@@ -828,6 +935,20 @@ __hypervisor_xcall_flush_tlb_kernel_range: /* 28 insns */ ...@@ -828,6 +935,20 @@ __hypervisor_xcall_flush_tlb_kernel_range: /* 28 insns */
1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 1: sethi %hi(__hypervisor_tlb_xcall_error), %g4
jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
nop nop
2: mov %o3, %g1
mov %o5, %g3
mov 0, %o0 /* ARG0: CPU lists unimplemented */
mov 0, %o1 /* ARG1: CPU lists unimplemented */
mov 0, %o2 /* ARG2: mmu context == nucleus */
mov HV_MMU_ALL, %o3 /* ARG3: flags */
mov HV_FAST_MMU_DEMAP_CTX, %o5
ta HV_FAST_TRAP
mov %g1, %o3
brz,pt %o0, 5b
mov %g3, %o5
mov HV_FAST_MMU_DEMAP_CTX, %g6
ba,pt %xcc, 1b
clr %g5
/* These just get rescheduled to PIL vectors. */ /* These just get rescheduled to PIL vectors. */
.globl xcall_call_function .globl xcall_call_function
...@@ -864,6 +985,58 @@ xcall_kgdb_capture: ...@@ -864,6 +985,58 @@ xcall_kgdb_capture:
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
.globl cheetah_patch_cachetlbops
cheetah_patch_cachetlbops:
save %sp, -128, %sp
sethi %hi(__flush_tlb_mm), %o0
or %o0, %lo(__flush_tlb_mm), %o0
sethi %hi(__cheetah_flush_tlb_mm), %o1
or %o1, %lo(__cheetah_flush_tlb_mm), %o1
call tlb_patch_one
mov 19, %o2
sethi %hi(__flush_tlb_page), %o0
or %o0, %lo(__flush_tlb_page), %o0
sethi %hi(__cheetah_flush_tlb_page), %o1
or %o1, %lo(__cheetah_flush_tlb_page), %o1
call tlb_patch_one
mov 22, %o2
sethi %hi(__flush_tlb_pending), %o0
or %o0, %lo(__flush_tlb_pending), %o0
sethi %hi(__cheetah_flush_tlb_pending), %o1
or %o1, %lo(__cheetah_flush_tlb_pending), %o1
call tlb_patch_one
mov 27, %o2
sethi %hi(__flush_tlb_kernel_range), %o0
or %o0, %lo(__flush_tlb_kernel_range), %o0
sethi %hi(__cheetah_flush_tlb_kernel_range), %o1
or %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1
call tlb_patch_one
mov 31, %o2
#ifdef DCACHE_ALIASING_POSSIBLE
sethi %hi(__flush_dcache_page), %o0
or %o0, %lo(__flush_dcache_page), %o0
sethi %hi(__cheetah_flush_dcache_page), %o1
or %o1, %lo(__cheetah_flush_dcache_page), %o1
call tlb_patch_one
mov 11, %o2
#endif /* DCACHE_ALIASING_POSSIBLE */
#ifdef CONFIG_SMP
sethi %hi(xcall_flush_tlb_kernel_range), %o0
or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
sethi %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1
or %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1
call tlb_patch_one
mov 44, %o2
#endif /* CONFIG_SMP */
ret
restore
.globl hypervisor_patch_cachetlbops .globl hypervisor_patch_cachetlbops
hypervisor_patch_cachetlbops: hypervisor_patch_cachetlbops:
...@@ -895,7 +1068,7 @@ hypervisor_patch_cachetlbops: ...@@ -895,7 +1068,7 @@ hypervisor_patch_cachetlbops:
sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1
or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
call tlb_patch_one call tlb_patch_one
mov 19, %o2 mov 31, %o2
#ifdef DCACHE_ALIASING_POSSIBLE #ifdef DCACHE_ALIASING_POSSIBLE
sethi %hi(__flush_dcache_page), %o0 sethi %hi(__flush_dcache_page), %o0
...@@ -926,7 +1099,7 @@ hypervisor_patch_cachetlbops: ...@@ -926,7 +1099,7 @@ hypervisor_patch_cachetlbops:
sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
call tlb_patch_one call tlb_patch_one
mov 28, %o2 mov 44, %o2
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
ret ret
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment