Commit ed4076ee authored by David S. Miller's avatar David S. Miller

[SPARC64]: Make clear_user_page more leight weight.

- Do not save/restore existing TLB entries, that is
  expensive, complicated, and does not really help
  performance much at all.
- Only one block store per loop iteration, doing more
  does not make things run any faster.
Signed-off-by: default avatarDavid S. Miller <davem@redhat.com>
parent 397a7ba6
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
EXTRA_AFLAGS := -ansi EXTRA_AFLAGS := -ansi
EXTRA_CFLAGS := -Werror EXTRA_CFLAGS := -Werror
lib-y := PeeCeeI.o blockops.o strlen.o strncmp.o \ lib-y := PeeCeeI.o blockops.o clear_page.o strlen.o strncmp.o \
memscan.o strncpy_from_user.o strlen_user.o memcmp.o checksum.o \ memscan.o strncpy_from_user.o strlen_user.o memcmp.o checksum.o \
VIScopy.o VISbzero.o VISmemset.o VIScsum.o VIScsumcopy.o \ VIScopy.o VISbzero.o VISmemset.o VIScsum.o VIScsumcopy.o \
VIScsumcopyusr.o VISsave.o atomic.o rwlock.o bitops.o \ VIScsumcopyusr.o VISsave.o atomic.o rwlock.o bitops.o \
......
...@@ -338,97 +338,6 @@ copy_page_using_blkcommit: ...@@ -338,97 +338,6 @@ copy_page_using_blkcommit:
stda %f16, [%o0] ASI_BLK_COMMIT_P stda %f16, [%o0] ASI_BLK_COMMIT_P
#endif #endif
.align 32
.globl _clear_page
.type _clear_page,@function
_clear_page: /* %o0=dest */
VISEntryHalf
ba,pt %xcc, clear_page_common
clr %o4
.align 32
.globl clear_user_page
.type clear_user_page,@function
clear_user_page: /* %o0=dest, %o1=vaddr */
VISEntryHalf
sethi %hi(PAGE_SIZE), %g3
sethi %uhi(PAGE_OFFSET), %g2
sllx %g2, 32, %g2
sub %o0, %g2, %g1
and %o1, %g3, %o0
mov TLB_TAG_ACCESS, %o2
sethi %uhi(_PAGE_VALID | _PAGE_SZBITS), %g3
sethi %hi(TLBTEMP_BASE), %o3
sllx %g3, 32, %g3
or %g3, (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W), %g3
or %g1, %g3, %g1
add %o0, %o3, %o0
#define FIX_INSN_2 0x96102068 /* mov (13 << 3), %o3 */
cheetah_patch_2:
mov TLBTEMP_ENT2, %o3
rdpr %pstate, %g3
wrpr %g3, PSTATE_IE, %pstate
/* Spitfire Errata #32 workaround */
mov PRIMARY_CONTEXT, %g5
stxa %g0, [%g5] ASI_DMMU
membar #Sync
ldxa [%o3] ASI_DTLB_TAG_READ, %g5
/* Spitfire Errata #32 workaround */
mov PRIMARY_CONTEXT, %g7
stxa %g0, [%g7] ASI_DMMU
membar #Sync
ldxa [%o3] ASI_DTLB_DATA_ACCESS, %g0
ldxa [%o3] ASI_DTLB_DATA_ACCESS, %g7
stxa %o0, [%o2] ASI_DMMU
stxa %g1, [%o3] ASI_DTLB_DATA_ACCESS
membar #Sync
mov 1, %o4
clear_page_common:
membar #StoreLoad | #StoreStore | #LoadStore ! LSU Group
fzero %f0 ! FPA Group
sethi %hi(PAGE_SIZE/256), %o1 ! IEU0
fzero %f2 ! FPA Group
or %o1, %lo(PAGE_SIZE/256), %o1 ! IEU0
faddd %f0, %f2, %f4 ! FPA Group
fmuld %f0, %f2, %f6 ! FPM
faddd %f0, %f2, %f8 ! FPA Group
fmuld %f0, %f2, %f10 ! FPM
faddd %f0, %f2, %f12 ! FPA Group
fmuld %f0, %f2, %f14 ! FPM
1: stda %f0, [%o0 + %g0] ASI_BLK_P ! Store Group
add %o0, 0x40, %o0 ! IEU0
stda %f0, [%o0 + %g0] ASI_BLK_P ! Store Group
add %o0, 0x40, %o0 ! IEU0
stda %f0, [%o0 + %g0] ASI_BLK_P ! Store Group
add %o0, 0x40, %o0 ! IEU0 Group
stda %f0, [%o0 + %g0] ASI_BLK_P ! Store Group
subcc %o1, 1, %o1 ! IEU1
bne,pt %icc, 1b ! CTI
add %o0, 0x40, %o0 ! IEU0 Group
membar #Sync ! LSU Group
VISExitHalf
brnz,pt %o4, 1f
nop
retl
nop
1:
stxa %g5, [%o2] ASI_DMMU
stxa %g7, [%o3] ASI_DTLB_DATA_ACCESS
membar #Sync
jmpl %o7 + 0x8, %g0
wrpr %g3, 0x0, %pstate
.globl cheetah_patch_pgcopyops .globl cheetah_patch_pgcopyops
cheetah_patch_pgcopyops: cheetah_patch_pgcopyops:
sethi %hi(FIX_INSN_1), %g1 sethi %hi(FIX_INSN_1), %g1
...@@ -437,15 +346,8 @@ cheetah_patch_pgcopyops: ...@@ -437,15 +346,8 @@ cheetah_patch_pgcopyops:
or %g2, %lo(cheetah_patch_1), %g2 or %g2, %lo(cheetah_patch_1), %g2
stw %g1, [%g2] stw %g1, [%g2]
flush %g2 flush %g2
sethi %hi(FIX_INSN_2), %g1
or %g1, %lo(FIX_INSN_2), %g1
sethi %hi(cheetah_patch_2), %g2
or %g2, %lo(cheetah_patch_2), %g2
stw %g1, [%g2]
flush %g2
retl retl
nop nop
#undef FIX_INSN1 #undef FIX_INSN_1
#undef FIX_INSN2
#undef PAGE_SIZE_REM #undef PAGE_SIZE_REM
/* clear_page.S: UltraSparc optimized clear page.
*
* Copyright (C) 1996, 1998, 1999, 2000, 2004 David S. Miller (davem@redhat.com)
* Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com)
*/
#include <asm/visasm.h>
#include <asm/thread_info.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/spitfire.h>
/* What we used to do was lock a TLB entry into a specific
* TLB slot, clear the page with interrupts disabled, then
* restore the original TLB entry. This was great for
* disturbing the TLB as little as possible, but it meant
* we had to keep interrupts disabled for a long time.
*
* Now, we simply use the normal TLB loading mechanism,
* and this makes the cpu choose a slot all by itself.
* Then we do a normal TLB flush on exit. We need only
* disable preemption during the clear.
*/
#define TTE_BITS_TOP (_PAGE_VALID | _PAGE_SZBITS)
#define TTE_BITS_BOTTOM (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W)
.text
.globl _clear_page
_clear_page: /* %o0=dest */
ba,pt %xcc, clear_page_common
clr %o4
/* This thing is pretty important, it shows up
* on the profiles via do_anonymous_page().
*/
.align 32
.globl clear_user_page
clear_user_page: /* %o0=dest, %o1=vaddr */
lduw [%g6 + TI_PRE_COUNT], %o2
sethi %uhi(PAGE_OFFSET), %g2
sethi %hi(PAGE_SIZE), %o4
sllx %g2, 32, %g2
sethi %uhi(TTE_BITS_TOP), %g3
sllx %g3, 32, %g3
sub %o0, %g2, %g1 ! paddr
or %g3, TTE_BITS_BOTTOM, %g3
and %o1, %o4, %o0 ! vaddr D-cache alias bit
or %g1, %g3, %g1 ! TTE data
sethi %hi(TLBTEMP_BASE), %o3
add %o2, 1, %o4
add %o0, %o3, %o0 ! TTE vaddr
/* Disable preemption. */
mov TLB_TAG_ACCESS, %g3
stw %o4, [%g6 + TI_PRE_COUNT]
/* Load TLB entry. */
rdpr %pstate, %o4
wrpr %o4, PSTATE_IE, %pstate
stxa %o0, [%g3] ASI_DMMU
stxa %g1, [%g0] ASI_DTLB_DATA_IN
flush %g6
wrpr %o4, 0x0, %pstate
mov 1, %o4
clear_page_common:
VISEntryHalf
membar #StoreLoad | #StoreStore | #LoadStore
fzero %f0
sethi %hi(PAGE_SIZE/64), %o1
mov %o0, %g1 ! remember vaddr for tlbflush
fzero %f2
or %o1, %lo(PAGE_SIZE/64), %o1
faddd %f0, %f2, %f4
fmuld %f0, %f2, %f6
faddd %f0, %f2, %f8
fmuld %f0, %f2, %f10
faddd %f0, %f2, %f12
fmuld %f0, %f2, %f14
1: stda %f0, [%o0 + %g0] ASI_BLK_P
subcc %o1, 1, %o1
bne,pt %icc, 1b
add %o0, 0x40, %o0
membar #Sync
VISExitHalf
brz,pn %o4, out
nop
stxa %g0, [%g1] ASI_DMMU_DEMAP
membar #Sync
stw %o2, [%g6 + TI_PRE_COUNT]
out: retl
nop
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment