Commit cf5adce1 authored by David S. Miller's avatar David S. Miller

[SPARC64]: Niagara-2 optimized copies.

The bzero/memset implementation stays the same as Niagara-1.
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 0fdb7f96
...@@ -501,7 +501,7 @@ niagara_tlb_fixup: ...@@ -501,7 +501,7 @@ niagara_tlb_fixup:
cmp %g1, SUN4V_CHIP_NIAGARA1 cmp %g1, SUN4V_CHIP_NIAGARA1
be,pt %xcc, niagara_patch be,pt %xcc, niagara_patch
cmp %g1, SUN4V_CHIP_NIAGARA2 cmp %g1, SUN4V_CHIP_NIAGARA2
be,pt %xcc, niagara_patch be,pt %xcc, niagara2_patch
nop nop
call generic_patch_copyops call generic_patch_copyops
...@@ -512,6 +512,15 @@ niagara_tlb_fixup: ...@@ -512,6 +512,15 @@ niagara_tlb_fixup:
nop nop
ba,a,pt %xcc, 80f ba,a,pt %xcc, 80f
niagara2_patch:
call niagara2_patch_copyops
nop
call niagara_patch_bzero
nop
call niagara2_patch_pageops
nop
ba,a,pt %xcc, 80f
niagara_patch: niagara_patch:
call niagara_patch_copyops call niagara_patch_copyops
......
...@@ -13,6 +13,8 @@ lib-y := PeeCeeI.o copy_page.o clear_page.o strlen.o strncmp.o \ ...@@ -13,6 +13,8 @@ lib-y := PeeCeeI.o copy_page.o clear_page.o strlen.o strncmp.o \
U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \ U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \
NGmemcpy.o NGcopy_from_user.o NGcopy_to_user.o NGpatch.o \ NGmemcpy.o NGcopy_from_user.o NGcopy_to_user.o NGpatch.o \
NGpage.o NGbzero.o \ NGpage.o NGbzero.o \
NG2memcpy.o NG2copy_from_user.o NG2copy_to_user.o NG2patch.o \
NG2page.o \
GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o GENpatch.o \ GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o GENpatch.o \
GENpage.o GENbzero.o \ GENpage.o GENbzero.o \
copy_in_user.o user_fixup.o memmove.o \ copy_in_user.o user_fixup.o memmove.o \
......
/* NG2copy_from_user.S: Niagara-2 optimized copy from userspace.
*
* Copyright (C) 2007 David S. Miller (davem@davemloft.net)
*/
#define EX_LD(x) \
98: x; \
.section .fixup; \
.align 4; \
99: wr %g0, ASI_AIUS, %asi;\
retl; \
mov 1, %o0; \
.section __ex_table,"a";\
.align 4; \
.word 98b, 99b; \
.text; \
.align 4;
#ifndef ASI_AIUS
#define ASI_AIUS 0x11
#endif
#ifndef ASI_BLK_AIUS_4V
#define ASI_BLK_AIUS_4V 0x17
#endif
#define FUNC_NAME NG2copy_from_user
#define LOAD(type,addr,dest) type##a [addr] %asi, dest
#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_AIUS_4V, dest
#define EX_RETVAL(x) 0
#ifdef __KERNEL__
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
bne,pn %icc, memcpy_user_stub; \
nop
#endif
#include "NG2memcpy.S"
/* NG2copy_to_user.S: Niagara-2 optimized copy to userspace.
*
* Copyright (C) 2007 David S. Miller (davem@davemloft.net)
*/
#define EX_ST(x) \
98: x; \
.section .fixup; \
.align 4; \
99: wr %g0, ASI_AIUS, %asi;\
retl; \
mov 1, %o0; \
.section __ex_table,"a";\
.align 4; \
.word 98b, 99b; \
.text; \
.align 4;
#ifndef ASI_AIUS
#define ASI_AIUS 0x11
#endif
#ifndef ASI_BLK_AIUS_4V
#define ASI_BLK_AIUS_4V 0x17
#endif
#ifndef ASI_BLK_INIT_QUAD_LDD_AIUS
#define ASI_BLK_INIT_QUAD_LDD_AIUS 0x23
#endif
#define FUNC_NAME NG2copy_to_user
#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS
#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_AIUS
#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS_4V
#define EX_RETVAL(x) 0
#ifdef __KERNEL__
/* Writing to %asi is _expensive_ so we hardcode it.
* Reading %asi to check for KERNEL_DS is comparatively
* cheap.
*/
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
bne,pn %icc, memcpy_user_stub; \
nop
#endif
#include "NG2memcpy.S"
This diff is collapsed.
/* NG2page.S: Niagara-2 optimized clear and copy page.
*
* Copyright (C) 2007 (davem@davemloft.net)
*/
#include <asm/asi.h>
#include <asm/page.h>
#include <asm/visasm.h>
.text
.align 32
/* This is heavily simplified from the sun4u variants
* because Niagara-2 does not have any D-cache aliasing issues.
*/
NG2copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
prefetch [%o1 + 0x00], #one_read
prefetch [%o1 + 0x40], #one_read
VISEntryHalf
set PAGE_SIZE, %g7
sub %o0, %o1, %g3
1: stxa %g0, [%o1 + %g3] ASI_BLK_INIT_QUAD_LDD_P
subcc %g7, 64, %g7
ldda [%o1] ASI_BLK_P, %f0
stda %f0, [%o1 + %g3] ASI_BLK_P
add %o1, 64, %o1
bne,pt %xcc, 1b
prefetch [%o1 + 0x40], #one_read
membar #Sync
VISExitHalf
retl
nop
#define BRANCH_ALWAYS 0x10680000
#define NOP 0x01000000
#define NG_DO_PATCH(OLD, NEW) \
sethi %hi(NEW), %g1; \
or %g1, %lo(NEW), %g1; \
sethi %hi(OLD), %g2; \
or %g2, %lo(OLD), %g2; \
sub %g1, %g2, %g1; \
sethi %hi(BRANCH_ALWAYS), %g3; \
sll %g1, 11, %g1; \
srl %g1, 11 + 2, %g1; \
or %g3, %lo(BRANCH_ALWAYS), %g3; \
or %g3, %g1, %g3; \
stw %g3, [%g2]; \
sethi %hi(NOP), %g3; \
or %g3, %lo(NOP), %g3; \
stw %g3, [%g2 + 0x4]; \
flush %g2;
.globl niagara2_patch_pageops
.type niagara2_patch_pageops,#function
niagara2_patch_pageops:
NG_DO_PATCH(copy_user_page, NG2copy_user_page)
NG_DO_PATCH(_clear_page, NGclear_page)
NG_DO_PATCH(clear_user_page, NGclear_user_page)
retl
nop
.size niagara2_patch_pageops,.-niagara2_patch_pageops
/* NG2patch.S: Patch Ultra-I routines with Niagara-2 variant.
*
* Copyright (C) 2007 David S. Miller <davem@davemloft.net>
*/
#define BRANCH_ALWAYS 0x10680000
#define NOP 0x01000000
#define NG_DO_PATCH(OLD, NEW) \
sethi %hi(NEW), %g1; \
or %g1, %lo(NEW), %g1; \
sethi %hi(OLD), %g2; \
or %g2, %lo(OLD), %g2; \
sub %g1, %g2, %g1; \
sethi %hi(BRANCH_ALWAYS), %g3; \
sll %g1, 11, %g1; \
srl %g1, 11 + 2, %g1; \
or %g3, %lo(BRANCH_ALWAYS), %g3; \
or %g3, %g1, %g3; \
stw %g3, [%g2]; \
sethi %hi(NOP), %g3; \
or %g3, %lo(NOP), %g3; \
stw %g3, [%g2 + 0x4]; \
flush %g2;
.globl niagara2_patch_copyops
.type niagara2_patch_copyops,#function
niagara2_patch_copyops:
NG_DO_PATCH(memcpy, NG2memcpy)
NG_DO_PATCH(___copy_from_user, NG2copy_from_user)
NG_DO_PATCH(___copy_to_user, NG2copy_to_user)
retl
nop
.size niagara2_patch_copyops,.-niagara2_patch_copyops
...@@ -45,6 +45,7 @@ NGcopy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ ...@@ -45,6 +45,7 @@ NGcopy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
retl retl
nop nop
.globl NGclear_page, NGclear_user_page
NGclear_page: /* %o0=dest */ NGclear_page: /* %o0=dest */
NGclear_user_page: /* %o0=dest, %o1=vaddr */ NGclear_user_page: /* %o0=dest, %o1=vaddr */
mov 8, %g1 mov 8, %g1
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment