Skip to content

L
linux
Commit a8e0e8ab authored by Linus Torvalds's avatar Linus Torvalds
Browse files
Options

Merge bk://ppc.bkbits.net/for-linus-ppc 

into ppc970.osdl.org:/home/torvalds/v2.6/linux
parents 4f449b97 74ba310d
Hide whitespace changes
Inline Side-by-side
Showing with 1008 additions and 1903 deletions
arch/arm/mach-ixp4xx/common-pci.c
View file @ a8e0e8ab
...@@ -540,4 +540,6 @@ pci_set_consistent_dma_mask(struct pci_dev *dev, u64 mask) ...@@ -540,4 +540,6 @@ pci_set_consistent_dma_mask(struct pci_dev *dev, u64 mask)
EXPORT_SYMBOL(pci_set_dma_mask); EXPORT_SYMBOL(pci_set_dma_mask);
EXPORT_SYMBOL(pci_dac_set_dma_mask); EXPORT_SYMBOL(pci_dac_set_dma_mask);
EXPORT_SYMBOL(pci_set_consistent_dma_mask); EXPORT_SYMBOL(pci_set_consistent_dma_mask);
EXPORT_SYMBOL(ixp4xx_pci_read);
EXPORT_SYMBOL(ixp4xx_pci_write);
arch/arm/mach-ixp4xx/coyote-setup.c
View file @ a8e0e8ab
...@@ -63,7 +63,7 @@ static struct resource coyote_flash_resource = { ...@@ -63,7 +63,7 @@ static struct resource coyote_flash_resource = {
.flags = IORESOURCE_MEM, .flags = IORESOURCE_MEM,
}; };
static struct platform_device coyote_flash_device = { static struct platform_device coyote_flash = {
.name = "IXP4XX-Flash", .name = "IXP4XX-Flash",
.id = 0, .id = 0,
.dev = { .dev = {
...@@ -73,9 +73,13 @@ static struct platform_device coyote_flash_device = { ...@@ -73,9 +73,13 @@ static struct platform_device coyote_flash_device = {
.resource = &coyote_flash_resource, .resource = &coyote_flash_resource,
}; };
static struct platform_device *coyote_devices[] __initdata = {
&coyote_flash
};
static void __init coyote_init(void) static void __init coyote_init(void)
{ {
platform_add_device(&coyote_flash_device); platform_add_devices(&coyote_devices, ARRAY_SIZE(coyote_devices));
} }
MACHINE_START(ADI_COYOTE, "ADI Engineering IXP4XX Coyote Development Platform") MACHINE_START(ADI_COYOTE, "ADI Engineering IXP4XX Coyote Development Platform")
......
arch/arm/mach-ixp4xx/ixdp425-setup.c
View file @ a8e0e8ab
...@@ -77,7 +77,7 @@ static struct resource ixdp425_flash_resource = { ...@@ -77,7 +77,7 @@ static struct resource ixdp425_flash_resource = {
.flags = IORESOURCE_MEM, .flags = IORESOURCE_MEM,
}; };
static struct platform_device ixdp425_flash_device = { static struct platform_device ixdp425_flash = {
.name = "IXP4XX-Flash", .name = "IXP4XX-Flash",
.id = 0, .id = 0,
.dev = { .dev = {
...@@ -101,10 +101,14 @@ static struct platform_device ixdp425_i2c_controller = { ...@@ -101,10 +101,14 @@ static struct platform_device ixdp425_i2c_controller = {
.num_resources = 0 .num_resources = 0
}; };
static struct platform_device *ixdp425_devices[] __initdata = {
&ixdp425_i2c_controller,
&ixdp425_flash
};
static void __init ixdp425_init(void) static void __init ixdp425_init(void)
{ {
platform_add_device(&ixdp425_flash_device); platform_add_devices(&ixdp425_devices, ARRAY_SIZE(ixdp425_devices));
platform_add_device(&ixdp425_i2c_controller);
} }
MACHINE_START(IXDP425, "Intel IXDP425 Development Platform") MACHINE_START(IXDP425, "Intel IXDP425 Development Platform")
......
arch/arm/mach-ixp4xx/prpmc1100-setup.c
View file @ a8e0e8ab
...@@ -63,7 +63,7 @@ static struct resource prpmc1100_flash_resource = { ...@@ -63,7 +63,7 @@ static struct resource prpmc1100_flash_resource = {
.flags = IORESOURCE_MEM, .flags = IORESOURCE_MEM,
}; };
static struct platform_device prpmc1100_flash_device = { static struct platform_device prpmc1100_flash = {
.name = "IXP4XX-Flash", .name = "IXP4XX-Flash",
.id = 0, .id = 0,
.dev = { .dev = {
...@@ -73,9 +73,13 @@ static struct platform_device prpmc1100_flash_device = { ...@@ -73,9 +73,13 @@ static struct platform_device prpmc1100_flash_device = {
.resource = &prpmc1100_flash_resource, .resource = &prpmc1100_flash_resource,
}; };
static struct platform_device *prpmc1100_devices[] __initdata = {
&prpmc1100_flash
};
static void __init prpmc1100_init(void) static void __init prpmc1100_init(void)
{ {
platform_add_device(&prpmc1100_flash_device); platform_add_devices(&prpmc1100_devices, ARRAY_SIZE(prpmc1100_devices));
} }
MACHINE_START(PRPMC1100, "Motorola PrPMC1100") MACHINE_START(PRPMC1100, "Motorola PrPMC1100")
......
arch/ia64/ia32/ia32_entry.S
View file @ a8e0e8ab
...@@ -478,8 +478,8 @@ ia32_syscall_table: ...@@ -478,8 +478,8 @@ ia32_syscall_table:
data8 compat_clock_gettime /* 265 */ data8 compat_clock_gettime /* 265 */
data8 compat_clock_getres data8 compat_clock_getres
data8 compat_clock_nanosleep data8 compat_clock_nanosleep
data8 sys_statfs64 data8 compat_statfs64
data8 sys_fstatfs64 data8 compat_fstatfs64
data8 sys_tgkill /* 270 */ data8 sys_tgkill /* 270 */
data8 compat_sys_utimes data8 compat_sys_utimes
data8 sys32_fadvise64_64 data8 sys32_fadvise64_64
......
arch/sparc64/defconfig
View file @ a8e0e8ab
...@@ -90,7 +90,6 @@ CONFIG_BINFMT_ELF32=y ...@@ -90,7 +90,6 @@ CONFIG_BINFMT_ELF32=y
# CONFIG_BINFMT_AOUT32 is not set # CONFIG_BINFMT_AOUT32 is not set
CONFIG_BINFMT_ELF=y CONFIG_BINFMT_ELF=y
CONFIG_BINFMT_MISC=m CONFIG_BINFMT_MISC=m
# CONFIG_SUNOS_EMUL is not set
CONFIG_SOLARIS_EMUL=m CONFIG_SOLARIS_EMUL=m
# #
...@@ -122,7 +121,6 @@ CONFIG_FB=y ...@@ -122,7 +121,6 @@ CONFIG_FB=y
# CONFIG_FB_CIRRUS is not set # CONFIG_FB_CIRRUS is not set
CONFIG_FB_PM2=y CONFIG_FB_PM2=y
# CONFIG_FB_PM2_FIFO_DISCONNECT is not set # CONFIG_FB_PM2_FIFO_DISCONNECT is not set
# CONFIG_FB_CYBER2000 is not set
# CONFIG_FB_ASILIANT is not set # CONFIG_FB_ASILIANT is not set
# CONFIG_FB_IMSTT is not set # CONFIG_FB_IMSTT is not set
# CONFIG_FB_BW2 is not set # CONFIG_FB_BW2 is not set
...@@ -180,7 +178,6 @@ CONFIG_LOGO_SUN_CLUT224=y ...@@ -180,7 +178,6 @@ CONFIG_LOGO_SUN_CLUT224=y
# #
# Serial drivers # Serial drivers
# #
# CONFIG_SERIAL_8250 is not set
# #
# Non-8250 serial port support # Non-8250 serial port support
...@@ -342,7 +339,6 @@ CONFIG_SCSI_SATA_SIL=m ...@@ -342,7 +339,6 @@ CONFIG_SCSI_SATA_SIL=m
CONFIG_SCSI_SATA_SIS=m CONFIG_SCSI_SATA_SIS=m
CONFIG_SCSI_SATA_VIA=m CONFIG_SCSI_SATA_VIA=m
CONFIG_SCSI_SATA_VITESSE=m CONFIG_SCSI_SATA_VITESSE=m
# CONFIG_SCSI_BUSLOGIC is not set
CONFIG_SCSI_DMX3191D=m CONFIG_SCSI_DMX3191D=m
CONFIG_SCSI_EATA_PIO=m CONFIG_SCSI_EATA_PIO=m
# CONFIG_SCSI_FUTURE_DOMAIN is not set # CONFIG_SCSI_FUTURE_DOMAIN is not set
...@@ -679,6 +675,9 @@ CONFIG_NET_DIVERT=y ...@@ -679,6 +675,9 @@ CONFIG_NET_DIVERT=y
# QoS and/or fair queueing # QoS and/or fair queueing
# #
CONFIG_NET_SCHED=y CONFIG_NET_SCHED=y
# CONFIG_NET_SCH_CLK_JIFFIES is not set
# CONFIG_NET_SCH_CLK_GETTIMEOFDAY is not set
CONFIG_NET_SCH_CLK_CPU=y
CONFIG_NET_SCH_CBQ=m CONFIG_NET_SCH_CBQ=m
CONFIG_NET_SCH_HTB=m CONFIG_NET_SCH_HTB=m
CONFIG_NET_SCH_HFSC=m CONFIG_NET_SCH_HFSC=m
...@@ -1522,11 +1521,6 @@ CONFIG_SND_VX222=m ...@@ -1522,11 +1521,6 @@ CONFIG_SND_VX222=m
CONFIG_SND_SUN_AMD7930=m CONFIG_SND_SUN_AMD7930=m
CONFIG_SND_SUN_CS4231=m CONFIG_SND_SUN_CS4231=m
#
# Open Sound System
#
# CONFIG_SOUND_PRIME is not set
# #
# USB support # USB support
# #
......
arch/sparc64/kernel/sparc64_ksyms.c
View file @ a8e0e8ab
...@@ -135,6 +135,7 @@ EXPORT_SYMBOL(__write_lock); ...@@ -135,6 +135,7 @@ EXPORT_SYMBOL(__write_lock);
EXPORT_SYMBOL(__write_unlock); EXPORT_SYMBOL(__write_unlock);
EXPORT_SYMBOL(__write_trylock); EXPORT_SYMBOL(__write_trylock);
/* Out of line spin-locking implementation. */ /* Out of line spin-locking implementation. */
EXPORT_SYMBOL(_raw_spin_lock);
EXPORT_SYMBOL(_raw_spin_lock_flags); EXPORT_SYMBOL(_raw_spin_lock_flags);
#endif #endif
...@@ -333,7 +334,6 @@ EXPORT_SYMBOL(sys_close); ...@@ -333,7 +334,6 @@ EXPORT_SYMBOL(sys_close);
#endif #endif
/* Special internal versions of library functions. */ /* Special internal versions of library functions. */
EXPORT_SYMBOL(__memcpy);
EXPORT_SYMBOL(__memset); EXPORT_SYMBOL(__memset);
EXPORT_SYMBOL(_clear_page); EXPORT_SYMBOL(_clear_page);
EXPORT_SYMBOL(clear_user_page); EXPORT_SYMBOL(clear_user_page);
......
arch/sparc64/kernel/sys_sparc32.c
View file @ a8e0e8ab
...@@ -867,138 +867,6 @@ asmlinkage long sys32_ftruncate64(unsigned int fd, unsigned long high, unsigned ...@@ -867,138 +867,6 @@ asmlinkage long sys32_ftruncate64(unsigned int fd, unsigned long high, unsigned
return sys_ftruncate(fd, (high << 32) | low); return sys_ftruncate(fd, (high << 32) | low);
} }
/* readdir & getdents */
#define NAME_OFFSET(de) ((int) ((de)->d_name - (char __user *) (de)))
#define ROUND_UP(x) (((x)+sizeof(u32)-1) & ~(sizeof(u32)-1))
struct old_linux_dirent32 {
u32 d_ino;
u32 d_offset;
unsigned short d_namlen;
char d_name[1];
};
struct readdir_callback32 {
struct old_linux_dirent32 __user * dirent;
int count;
};
static int fillonedir(void * __buf, const char * name, int namlen,
loff_t offset, ino_t ino, unsigned int d_type)
{
struct readdir_callback32 * buf = (struct readdir_callback32 *) __buf;
struct old_linux_dirent32 __user * dirent;
if (buf->count)
return -EINVAL;
buf->count++;
dirent = buf->dirent;
put_user(ino, &dirent->d_ino);
put_user(offset, &dirent->d_offset);
put_user(namlen, &dirent->d_namlen);
copy_to_user(dirent->d_name, name, namlen);
put_user(0, dirent->d_name + namlen);
return 0;
}
asmlinkage long old32_readdir(unsigned int fd, struct old_linux_dirent32 __user *dirent, unsigned int count)
{
int error = -EBADF;
struct file * file;
struct readdir_callback32 buf;
file = fget(fd);
if (!file)
goto out;
buf.count = 0;
buf.dirent = dirent;
error = vfs_readdir(file, fillonedir, &buf);
if (error < 0)
goto out_putf;
error = buf.count;
out_putf:
fput(file);
out:
return error;
}
struct linux_dirent32 {
u32 d_ino;
u32 d_off;
unsigned short d_reclen;
char d_name[1];
};
struct getdents_callback32 {
struct linux_dirent32 __user *current_dir;
struct linux_dirent32 __user *previous;
int count;
int error;
};
static int filldir(void * __buf, const char * name, int namlen, loff_t offset, ino_t ino,
unsigned int d_type)
{
struct linux_dirent32 __user * dirent;
struct getdents_callback32 * buf = (struct getdents_callback32 *) __buf;
int reclen = ROUND_UP(NAME_OFFSET(dirent) + namlen + 2);
buf->error = -EINVAL; /* only used if we fail.. */
if (reclen > buf->count)
return -EINVAL;
dirent = buf->previous;
if (dirent)
put_user(offset, &dirent->d_off);
dirent = buf->current_dir;
buf->previous = dirent;
put_user(ino, &dirent->d_ino);
put_user(reclen, &dirent->d_reclen);
copy_to_user(dirent->d_name, name, namlen);
put_user(0, dirent->d_name + namlen);
put_user(d_type, (char __user *) dirent + reclen - 1);
dirent = (void __user *) dirent + reclen;
buf->current_dir = dirent;
buf->count -= reclen;
return 0;
}
asmlinkage long sys32_getdents(unsigned int fd, struct linux_dirent32 __user *dirent, unsigned int count)
{
struct file * file;
struct linux_dirent32 __user *lastdirent;
struct getdents_callback32 buf;
int error = -EBADF;
file = fget(fd);
if (!file)
goto out;
buf.current_dir = dirent;
buf.previous = NULL;
buf.count = count;
buf.error = 0;
error = vfs_readdir(file, filldir, &buf);
if (error < 0)
goto out_putf;
lastdirent = buf.previous;
error = buf.error;
if (lastdirent) {
put_user(file->f_pos, &lastdirent->d_off);
error = count - buf.count;
}
out_putf:
fput(file);
out:
return error;
}
/* end of readdir & getdents */
int cp_compat_stat(struct kstat *stat, struct compat_stat __user *statbuf) int cp_compat_stat(struct kstat *stat, struct compat_stat __user *statbuf)
{ {
int err; int err;
......
arch/sparc64/kernel/systbls.S
View file @ a8e0e8ab
...@@ -54,13 +54,13 @@ sys_call_table32: ...@@ -54,13 +54,13 @@ sys_call_table32:
.word compat_sys_fcntl64, sys_ni_syscall, compat_sys_statfs, compat_sys_fstatfs, sys_oldumount .word compat_sys_fcntl64, sys_ni_syscall, compat_sys_statfs, compat_sys_fstatfs, sys_oldumount
/*160*/ .word compat_sys_sched_setaffinity, compat_sys_sched_getaffinity, sys32_getdomainname, sys32_setdomainname, sys_nis_syscall /*160*/ .word compat_sys_sched_setaffinity, compat_sys_sched_getaffinity, sys32_getdomainname, sys32_setdomainname, sys_nis_syscall
.word sys_quotactl, sys_set_tid_address, compat_sys_mount, sys_ustat, sys32_setxattr .word sys_quotactl, sys_set_tid_address, compat_sys_mount, sys_ustat, sys32_setxattr
/*170*/ .word sys32_lsetxattr, sys32_fsetxattr, sys_getxattr, sys_lgetxattr, sys32_getdents /*170*/ .word sys32_lsetxattr, sys32_fsetxattr, sys_getxattr, sys_lgetxattr, compat_sys_getdents
.word sys_setsid, sys_fchdir, sys32_fgetxattr, sys_listxattr, sys_llistxattr .word sys_setsid, sys_fchdir, sys32_fgetxattr, sys_listxattr, sys_llistxattr
/*180*/ .word sys32_flistxattr, sys_removexattr, sys_lremovexattr, compat_sys_sigpending, sys_ni_syscall /*180*/ .word sys32_flistxattr, sys_removexattr, sys_lremovexattr, compat_sys_sigpending, sys_ni_syscall
.word sys32_setpgid, sys32_fremovexattr, sys32_tkill, sys32_exit_group, sparc64_newuname .word sys32_setpgid, sys32_fremovexattr, sys32_tkill, sys32_exit_group, sparc64_newuname
/*190*/ .word sys32_init_module, sparc64_personality, sys_remap_file_pages, sys32_epoll_create, sys32_epoll_ctl /*190*/ .word sys32_init_module, sparc64_personality, sys_remap_file_pages, sys32_epoll_create, sys32_epoll_ctl
.word sys32_epoll_wait, sys_nis_syscall, sys_getppid, sys32_sigaction, sys_sgetmask .word sys32_epoll_wait, sys_nis_syscall, sys_getppid, sys32_sigaction, sys_sgetmask
/*200*/ .word sys32_ssetmask, sys_sigsuspend, compat_sys_newlstat, sys_uselib, old32_readdir /*200*/ .word sys32_ssetmask, sys_sigsuspend, compat_sys_newlstat, sys_uselib, compat_old_readdir
.word sys32_readahead, sys32_socketcall, sys32_syslog, sys32_lookup_dcookie, sys32_fadvise64 .word sys32_readahead, sys32_socketcall, sys32_syslog, sys32_lookup_dcookie, sys32_fadvise64
/*210*/ .word sys32_fadvise64_64, sys32_tgkill, sys32_waitpid, sys_swapoff, sys32_sysinfo /*210*/ .word sys32_fadvise64_64, sys32_tgkill, sys32_waitpid, sys_swapoff, sys32_sysinfo
.word sys32_ipc, sys32_sigreturn, sys_clone, sys_nis_syscall, sys32_adjtimex .word sys32_ipc, sys32_sigreturn, sys_clone, sys_nis_syscall, sys32_adjtimex
......
arch/sparc64/lib/U3copy_from_user.S
View file @ a8e0e8ab
/* $Id: U3copy_from_user.S,v 1.4 2002/01/15 07:16:26 davem Exp $ /* U3copy_from_user.S: UltraSparc-III optimized copy from userspace.
* U3memcpy.S: UltraSparc-III optimized copy from userspace.
* *
* Copyright (C) 1999, 2000 David S. Miller (davem@redhat.com) * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
*/ */
#ifdef __KERNEL__
#include <asm/visasm.h> #include <asm/visasm.h>
#include <asm/asi.h> #include <asm/asi.h>
#include <asm/dcu.h> #include <asm/dcu.h>
#include <asm/spitfire.h> #include <asm/spitfire.h>
#undef SMALL_COPY_USES_FPU
#define XCC xcc
#define EXNV_RAW(x,y,a,b) \
98: x,y; \
.section .fixup; \
.align 4; \
99: ba U3cfu_fixup; \
a, b, %o1; \
.section __ex_table; \
.align 4; \
.word 98b, 99b; \
.text; \
.align 4;
#define EXNV(x,y,a,b) \ #define EXNV(x,y,a,b) \
98: x,y; \ 98: x,y; \
.section .fixup; \ .section .fixup; \
.align 4; \ .align 4; \
99: VISExitHalf; \ 99: add %o1, %o3, %o0; \
ba U3cfu_fixup; \ ba U3cfu_fixup; \
a, b, %o1; \ a, b, %o1; \
.section __ex_table; \ .section __ex_table; \
...@@ -22,6 +33,32 @@ ...@@ -22,6 +33,32 @@
.word 98b, 99b; \ .word 98b, 99b; \
.text; \ .text; \
.align 4; .align 4;
#define EXNV4(x,y,a,b) \
98: x,y; \
.section .fixup; \
.align 4; \
99: add %o1, %o3, %o0; \
a, b, %o1; \
ba U3cfu_fixup; \
add %o1, 4, %o1; \
.section __ex_table; \
.align 4; \
.word 98b, 99b; \
.text; \
.align 4;
#define EXNV8(x,y,a,b) \
98: x,y; \
.section .fixup; \
.align 4; \
99: add %o1, %o3, %o0; \
a, b, %o1; \
ba U3cfu_fixup; \
add %o1, 8, %o1; \
.section __ex_table; \
.align 4; \
.word 98b, 99b; \
.text; \
.align 4;
#define EX(x,y,a,b) \ #define EX(x,y,a,b) \
98: x,y; \ 98: x,y; \
.section .fixup; \ .section .fixup; \
...@@ -77,18 +114,9 @@ ...@@ -77,18 +114,9 @@
.word 98b, 99b; \ .word 98b, 99b; \
.text; \ .text; \
.align 4; .align 4;
#else
#define ASI_BLK_P 0xf0 .register %g2,#scratch
#define FPRS_FEF 0x04 .register %g3,#scratch
#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
#define SMALL_COPY_USES_FPU
#define EXNV(x,y,a,b) x,y;
#define EX(x,y,a,b) x,y;
#define EX2(x,y) x,y;
#define EX3(x,y) x,y;
#define EX4(x,y) x,y;
#endif
/* Special/non-trivial issues of this code: /* Special/non-trivial issues of this code:
* *
...@@ -109,81 +137,55 @@ ...@@ -109,81 +137,55 @@
* of up to 2.4GB per second. * of up to 2.4GB per second.
*/ */
.globl U3copy_from_user .globl U3copy_from_user
U3copy_from_user: /* %o0=dst, %o1=src, %o2=len */ U3copy_from_user: /* %o0=dst, %o1=src, %o2=len */
#ifndef __KERNEL__ cmp %o2, 0
/* Save away original 'dst' for memcpy return value. */ be,pn %XCC, out
mov %o0, %g3 ! A0 Group or %o0, %o1, %o3
#endif cmp %o2, 16
/* Anything to copy at all? */ bleu,a,pn %XCC, small_copy
cmp %o2, 0 ! A1 or %o3, %o2, %o3
ble,pn %icc, U3copy_from_user_short_ret! BR
cmp %o2, 256
/* Extremely small copy? */ blu,pt %XCC, medium_copy
cmp %o2, 31 ! A0 Group andcc %o3, 0x7, %g0
ble,pn %icc, U3copy_from_user_short ! BR
ba,pt %xcc, enter
/* Large enough to use unrolled prefetch loops? */ andcc %o0, 0x3f, %g2
cmp %o2, 0x100 ! A1
bge,a,pt %icc, U3copy_from_user_enter ! BR Group /* Here len >= 256 and condition codes reflect execution
andcc %o0, 0x3f, %g2 ! A0
ba,pt %xcc, U3copy_from_user_toosmall ! BR Group
andcc %o0, 0x7, %g2 ! A0
.align 32
U3copy_from_user_short:
/* Copy %o2 bytes from src to dst, one byte at a time. */
EXNV(lduba [%o1 + 0x00] %asi, %o3, add %o2, %g0)! MS Group
add %o1, 0x1, %o1 ! A0
add %o0, 0x1, %o0 ! A1
subcc %o2, 1, %o2 ! A0 Group
bg,pt %icc, U3copy_from_user_short ! BR
stb %o3, [%o0 + -1] ! MS Group (1-cycle stall)
U3copy_from_user_short_ret:
#ifdef __KERNEL__
retl ! BR Group (0-4 cycle stall)
clr %o0 ! A0
#else
retl ! BR Group (0-4 cycle stall)
mov %g3, %o0 ! A0
#endif
/* Here len >= (6 * 64) and condition codes reflect execution
* of "andcc %o0, 0x7, %g2", done by caller. * of "andcc %o0, 0x7, %g2", done by caller.
*/ */
.align 64 .align 64
U3copy_from_user_enter: enter:
/* Is 'dst' already aligned on an 64-byte boundary? */ /* Is 'dst' already aligned on an 64-byte boundary? */
be,pt %xcc, 2f ! BR be,pt %XCC, 2f
/* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number /* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number
* of bytes to copy to make 'dst' 64-byte aligned. We pre- * of bytes to copy to make 'dst' 64-byte aligned. We pre-
* subtract this from 'len'. * subtract this from 'len'.
*/ */
sub %g2, 0x40, %g2 ! A0 Group sub %g2, 0x40, %g2
sub %g0, %g2, %g2 ! A0 Group sub %g0, %g2, %g2
sub %o2, %g2, %o2 ! A0 Group sub %o2, %g2, %o2
/* Copy %g2 bytes from src to dst, one byte at a time. */ /* Copy %g2 bytes from src to dst, one byte at a time. */
1: EXNV(lduba [%o1 + 0x00] %asi, %o3, add %o2, %g2)! MS (Group) 1: EXNV_RAW(lduba [%o1 + 0x00] %asi, %o3, add %o2, %g2)
add %o1, 0x1, %o1 ! A1 add %o1, 0x1, %o1
add %o0, 0x1, %o0 ! A0 Group add %o0, 0x1, %o0
subcc %g2, 0x1, %g2 ! A1 subcc %g2, 0x1, %g2
bg,pt %icc, 1b ! BR Group bg,pt %XCC, 1b
stb %o3, [%o0 + -1] ! MS Group stb %o3, [%o0 + -1]
2: VISEntryHalf ! MS+MS 2: VISEntryHalf
and %o1, 0x7, %g1 ! A1 and %o1, 0x7, %g1
ba,pt %xcc, U3copy_from_user_begin ! BR ba,pt %xcc, begin
alignaddr %o1, %g0, %o1 ! MS (Break-after) alignaddr %o1, %g0, %o1
.align 64 .align 64
U3copy_from_user_begin: begin:
#ifdef __KERNEL__
.globl U3copy_from_user_nop_1_6 .globl U3copy_from_user_nop_1_6
U3copy_from_user_nop_1_6: U3copy_from_user_nop_1_6:
ldxa [%g0] ASI_DCU_CONTROL_REG, %g3 ldxa [%g0] ASI_DCU_CONTROL_REG, %g3
...@@ -192,315 +194,225 @@ U3copy_from_user_nop_1_6: ...@@ -192,315 +194,225 @@ U3copy_from_user_nop_1_6:
or %g3, %o3, %o3 or %g3, %o3, %o3
stxa %o3, [%g0] ASI_DCU_CONTROL_REG ! Enable P-cache stxa %o3, [%g0] ASI_DCU_CONTROL_REG ! Enable P-cache
membar #Sync membar #Sync
#endif
prefetcha [%o1 + 0x000] %asi, #one_read ! MS Group1 prefetcha [%o1 + 0x000] %asi, #one_read
prefetcha [%o1 + 0x040] %asi, #one_read ! MS Group2 prefetcha [%o1 + 0x040] %asi, #one_read
andn %o2, (0x40 - 1), %o4 ! A0 andn %o2, (0x40 - 1), %o4
prefetcha [%o1 + 0x080] %asi, #one_read ! MS Group3 prefetcha [%o1 + 0x080] %asi, #one_read
cmp %o4, 0x140 ! A0 prefetcha [%o1 + 0x0c0] %asi, #one_read
prefetcha [%o1 + 0x0c0] %asi, #one_read ! MS Group4 EX(ldda [%o1 + 0x000] %asi, %f0, add %o2, %g0)
EX(ldda [%o1 + 0x000] %asi, %f0, add %o2, %g0) ! MS Group5 (%f0 results at G8) prefetcha [%o1 + 0x100] %asi, #one_read
bge,a,pt %icc, 1f ! BR EX(ldda [%o1 + 0x008] %asi, %f2, add %o2, %g0)
prefetcha [%o1 + 0x140] %asi, #one_read
prefetcha [%o1 + 0x100] %asi, #one_read ! MS Group6 EX(ldda [%o1 + 0x010] %asi, %f4, add %o2, %g0)
1: EX(ldda [%o1 + 0x008] %asi, %f2, add %o2, %g0) ! AX (%f2 results at G9) prefetcha [%o1 + 0x180] %asi, #one_read
cmp %o4, 0x180 ! A1 faligndata %f0, %f2, %f16
bge,a,pt %icc, 1f ! BR EX(ldda [%o1 + 0x018] %asi, %f6, add %o2, %g0)
prefetcha [%o1 + 0x140] %asi, #one_read ! MS Group7 faligndata %f2, %f4, %f18
1: EX(ldda [%o1 + 0x010] %asi, %f4, add %o2, %g0) ! AX (%f4 results at G10) EX(ldda [%o1 + 0x020] %asi, %f8, add %o2, %g0)
cmp %o4, 0x1c0 ! A1 faligndata %f4, %f6, %f20
bge,a,pt %icc, 1f ! BR EX(ldda [%o1 + 0x028] %asi, %f10, add %o2, %g0)
faligndata %f6, %f8, %f22
prefetcha [%o1 + 0x180] %asi, #one_read ! MS Group8
1: faligndata %f0, %f2, %f16 ! FGA Group9 (%f16 at G12) EX(ldda [%o1 + 0x030] %asi, %f12, add %o2, %g0)
EX(ldda [%o1 + 0x018] %asi, %f6, add %o2, %g0) ! AX (%f6 results at G12) faligndata %f8, %f10, %f24
faligndata %f2, %f4, %f18 ! FGA Group10 (%f18 results at G13) EX(ldda [%o1 + 0x038] %asi, %f14, add %o2, %g0)
EX(ldda [%o1 + 0x020] %asi, %f8, add %o2, %g0) ! MS (%f8 results at G13) faligndata %f10, %f12, %f26
faligndata %f4, %f6, %f20 ! FGA Group12 (1-cycle stall,%f20 at G15) EX(ldda [%o1 + 0x040] %asi, %f0, add %o2, %g0)
EX(ldda [%o1 + 0x028] %asi, %f10, add %o2, %g0) ! MS (%f10 results at G15)
faligndata %f6, %f8, %f22 ! FGA Group13 (%f22 results at G16) sub %o4, 0x80, %o4
add %o1, 0x40, %o1
EX(ldda [%o1 + 0x030] %asi, %f12, add %o2, %g0) ! MS (%f12 results at G16) ba,pt %xcc, loop
faligndata %f8, %f10, %f24 ! FGA Group15 (1-cycle stall,%f24 at G18) srl %o4, 6, %o3
EX(ldda [%o1 + 0x038] %asi, %f14, add %o2, %g0) ! MS (%f14 results at G18)
faligndata %f10, %f12, %f26 ! FGA Group16 (%f26 results at G19) .align 64
EX(ldda [%o1 + 0x040] %asi, %f0, add %o2, %g0) ! MS (%f0 results at G19) loop:
EX3(ldda [%o1 + 0x008] %asi, %f2)
/* We only use the first loop if len > (7 * 64). */ faligndata %f12, %f14, %f28
subcc %o4, 0x1c0, %o4 ! A0 Group17 EX3(ldda [%o1 + 0x010] %asi, %f4)
bg,pt %icc, U3copy_from_user_loop1 ! BR faligndata %f14, %f0, %f30
add %o1, 0x40, %o1 ! A1 stda %f16, [%o0] ASI_BLK_P
EX3(ldda [%o1 + 0x018] %asi, %f6)
add %o4, 0x140, %o4 ! A0 Group18 faligndata %f0, %f2, %f16
ba,pt %xcc, U3copy_from_user_loop2 ! BR
srl %o4, 6, %o3 ! A0 Group19 EX3(ldda [%o1 + 0x020] %asi, %f8)
nop faligndata %f2, %f4, %f18
nop EX3(ldda [%o1 + 0x028] %asi, %f10)
nop faligndata %f4, %f6, %f20
nop EX3(ldda [%o1 + 0x030] %asi, %f12)
nop faligndata %f6, %f8, %f22
EX3(ldda [%o1 + 0x038] %asi, %f14)
nop faligndata %f8, %f10, %f24
nop
EX3(ldda [%o1 + 0x040] %asi, %f0)
/* This loop performs the copy and queues new prefetches. prefetcha [%o1 + 0x180] %asi, #one_read
* We drop into the second loop when len <= (5 * 64). Note faligndata %f10, %f12, %f26
* that this (5 * 64) factor has been subtracted from len subcc %o3, 0x01, %o3
* already. add %o1, 0x40, %o1
*/ bg,pt %XCC, loop
U3copy_from_user_loop1: add %o0, 0x40, %o0
EX2(ldda [%o1 + 0x008] %asi, %f2) ! MS Group2 (%f2 results at G5)
faligndata %f12, %f14, %f28 ! FGA (%f28 results at G5)
EX2(ldda [%o1 + 0x010] %asi, %f4) ! MS Group3 (%f4 results at G6)
faligndata %f14, %f0, %f30 ! FGA Group4 (1-cycle stall, %f30 at G7)
stda %f16, [%o0] ASI_BLK_P ! MS
EX2(ldda [%o1 + 0x018] %asi, %f6) ! AX (%f6 results at G7)
faligndata %f0, %f2, %f16 ! FGA Group12 (7-cycle stall)
EX2(ldda [%o1 + 0x020] %asi, %f8) ! MS (%f8 results at G15)
faligndata %f2, %f4, %f18 ! FGA Group13 (%f18 results at G16)
EX2(ldda [%o1 + 0x028] %asi, %f10) ! MS (%f10 results at G16)
faligndata %f4, %f6, %f20 ! FGA Group14 (%f20 results at G17)
EX2(ldda [%o1 + 0x030] %asi, %f12) ! MS (%f12 results at G17)
faligndata %f6, %f8, %f22 ! FGA Group15 (%f22 results at G18)
EX2(ldda [%o1 + 0x038] %asi, %f14) ! MS (%f14 results at G18)
faligndata %f8, %f10, %f24 ! FGA Group16 (%f24 results at G19)
EX2(ldda [%o1 + 0x040] %asi, %f0) ! AX (%f0 results at G19)
prefetcha [%o1 + 0x180] %asi, #one_read ! MS
faligndata %f10, %f12, %f26 ! FGA Group17 (%f26 results at G20)
subcc %o4, 0x40, %o4 ! A0
add %o1, 0x40, %o1 ! A1
bg,pt %xcc, U3copy_from_user_loop1 ! BR
add %o0, 0x40, %o0 ! A0 Group18
U3copy_from_user_loop2_enter:
mov 5, %o3 ! A1
/* This loop performs on the copy, no new prefetches are
* queued. We do things this way so that we do not perform
* any spurious prefetches past the end of the src buffer.
*/
U3copy_from_user_loop2:
EX3(ldda [%o1 + 0x008] %asi, %f2) ! MS
faligndata %f12, %f14, %f28 ! FGA Group2
EX3(ldda [%o1 + 0x010] %asi, %f4) ! MS
faligndata %f14, %f0, %f30 ! FGA Group4 (1-cycle stall)
stda %f16, [%o0] ASI_BLK_P ! MS
EX3(ldda [%o1 + 0x018] %asi, %f6) ! AX
faligndata %f0, %f2, %f16 ! FGA Group12 (7-cycle stall)
EX3(ldda [%o1 + 0x020] %asi, %f8) ! MS
faligndata %f2, %f4, %f18 ! FGA Group13
EX3(ldda [%o1 + 0x028] %asi, %f10) ! MS
faligndata %f4, %f6, %f20 ! FGA Group14
EX3(ldda [%o1 + 0x030] %asi, %f12) ! MS
faligndata %f6, %f8, %f22 ! FGA Group15
EX3(ldda [%o1 + 0x038] %asi, %f14) ! MS
faligndata %f8, %f10, %f24 ! FGA Group16
EX3(ldda [%o1 + 0x040] %asi, %f0) ! AX
faligndata %f10, %f12, %f26 ! FGA Group17
subcc %o3, 0x01, %o3 ! A0
add %o1, 0x40, %o1 ! A1
bg,pt %xcc, U3copy_from_user_loop2 ! BR
add %o0, 0x40, %o0 ! A0 Group18
/* Finally we copy the last full 64-byte block. */ /* Finally we copy the last full 64-byte block. */
U3copy_from_user_loopfini: loopfini:
EX3(ldda [%o1 + 0x008] %asi, %f2) ! MS EX3(ldda [%o1 + 0x008] %asi, %f2)
faligndata %f12, %f14, %f28 ! FGA faligndata %f12, %f14, %f28
EX3(ldda [%o1 + 0x010] %asi, %f4) ! MS Group19 EX3(ldda [%o1 + 0x010] %asi, %f4)
faligndata %f14, %f0, %f30 ! FGA faligndata %f14, %f0, %f30
stda %f16, [%o0] ASI_BLK_P ! MS Group20 stda %f16, [%o0] ASI_BLK_P
EX3(ldda [%o1 + 0x018] %asi, %f6) ! AX EX3(ldda [%o1 + 0x018] %asi, %f6)
faligndata %f0, %f2, %f16 ! FGA Group11 (7-cycle stall) faligndata %f0, %f2, %f16
EX3(ldda [%o1 + 0x020] %asi, %f8) ! MS EX3(ldda [%o1 + 0x020] %asi, %f8)
faligndata %f2, %f4, %f18 ! FGA Group12 faligndata %f2, %f4, %f18
EX3(ldda [%o1 + 0x028] %asi, %f10) ! MS EX3(ldda [%o1 + 0x028] %asi, %f10)
faligndata %f4, %f6, %f20 ! FGA Group13 faligndata %f4, %f6, %f20
EX3(ldda [%o1 + 0x030] %asi, %f12) ! MS EX3(ldda [%o1 + 0x030] %asi, %f12)
faligndata %f6, %f8, %f22 ! FGA Group14 faligndata %f6, %f8, %f22
EX3(ldda [%o1 + 0x038] %asi, %f14) ! MS EX3(ldda [%o1 + 0x038] %asi, %f14)
faligndata %f8, %f10, %f24 ! FGA Group15 faligndata %f8, %f10, %f24
cmp %g1, 0 ! A0 cmp %g1, 0
be,pt %icc, 1f ! BR be,pt %XCC, 1f
add %o0, 0x40, %o0 ! A1 add %o0, 0x40, %o0
EX4(ldda [%o1 + 0x040] %asi, %f0) ! MS EX4(ldda [%o1 + 0x040] %asi, %f0)
1: faligndata %f10, %f12, %f26 ! FGA Group16 1: faligndata %f10, %f12, %f26
faligndata %f12, %f14, %f28 ! FGA Group17 faligndata %f12, %f14, %f28
faligndata %f14, %f0, %f30 ! FGA Group18 faligndata %f14, %f0, %f30
stda %f16, [%o0] ASI_BLK_P ! MS stda %f16, [%o0] ASI_BLK_P
add %o0, 0x40, %o0 ! A0 add %o0, 0x40, %o0
add %o1, 0x40, %o1 ! A1 add %o1, 0x40, %o1
#ifdef __KERNEL__
.globl U3copy_from_user_nop_2_3 .globl U3copy_from_user_nop_2_3
U3copy_from_user_nop_2_3: U3copy_from_user_nop_2_3:
mov PRIMARY_CONTEXT, %o3 mov PRIMARY_CONTEXT, %o3
stxa %g0, [%o3] ASI_DMMU ! Flush P-cache stxa %g0, [%o3] ASI_DMMU ! Flush P-cache
stxa %g3, [%g0] ASI_DCU_CONTROL_REG ! Disable P-cache stxa %g3, [%g0] ASI_DCU_CONTROL_REG ! Disable P-cache
#endif
membar #Sync ! MS Group26 (7-cycle stall) membar #Sync
/* Now we copy the (len modulo 64) bytes at the end. /* Now we copy the (len modulo 64) bytes at the end.
* Note how we borrow the %f0 loaded above. * Note how we borrow the %f0 loaded above.
* *
* Also notice how this code is careful not to perform a * Also notice how this code is careful not to perform a
* load past the end of the src buffer just like similar * load past the end of the src buffer.
* code found in U3copy_from_user_toosmall processing.
*/ */
U3copy_from_user_loopend: loopend:
and %o2, 0x3f, %o2 ! A0 Group and %o2, 0x3f, %o2
andcc %o2, 0x38, %g2 ! A0 Group andcc %o2, 0x38, %g2
be,pn %icc, U3copy_from_user_endcruft ! BR be,pn %XCC, endcruft
subcc %g2, 0x8, %g2 ! A1 subcc %g2, 0x8, %g2
be,pn %icc, U3copy_from_user_endcruft ! BR Group be,pn %XCC, endcruft
cmp %g1, 0 ! A0 cmp %g1, 0
be,a,pt %icc, 1f ! BR Group be,a,pt %XCC, 1f
EX(ldda [%o1 + 0x00] %asi, %f0, add %o2, %g0) ! MS EX(ldda [%o1 + 0x00] %asi, %f0, add %o2, %g0)
1: EX(ldda [%o1 + 0x08] %asi, %f2, add %o2, %g0) ! MS Group 1: EX(ldda [%o1 + 0x08] %asi, %f2, add %o2, %g0)
add %o1, 0x8, %o1 ! A0 add %o1, 0x8, %o1
sub %o2, 0x8, %o2 ! A1 sub %o2, 0x8, %o2
subcc %g2, 0x8, %g2 ! A0 Group subcc %g2, 0x8, %g2
faligndata %f0, %f2, %f8 ! FGA Group faligndata %f0, %f2, %f8
std %f8, [%o0 + 0x00] ! MS (XXX does it stall here? XXX) std %f8, [%o0 + 0x00]
be,pn %icc, U3copy_from_user_endcruft ! BR be,pn %XCC, endcruft
add %o0, 0x8, %o0 ! A0 add %o0, 0x8, %o0
EX(ldda [%o1 + 0x08] %asi, %f0, add %o2, %g0) ! MS Group EX(ldda [%o1 + 0x08] %asi, %f0, add %o2, %g0)
add %o1, 0x8, %o1 ! A0 add %o1, 0x8, %o1
sub %o2, 0x8, %o2 ! A1 sub %o2, 0x8, %o2
subcc %g2, 0x8, %g2 ! A0 Group subcc %g2, 0x8, %g2
faligndata %f2, %f0, %f8 ! FGA faligndata %f2, %f0, %f8
std %f8, [%o0 + 0x00] ! MS (XXX does it stall here? XXX) std %f8, [%o0 + 0x00]
bne,pn %icc, 1b ! BR bne,pn %XCC, 1b
add %o0, 0x8, %o0 ! A0 Group add %o0, 0x8, %o0
/* If anything is left, we copy it one byte at a time. /* If anything is left, we copy it one byte at a time.
* Note that %g1 is (src & 0x3) saved above before the * Note that %g1 is (src & 0x3) saved above before the
* alignaddr was performed. * alignaddr was performed.
*/ */
U3copy_from_user_endcruft: endcruft:
cmp %o2, 0 cmp %o2, 0
add %o1, %g1, %o1 add %o1, %g1, %o1
VISExitHalf VISExitHalf
be,pn %icc, U3copy_from_user_short_ret be,pn %XCC, out
nop sub %o0, %o1, %o3
ba,a,pt %xcc, U3copy_from_user_short
/* If we get here, then 32 <= len < (6 * 64) */
U3copy_from_user_toosmall:
#ifdef SMALL_COPY_USES_FPU
/* Is 'dst' already aligned on an 8-byte boundary? */
be,pt %xcc, 2f ! BR Group
/* Compute abs((dst & 7) - 8) into %g2. This is the number
* of bytes to copy to make 'dst' 8-byte aligned. We pre-
* subtract this from 'len'.
*/
sub %g2, 0x8, %g2 ! A0
sub %g0, %g2, %g2 ! A0 Group (reg-dep)
sub %o2, %g2, %o2 ! A0 Group (reg-dep)
/* Copy %g2 bytes from src to dst, one byte at a time. */ andcc %g1, 0x7, %g0
1: EXNV(lduba [%o1 + 0x00] %asi, %o3, add %o2, %g2)! MS (Group) (%o3 in 3 cycles) bne,pn %icc, small_copy_unaligned
add %o1, 0x1, %o1 ! A1 andcc %o2, 0x8, %g0
add %o0, 0x1, %o0 ! A0 Group be,pt %icc, 1f
subcc %g2, 0x1, %g2 ! A1 nop
EXNV(ldxa [%o1] %asi, %o5, add %o2, %g0)
bg,pt %icc, 1b ! BR Group stx %o5, [%o1 + %o3]
stb %o3, [%o0 + -1] ! MS Group add %o1, 0x8, %o1
2: VISEntryHalf ! MS+MS 1: andcc %o2, 0x4, %g0
be,pt %icc, 1f
nop
EXNV(lduwa [%o1] %asi, %o5, and %o2, 0x7)
stw %o5, [%o1 + %o3]
add %o1, 0x4, %o1
/* Compute (len - (len % 8)) into %g2. This is guaranteed 1: andcc %o2, 0x2, %g0
* to be nonzero. be,pt %icc, 1f
*/ nop
andn %o2, 0x7, %g2 ! A0 Group EXNV(lduha [%o1] %asi, %o5, and %o2, 0x3)
sth %o5, [%o1 + %o3]
/* You may read this and believe that it allows reading add %o1, 0x2, %o1
* one 8-byte longword past the end of src. It actually
* does not, as %g2 is subtracted as loads are done from
* src, so we always stop before running off the end.
* Also, we are guaranteed to have at least 0x10 bytes
* to move here.
*/
sub %g2, 0x8, %g2 ! A0 Group (reg-dep)
alignaddr %o1, %g0, %g1 ! MS (Break-after)
EX(ldda [%g1 + 0x00] %asi, %f0, add %o2, %g0) ! MS Group (1-cycle stall)
add %g1, 0x8, %g1 ! A0
1: EX(ldda [%g1 + 0x00] %asi, %f2, add %o2, %g0) ! MS Group
add %g1, 0x8, %g1 ! A0
sub %o2, 0x8, %o2 ! A1
subcc %g2, 0x8, %g2 ! A0 Group
faligndata %f0, %f2, %f8 ! FGA Group (1-cycle stall)
std %f8, [%o0 + 0x00] ! MS Group (2-cycle stall)
add %o1, 0x8, %o1 ! A0
be,pn %icc, 2f ! BR
add %o0, 0x8, %o0 ! A1
EX(ldda [%g1 + 0x00] %asi, %f0, add %o2, %g0) ! MS Group
add %g1, 0x8, %g1 ! A0
sub %o2, 0x8, %o2 ! A1
subcc %g2, 0x8, %g2 ! A0 Group
faligndata %f2, %f0, %f8 ! FGA Group (1-cycle stall)
std %f8, [%o0 + 0x00] ! MS Group (2-cycle stall)
add %o1, 0x8, %o1 ! A0
bne,pn %icc, 1b ! BR
add %o0, 0x8, %o0 ! A1
/* Nothing left to copy? */
2: cmp %o2, 0 ! A0 Group
VISExitHalf ! A0+MS
be,pn %icc, U3copy_from_user_short_ret! BR Group
nop ! A0
ba,a,pt %xcc, U3copy_from_user_short ! BR Group
#else /* !(SMALL_COPY_USES_FPU) */
xor %o1, %o0, %g2
andcc %g2, 0x7, %g0
bne,pn %icc, U3copy_from_user_short
andcc %o1, 0x7, %g2
be,pt %xcc, 2f
sub %g2, 0x8, %g2
sub %g0, %g2, %g2
sub %o2, %g2, %o2
1: EXNV(lduba [%o1 + 0x00] %asi, %o3, add %o2, %g2) 1: andcc %o2, 0x1, %g0
add %o1, 0x1, %o1 be,pt %icc, out
add %o0, 0x1, %o0 nop
subcc %g2, 0x1, %g2 EXNV(lduba [%o1] %asi, %o5, and %o2, 0x1)
bg,pt %icc, 1b ba,pt %xcc, out
stb %o3, [%o0 + -1] stb %o5, [%o1 + %o3]
medium_copy: /* 16 < len <= 64 */
bne,pn %XCC, small_copy_unaligned
sub %o0, %o1, %o3
medium_copy_aligned:
andn %o2, 0x7, %o4
and %o2, 0x7, %o2
1: subcc %o4, 0x8, %o4
EXNV8(ldxa [%o1] %asi, %o5, add %o2, %o4)
stx %o5, [%o1 + %o3]
bgu,pt %XCC, 1b
add %o1, 0x8, %o1
andcc %o2, 0x4, %g0
be,pt %XCC, 1f
nop
sub %o2, 0x4, %o2
EXNV4(lduwa [%o1] %asi, %o5, add %o2, %g0)
stw %o5, [%o1 + %o3]
add %o1, 0x4, %o1
1: cmp %o2, 0
be,pt %XCC, out
nop
ba,pt %xcc, small_copy_unaligned
nop
2: andn %o2, 0x7, %g2 small_copy: /* 0 < len <= 16 */
sub %o2, %g2, %o2 andcc %o3, 0x3, %g0
bne,pn %XCC, small_copy_unaligned
sub %o0, %o1, %o3
3: EXNV(ldxa [%o1 + 0x00] %asi, %o3, add %o2, %g2) small_copy_aligned:
add %o1, 0x8, %o1 subcc %o2, 4, %o2
add %o0, 0x8, %o0 EXNV(lduwa [%o1] %asi, %g1, add %o2, %g0)
subcc %g2, 0x8, %g2 stw %g1, [%o1 + %o3]
bg,pt %icc, 3b bgu,pt %XCC, small_copy_aligned
stx %o3, [%o0 + -8] add %o1, 4, %o1
cmp %o2, 0 out: retl
bne,pn %icc, U3copy_from_user_short clr %o0
nop
ba,a,pt %xcc, U3copy_from_user_short_ret
#endif /* !(SMALL_COPY_USES_FPU) */ .align 32
small_copy_unaligned:
subcc %o2, 1, %o2
EXNV(lduba [%o1] %asi, %g1, add %o2, %g0)
stb %g1, [%o1 + %o3]
bgu,pt %XCC, small_copy_unaligned
add %o1, 1, %o1
retl
clr %o0
#ifdef __KERNEL__
.globl U3cfu_fixup
U3cfu_fixup: U3cfu_fixup:
/* Since this is copy_from_user(), zero out the rest of the /* Since this is copy_from_user(), zero out the rest of the
* kernel buffer. * kernel buffer.
...@@ -516,4 +428,3 @@ U3cfu_fixup: ...@@ -516,4 +428,3 @@ U3cfu_fixup:
2: retl 2: retl
mov %o1, %o0 mov %o1, %o0
#endif
arch/sparc64/lib/U3copy_in_user.S
View file @ a8e0e8ab
/* $Id: U3copy_in_user.S,v 1.4 2001/03/21 05:58:47 davem Exp $ /* U3copy_in_user.S: UltraSparc-III optimized memcpy.
* U3memcpy.S: UltraSparc-III optimized copy within userspace.
* *
* Copyright (C) 1999, 2000 David S. Miller (davem@redhat.com) * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
*/ */
#ifdef __KERNEL__
#include <asm/visasm.h> #include <asm/visasm.h>
#include <asm/asi.h> #include <asm/asi.h>
#undef SMALL_COPY_USES_FPU #include <asm/dcu.h>
#include <asm/spitfire.h>
#define XCC xcc
#define EXNV(x,y,a,b) \ #define EXNV(x,y,a,b) \
98: x,y; \ 98: x,y; \
.section .fixup; \ .section .fixup; \
...@@ -19,7 +21,7 @@ ...@@ -19,7 +21,7 @@
.word 98b, 99b; \ .word 98b, 99b; \
.text; \ .text; \
.align 4; .align 4;
#define EXNV2(x,y,a,b) \ #define EXNV1(x,y,a,b) \
98: x,y; \ 98: x,y; \
.section .fixup; \ .section .fixup; \
.align 4; \ .align 4; \
...@@ -31,501 +33,108 @@ ...@@ -31,501 +33,108 @@
.word 98b, 99b; \ .word 98b, 99b; \
.text; \ .text; \
.align 4; .align 4;
#define EXNV3(x,y,a,b) \ #define EXNV4(x,y,a,b) \
98: x,y; \ 98: x,y; \
.section .fixup; \ .section .fixup; \
.align 4; \ .align 4; \
99: a, b, %o0; \ 99: a, b, %o0; \
retl; \ retl; \
add %o0, 8, %o0; \ add %o0, 4, %o0; \
.section __ex_table; \
.align 4; \
.word 98b, 99b; \
.text; \
.align 4;
#define EX(x,y,a,b) \
98: x,y; \
.section .fixup; \
.align 4; \
99: VISExitHalf; \
retl; \
a, b, %o0; \
.section __ex_table; \ .section __ex_table; \
.align 4; \ .align 4; \
.word 98b, 99b; \ .word 98b, 99b; \
.text; \ .text; \
.align 4; .align 4;
#define EXBLK1(x,y) \ #define EXNV8(x,y,a,b) \
98: x,y; \ 98: x,y; \
.section .fixup; \ .section .fixup; \
.align 4; \ .align 4; \
99: VISExitHalf; \ 99: a, b, %o0; \
add %o4, 0x1c0, %o1; \
and %o2, (0x40 - 1), %o2; \
retl; \
add %o1, %o2, %o0; \
.section __ex_table; \
.align 4; \
.word 98b, 99b; \
.text; \
.align 4;
#define EXBLK2(x,y) \
98: x,y; \
.section .fixup; \
.align 4; \
99: VISExitHalf; \
sll %o3, 6, %o3; \
and %o2, (0x40 - 1), %o2; \
add %o3, 0x80, %o1; \
retl; \
add %o1, %o2, %o0; \
.section __ex_table; \
.align 4; \
.word 98b, 99b; \
.text; \
.align 4;
#define EXBLK3(x,y) \
98: x,y; \
.section .fixup; \
.align 4; \
99: VISExitHalf; \
and %o2, (0x40 - 1), %o2; \
retl; \
add %o2, 0x80, %o0; \
.section __ex_table; \
.align 4; \
.word 98b, 99b; \
.text; \
.align 4;
#define EXBLK4(x,y) \
98: x,y; \
.section .fixup; \
.align 4; \
99: VISExitHalf; \
and %o2, (0x40 - 1), %o2; \
retl; \ retl; \
add %o2, 0x40, %o0; \ add %o0, 8, %o0; \
.section __ex_table; \ .section __ex_table; \
.align 4; \ .align 4; \
.word 98b, 99b; \ .word 98b, 99b; \
.text; \ .text; \
.align 4; .align 4;
#else
#define ASI_AIUS 0x80
#define ASI_BLK_AIUS 0xf0
#define FPRS_FEF 0x04
#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
#define SMALL_COPY_USES_FPU
#define EXNV(x,y,a,b) x,y;
#define EXNV2(x,y,a,b) x,y;
#define EXNV3(x,y,a,b) x,y;
#define EX(x,y,a,b) x,y;
#define EXBLK1(x,y) x,y;
#define EXBLK2(x,y) x,y;
#define EXBLK3(x,y) x,y;
#define EXBLK4(x,y) x,y;
#endif
/* Special/non-trivial issues of this code: .register %g2,#scratch
* .register %g3,#scratch
* 1) %o5 is preserved from VISEntryHalf to VISExitHalf
* 2) Only low 32 FPU registers are used so that only the
* lower half of the FPU register set is dirtied by this
* code. This is especially important in the kernel.
* 3) This code never prefetches cachelines past the end
* of the source buffer.
*
* XXX Actually, Cheetah can buffer up to 8 concurrent
* XXX prefetches, revisit this...
*/
.text .text
.align 32 .align 32
/* The cheetah's flexible spine, oversized liver, enlarged heart, /* Don't try to get too fancy here, just nice and
* slender muscular body, and claws make it the swiftest hunter * simple. This is predominantly used for well aligned
* in Africa and the fastest animal on land. Can reach speeds * small copies in the compat layer. It is also used
* of up to 2.4GB per second. * to copy register windows around during thread cloning.
*/ */
.globl U3copy_in_user .globl U3copy_in_user
U3copy_in_user: /* %o0=dst, %o1=src, %o2=len */ U3copy_in_user: /* %o0=dst, %o1=src, %o2=len */
/* Writing to %asi is _expensive_ so we hardcode it. /* Writing to %asi is _expensive_ so we hardcode it.
* Reading %asi to check for KERNEL_DS is comparatively * Reading %asi to check for KERNEL_DS is comparatively
* cheap. * cheap.
*/ */
rd %asi, %g1 ! MS Group (4 cycles) rd %asi, %g1
cmp %g1, ASI_AIUS ! A0 Group cmp %g1, ASI_AIUS
bne U3memcpy ! BR bne,pn %icc, U3memcpy_user_stub
nop ! A1 nop
#ifndef __KERNEL__
/* Save away original 'dst' for memcpy return value. */
mov %o0, %g3 ! A0 Group
#endif
/* Anything to copy at all? */
cmp %o2, 0 ! A1
ble,pn %icc, U3copy_in_user_short_ret ! BR
/* Extremely small copy? */
cmp %o2, 31 ! A0 Group
ble,pn %icc, U3copy_in_user_short ! BR
/* Large enough to use unrolled prefetch loops? */
cmp %o2, 0x100 ! A1
bge,a,pt %icc, U3copy_in_user_enter ! BR Group
andcc %o0, 0x3f, %g2 ! A0
ba,pt %xcc, U3copy_in_user_toosmall ! BR Group
andcc %o0, 0x7, %g2 ! A0
.align 32
U3copy_in_user_short:
/* Copy %o2 bytes from src to dst, one byte at a time. */
EXNV(lduba [%o1 + 0x00] %asi, %o3, add %o2, %g0)! MS Group
add %o1, 0x1, %o1 ! A0
add %o0, 0x1, %o0 ! A1
subcc %o2, 1, %o2 ! A0 Group
bg,pt %icc, U3copy_in_user_short ! BR
EXNV(stba %o3, [%o0 + -1] %asi, add %o2, 1) ! MS Group (1-cycle stall)
U3copy_in_user_short_ret:
#ifdef __KERNEL__
retl ! BR Group (0-4 cycle stall)
clr %o0 ! A0
#else
retl ! BR Group (0-4 cycle stall)
mov %g3, %o0 ! A0
#endif
/* Here len >= (6 * 64) and condition codes reflect execution
* of "andcc %o0, 0x7, %g2", done by caller.
*/
.align 64
U3copy_in_user_enter:
/* Is 'dst' already aligned on an 64-byte boundary? */
be,pt %xcc, 2f ! BR
/* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number
* of bytes to copy to make 'dst' 64-byte aligned. We pre-
* subtract this from 'len'.
*/
sub %g2, 0x40, %g2 ! A0 Group
sub %g0, %g2, %g2 ! A0 Group
sub %o2, %g2, %o2 ! A0 Group
/* Copy %g2 bytes from src to dst, one byte at a time. */
1: EXNV(lduba [%o1 + 0x00] %asi, %o3, add %o2, %g2)! MS (Group)
add %o1, 0x1, %o1 ! A1
add %o0, 0x1, %o0 ! A0 Group
subcc %g2, 0x1, %g2 ! A1
bg,pt %icc, 1b ! BR Group
EXNV2(stba %o3, [%o0 + -1] %asi, add %o2, %g2) ! MS Group
2: VISEntryHalf ! MS+MS
and %o1, 0x7, %g1 ! A1
ba,pt %xcc, U3copy_in_user_begin ! BR
alignaddr %o1, %g0, %o1 ! MS (Break-after)
.align 64
U3copy_in_user_begin:
prefetcha [%o1 + 0x000] %asi, #one_read ! MS Group1
prefetcha [%o1 + 0x040] %asi, #one_read ! MS Group2
andn %o2, (0x40 - 1), %o4 ! A0
prefetcha [%o1 + 0x080] %asi, #one_read ! MS Group3
cmp %o4, 0x140 ! A0
prefetcha [%o1 + 0x0c0] %asi, #one_read ! MS Group4
EX(ldda [%o1 + 0x000] %asi, %f0, add %o2, %g0) ! MS Group5 (%f0 results at G8)
bge,a,pt %icc, 1f ! BR
prefetcha [%o1 + 0x100] %asi, #one_read ! MS Group6
1: EX(ldda [%o1 + 0x008] %asi, %f2, add %o2, %g0) ! AX (%f2 results at G9)
cmp %o4, 0x180 ! A1
bge,a,pt %icc, 1f ! BR
prefetcha [%o1 + 0x140] %asi, #one_read ! MS Group7
1: EX(ldda [%o1 + 0x010] %asi, %f4, add %o2, %g0) ! AX (%f4 results at G10)
cmp %o4, 0x1c0 ! A1
bge,a,pt %icc, 1f ! BR
prefetcha [%o1 + 0x180] %asi, #one_read ! MS Group8
1: faligndata %f0, %f2, %f16 ! FGA Group9 (%f16 at G12)
EX(ldda [%o1 + 0x018] %asi, %f6, add %o2, %g0) ! AX (%f6 results at G12)
faligndata %f2, %f4, %f18 ! FGA Group10 (%f18 results at G13)
EX(ldda [%o1 + 0x020] %asi, %f8, add %o2, %g0) ! MS (%f8 results at G13)
faligndata %f4, %f6, %f20 ! FGA Group12 (1-cycle stall,%f20 at G15)
EX(ldda [%o1 + 0x028] %asi, %f10, add %o2, %g0) ! MS (%f10 results at G15)
faligndata %f6, %f8, %f22 ! FGA Group13 (%f22 results at G16)
EX(ldda [%o1 + 0x030] %asi, %f12, add %o2, %g0) ! MS (%f12 results at G16)
faligndata %f8, %f10, %f24 ! FGA Group15 (1-cycle stall,%f24 at G18)
EX(ldda [%o1 + 0x038] %asi, %f14, add %o2, %g0) ! MS (%f14 results at G18)
faligndata %f10, %f12, %f26 ! FGA Group16 (%f26 results at G19)
EX(ldda [%o1 + 0x040] %asi, %f0, add %o2, %g0) ! MS (%f0 results at G19)
/* We only use the first loop if len > (7 * 64). */
subcc %o4, 0x1c0, %o4 ! A0 Group17
bg,pt %icc, U3copy_in_user_loop1 ! BR
add %o1, 0x40, %o1 ! A1
add %o4, 0x140, %o4 ! A0 Group18
ba,pt %xcc, U3copy_in_user_loop2 ! BR
srl %o4, 6, %o3 ! A0 Group19
nop
nop
nop
nop
nop
nop
nop
/* This loop performs the copy and queues new prefetches.
* We drop into the second loop when len <= (5 * 64). Note
* that this (5 * 64) factor has been subtracted from len
* already.
*/
U3copy_in_user_loop1:
EXBLK1(ldda [%o1 + 0x008] %asi, %f2) ! MS Group2 (%f2 results at G5)
faligndata %f12, %f14, %f28 ! FGA (%f28 results at G5)
EXBLK1(ldda [%o1 + 0x010] %asi, %f4) ! MS Group3 (%f4 results at G6)
faligndata %f14, %f0, %f30 ! FGA Group4 (1-cycle stall, %f30 at G7)
EXBLK1(stda %f16, [%o0] ASI_BLK_AIUS) ! MS
EXBLK1(ldda [%o1 + 0x018] %asi, %f6) ! AX (%f6 results at G7)
faligndata %f0, %f2, %f16 ! FGA Group12 (7-cycle stall)
EXBLK1(ldda [%o1 + 0x020] %asi, %f8) ! MS (%f8 results at G15)
faligndata %f2, %f4, %f18 ! FGA Group13 (%f18 results at G16)
EXBLK1(ldda [%o1 + 0x028] %asi, %f10) ! MS (%f10 results at G16)
faligndata %f4, %f6, %f20 ! FGA Group14 (%f20 results at G17)
EXBLK1(ldda [%o1 + 0x030] %asi, %f12) ! MS (%f12 results at G17)
faligndata %f6, %f8, %f22 ! FGA Group15 (%f22 results at G18)
EXBLK1(ldda [%o1 + 0x038] %asi, %f14) ! MS (%f14 results at G18)
faligndata %f8, %f10, %f24 ! FGA Group16 (%f24 results at G19)
EXBLK1(ldda [%o1 + 0x040] %asi, %f0) ! AX (%f0 results at G19)
prefetcha [%o1 + 0x180] %asi, #one_read ! MS
faligndata %f10, %f12, %f26 ! FGA Group17 (%f26 results at G20)
subcc %o4, 0x40, %o4 ! A0
add %o1, 0x40, %o1 ! A1
bg,pt %xcc, U3copy_in_user_loop1 ! BR
add %o0, 0x40, %o0 ! A0 Group18
U3copy_in_user_loop2_enter:
mov 5, %o3 ! A1
/* This loop performs on the copy, no new prefetches are
* queued. We do things this way so that we do not perform
* any spurious prefetches past the end of the src buffer.
*/
U3copy_in_user_loop2:
EXBLK2(ldda [%o1 + 0x008] %asi, %f2) ! MS
faligndata %f12, %f14, %f28 ! FGA Group2
EXBLK2(ldda [%o1 + 0x010] %asi, %f4) ! MS
faligndata %f14, %f0, %f30 ! FGA Group4 (1-cycle stall)
EXBLK2(stda %f16, [%o0] ASI_BLK_AIUS) ! MS
EXBLK2(ldda [%o1 + 0x018] %asi, %f6) ! AX
faligndata %f0, %f2, %f16 ! FGA Group12 (7-cycle stall)
EXBLK2(ldda [%o1 + 0x020] %asi, %f8) ! MS
faligndata %f2, %f4, %f18 ! FGA Group13
EXBLK2(ldda [%o1 + 0x028] %asi, %f10) ! MS
faligndata %f4, %f6, %f20 ! FGA Group14
EXBLK2(ldda [%o1 + 0x030] %asi, %f12) ! MS
faligndata %f6, %f8, %f22 ! FGA Group15
EXBLK2(ldda [%o1 + 0x038] %asi, %f14) ! MS
faligndata %f8, %f10, %f24 ! FGA Group16
EXBLK2(ldda [%o1 + 0x040] %asi, %f0) ! AX
faligndata %f10, %f12, %f26 ! FGA Group17
subcc %o3, 0x01, %o3 ! A0
add %o1, 0x40, %o1 ! A1
bg,pt %xcc, U3copy_in_user_loop2 ! BR
add %o0, 0x40, %o0 ! A0 Group18
/* Finally we copy the last full 64-byte block. */
U3copy_in_user_loopfini:
EXBLK3(ldda [%o1 + 0x008] %asi, %f2) ! MS
faligndata %f12, %f14, %f28 ! FGA
EXBLK3(ldda [%o1 + 0x010] %asi, %f4) ! MS Group19
faligndata %f14, %f0, %f30 ! FGA
EXBLK3(stda %f16, [%o0] ASI_BLK_AIUS) ! MS Group20
EXBLK4(ldda [%o1 + 0x018] %asi, %f6) ! AX
faligndata %f0, %f2, %f16 ! FGA Group11 (7-cycle stall)
EXBLK4(ldda [%o1 + 0x020] %asi, %f8) ! MS
faligndata %f2, %f4, %f18 ! FGA Group12
EXBLK4(ldda [%o1 + 0x028] %asi, %f10) ! MS
faligndata %f4, %f6, %f20 ! FGA Group13
EXBLK4(ldda [%o1 + 0x030] %asi, %f12) ! MS
faligndata %f6, %f8, %f22 ! FGA Group14
EXBLK4(ldda [%o1 + 0x038] %asi, %f14) ! MS
faligndata %f8, %f10, %f24 ! FGA Group15
cmp %g1, 0 ! A0
be,pt %icc, 1f ! BR
add %o0, 0x40, %o0 ! A1
EXBLK4(ldda [%o1 + 0x040] %asi, %f0) ! MS
1: faligndata %f10, %f12, %f26 ! FGA Group16
faligndata %f12, %f14, %f28 ! FGA Group17
faligndata %f14, %f0, %f30 ! FGA Group18
EXBLK4(stda %f16, [%o0] ASI_BLK_AIUS) ! MS
add %o0, 0x40, %o0 ! A0
add %o1, 0x40, %o1 ! A1
membar #Sync ! MS Group26 (7-cycle stall)
/* Now we copy the (len modulo 64) bytes at the end.
* Note how we borrow the %f0 loaded above.
*
* Also notice how this code is careful not to perform a
* load past the end of the src buffer just like similar
* code found in U3copy_in_user_toosmall processing.
*/
U3copy_in_user_loopend:
and %o2, 0x3f, %o2 ! A0 Group
andcc %o2, 0x38, %g2 ! A0 Group
be,pn %icc, U3copy_in_user_endcruft ! BR
subcc %g2, 0x8, %g2 ! A1
be,pn %icc, U3copy_in_user_endcruft ! BR Group
cmp %g1, 0 ! A0
be,a,pt %icc, 1f ! BR Group
EX(ldda [%o1 + 0x00] %asi, %f0, add %o2, %g0) ! MS
1: EX(ldda [%o1 + 0x08] %asi, %f2, add %o2, %g0) ! MS Group
add %o1, 0x8, %o1 ! A0
sub %o2, 0x8, %o2 ! A1
subcc %g2, 0x8, %g2 ! A0 Group
faligndata %f0, %f2, %f8 ! FGA Group
EX(stda %f8, [%o0 + 0x00] %asi, add %o2, 0x8) ! MS (XXX does it stall here? XXX)
be,pn %icc, U3copy_in_user_endcruft ! BR
add %o0, 0x8, %o0 ! A0
EX(ldda [%o1 + 0x08] %asi, %f0, add %o2, %g0) ! MS Group
add %o1, 0x8, %o1 ! A0
sub %o2, 0x8, %o2 ! A1
subcc %g2, 0x8, %g2 ! A0 Group
faligndata %f2, %f0, %f8 ! FGA
EX(stda %f8, [%o0 + 0x00] %asi, add %o2, 0x8) ! MS (XXX does it stall here? XXX)
bne,pn %icc, 1b ! BR
add %o0, 0x8, %o0 ! A0 Group
/* If anything is left, we copy it one byte at a time.
* Note that %g1 is (src & 0x3) saved above before the
* alignaddr was performed.
*/
U3copy_in_user_endcruft:
cmp %o2, 0 cmp %o2, 0
add %o1, %g1, %o1 be,pn %XCC, out
VISExitHalf or %o0, %o1, %o3
be,pn %icc, U3copy_in_user_short_ret cmp %o2, 16
bleu,a,pn %XCC, small_copy
or %o3, %o2, %o3
medium_copy: /* 16 < len <= 64 */
andcc %o3, 0x7, %g0
bne,pn %XCC, small_copy_unaligned
sub %o0, %o1, %o3
medium_copy_aligned:
andn %o2, 0x7, %o4
and %o2, 0x7, %o2
1: subcc %o4, 0x8, %o4
EXNV8(ldxa [%o1] %asi, %o5, add %o4, %o2)
EXNV8(stxa %o5, [%o1 + %o3] ASI_AIUS, add %o4, %o2)
bgu,pt %XCC, 1b
add %o1, 0x8, %o1
andcc %o2, 0x4, %g0
be,pt %XCC, 1f
nop
sub %o2, 0x4, %o2
EXNV4(lduwa [%o1] %asi, %o5, add %o4, %o2)
EXNV4(stwa %o5, [%o1 + %o3] ASI_AIUS, add %o4, %o2)
add %o1, 0x4, %o1
1: cmp %o2, 0
be,pt %XCC, out
nop
ba,pt %xcc, small_copy_unaligned
nop nop
ba,a,pt %xcc, U3copy_in_user_short
/* If we get here, then 32 <= len < (6 * 64) */
U3copy_in_user_toosmall:
#ifdef SMALL_COPY_USES_FPU
/* Is 'dst' already aligned on an 8-byte boundary? */
be,pt %xcc, 2f ! BR Group
/* Compute abs((dst & 7) - 8) into %g2. This is the number
* of bytes to copy to make 'dst' 8-byte aligned. We pre-
* subtract this from 'len'.
*/
sub %g2, 0x8, %g2 ! A0
sub %g0, %g2, %g2 ! A0 Group (reg-dep)
sub %o2, %g2, %o2 ! A0 Group (reg-dep)
/* Copy %g2 bytes from src to dst, one byte at a time. */
1: EXNV2(lduba [%o1 + 0x00] %asi, %o3, add %o2, %g2)! MS (Group) (%o3 in 3 cycles)
add %o1, 0x1, %o1 ! A1
add %o0, 0x1, %o0 ! A0 Group
subcc %g2, 0x1, %g2 ! A1
bg,pt %icc, 1b ! BR Group
EXNV2(stba %o3, [%o0 + -1] %asi, add %o2, %g2) ! MS Group
2: VISEntryHalf ! MS+MS
/* Compute (len - (len % 8)) into %g2. This is guaranteed
* to be nonzero.
*/
andn %o2, 0x7, %g2 ! A0 Group
/* You may read this and believe that it allows reading
* one 8-byte longword past the end of src. It actually
* does not, as %g2 is subtracted as loads are done from
* src, so we always stop before running off the end.
* Also, we are guaranteed to have at least 0x10 bytes
* to move here.
*/
sub %g2, 0x8, %g2 ! A0 Group (reg-dep)
alignaddr %o1, %g0, %g1 ! MS (Break-after)
EX(ldda [%g1 + 0x00] %asi, %f0, add %o2, %g0) ! MS Group (1-cycle stall)
add %g1, 0x8, %g1 ! A0
1: EX(ldda [%g1 + 0x00] %asi, %f2, add %o2, %g0) ! MS Group
add %g1, 0x8, %g1 ! A0
sub %o2, 0x8, %o2 ! A1
subcc %g2, 0x8, %g2 ! A0 Group
faligndata %f0, %f2, %f8 ! FGA Group (1-cycle stall)
EX(stda %f8, [%o0 + 0x00] %asi, add %o2, 0x8) ! MS Group (2-cycle stall)
add %o1, 0x8, %o1 ! A0
be,pn %icc, 2f ! BR
add %o0, 0x8, %o0 ! A1
EX(ldda [%g1 + 0x00] %asi, %f0, add %o2, %g0) ! MS Group
add %g1, 0x8, %g1 ! A0
sub %o2, 0x8, %o2 ! A1
subcc %g2, 0x8, %g2 ! A0 Group
faligndata %f2, %f0, %f8 ! FGA Group (1-cycle stall)
EX(stda %f8, [%o0 + 0x00] %asi, add %o2, 0x8) ! MS Group (2-cycle stall)
add %o1, 0x8, %o1 ! A0
bne,pn %icc, 1b ! BR
add %o0, 0x8, %o0 ! A1
/* Nothing left to copy? */
2: cmp %o2, 0 ! A0 Group
VISExitHalf ! A0+MS
be,pn %icc, U3copy_in_user_short_ret ! BR Group
nop ! A0
ba,a,pt %xcc, U3copy_in_user_short ! BR Group
#else /* !(SMALL_COPY_USES_FPU) */
xor %o1, %o0, %g2
andcc %g2, 0x7, %g0
bne,pn %icc, U3copy_in_user_short
andcc %o1, 0x7, %g2
be,pt %xcc, 2f
sub %g2, 0x8, %g2
sub %g0, %g2, %g2
sub %o2, %g2, %o2
1: EXNV2(lduba [%o1 + 0x00] %asi, %o3, add %o2, %g2)
add %o1, 0x1, %o1
add %o0, 0x1, %o0
subcc %g2, 0x1, %g2
bg,pt %icc, 1b
EXNV2(stba %o3, [%o0 + -1] %asi, add %o2, %g2)
2: andn %o2, 0x7, %g2 small_copy: /* 0 < len <= 16 */
sub %o2, %g2, %o2 andcc %o3, 0x3, %g0
bne,pn %XCC, small_copy_unaligned
sub %o0, %o1, %o3
3: EXNV3(ldxa [%o1 + 0x00] %asi, %o3, add %o2, %g2) small_copy_aligned:
add %o1, 0x8, %o1 subcc %o2, 4, %o2
add %o0, 0x8, %o0 EXNV4(lduwa [%o1] %asi, %g1, add %o2, %g0)
subcc %g2, 0x8, %g2 EXNV4(stwa %g1, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
bg,pt %icc, 3b bgu,pt %XCC, small_copy_aligned
EXNV3(stxa %o3, [%o0 + -8] %asi, add %o2, %g2) add %o1, 4, %o1
cmp %o2, 0 out: retl
bne,pn %icc, U3copy_in_user_short clr %o0
nop
ba,a,pt %xcc, U3copy_in_user_short_ret
#endif /* !(SMALL_COPY_USES_FPU) */ .align 32
small_copy_unaligned:
subcc %o2, 1, %o2
EXNV1(lduba [%o1] %asi, %g1, add %o2, %g0)
EXNV1(stba %g1, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
bgu,pt %XCC, small_copy_unaligned
add %o1, 1, %o1
retl
clr %o0
arch/sparc64/lib/U3copy_to_user.S
View file @ a8e0e8ab
/* $Id: U3copy_to_user.S,v 1.3 2000/11/01 09:29:19 davem Exp $ /* U3copy_to_user.S: UltraSparc-III optimized memcpy.
* U3memcpy.S: UltraSparc-III optimized copy to userspace.
* *
* Copyright (C) 1999, 2000 David S. Miller (davem@redhat.com) * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
*/ */
#ifdef __KERNEL__
#include <asm/visasm.h> #include <asm/visasm.h>
#include <asm/asi.h> #include <asm/asi.h>
#include <asm/dcu.h> #include <asm/dcu.h>
#include <asm/spitfire.h> #include <asm/spitfire.h>
#undef SMALL_COPY_USES_FPU
#define XCC xcc
#define EXNV(x,y,a,b) \ #define EXNV(x,y,a,b) \
98: x,y; \ 98: x,y; \
.section .fixup; \ .section .fixup; \
...@@ -34,6 +34,18 @@ ...@@ -34,6 +34,18 @@
.text; \ .text; \
.align 4; .align 4;
#define EXNV3(x,y,a,b) \ #define EXNV3(x,y,a,b) \
98: x,y; \
.section .fixup; \
.align 4; \
99: a, b, %o0; \
retl; \
add %o0, 4, %o0; \
.section __ex_table; \
.align 4; \
.word 98b, 99b; \
.text; \
.align 4;
#define EXNV4(x,y,a,b) \
98: x,y; \ 98: x,y; \
.section .fixup; \ .section .fixup; \
.align 4; \ .align 4; \
...@@ -112,22 +124,9 @@ ...@@ -112,22 +124,9 @@
.word 98b, 99b; \ .word 98b, 99b; \
.text; \ .text; \
.align 4; .align 4;
#else
#define ASI_AIUS 0x80 .register %g2,#scratch
#define ASI_BLK_AIUS 0xf0 .register %g3,#scratch
#define FPRS_FEF 0x04
#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
#define SMALL_COPY_USES_FPU
#define EXNV(x,y,a,b) x,y;
#define EXNV2(x,y,a,b) x,y;
#define EXNV3(x,y,a,b) x,y;
#define EX(x,y,a,b) x,y;
#define EXBLK1(x,y) x,y;
#define EXBLK2(x,y) x,y;
#define EXBLK3(x,y) x,y;
#define EXBLK4(x,y) x,y;
#endif
/* Special/non-trivial issues of this code: /* Special/non-trivial issues of this code:
* *
...@@ -148,89 +147,64 @@ ...@@ -148,89 +147,64 @@
* of up to 2.4GB per second. * of up to 2.4GB per second.
*/ */
.globl U3copy_to_user .globl U3copy_to_user
U3copy_to_user: /* %o0=dst, %o1=src, %o2=len */ U3copy_to_user: /* %o0=dst, %o1=src, %o2=len */
/* Writing to %asi is _expensive_ so we hardcode it. /* Writing to %asi is _expensive_ so we hardcode it.
* Reading %asi to check for KERNEL_DS is comparatively * Reading %asi to check for KERNEL_DS is comparatively
* cheap. * cheap.
*/ */
rd %asi, %g1 ! MS Group (4 cycles) rd %asi, %g1
cmp %g1, ASI_AIUS ! A0 Group cmp %g1, ASI_AIUS
bne U3memcpy ! BR bne,pn %icc, U3memcpy_user_stub
nop ! A1 nop
#ifndef __KERNEL__
/* Save away original 'dst' for memcpy return value. */ cmp %o2, 0
mov %o0, %g3 ! A0 Group be,pn %XCC, out
#endif or %o0, %o1, %o3
/* Anything to copy at all? */ cmp %o2, 16
cmp %o2, 0 ! A1 bleu,a,pn %XCC, small_copy
ble,pn %icc, U3copy_to_user_short_ret ! BR or %o3, %o2, %o3
/* Extremely small copy? */ cmp %o2, 256
cmp %o2, 31 ! A0 Group blu,pt %XCC, medium_copy
ble,pn %icc, U3copy_to_user_short ! BR andcc %o3, 0x7, %g0
/* Large enough to use unrolled prefetch loops? */ ba,pt %xcc, enter
cmp %o2, 0x100 ! A1 andcc %o0, 0x3f, %g2
bge,a,pt %icc, U3copy_to_user_enter ! BR Group
andcc %o0, 0x3f, %g2 ! A0 /* Here len >= 256 and condition codes reflect execution
ba,pt %xcc, U3copy_to_user_toosmall ! BR Group
andcc %o0, 0x7, %g2 ! A0
.align 32
U3copy_to_user_short:
/* Copy %o2 bytes from src to dst, one byte at a time. */
ldub [%o1 + 0x00], %o3 ! MS Group
add %o1, 0x1, %o1 ! A0
add %o0, 0x1, %o0 ! A1
subcc %o2, 1, %o2 ! A0 Group
bg,pt %icc, U3copy_to_user_short ! BR
EXNV(stba %o3, [%o0 + -1] %asi, add %o2, 1) ! MS Group (1-cycle stall)
U3copy_to_user_short_ret:
#ifdef __KERNEL__
retl ! BR Group (0-4 cycle stall)
clr %o0 ! A0
#else
retl ! BR Group (0-4 cycle stall)
mov %g3, %o0 ! A0
#endif
/* Here len >= (6 * 64) and condition codes reflect execution
* of "andcc %o0, 0x7, %g2", done by caller. * of "andcc %o0, 0x7, %g2", done by caller.
*/ */
.align 64 .align 64
U3copy_to_user_enter: enter:
/* Is 'dst' already aligned on an 64-byte boundary? */ /* Is 'dst' already aligned on an 64-byte boundary? */
be,pt %xcc, 2f ! BR be,pt %XCC, 2f
/* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number /* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number
* of bytes to copy to make 'dst' 64-byte aligned. We pre- * of bytes to copy to make 'dst' 64-byte aligned. We pre-
* subtract this from 'len'. * subtract this from 'len'.
*/ */
sub %g2, 0x40, %g2 ! A0 Group sub %g2, 0x40, %g2
sub %g0, %g2, %g2 ! A0 Group sub %g0, %g2, %g2
sub %o2, %g2, %o2 ! A0 Group sub %o2, %g2, %o2
/* Copy %g2 bytes from src to dst, one byte at a time. */ /* Copy %g2 bytes from src to dst, one byte at a time. */
1: ldub [%o1 + 0x00], %o3 ! MS (Group) 1: ldub [%o1 + 0x00], %o3
add %o1, 0x1, %o1 ! A1 add %o1, 0x1, %o1
add %o0, 0x1, %o0 ! A0 Group add %o0, 0x1, %o0
subcc %g2, 0x1, %g2 ! A1 subcc %g2, 0x1, %g2
bg,pt %icc, 1b ! BR Group bg,pt %XCC, 1b
EXNV2(stba %o3, [%o0 + -1] %asi, add %o2, %g2) ! MS Group EXNV2(stba %o3, [%o0 + -1] %asi, add %o2, %g2)
2: VISEntryHalf ! MS+MS 2: VISEntryHalf
and %o1, 0x7, %g1 ! A1 and %o1, 0x7, %g1
ba,pt %xcc, U3copy_to_user_begin ! BR ba,pt %xcc, begin
alignaddr %o1, %g0, %o1 ! MS (Break-after) alignaddr %o1, %g0, %o1
.align 64 .align 64
U3copy_to_user_begin: begin:
#ifdef __KERNEL__
.globl U3copy_to_user_nop_1_6 .globl U3copy_to_user_nop_1_6
U3copy_to_user_nop_1_6: U3copy_to_user_nop_1_6:
ldxa [%g0] ASI_DCU_CONTROL_REG, %g3 ldxa [%g0] ASI_DCU_CONTROL_REG, %g3
...@@ -239,309 +213,221 @@ U3copy_to_user_nop_1_6: ...@@ -239,309 +213,221 @@ U3copy_to_user_nop_1_6:
or %g3, %o3, %o3 or %g3, %o3, %o3
stxa %o3, [%g0] ASI_DCU_CONTROL_REG ! Enable P-cache stxa %o3, [%g0] ASI_DCU_CONTROL_REG ! Enable P-cache
membar #Sync membar #Sync
#endif
prefetch [%o1 + 0x000], #one_read ! MS Group1 prefetch [%o1 + 0x000], #one_read
prefetch [%o1 + 0x040], #one_read ! MS Group2 prefetch [%o1 + 0x040], #one_read
andn %o2, (0x40 - 1), %o4 ! A0 andn %o2, (0x40 - 1), %o4
prefetch [%o1 + 0x080], #one_read ! MS Group3 prefetch [%o1 + 0x080], #one_read
cmp %o4, 0x140 ! A0 prefetch [%o1 + 0x0c0], #one_read
prefetch [%o1 + 0x0c0], #one_read ! MS Group4 ldd [%o1 + 0x000], %f0
ldd [%o1 + 0x000], %f0 ! MS Group5 (%f0 results at G8) prefetch [%o1 + 0x100], #one_read
bge,a,pt %icc, 1f ! BR ldd [%o1 + 0x008], %f2
prefetch [%o1 + 0x140], #one_read
prefetch [%o1 + 0x100], #one_read ! MS Group6 ldd [%o1 + 0x010], %f4
1: ldd [%o1 + 0x008], %f2 ! AX (%f2 results at G9) prefetch [%o1 + 0x180], #one_read
cmp %o4, 0x180 ! A1 faligndata %f0, %f2, %f16
bge,a,pt %icc, 1f ! BR ldd [%o1 + 0x018], %f6
prefetch [%o1 + 0x140], #one_read ! MS Group7 faligndata %f2, %f4, %f18
1: ldd [%o1 + 0x010], %f4 ! AX (%f4 results at G10) ldd [%o1 + 0x020], %f8
cmp %o4, 0x1c0 ! A1 faligndata %f4, %f6, %f20
bge,a,pt %icc, 1f ! BR ldd [%o1 + 0x028], %f10
faligndata %f6, %f8, %f22
prefetch [%o1 + 0x180], #one_read ! MS Group8
1: faligndata %f0, %f2, %f16 ! FGA Group9 (%f16 at G12) ldd [%o1 + 0x030], %f12
ldd [%o1 + 0x018], %f6 ! AX (%f6 results at G12) faligndata %f8, %f10, %f24
faligndata %f2, %f4, %f18 ! FGA Group10 (%f18 results at G13) ldd [%o1 + 0x038], %f14
ldd [%o1 + 0x020], %f8 ! MS (%f8 results at G13) faligndata %f10, %f12, %f26
faligndata %f4, %f6, %f20 ! FGA Group12 (1-cycle stall,%f20 at G15) ldd [%o1 + 0x040], %f0
ldd [%o1 + 0x028], %f10 ! MS (%f10 results at G15)
faligndata %f6, %f8, %f22 ! FGA Group13 (%f22 results at G16) sub %o4, 0x80, %o4
add %o1, 0x40, %o1
ldd [%o1 + 0x030], %f12 ! MS (%f12 results at G16) ba,pt %xcc, loop
faligndata %f8, %f10, %f24 ! FGA Group15 (1-cycle stall,%f24 at G18) srl %o4, 6, %o3
ldd [%o1 + 0x038], %f14 ! MS (%f14 results at G18)
faligndata %f10, %f12, %f26 ! FGA Group16 (%f26 results at G19) .align 64
ldd [%o1 + 0x040], %f0 ! MS (%f0 results at G19) loop:
ldd [%o1 + 0x008], %f2
/* We only use the first loop if len > (7 * 64). */ faligndata %f12, %f14, %f28
subcc %o4, 0x1c0, %o4 ! A0 Group17 ldd [%o1 + 0x010], %f4
bg,pt %icc, U3copy_to_user_loop1 ! BR faligndata %f14, %f0, %f30
add %o1, 0x40, %o1 ! A1 EXBLK2(stda %f16, [%o0] ASI_BLK_AIUS)
ldd [%o1 + 0x018], %f6
add %o4, 0x140, %o4 ! A0 Group18 faligndata %f0, %f2, %f16
ba,pt %xcc, U3copy_to_user_loop2 ! BR
srl %o4, 6, %o3 ! A0 Group19 ldd [%o1 + 0x020], %f8
nop faligndata %f2, %f4, %f18
nop ldd [%o1 + 0x028], %f10
nop faligndata %f4, %f6, %f20
nop ldd [%o1 + 0x030], %f12
nop faligndata %f6, %f8, %f22
ldd [%o1 + 0x038], %f14
nop faligndata %f8, %f10, %f24
nop
ldd [%o1 + 0x040], %f0
/* This loop performs the copy and queues new prefetches. prefetch [%o1 + 0x180], #one_read
* We drop into the second loop when len <= (5 * 64). Note faligndata %f10, %f12, %f26
* that this (5 * 64) factor has been subtracted from len subcc %o3, 0x01, %o3
* already. add %o1, 0x40, %o1
*/ bg,pt %XCC, loop
U3copy_to_user_loop1: add %o0, 0x40, %o0
ldd [%o1 + 0x008], %f2 ! MS Group2 (%f2 results at G5)
faligndata %f12, %f14, %f28 ! FGA (%f28 results at G5)
ldd [%o1 + 0x010], %f4 ! MS Group3 (%f4 results at G6)
faligndata %f14, %f0, %f30 ! FGA Group4 (1-cycle stall, %f30 at G7)
EXBLK1(stda %f16, [%o0] ASI_BLK_AIUS) ! MS
ldd [%o1 + 0x018], %f6 ! AX (%f6 results at G7)
faligndata %f0, %f2, %f16 ! FGA Group12 (7-cycle stall)
ldd [%o1 + 0x020], %f8 ! MS (%f8 results at G15)
faligndata %f2, %f4, %f18 ! FGA Group13 (%f18 results at G16)
ldd [%o1 + 0x028], %f10 ! MS (%f10 results at G16)
faligndata %f4, %f6, %f20 ! FGA Group14 (%f20 results at G17)
ldd [%o1 + 0x030], %f12 ! MS (%f12 results at G17)
faligndata %f6, %f8, %f22 ! FGA Group15 (%f22 results at G18)
ldd [%o1 + 0x038], %f14 ! MS (%f14 results at G18)
faligndata %f8, %f10, %f24 ! FGA Group16 (%f24 results at G19)
ldd [%o1 + 0x040], %f0 ! AX (%f0 results at G19)
prefetch [%o1 + 0x180], #one_read ! MS
faligndata %f10, %f12, %f26 ! FGA Group17 (%f26 results at G20)
subcc %o4, 0x40, %o4 ! A0
add %o1, 0x40, %o1 ! A1
bg,pt %xcc, U3copy_to_user_loop1 ! BR
add %o0, 0x40, %o0 ! A0 Group18
U3copy_to_user_loop2_enter:
mov 5, %o3 ! A1
/* This loop performs on the copy, no new prefetches are
* queued. We do things this way so that we do not perform
* any spurious prefetches past the end of the src buffer.
*/
U3copy_to_user_loop2:
ldd [%o1 + 0x008], %f2 ! MS
faligndata %f12, %f14, %f28 ! FGA Group2
ldd [%o1 + 0x010], %f4 ! MS
faligndata %f14, %f0, %f30 ! FGA Group4 (1-cycle stall)
EXBLK2(stda %f16, [%o0] ASI_BLK_AIUS) ! MS
ldd [%o1 + 0x018], %f6 ! AX
faligndata %f0, %f2, %f16 ! FGA Group12 (7-cycle stall)
ldd [%o1 + 0x020], %f8 ! MS
faligndata %f2, %f4, %f18 ! FGA Group13
ldd [%o1 + 0x028], %f10 ! MS
faligndata %f4, %f6, %f20 ! FGA Group14
ldd [%o1 + 0x030], %f12 ! MS
faligndata %f6, %f8, %f22 ! FGA Group15
ldd [%o1 + 0x038], %f14 ! MS
faligndata %f8, %f10, %f24 ! FGA Group16
ldd [%o1 + 0x040], %f0 ! AX
faligndata %f10, %f12, %f26 ! FGA Group17
subcc %o3, 0x01, %o3 ! A0
add %o1, 0x40, %o1 ! A1
bg,pt %xcc, U3copy_to_user_loop2 ! BR
add %o0, 0x40, %o0 ! A0 Group18
/* Finally we copy the last full 64-byte block. */ /* Finally we copy the last full 64-byte block. */
U3copy_to_user_loopfini: loopfini:
ldd [%o1 + 0x008], %f2 ! MS ldd [%o1 + 0x008], %f2
faligndata %f12, %f14, %f28 ! FGA faligndata %f12, %f14, %f28
ldd [%o1 + 0x010], %f4 ! MS Group19 ldd [%o1 + 0x010], %f4
faligndata %f14, %f0, %f30 ! FGA faligndata %f14, %f0, %f30
EXBLK3(stda %f16, [%o0] ASI_BLK_AIUS) ! MS Group20 EXBLK3(stda %f16, [%o0] ASI_BLK_AIUS)
ldd [%o1 + 0x018], %f6 ! AX ldd [%o1 + 0x018], %f6
faligndata %f0, %f2, %f16 ! FGA Group11 (7-cycle stall) faligndata %f0, %f2, %f16
ldd [%o1 + 0x020], %f8 ! MS ldd [%o1 + 0x020], %f8
faligndata %f2, %f4, %f18 ! FGA Group12 faligndata %f2, %f4, %f18
ldd [%o1 + 0x028], %f10 ! MS ldd [%o1 + 0x028], %f10
faligndata %f4, %f6, %f20 ! FGA Group13 faligndata %f4, %f6, %f20
ldd [%o1 + 0x030], %f12 ! MS ldd [%o1 + 0x030], %f12
faligndata %f6, %f8, %f22 ! FGA Group14 faligndata %f6, %f8, %f22
ldd [%o1 + 0x038], %f14 ! MS ldd [%o1 + 0x038], %f14
faligndata %f8, %f10, %f24 ! FGA Group15 faligndata %f8, %f10, %f24
cmp %g1, 0 ! A0 cmp %g1, 0
be,pt %icc, 1f ! BR be,pt %XCC, 1f
add %o0, 0x40, %o0 ! A1 add %o0, 0x40, %o0
ldd [%o1 + 0x040], %f0 ! MS ldd [%o1 + 0x040], %f0
1: faligndata %f10, %f12, %f26 ! FGA Group16 1: faligndata %f10, %f12, %f26
faligndata %f12, %f14, %f28 ! FGA Group17 faligndata %f12, %f14, %f28
faligndata %f14, %f0, %f30 ! FGA Group18 faligndata %f14, %f0, %f30
EXBLK4(stda %f16, [%o0] ASI_BLK_AIUS) ! MS EXBLK4(stda %f16, [%o0] ASI_BLK_AIUS)
add %o0, 0x40, %o0 ! A0 add %o0, 0x40, %o0
add %o1, 0x40, %o1 ! A1 add %o1, 0x40, %o1
#ifdef __KERNEL__
.globl U3copy_to_user_nop_2_3 .globl U3copy_to_user_nop_2_3
U3copy_to_user_nop_2_3: U3copy_to_user_nop_2_3:
mov PRIMARY_CONTEXT, %o3 mov PRIMARY_CONTEXT, %o3
stxa %g0, [%o3] ASI_DMMU ! Flush P-cache stxa %g0, [%o3] ASI_DMMU ! Flush P-cache
stxa %g3, [%g0] ASI_DCU_CONTROL_REG ! Disable P-cache stxa %g3, [%g0] ASI_DCU_CONTROL_REG ! Disable P-cache
#endif
membar #Sync ! MS Group26 (7-cycle stall) membar #Sync
/* Now we copy the (len modulo 64) bytes at the end. /* Now we copy the (len modulo 64) bytes at the end.
* Note how we borrow the %f0 loaded above. * Note how we borrow the %f0 loaded above.
* *
* Also notice how this code is careful not to perform a * Also notice how this code is careful not to perform a
* load past the end of the src buffer just like similar * load past the end of the src buffer.
* code found in U3copy_to_user_toosmall processing.
*/ */
U3copy_to_user_loopend: loopend:
and %o2, 0x3f, %o2 ! A0 Group and %o2, 0x3f, %o2
andcc %o2, 0x38, %g2 ! A0 Group andcc %o2, 0x38, %g2
be,pn %icc, U3copy_to_user_endcruft ! BR be,pn %XCC, endcruft
subcc %g2, 0x8, %g2 ! A1 subcc %g2, 0x8, %g2
be,pn %icc, U3copy_to_user_endcruft ! BR Group be,pn %XCC, endcruft
cmp %g1, 0 ! A0 cmp %g1, 0
be,a,pt %icc, 1f ! BR Group be,a,pt %XCC, 1f
ldd [%o1 + 0x00], %f0 ! MS ldd [%o1 + 0x00], %f0
1: ldd [%o1 + 0x08], %f2 ! MS Group 1: ldd [%o1 + 0x08], %f2
add %o1, 0x8, %o1 ! A0 add %o1, 0x8, %o1
sub %o2, 0x8, %o2 ! A1 sub %o2, 0x8, %o2
subcc %g2, 0x8, %g2 ! A0 Group subcc %g2, 0x8, %g2
faligndata %f0, %f2, %f8 ! FGA Group faligndata %f0, %f2, %f8
EX(stda %f8, [%o0 + 0x00] %asi, add %o2, 0x8) ! MS (XXX does it stall here? XXX) EX(stda %f8, [%o0 + 0x00] %asi, add %o2, 0x8)
be,pn %icc, U3copy_to_user_endcruft ! BR be,pn %XCC, endcruft
add %o0, 0x8, %o0 ! A0 add %o0, 0x8, %o0
ldd [%o1 + 0x08], %f0 ! MS Group ldd [%o1 + 0x08], %f0
add %o1, 0x8, %o1 ! A0 add %o1, 0x8, %o1
sub %o2, 0x8, %o2 ! A1 sub %o2, 0x8, %o2
subcc %g2, 0x8, %g2 ! A0 Group subcc %g2, 0x8, %g2
faligndata %f2, %f0, %f8 ! FGA faligndata %f2, %f0, %f8
EX(stda %f8, [%o0 + 0x00] %asi, add %o2, 0x8) ! MS (XXX does it stall here? XXX) EX(stda %f8, [%o0 + 0x00] %asi, add %o2, 0x8)
bne,pn %icc, 1b ! BR bne,pn %XCC, 1b
add %o0, 0x8, %o0 ! A0 Group add %o0, 0x8, %o0
/* If anything is left, we copy it one byte at a time. /* If anything is left, we copy it one byte at a time.
* Note that %g1 is (src & 0x3) saved above before the * Note that %g1 is (src & 0x3) saved above before the
* alignaddr was performed. * alignaddr was performed.
*/ */
U3copy_to_user_endcruft: endcruft:
cmp %o2, 0 cmp %o2, 0
add %o1, %g1, %o1 add %o1, %g1, %o1
VISExitHalf VISExitHalf
be,pn %icc, U3copy_to_user_short_ret be,pn %XCC, out
nop sub %o0, %o1, %o3
ba,a,pt %xcc, U3copy_to_user_short
/* If we get here, then 32 <= len < (6 * 64) */
U3copy_to_user_toosmall:
#ifdef SMALL_COPY_USES_FPU
/* Is 'dst' already aligned on an 8-byte boundary? */
be,pt %xcc, 2f ! BR Group
/* Compute abs((dst & 7) - 8) into %g2. This is the number
* of bytes to copy to make 'dst' 8-byte aligned. We pre-
* subtract this from 'len'.
*/
sub %g2, 0x8, %g2 ! A0
sub %g0, %g2, %g2 ! A0 Group (reg-dep)
sub %o2, %g2, %o2 ! A0 Group (reg-dep)
/* Copy %g2 bytes from src to dst, one byte at a time. */
1: ldub [%o1 + 0x00], %o3 ! MS (Group) (%o3 in 3 cycles)
add %o1, 0x1, %o1 ! A1
add %o0, 0x1, %o0 ! A0 Group
subcc %g2, 0x1, %g2 ! A1
bg,pt %icc, 1b ! BR Group andcc %g1, 0x7, %g0
EXNV2(stba %o3, [%o0 + -1] %asi, add %o2, %g2) ! MS Group bne,pn %icc, small_copy_unaligned
andcc %o2, 0x8, %g0
be,pt %icc, 1f
nop
ldx [%o1], %o5
EXNV(stxa %o5, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
add %o1, 0x8, %o1
2: VISEntryHalf ! MS+MS 1: andcc %o2, 0x4, %g0
be,pt %icc, 1f
nop
lduw [%o1], %o5
EXNV(stwa %o5, [%o1 + %o3] ASI_AIUS, and %o2, 0x7)
add %o1, 0x4, %o1
/* Compute (len - (len % 8)) into %g2. This is guaranteed 1: andcc %o2, 0x2, %g0
* to be nonzero. be,pt %icc, 1f
*/ nop
andn %o2, 0x7, %g2 ! A0 Group lduh [%o1], %o5
EXNV(stha %o5, [%o1 + %o3] ASI_AIUS, and %o2, 0x3)
/* You may read this and believe that it allows reading add %o1, 0x2, %o1
* one 8-byte longword past the end of src. It actually
* does not, as %g2 is subtracted as loads are done from
* src, so we always stop before running off the end.
* Also, we are guaranteed to have at least 0x10 bytes
* to move here.
*/
sub %g2, 0x8, %g2 ! A0 Group (reg-dep)
alignaddr %o1, %g0, %g1 ! MS (Break-after)
ldd [%g1 + 0x00], %f0 ! MS Group (1-cycle stall)
add %g1, 0x8, %g1 ! A0
1: ldd [%g1 + 0x00], %f2 ! MS Group
add %g1, 0x8, %g1 ! A0
sub %o2, 0x8, %o2 ! A1
subcc %g2, 0x8, %g2 ! A0 Group
faligndata %f0, %f2, %f8 ! FGA Group (1-cycle stall)
EX(stda %f8, [%o0 + 0x00] %asi, add %o2, 0x8) ! MS Group (2-cycle stall)
add %o1, 0x8, %o1 ! A0
be,pn %icc, 2f ! BR
add %o0, 0x8, %o0 ! A1
ldd [%g1 + 0x00], %f0 ! MS Group
add %g1, 0x8, %g1 ! A0
sub %o2, 0x8, %o2 ! A1
subcc %g2, 0x8, %g2 ! A0 Group
faligndata %f2, %f0, %f8 ! FGA Group (1-cycle stall)
EX(stda %f8, [%o0 + 0x00] %asi, add %o2, 0x8) ! MS Group (2-cycle stall)
add %o1, 0x8, %o1 ! A0
bne,pn %icc, 1b ! BR
add %o0, 0x8, %o0 ! A1
/* Nothing left to copy? */
2: cmp %o2, 0 ! A0 Group
VISExitHalf ! A0+MS
be,pn %icc, U3copy_to_user_short_ret ! BR Group
nop ! A0
ba,a,pt %xcc, U3copy_to_user_short ! BR Group
#else /* !(SMALL_COPY_USES_FPU) */
xor %o1, %o0, %g2
andcc %g2, 0x7, %g0
bne,pn %icc, U3copy_to_user_short
andcc %o1, 0x7, %g2
be,pt %xcc, 2f
sub %g2, 0x8, %g2
sub %g0, %g2, %g2
sub %o2, %g2, %o2
1: ldub [%o1 + 0x00], %o3 1: andcc %o2, 0x1, %g0
add %o1, 0x1, %o1 be,pt %icc, out
add %o0, 0x1, %o0 nop
subcc %g2, 0x1, %g2 ldub [%o1], %o5
bg,pt %icc, 1b ba,pt %xcc, out
EXNV2(stba %o3, [%o0 + -1] %asi, add %o2, %g2) EXNV(stba %o5, [%o1 + %o3] ASI_AIUS, and %o2, 0x1)
medium_copy: /* 16 < len <= 64 */
bne,pn %XCC, small_copy_unaligned
sub %o0, %o1, %o3
medium_copy_aligned:
andn %o2, 0x7, %o4
and %o2, 0x7, %o2
1: subcc %o4, 0x8, %o4
ldx [%o1], %o5
EXNV4(stxa %o5, [%o1 + %o3] ASI_AIUS, add %o2, %o4)
bgu,pt %XCC, 1b
add %o1, 0x8, %o1
andcc %o2, 0x4, %g0
be,pt %XCC, 1f
nop
sub %o2, 0x4, %o2
lduw [%o1], %o5
EXNV3(stwa %o5, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
add %o1, 0x4, %o1
1: cmp %o2, 0
be,pt %XCC, out
nop
ba,pt %xcc, small_copy_unaligned
nop
2: andn %o2, 0x7, %g2 small_copy: /* 0 < len <= 16 */
sub %o2, %g2, %o2 andcc %o3, 0x3, %g0
bne,pn %XCC, small_copy_unaligned
sub %o0, %o1, %o3
3: ldx [%o1 + 0x00], %o3 small_copy_aligned:
add %o1, 0x8, %o1 subcc %o2, 4, %o2
add %o0, 0x8, %o0 lduw [%o1], %g1
subcc %g2, 0x8, %g2 EXNV3(stwa %g1, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
bg,pt %icc, 3b bgu,pt %XCC, small_copy_aligned
EXNV3(stxa %o3, [%o0 + -8] %asi, add %o2, %g2) add %o1, 4, %o1
cmp %o2, 0 out: retl
bne,pn %icc, U3copy_to_user_short clr %o0
nop
ba,a,pt %xcc, U3copy_to_user_short_ret
#endif /* !(SMALL_COPY_USES_FPU) */ .align 32
small_copy_unaligned:
subcc %o2, 1, %o2
ldub [%o1], %g1
EXNV2(stba %g1, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
bgu,pt %XCC, small_copy_unaligned
add %o1, 1, %o1
retl
clr %o0
arch/sparc64/lib/U3memcpy.S
View file @ a8e0e8ab
/* $Id: U3memcpy.S,v 1.2 2000/11/01 09:29:19 davem Exp $ /* U3memcpy.S: UltraSparc-III optimized memcpy.
* U3memcpy.S: UltraSparc-III optimized memcpy.
* *
* Copyright (C) 1999, 2000 David S. Miller (davem@redhat.com) * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
*/ */
#ifdef __KERNEL__ #ifdef __KERNEL__
...@@ -9,15 +8,20 @@ ...@@ -9,15 +8,20 @@
#include <asm/asi.h> #include <asm/asi.h>
#include <asm/dcu.h> #include <asm/dcu.h>
#include <asm/spitfire.h> #include <asm/spitfire.h>
#undef SMALL_COPY_USES_FPU
#else #else
#define ASI_BLK_P 0xf0 #define ASI_BLK_P 0xf0
#define FPRS_FEF 0x04 #define FPRS_FEF 0x04
#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs #define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
#define SMALL_COPY_USES_FPU
#endif #endif
#ifndef XCC
#define XCC xcc
#endif
.register %g2,#scratch
.register %g3,#scratch
/* Special/non-trivial issues of this code: /* Special/non-trivial issues of this code:
* *
* 1) %o5 is preserved from VISEntryHalf to VISExitHalf * 1) %o5 is preserved from VISEntryHalf to VISExitHalf
...@@ -37,80 +41,55 @@ ...@@ -37,80 +41,55 @@
* of up to 2.4GB per second. * of up to 2.4GB per second.
*/ */
.globl U3memcpy .globl U3memcpy
U3memcpy: /* %o0=dst, %o1=src, %o2=len */ U3memcpy: /* %o0=dst, %o1=src, %o2=len */
#ifndef __KERNEL__ mov %o0, %g5
/* Save away original 'dst' for memcpy return value. */ cmp %o2, 0
mov %o0, %g3 ! A0 Group be,pn %XCC, out
#endif or %o0, %o1, %o3
/* Anything to copy at all? */ cmp %o2, 16
cmp %o2, 0 ! A1 bleu,a,pn %XCC, small_copy
ble,pn %icc, U3memcpy_short_ret ! BR or %o3, %o2, %o3
/* Extremely small copy? */
cmp %o2, 31 ! A0 Group
ble,pn %icc, U3memcpy_short ! BR
/* Large enough to use unrolled prefetch loops? */
cmp %o2, 0x100 ! A1
bge,a,pt %icc, U3memcpy_enter ! BR Group
andcc %o0, 0x3f, %g2 ! A0
ba,pt %xcc, U3memcpy_toosmall ! BR Group
andcc %o0, 0x7, %g2 ! A0
.align 32
U3memcpy_short:
/* Copy %o2 bytes from src to dst, one byte at a time. */
ldub [%o1 + 0x00], %o3 ! MS Group
add %o1, 0x1, %o1 ! A0
add %o0, 0x1, %o0 ! A1
subcc %o2, 1, %o2 ! A0 Group
bg,pt %icc, U3memcpy_short ! BR
stb %o3, [%o0 + -1] ! MS Group (1-cycle stall)
U3memcpy_short_ret:
#ifdef __KERNEL__
retl ! BR Group (0-4 cycle stall)
clr %o0 ! A0
#else
retl ! BR Group (0-4 cycle stall)
mov %g3, %o0 ! A0
#endif
/* Here len >= (6 * 64) and condition codes reflect execution cmp %o2, 256
blu,pt %XCC, medium_copy
andcc %o3, 0x7, %g0
ba,pt %xcc, enter
andcc %o0, 0x3f, %g2
/* Here len >= 256 and condition codes reflect execution
* of "andcc %o0, 0x7, %g2", done by caller. * of "andcc %o0, 0x7, %g2", done by caller.
*/ */
.align 64 .align 64
U3memcpy_enter: enter:
/* Is 'dst' already aligned on an 64-byte boundary? */ /* Is 'dst' already aligned on an 64-byte boundary? */
be,pt %xcc, 2f ! BR be,pt %XCC, 2f
/* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number /* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number
* of bytes to copy to make 'dst' 64-byte aligned. We pre- * of bytes to copy to make 'dst' 64-byte aligned. We pre-
* subtract this from 'len'. * subtract this from 'len'.
*/ */
sub %g2, 0x40, %g2 ! A0 Group sub %g2, 0x40, %g2
sub %g0, %g2, %g2 ! A0 Group sub %g0, %g2, %g2
sub %o2, %g2, %o2 ! A0 Group sub %o2, %g2, %o2
/* Copy %g2 bytes from src to dst, one byte at a time. */ /* Copy %g2 bytes from src to dst, one byte at a time. */
1: ldub [%o1 + 0x00], %o3 ! MS (Group) 1: ldub [%o1 + 0x00], %o3
add %o1, 0x1, %o1 ! A1 add %o1, 0x1, %o1
add %o0, 0x1, %o0 ! A0 Group add %o0, 0x1, %o0
subcc %g2, 0x1, %g2 ! A1 subcc %g2, 0x1, %g2
bg,pt %icc, 1b ! BR Group bg,pt %XCC, 1b
stb %o3, [%o0 + -1] ! MS Group stb %o3, [%o0 + -1]
2: VISEntryHalf ! MS+MS 2: VISEntryHalf
and %o1, 0x7, %g1 ! A1 and %o1, 0x7, %g1
ba,pt %xcc, U3memcpy_begin ! BR ba,pt %xcc, begin
alignaddr %o1, %g0, %o1 ! MS (Break-after) alignaddr %o1, %g0, %o1
.align 64 .align 64
U3memcpy_begin: begin:
#ifdef __KERNEL__ #ifdef __KERNEL__
.globl U3memcpy_nop_1_6 .globl U3memcpy_nop_1_6
U3memcpy_nop_1_6: U3memcpy_nop_1_6:
...@@ -121,146 +100,90 @@ U3memcpy_nop_1_6: ...@@ -121,146 +100,90 @@ U3memcpy_nop_1_6:
stxa %o3, [%g0] ASI_DCU_CONTROL_REG ! Enable P-cache stxa %o3, [%g0] ASI_DCU_CONTROL_REG ! Enable P-cache
membar #Sync membar #Sync
#endif #endif
prefetch [%o1 + 0x000], #one_read ! MS Group1 prefetch [%o1 + 0x000], #one_read
prefetch [%o1 + 0x040], #one_read ! MS Group2 prefetch [%o1 + 0x040], #one_read
andn %o2, (0x40 - 1), %o4 ! A0 andn %o2, (0x40 - 1), %o4
prefetch [%o1 + 0x080], #one_read ! MS Group3 prefetch [%o1 + 0x080], #one_read
cmp %o4, 0x140 ! A0 prefetch [%o1 + 0x0c0], #one_read
prefetch [%o1 + 0x0c0], #one_read ! MS Group4 ldd [%o1 + 0x000], %f0
ldd [%o1 + 0x000], %f0 ! MS Group5 (%f0 results at G8) prefetch [%o1 + 0x100], #one_read
bge,a,pt %icc, 1f ! BR ldd [%o1 + 0x008], %f2
prefetch [%o1 + 0x140], #one_read
prefetch [%o1 + 0x100], #one_read ! MS Group6 ldd [%o1 + 0x010], %f4
1: ldd [%o1 + 0x008], %f2 ! AX (%f2 results at G9) prefetch [%o1 + 0x180], #one_read
cmp %o4, 0x180 ! A1 faligndata %f0, %f2, %f16
bge,a,pt %icc, 1f ! BR ldd [%o1 + 0x018], %f6
prefetch [%o1 + 0x140], #one_read ! MS Group7 faligndata %f2, %f4, %f18
1: ldd [%o1 + 0x010], %f4 ! AX (%f4 results at G10) ldd [%o1 + 0x020], %f8
cmp %o4, 0x1c0 ! A1 faligndata %f4, %f6, %f20
bge,a,pt %icc, 1f ! BR ldd [%o1 + 0x028], %f10
faligndata %f6, %f8, %f22
prefetch [%o1 + 0x180], #one_read ! MS Group8
1: faligndata %f0, %f2, %f16 ! FGA Group9 (%f16 at G12) ldd [%o1 + 0x030], %f12
ldd [%o1 + 0x018], %f6 ! AX (%f6 results at G12) faligndata %f8, %f10, %f24
faligndata %f2, %f4, %f18 ! FGA Group10 (%f18 results at G13) ldd [%o1 + 0x038], %f14
ldd [%o1 + 0x020], %f8 ! MS (%f8 results at G13) faligndata %f10, %f12, %f26
faligndata %f4, %f6, %f20 ! FGA Group12 (1-cycle stall,%f20 at G15) ldd [%o1 + 0x040], %f0
ldd [%o1 + 0x028], %f10 ! MS (%f10 results at G15)
faligndata %f6, %f8, %f22 ! FGA Group13 (%f22 results at G16) sub %o4, 0x80, %o4
add %o1, 0x40, %o1
ldd [%o1 + 0x030], %f12 ! MS (%f12 results at G16) ba,pt %xcc, loop
faligndata %f8, %f10, %f24 ! FGA Group15 (1-cycle stall,%f24 at G18) srl %o4, 6, %o3
ldd [%o1 + 0x038], %f14 ! MS (%f14 results at G18)
faligndata %f10, %f12, %f26 ! FGA Group16 (%f26 results at G19) .align 64
ldd [%o1 + 0x040], %f0 ! MS (%f0 results at G19) loop:
ldd [%o1 + 0x008], %f2
/* We only use the first loop if len > (7 * 64). */ faligndata %f12, %f14, %f28
subcc %o4, 0x1c0, %o4 ! A0 Group17 ldd [%o1 + 0x010], %f4
bg,pt %icc, U3memcpy_loop1 ! BR faligndata %f14, %f0, %f30
add %o1, 0x40, %o1 ! A1 stda %f16, [%o0] ASI_BLK_P
ldd [%o1 + 0x018], %f6
add %o4, 0x140, %o4 ! A0 Group18 faligndata %f0, %f2, %f16
ba,pt %xcc, U3memcpy_loop2 ! BR
srl %o4, 6, %o3 ! A0 Group19 ldd [%o1 + 0x020], %f8
nop faligndata %f2, %f4, %f18
nop ldd [%o1 + 0x028], %f10
nop faligndata %f4, %f6, %f20
nop ldd [%o1 + 0x030], %f12
nop faligndata %f6, %f8, %f22
ldd [%o1 + 0x038], %f14
nop faligndata %f8, %f10, %f24
nop
ldd [%o1 + 0x040], %f0
/* This loop performs the copy and queues new prefetches. prefetch [%o1 + 0x180], #one_read
* We drop into the second loop when len <= (5 * 64). Note faligndata %f10, %f12, %f26
* that this (5 * 64) factor has been subtracted from len subcc %o3, 0x01, %o3
* already. add %o1, 0x40, %o1
*/ bg,pt %XCC, loop
U3memcpy_loop1: add %o0, 0x40, %o0
ldd [%o1 + 0x008], %f2 ! MS Group2 (%f2 results at G5)
faligndata %f12, %f14, %f28 ! FGA (%f28 results at G5)
ldd [%o1 + 0x010], %f4 ! MS Group3 (%f4 results at G6)
faligndata %f14, %f0, %f30 ! FGA Group4 (1-cycle stall, %f30 at G7)
stda %f16, [%o0] ASI_BLK_P ! MS
ldd [%o1 + 0x018], %f6 ! AX (%f6 results at G7)
faligndata %f0, %f2, %f16 ! FGA Group12 (7-cycle stall)
ldd [%o1 + 0x020], %f8 ! MS (%f8 results at G15)
faligndata %f2, %f4, %f18 ! FGA Group13 (%f18 results at G16)
ldd [%o1 + 0x028], %f10 ! MS (%f10 results at G16)
faligndata %f4, %f6, %f20 ! FGA Group14 (%f20 results at G17)
ldd [%o1 + 0x030], %f12 ! MS (%f12 results at G17)
faligndata %f6, %f8, %f22 ! FGA Group15 (%f22 results at G18)
ldd [%o1 + 0x038], %f14 ! MS (%f14 results at G18)
faligndata %f8, %f10, %f24 ! FGA Group16 (%f24 results at G19)
ldd [%o1 + 0x040], %f0 ! AX (%f0 results at G19)
prefetch [%o1 + 0x180], #one_read ! MS
faligndata %f10, %f12, %f26 ! FGA Group17 (%f26 results at G20)
subcc %o4, 0x40, %o4 ! A0
add %o1, 0x40, %o1 ! A1
bg,pt %xcc, U3memcpy_loop1 ! BR
add %o0, 0x40, %o0 ! A0 Group18
U3memcpy_loop2_enter:
mov 5, %o3 ! A1
/* This loop performs on the copy, no new prefetches are
* queued. We do things this way so that we do not perform
* any spurious prefetches past the end of the src buffer.
*/
U3memcpy_loop2:
ldd [%o1 + 0x008], %f2 ! MS
faligndata %f12, %f14, %f28 ! FGA Group2
ldd [%o1 + 0x010], %f4 ! MS
faligndata %f14, %f0, %f30 ! FGA Group4 (1-cycle stall)
stda %f16, [%o0] ASI_BLK_P ! MS
ldd [%o1 + 0x018], %f6 ! AX
faligndata %f0, %f2, %f16 ! FGA Group12 (7-cycle stall)
ldd [%o1 + 0x020], %f8 ! MS
faligndata %f2, %f4, %f18 ! FGA Group13
ldd [%o1 + 0x028], %f10 ! MS
faligndata %f4, %f6, %f20 ! FGA Group14
ldd [%o1 + 0x030], %f12 ! MS
faligndata %f6, %f8, %f22 ! FGA Group15
ldd [%o1 + 0x038], %f14 ! MS
faligndata %f8, %f10, %f24 ! FGA Group16
ldd [%o1 + 0x040], %f0 ! AX
faligndata %f10, %f12, %f26 ! FGA Group17
subcc %o3, 0x01, %o3 ! A0
add %o1, 0x40, %o1 ! A1
bg,pt %xcc, U3memcpy_loop2 ! BR
add %o0, 0x40, %o0 ! A0 Group18
/* Finally we copy the last full 64-byte block. */ /* Finally we copy the last full 64-byte block. */
U3memcpy_loopfini: loopfini:
ldd [%o1 + 0x008], %f2 ! MS ldd [%o1 + 0x008], %f2
faligndata %f12, %f14, %f28 ! FGA faligndata %f12, %f14, %f28
ldd [%o1 + 0x010], %f4 ! MS Group19 ldd [%o1 + 0x010], %f4
faligndata %f14, %f0, %f30 ! FGA faligndata %f14, %f0, %f30
stda %f16, [%o0] ASI_BLK_P ! MS Group20 stda %f16, [%o0] ASI_BLK_P
ldd [%o1 + 0x018], %f6 ! AX ldd [%o1 + 0x018], %f6
faligndata %f0, %f2, %f16 ! FGA Group11 (7-cycle stall) faligndata %f0, %f2, %f16
ldd [%o1 + 0x020], %f8 ! MS ldd [%o1 + 0x020], %f8
faligndata %f2, %f4, %f18 ! FGA Group12 faligndata %f2, %f4, %f18
ldd [%o1 + 0x028], %f10 ! MS ldd [%o1 + 0x028], %f10
faligndata %f4, %f6, %f20 ! FGA Group13 faligndata %f4, %f6, %f20
ldd [%o1 + 0x030], %f12 ! MS ldd [%o1 + 0x030], %f12
faligndata %f6, %f8, %f22 ! FGA Group14 faligndata %f6, %f8, %f22
ldd [%o1 + 0x038], %f14 ! MS ldd [%o1 + 0x038], %f14
faligndata %f8, %f10, %f24 ! FGA Group15 faligndata %f8, %f10, %f24
cmp %g1, 0 ! A0 cmp %g1, 0
be,pt %icc, 1f ! BR be,pt %XCC, 1f
add %o0, 0x40, %o0 ! A1 add %o0, 0x40, %o0
ldd [%o1 + 0x040], %f0 ! MS ldd [%o1 + 0x040], %f0
1: faligndata %f10, %f12, %f26 ! FGA Group16 1: faligndata %f10, %f12, %f26
faligndata %f12, %f14, %f28 ! FGA Group17 faligndata %f12, %f14, %f28
faligndata %f14, %f0, %f30 ! FGA Group18 faligndata %f14, %f0, %f30
stda %f16, [%o0] ASI_BLK_P ! MS stda %f16, [%o0] ASI_BLK_P
add %o0, 0x40, %o0 ! A0 add %o0, 0x40, %o0
add %o1, 0x40, %o1 ! A1 add %o1, 0x40, %o1
#ifdef __KERNEL__ #ifdef __KERNEL__
.globl U3memcpy_nop_2_3 .globl U3memcpy_nop_2_3
U3memcpy_nop_2_3: U3memcpy_nop_2_3:
...@@ -268,161 +191,143 @@ U3memcpy_nop_2_3: ...@@ -268,161 +191,143 @@ U3memcpy_nop_2_3:
stxa %g0, [%o3] ASI_DMMU ! Flush P-cache stxa %g0, [%o3] ASI_DMMU ! Flush P-cache
stxa %g3, [%g0] ASI_DCU_CONTROL_REG ! Disable P-cache stxa %g3, [%g0] ASI_DCU_CONTROL_REG ! Disable P-cache
#endif #endif
membar #Sync ! MS Group26 (7-cycle stall) membar #Sync
/* Now we copy the (len modulo 64) bytes at the end. /* Now we copy the (len modulo 64) bytes at the end.
* Note how we borrow the %f0 loaded above. * Note how we borrow the %f0 loaded above.
* *
* Also notice how this code is careful not to perform a * Also notice how this code is careful not to perform a
* load past the end of the src buffer just like similar * load past the end of the src buffer.
* code found in U3memcpy_toosmall processing.
*/ */
U3memcpy_loopend: loopend:
and %o2, 0x3f, %o2 ! A0 Group and %o2, 0x3f, %o2
andcc %o2, 0x38, %g2 ! A0 Group andcc %o2, 0x38, %g2
be,pn %icc, U3memcpy_endcruft ! BR be,pn %XCC, endcruft
subcc %g2, 0x8, %g2 ! A1 subcc %g2, 0x8, %g2
be,pn %icc, U3memcpy_endcruft ! BR Group be,pn %XCC, endcruft
cmp %g1, 0 ! A0 cmp %g1, 0
be,a,pt %icc, 1f ! BR Group be,a,pt %XCC, 1f
ldd [%o1 + 0x00], %f0 ! MS ldd [%o1 + 0x00], %f0
1: ldd [%o1 + 0x08], %f2 ! MS Group 1: ldd [%o1 + 0x08], %f2
add %o1, 0x8, %o1 ! A0 add %o1, 0x8, %o1
sub %o2, 0x8, %o2 ! A1 sub %o2, 0x8, %o2
subcc %g2, 0x8, %g2 ! A0 Group subcc %g2, 0x8, %g2
faligndata %f0, %f2, %f8 ! FGA Group faligndata %f0, %f2, %f8
std %f8, [%o0 + 0x00] ! MS (XXX does it stall here? XXX) std %f8, [%o0 + 0x00]
be,pn %icc, U3memcpy_endcruft ! BR be,pn %XCC, endcruft
add %o0, 0x8, %o0 ! A0 add %o0, 0x8, %o0
ldd [%o1 + 0x08], %f0 ! MS Group ldd [%o1 + 0x08], %f0
add %o1, 0x8, %o1 ! A0 add %o1, 0x8, %o1
sub %o2, 0x8, %o2 ! A1 sub %o2, 0x8, %o2
subcc %g2, 0x8, %g2 ! A0 Group subcc %g2, 0x8, %g2
faligndata %f2, %f0, %f8 ! FGA faligndata %f2, %f0, %f8
std %f8, [%o0 + 0x00] ! MS (XXX does it stall here? XXX) std %f8, [%o0 + 0x00]
bne,pn %icc, 1b ! BR bne,pn %XCC, 1b
add %o0, 0x8, %o0 ! A0 Group add %o0, 0x8, %o0
/* If anything is left, we copy it one byte at a time. /* If anything is left, we copy it one byte at a time.
* Note that %g1 is (src & 0x3) saved above before the * Note that %g1 is (src & 0x3) saved above before the
* alignaddr was performed. * alignaddr was performed.
*/ */
U3memcpy_endcruft: endcruft:
cmp %o2, 0 cmp %o2, 0
add %o1, %g1, %o1 add %o1, %g1, %o1
VISExitHalf VISExitHalf
be,pn %icc, U3memcpy_short_ret be,pn %XCC, out
nop sub %o0, %o1, %o3
ba,a,pt %xcc, U3memcpy_short
/* If we get here, then 32 <= len < (6 * 64) */ andcc %g1, 0x7, %g0
U3memcpy_toosmall: bne,pn %icc, small_copy_unaligned
andcc %o2, 0x8, %g0
be,pt %icc, 1f
nop
ldx [%o1], %o5
stx %o5, [%o1 + %o3]
add %o1, 0x8, %o1
#ifdef SMALL_COPY_USES_FPU 1: andcc %o2, 0x4, %g0
be,pt %icc, 1f
nop
lduw [%o1], %o5
stw %o5, [%o1 + %o3]
add %o1, 0x4, %o1
/* Is 'dst' already aligned on an 8-byte boundary? */ 1: andcc %o2, 0x2, %g0
be,pt %xcc, 2f ! BR Group be,pt %icc, 1f
nop
lduh [%o1], %o5
sth %o5, [%o1 + %o3]
add %o1, 0x2, %o1
/* Compute abs((dst & 7) - 8) into %g2. This is the number 1: andcc %o2, 0x1, %g0
* of bytes to copy to make 'dst' 8-byte aligned. We pre- be,pt %icc, out
* subtract this from 'len'. nop
*/ ldub [%o1], %o5
sub %g2, 0x8, %g2 ! A0 ba,pt %xcc, out
sub %g0, %g2, %g2 ! A0 Group (reg-dep) stb %o5, [%o1 + %o3]
sub %o2, %g2, %o2 ! A0 Group (reg-dep)
medium_copy: /* 16 < len <= 64 */
bne,pn %XCC, small_copy_unaligned
sub %o0, %o1, %o3
medium_copy_aligned:
andn %o2, 0x7, %o4
and %o2, 0x7, %o2
1: subcc %o4, 0x8, %o4
ldx [%o1], %o5
stx %o5, [%o1 + %o3]
bgu,pt %XCC, 1b
add %o1, 0x8, %o1
andcc %o2, 0x4, %g0
be,pt %XCC, 1f
nop
sub %o2, 0x4, %o2
lduw [%o1], %o5
stw %o5, [%o1 + %o3]
add %o1, 0x4, %o1
1: cmp %o2, 0
be,pt %XCC, out
nop
ba,pt %xcc, small_copy_unaligned
nop
/* Copy %g2 bytes from src to dst, one byte at a time. */ small_copy: /* 0 < len <= 16 */
1: ldub [%o1 + 0x00], %o3 ! MS (Group) (%o3 in 3 cycles) andcc %o3, 0x3, %g0
add %o1, 0x1, %o1 ! A1 bne,pn %XCC, small_copy_unaligned
add %o0, 0x1, %o0 ! A0 Group sub %o0, %o1, %o3
subcc %g2, 0x1, %g2 ! A1
bg,pt %icc, 1b ! BR Group small_copy_aligned:
stb %o3, [%o0 + -1] ! MS Group subcc %o2, 4, %o2
lduw [%o1], %g1
stw %g1, [%o1 + %o3]
bgu,pt %XCC, small_copy_aligned
add %o1, 4, %o1
2: VISEntryHalf ! MS+MS out: retl
mov %g5, %o0
/* Compute (len - (len % 8)) into %g2. This is guaranteed .align 32
* to be nonzero. small_copy_unaligned:
*/ subcc %o2, 1, %o2
andn %o2, 0x7, %g2 ! A0 Group ldub [%o1], %g1
stb %g1, [%o1 + %o3]
/* You may read this and believe that it allows reading bgu,pt %XCC, small_copy_unaligned
* one 8-byte longword past the end of src. It actually add %o1, 1, %o1
* does not, as %g2 is subtracted as loads are done from retl
* src, so we always stop before running off the end. mov %g5, %o0
* Also, we are guaranteed to have at least 0x10 bytes
* to move here. /* Act like copy_{to,in}_user(), ie. return zero instead
* of original destination pointer. This is invoked when
* copy_{to,in}_user() finds that %asi is kernel space.
*/ */
sub %g2, 0x8, %g2 ! A0 Group (reg-dep) .globl U3memcpy_user_stub
alignaddr %o1, %g0, %g1 ! MS (Break-after) U3memcpy_user_stub:
ldd [%g1 + 0x00], %f0 ! MS Group (1-cycle stall) save %sp, -192, %sp
add %g1, 0x8, %g1 ! A0 mov %i0, %o0
mov %i1, %o1
1: ldd [%g1 + 0x00], %f2 ! MS Group call U3memcpy
add %g1, 0x8, %g1 ! A0 mov %i2, %o2
sub %o2, 0x8, %o2 ! A1 ret
subcc %g2, 0x8, %g2 ! A0 Group restore %g0, %g0, %o0
faligndata %f0, %f2, %f8 ! FGA Group (1-cycle stall)
std %f8, [%o0 + 0x00] ! MS Group (2-cycle stall)
add %o1, 0x8, %o1 ! A0
be,pn %icc, 2f ! BR
add %o0, 0x8, %o0 ! A1
ldd [%g1 + 0x00], %f0 ! MS Group
add %g1, 0x8, %g1 ! A0
sub %o2, 0x8, %o2 ! A1
subcc %g2, 0x8, %g2 ! A0 Group
faligndata %f2, %f0, %f8 ! FGA Group (1-cycle stall)
std %f8, [%o0 + 0x00] ! MS Group (2-cycle stall)
add %o1, 0x8, %o1 ! A0
bne,pn %icc, 1b ! BR
add %o0, 0x8, %o0 ! A1
/* Nothing left to copy? */
2: cmp %o2, 0 ! A0 Group
VISExitHalf ! A0+MS
be,pn %icc, U3memcpy_short_ret ! BR Group
nop ! A0
ba,a,pt %xcc, U3memcpy_short ! BR Group
#else /* !(SMALL_COPY_USES_FPU) */
xor %o1, %o0, %g2
andcc %g2, 0x7, %g0
bne,pn %icc, U3memcpy_short
andcc %o1, 0x7, %g2
be,pt %xcc, 2f
sub %g2, 0x8, %g2
sub %g0, %g2, %g2
sub %o2, %g2, %o2
1: ldub [%o1 + 0x00], %o3
add %o1, 0x1, %o1
add %o0, 0x1, %o0
subcc %g2, 0x1, %g2
bg,pt %icc, 1b
stb %o3, [%o0 + -1]
2: andn %o2, 0x7, %g2
sub %o2, %g2, %o2
3: ldx [%o1 + 0x00], %o3
add %o1, 0x8, %o1
add %o0, 0x8, %o0
subcc %g2, 0x8, %g2
bg,pt %icc, 3b
stx %o3, [%o0 + -8]
cmp %o2, 0
bne,pn %icc, U3memcpy_short
nop
ba,a,pt %xcc, U3memcpy_short_ret
#endif /* !(SMALL_COPY_USES_FPU) */
arch/sparc64/lib/VIScopy.S
View file @ a8e0e8ab
...@@ -306,11 +306,7 @@ ...@@ -306,11 +306,7 @@
.globl __memcpy_begin .globl __memcpy_begin
__memcpy_begin: __memcpy_begin:
.globl __memcpy
.type __memcpy,@function
memcpy_private: memcpy_private:
__memcpy:
memcpy: mov ASI_P, asi_src ! IEU0 Group memcpy: mov ASI_P, asi_src ! IEU0 Group
brnz,pt %o2, __memcpy_entry ! CTI brnz,pt %o2, __memcpy_entry ! CTI
mov ASI_P, asi_dest ! IEU1 mov ASI_P, asi_dest ! IEU1
......
arch/sparc64/lib/splock.S
View file @ a8e0e8ab
...@@ -6,6 +6,18 @@ ...@@ -6,6 +6,18 @@
.text .text
.align 64 .align 64
.globl _raw_spin_lock
_raw_spin_lock: /* %o0 = lock_ptr */
1: ldstub [%o0], %g7
brnz,pn %g7, 2f
membar #StoreLoad | #StoreStore
retl
nop
2: ldub [%o0], %g7
brnz,pt %g7, 2b
membar #LoadLoad
ba,a,pt %xcc, 1b
.globl _raw_spin_lock_flags .globl _raw_spin_lock_flags
_raw_spin_lock_flags: /* %o0 = lock_ptr, %o1 = irq_flags */ _raw_spin_lock_flags: /* %o0 = lock_ptr, %o1 = irq_flags */
1: ldstub [%o0], %g7 1: ldstub [%o0], %g7
......
drivers/bluetooth/bluecard_cs.c
View file @ a8e0e8ab
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/types.h> #include <linux/types.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/delay.h>
#include <linux/timer.h> #include <linux/timer.h>
#include <linux/errno.h> #include <linux/errno.h>
#include <linux/ptrace.h> #include <linux/ptrace.h>
...@@ -776,8 +777,7 @@ int bluecard_open(bluecard_info_t *info) ...@@ -776,8 +777,7 @@ int bluecard_open(bluecard_info_t *info)
outb(0x80, iobase + 0x30); outb(0x80, iobase + 0x30);
/* Wait some time */ /* Wait some time */
set_current_state(TASK_INTERRUPTIBLE); msleep(10);
schedule_timeout(HZ / 100);
/* Turn FPGA on */ /* Turn FPGA on */
outb(0x00, iobase + 0x30); outb(0x00, iobase + 0x30);
...@@ -823,8 +823,7 @@ int bluecard_open(bluecard_info_t *info) ...@@ -823,8 +823,7 @@ int bluecard_open(bluecard_info_t *info)
outb((0x0f << RTS_LEVEL_SHIFT_BITS) | 1, iobase + REG_RX_CONTROL); outb((0x0f << RTS_LEVEL_SHIFT_BITS) | 1, iobase + REG_RX_CONTROL);
/* Timeout before it is safe to send the first HCI packet */ /* Timeout before it is safe to send the first HCI packet */
set_current_state(TASK_INTERRUPTIBLE); msleep(1250);
schedule_timeout((HZ * 5) / 4); // or set it to 3/2
/* Register HCI device */ /* Register HCI device */
if (hci_register_dev(hdev) < 0) { if (hci_register_dev(hdev) < 0) {
......
drivers/bluetooth/bt3c_cs.c
View file @ a8e0e8ab
...@@ -25,12 +25,11 @@ ...@@ -25,12 +25,11 @@
#include <linux/module.h> #include <linux/module.h>
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/kmod.h>
#include <linux/init.h> #include <linux/init.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/delay.h>
#include <linux/types.h> #include <linux/types.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/delay.h>
#include <linux/errno.h> #include <linux/errno.h>
#include <linux/ptrace.h> #include <linux/ptrace.h>
#include <linux/ioport.h> #include <linux/ioport.h>
...@@ -639,8 +638,7 @@ int bt3c_open(bt3c_info_t *info) ...@@ -639,8 +638,7 @@ int bt3c_open(bt3c_info_t *info)
} }
/* Timeout before it is safe to send the first HCI packet */ /* Timeout before it is safe to send the first HCI packet */
set_current_state(TASK_INTERRUPTIBLE); msleep(1000);
schedule_timeout(HZ);
/* Register HCI device */ /* Register HCI device */
err = hci_register_dev(hdev); err = hci_register_dev(hdev);
......
drivers/bluetooth/btuart_cs.c
View file @ a8e0e8ab
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/types.h> #include <linux/types.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/delay.h>
#include <linux/errno.h> #include <linux/errno.h>
#include <linux/ptrace.h> #include <linux/ptrace.h>
#include <linux/ioport.h> #include <linux/ioport.h>
...@@ -545,8 +546,7 @@ int btuart_open(btuart_info_t *info) ...@@ -545,8 +546,7 @@ int btuart_open(btuart_info_t *info)
btuart_change_speed(info, DEFAULT_BAUD_RATE); btuart_change_speed(info, DEFAULT_BAUD_RATE);
/* Timeout before it is safe to send the first HCI packet */ /* Timeout before it is safe to send the first HCI packet */
set_current_state(TASK_INTERRUPTIBLE); msleep(1000);
schedule_timeout(HZ);
/* Register HCI device */ /* Register HCI device */
if (hci_register_dev(hdev) < 0) { if (hci_register_dev(hdev) < 0) {
......
drivers/bluetooth/dtl1_cs.c
View file @ a8e0e8ab
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/types.h> #include <linux/types.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/delay.h>
#include <linux/errno.h> #include <linux/errno.h>
#include <linux/ptrace.h> #include <linux/ptrace.h>
#include <linux/ioport.h> #include <linux/ioport.h>
...@@ -524,8 +525,7 @@ int dtl1_open(dtl1_info_t *info) ...@@ -524,8 +525,7 @@ int dtl1_open(dtl1_info_t *info)
spin_unlock_irqrestore(&(info->lock), flags); spin_unlock_irqrestore(&(info->lock), flags);
/* Timeout before it is safe to send the first HCI packet */ /* Timeout before it is safe to send the first HCI packet */
set_current_state(TASK_INTERRUPTIBLE); msleep(2000);
schedule_timeout(HZ * 2);
/* Register HCI device */ /* Register HCI device */
if (hci_register_dev(hdev) < 0) { if (hci_register_dev(hdev) < 0) {
......
drivers/bluetooth/hci_bcsp.c
View file @ a8e0e8ab
...@@ -633,7 +633,8 @@ static void bcsp_timed_event(unsigned long arg) ...@@ -633,7 +633,8 @@ static void bcsp_timed_event(unsigned long arg)
struct sk_buff *skb; struct sk_buff *skb;
unsigned long flags; unsigned long flags;
BT_ERR("Timeout, retransmitting %u pkts", bcsp->unack.qlen); BT_DBG("hu %p retransmitting %u pkts", hu, bcsp->unack.qlen);
spin_lock_irqsave(&bcsp->unack.lock, flags); spin_lock_irqsave(&bcsp->unack.lock, flags);
while ((skb = __skb_dequeue_tail(&bcsp->unack)) != NULL) { while ((skb = __skb_dequeue_tail(&bcsp->unack)) != NULL) {
......
drivers/net/ethertap.c
View file @ a8e0e8ab
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
#include <linux/config.h> #include <linux/config.h>
#include <linux/module.h> #include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/jiffies.h> #include <linux/jiffies.h>
#include <linux/slab.h> #include <linux/slab.h>
...@@ -45,7 +45,7 @@ static void set_multicast_list(struct net_device *dev); ...@@ -45,7 +45,7 @@ static void set_multicast_list(struct net_device *dev);
static int ethertap_debug; static int ethertap_debug;
static int max_taps = 1; static int max_taps = 1;
MODULE_PARM(max_taps, "i"); module_param(max_taps, int, 0);
MODULE_PARM_DESC(max_taps,"Max number of ethernet tap devices"); MODULE_PARM_DESC(max_taps,"Max number of ethernet tap devices");
static struct net_device **tap_map; /* Returns the tap device for a given netlink */ static struct net_device **tap_map; /* Returns the tap device for a given netlink */
......
drivers/net/sungem.c
View file @ a8e0e8ab
...@@ -2024,8 +2024,7 @@ static void gem_stop_phy(struct gem *gp) ...@@ -2024,8 +2024,7 @@ static void gem_stop_phy(struct gem *gp)
/* Let the chip settle down a bit, it seems that helps /* Let the chip settle down a bit, it seems that helps
* for sleep mode on some models * for sleep mode on some models
*/ */
set_current_state(TASK_UNINTERRUPTIBLE); msleep(10);
schedule_timeout(HZ/100);
/* Make sure we aren't polling PHY status change. We /* Make sure we aren't polling PHY status change. We
* don't currently use that feature though * don't currently use that feature though
...@@ -2043,8 +2042,7 @@ static void gem_stop_phy(struct gem *gp) ...@@ -2043,8 +2042,7 @@ static void gem_stop_phy(struct gem *gp)
* dont wait a bit here, looks like the chip takes * dont wait a bit here, looks like the chip takes
* some time to really shut down * some time to really shut down
*/ */
set_current_state(TASK_UNINTERRUPTIBLE); msleep(10);
schedule_timeout(HZ/100);
} }
writel(0, gp->regs + MAC_TXCFG); writel(0, gp->regs + MAC_TXCFG);
......
drivers/sbus/char/bbc_envctrl.c
View file @ a8e0e8ab
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/delay.h>
#include <asm/oplib.h> #include <asm/oplib.h>
#include <asm/ebus.h> #include <asm/ebus.h>
#define __KERNEL_SYSCALLS__ #define __KERNEL_SYSCALLS__
...@@ -622,9 +623,7 @@ void bbc_envctrl_cleanup(void) ...@@ -622,9 +623,7 @@ void bbc_envctrl_cleanup(void)
read_unlock(&tasklist_lock); read_unlock(&tasklist_lock);
if (!found) if (!found)
break; break;
current->state = TASK_INTERRUPTIBLE; msleep(1000);
schedule_timeout(HZ);
current->state = TASK_RUNNING;
} }
kenvctrld_task = NULL; kenvctrld_task = NULL;
} }
......
drivers/sbus/char/envctrl.c
View file @ a8e0e8ab
...@@ -1181,8 +1181,7 @@ static void __exit envctrl_cleanup(void) ...@@ -1181,8 +1181,7 @@ static void __exit envctrl_cleanup(void)
if (!found) if (!found)
break; break;
current->state = TASK_INTERRUPTIBLE; msleep(1000);
schedule_timeout(HZ);
} }
kenvctrld_task = NULL; kenvctrld_task = NULL;
} }
......
include/asm-arm/bitops.h
View file @ a8e0e8ab
...@@ -227,7 +227,7 @@ extern int _test_and_change_bit_be(int nr, volatile unsigned long * p); ...@@ -227,7 +227,7 @@ extern int _test_and_change_bit_be(int nr, volatile unsigned long * p);
extern int _find_first_zero_bit_be(void * p, unsigned size); extern int _find_first_zero_bit_be(void * p, unsigned size);
extern int _find_next_zero_bit_be(void * p, int size, int offset); extern int _find_next_zero_bit_be(void * p, int size, int offset);
extern int _find_first_bit_be(const unsigned long *p, unsigned size); extern int _find_first_bit_be(const unsigned long *p, unsigned size);
extern int _find_next_bit_be(unsigned long *p, int size, int offset); extern int _find_next_bit_be(const unsigned long *p, int size, int offset);
/* /*
* The __* form of bitops are non-atomic and may be reordered. * The __* form of bitops are non-atomic and may be reordered.
......
include/asm-sparc/pci.h
View file @ a8e0e8ab
...@@ -87,12 +87,6 @@ extern dma_addr_t pci_map_page(struct pci_dev *hwdev, struct page *page, ...@@ -87,12 +87,6 @@ extern dma_addr_t pci_map_page(struct pci_dev *hwdev, struct page *page,
extern void pci_unmap_page(struct pci_dev *hwdev, extern void pci_unmap_page(struct pci_dev *hwdev,
dma_addr_t dma_address, size_t size, int direction); dma_addr_t dma_address, size_t size, int direction);
/* map_page and map_single cannot fail */
static inline int pci_dma_mapping_error(dma_addr_t dma_addr)
{
return 0;
}
/* Map a set of buffers described by scatterlist in streaming /* Map a set of buffers described by scatterlist in streaming
* mode for DMA. This is the scather-gather version of the * mode for DMA. This is the scather-gather version of the
* above pci_map_single interface. Here the scatter gather list * above pci_map_single interface. Here the scatter gather list
......
include/asm-sparc64/page.h
View file @ a8e0e8ab
...@@ -18,7 +18,7 @@ extern void _clear_page(void *page); ...@@ -18,7 +18,7 @@ extern void _clear_page(void *page);
#define clear_page(X) _clear_page((void *)(X)) #define clear_page(X) _clear_page((void *)(X))
struct page; struct page;
extern void clear_user_page(void *addr, unsigned long vaddr, struct page *page); extern void clear_user_page(void *addr, unsigned long vaddr, struct page *page);
#define copy_page(X,Y) __memcpy((void *)(X), (void *)(Y), PAGE_SIZE) #define copy_page(X,Y) memcpy((void *)(X), (void *)(Y), PAGE_SIZE)
extern void copy_user_page(void *to, void *from, unsigned long vaddr, struct page *topage); extern void copy_user_page(void *to, void *from, unsigned long vaddr, struct page *topage);
/* GROSS, defining this makes gcc pass these types as aggregates, /* GROSS, defining this makes gcc pass these types as aggregates,
......
include/asm-sparc64/spinlock.h
View file @ a8e0e8ab
...@@ -41,22 +41,8 @@ typedef unsigned char spinlock_t; ...@@ -41,22 +41,8 @@ typedef unsigned char spinlock_t;
do { membar("#LoadLoad"); \ do { membar("#LoadLoad"); \
} while(*((volatile unsigned char *)lock)) } while(*((volatile unsigned char *)lock))
static __inline__ void _raw_spin_lock(spinlock_t *lock) /* arch/sparc64/lib/spinlock.S */
{ extern void _raw_spin_lock(spinlock_t *lock);
__asm__ __volatile__(
"1: ldstub [%0], %%g7\n"
" brnz,pn %%g7, 2f\n"
" membar #StoreLoad | #StoreStore\n"
" .subsection 2\n"
"2: ldub [%0], %%g7\n"
" brnz,pt %%g7, 2b\n"
" membar #LoadLoad\n"
" b,a,pt %%xcc, 1b\n"
" .previous\n"
: /* no outputs */
: "r" (lock)
: "g7", "memory");
}
static __inline__ int _raw_spin_trylock(spinlock_t *lock) static __inline__ int _raw_spin_trylock(spinlock_t *lock)
{ {
......
include/asm-sparc64/string.h
View file @ a8e0e8ab
...@@ -16,9 +16,7 @@ ...@@ -16,9 +16,7 @@
#include <asm/asi.h> #include <asm/asi.h>
extern void __memmove(void *,const void *,__kernel_size_t); extern void __memmove(void *,const void *,__kernel_size_t);
extern __kernel_size_t __memcpy(void *,const void *,__kernel_size_t);
extern void *__memset(void *,int,__kernel_size_t); extern void *__memset(void *,int,__kernel_size_t);
extern void *__builtin_memcpy(void *,const void *,__kernel_size_t);
extern void *__builtin_memset(void *,int,__kernel_size_t); extern void *__builtin_memset(void *,int,__kernel_size_t);
#ifndef EXPORT_SYMTAB_STROPS #ifndef EXPORT_SYMTAB_STROPS
...@@ -37,29 +35,7 @@ extern void *__builtin_memset(void *,int,__kernel_size_t); ...@@ -37,29 +35,7 @@ extern void *__builtin_memset(void *,int,__kernel_size_t);
#define __HAVE_ARCH_MEMCPY #define __HAVE_ARCH_MEMCPY
static inline void *__constant_memcpy(void *to, const void *from, __kernel_size_t n) extern void * memcpy(void *,const void *,__kernel_size_t);
{
if(n) {
if(n <= 32) {
__builtin_memcpy(to, from, n);
} else {
__memcpy(to, from, n);
}
}
return to;
}
static inline void *__nonconstant_memcpy(void *to, const void *from, __kernel_size_t n)
{
__memcpy(to, from, n);
return to;
}
#undef memcpy
#define memcpy(t, f, n) \
(__builtin_constant_p(n) ? \
__constant_memcpy((t),(f),(n)) : \
__nonconstant_memcpy((t),(f),(n)))
#define __HAVE_ARCH_MEMSET #define __HAVE_ARCH_MEMSET
......
include/linux/icmpv6.h
View file @ a8e0e8ab
...@@ -86,17 +86,15 @@ struct icmp6hdr { ...@@ -86,17 +86,15 @@ struct icmp6hdr {
#define ICMPV6_MGM_REPORT 131 #define ICMPV6_MGM_REPORT 131
#define ICMPV6_MGM_REDUCTION 132 #define ICMPV6_MGM_REDUCTION 132
/* definitions for MLDv2 */ #define ICMPV6_NI_QUERY 139
#define ICMPV6_NI_REPLY 140
#define MLD2_MODE_IS_INCLUDE 1
#define MLD2_MODE_IS_EXCLUDE 2
#define MLD2_CHANGE_TO_INCLUDE 3
#define MLD2_CHANGE_TO_EXCLUDE 4
#define MLD2_ALLOW_NEW_SOURCES 5
#define MLD2_BLOCK_OLD_SOURCES 6
#define ICMPV6_MLD2_REPORT 143 #define ICMPV6_MLD2_REPORT 143
#define MLD2_ALL_MCR_INIT { { { 0xff,0x02,0,0,0,0,0,0,0,0,0,0,0,0,0,0x16 } } }
#define ICMPV6_DHAAD_REQUEST 144
#define ICMPV6_DHAAD_REPLY 145
#define ICMPV6_MOBILE_PREFIX_SOL 146
#define ICMPV6_MOBILE_PREFIX_ADV 147
/* /*
* Codes for Destination Unreachable * Codes for Destination Unreachable
...@@ -139,6 +137,18 @@ struct icmp6_filter { ...@@ -139,6 +137,18 @@ struct icmp6_filter {
__u32 data[8]; __u32 data[8];
}; };
/*
* Definitions for MLDv2
*/
#define MLD2_MODE_IS_INCLUDE 1
#define MLD2_MODE_IS_EXCLUDE 2
#define MLD2_CHANGE_TO_INCLUDE 3
#define MLD2_CHANGE_TO_EXCLUDE 4
#define MLD2_ALLOW_NEW_SOURCES 5
#define MLD2_BLOCK_OLD_SOURCES 6
#define MLD2_ALL_MCR_INIT { { { 0xff,0x02,0,0,0,0,0,0,0,0,0,0,0,0,0,0x16 } } }
#ifdef __KERNEL__ #ifdef __KERNEL__
#include <linux/netdevice.h> #include <linux/netdevice.h>
......
include/linux/pkt_cls.h
View file @ a8e0e8ab
...@@ -117,17 +117,8 @@ enum ...@@ -117,17 +117,8 @@ enum
struct tc_police struct tc_police
{ {
__u32 index; __u32 index;
#ifdef CONFIG_NET_CLS_ACT
int refcnt; int refcnt;
int bindcnt; int bindcnt;
#endif
/* Turned off because it requires new tc
* to work (for now maintain ABI)
*
#ifdef CONFIG_NET_CLS_ACT
__u32 capab;
#endif
*/
int action; int action;
#define TC_POLICE_UNSPEC TC_ACT_UNSPEC #define TC_POLICE_UNSPEC TC_ACT_UNSPEC
#define TC_POLICE_OK TC_ACT_OK #define TC_POLICE_OK TC_ACT_OK
...@@ -195,12 +186,8 @@ enum ...@@ -195,12 +186,8 @@ enum
TCA_U32_DIVISOR, TCA_U32_DIVISOR,
TCA_U32_SEL, TCA_U32_SEL,
TCA_U32_POLICE, TCA_U32_POLICE,
#ifdef CONFIG_NET_CLS_ACT
TCA_U32_ACT, TCA_U32_ACT,
#endif
#ifdef CONFIG_NET_CLS_IND
TCA_U32_INDEV, TCA_U32_INDEV,
#endif
__TCA_U32_MAX __TCA_U32_MAX
}; };
...@@ -212,9 +199,7 @@ struct tc_u32_key ...@@ -212,9 +199,7 @@ struct tc_u32_key
__u32 val; __u32 val;
int off; int off;
int offmask; int offmask;
#ifdef CONFIG_CLS_U32_PERF __u32 kcnt;
unsigned long kcnt;
#endif
}; };
struct tc_u32_sel struct tc_u32_sel
...@@ -229,11 +214,9 @@ struct tc_u32_sel ...@@ -229,11 +214,9 @@ struct tc_u32_sel
short hoff; short hoff;
__u32 hmask; __u32 hmask;
#ifdef CONFIG_CLS_U32_PERF struct tc_u32_key keys[0];
unsigned long rcnt; unsigned long rcnt;
unsigned long rhit; unsigned long rhit;
#endif
struct tc_u32_key keys[0];
}; };
/* Flags */ /* Flags */
...@@ -300,12 +283,8 @@ enum ...@@ -300,12 +283,8 @@ enum
TCA_FW_UNSPEC, TCA_FW_UNSPEC,
TCA_FW_CLASSID, TCA_FW_CLASSID,
TCA_FW_POLICE, TCA_FW_POLICE,
#ifdef CONFIG_NET_CLS_IND
TCA_FW_INDEV, TCA_FW_INDEV,
#endif
#ifdef CONFIG_NET_CLS_ACT
TCA_FW_ACT, TCA_FW_ACT,
#endif
__TCA_FW_MAX __TCA_FW_MAX
}; };
......
include/linux/rtnetlink.h
View file @ a8e0e8ab
...@@ -746,10 +746,6 @@ extern void rtmsg_ifinfo(int type, struct net_device *dev, unsigned change); ...@@ -746,10 +746,6 @@ extern void rtmsg_ifinfo(int type, struct net_device *dev, unsigned change);
extern struct semaphore rtnl_sem; extern struct semaphore rtnl_sem;
#define rtnl_exlock() do { } while(0)
#define rtnl_exunlock() do { } while(0)
#define rtnl_exlock_nowait() (0)
#define rtnl_shlock() down(&rtnl_sem) #define rtnl_shlock() down(&rtnl_sem)
#define rtnl_shlock_nowait() down_trylock(&rtnl_sem) #define rtnl_shlock_nowait() down_trylock(&rtnl_sem)
......
include/net/addrconf.h
View file @ a8e0e8ab
...@@ -160,8 +160,8 @@ static inline void in6_ifa_put(struct inet6_ifaddr *ifp) ...@@ -160,8 +160,8 @@ static inline void in6_ifa_put(struct inet6_ifaddr *ifp)
inet6_ifa_finish_destroy(ifp); inet6_ifa_finish_destroy(ifp);
} }
#define __in6_ifa_put(idev) atomic_dec(&(idev)->refcnt) #define __in6_ifa_put(ifp) atomic_dec(&(ifp)->refcnt)
#define in6_ifa_hold(idev) atomic_inc(&(idev)->refcnt) #define in6_ifa_hold(ifp) atomic_inc(&(ifp)->refcnt)
extern void addrconf_forwarding_on(void); extern void addrconf_forwarding_on(void);
......
include/net/pkt_sched.h
View file @ a8e0e8ab
...@@ -105,43 +105,15 @@ struct qdisc_rate_table ...@@ -105,43 +105,15 @@ struct qdisc_rate_table
int refcnt; int refcnt;
}; };
static inline void sch_tree_lock(struct Qdisc *q) extern void qdisc_lock_tree(struct net_device *dev);
{ extern void qdisc_unlock_tree(struct net_device *dev);
write_lock(&qdisc_tree_lock);
spin_lock_bh(&q->dev->queue_lock);
}
static inline void sch_tree_unlock(struct Qdisc *q)
{
spin_unlock_bh(&q->dev->queue_lock);
write_unlock(&qdisc_tree_lock);
}
static inline void tcf_tree_lock(struct tcf_proto *tp)
{
write_lock(&qdisc_tree_lock);
spin_lock_bh(&tp->q->dev->queue_lock);
}
static inline void tcf_tree_unlock(struct tcf_proto *tp)
{
spin_unlock_bh(&tp->q->dev->queue_lock);
write_unlock(&qdisc_tree_lock);
}
#define sch_tree_lock(q) qdisc_lock_tree((q)->dev)
#define sch_tree_unlock(q) qdisc_unlock_tree((q)->dev)
#define tcf_tree_lock(tp) qdisc_lock_tree((tp)->q->dev)
#define tcf_tree_unlock(tp) qdisc_unlock_tree((tp)->q->dev)
static inline unsigned long #define cls_set_class(tp, clp, cl) tcf_set_class(tp, clp, cl)
cls_set_class(struct tcf_proto *tp, unsigned long *clp, unsigned long cl)
{
unsigned long old_cl;
tcf_tree_lock(tp);
old_cl = *clp;
*clp = cl;
tcf_tree_unlock(tp);
return old_cl;
}
static inline unsigned long static inline unsigned long
__cls_set_class(unsigned long *clp, unsigned long cl) __cls_set_class(unsigned long *clp, unsigned long cl)
{ {
...@@ -407,6 +379,8 @@ extern int tcf_act_police_dump(struct sk_buff *, struct tc_action *, int, int); ...@@ -407,6 +379,8 @@ extern int tcf_act_police_dump(struct sk_buff *, struct tc_action *, int, int);
extern int tcf_act_police(struct sk_buff **skb, struct tc_action *a); extern int tcf_act_police(struct sk_buff **skb, struct tc_action *a);
#endif #endif
extern unsigned long tcf_set_class(struct tcf_proto *tp, unsigned long *clp,
unsigned long cl);
extern int tcf_police(struct sk_buff *skb, struct tcf_police *p); extern int tcf_police(struct sk_buff *skb, struct tcf_police *p);
extern int qdisc_copy_stats(struct sk_buff *skb, struct tc_stats *st, spinlock_t *lock); extern int qdisc_copy_stats(struct sk_buff *skb, struct tc_stats *st, spinlock_t *lock);
extern void tcf_police_destroy(struct tcf_police *p); extern void tcf_police_destroy(struct tcf_police *p);
...@@ -458,13 +432,6 @@ void qdisc_put_rtab(struct qdisc_rate_table *tab); ...@@ -458,13 +432,6 @@ void qdisc_put_rtab(struct qdisc_rate_table *tab);
extern int qdisc_restart(struct net_device *dev); extern int qdisc_restart(struct net_device *dev);
static inline void qdisc_run(struct net_device *dev)
{
while (!netif_queue_stopped(dev) &&
qdisc_restart(dev)<0)
/* NOTHING */;
}
/* Calculate maximal size of packet seen by hard_start_xmit /* Calculate maximal size of packet seen by hard_start_xmit
routine of this device. routine of this device.
*/ */
......
net/atm/lec.c
View file @ a8e0e8ab
...@@ -71,9 +71,9 @@ static int lec_start_xmit(struct sk_buff *skb, struct net_device *dev); ...@@ -71,9 +71,9 @@ static int lec_start_xmit(struct sk_buff *skb, struct net_device *dev);
static int lec_close(struct net_device *dev); static int lec_close(struct net_device *dev);
static struct net_device_stats *lec_get_stats(struct net_device *dev); static struct net_device_stats *lec_get_stats(struct net_device *dev);
static void lec_init(struct net_device *dev); static void lec_init(struct net_device *dev);
static inline struct lec_arp_table* lec_arp_find(struct lec_priv *priv, static struct lec_arp_table* lec_arp_find(struct lec_priv *priv,
unsigned char *mac_addr); unsigned char *mac_addr);
static inline int lec_arp_remove(struct lec_priv *priv, static int lec_arp_remove(struct lec_priv *priv,
struct lec_arp_table *to_remove); struct lec_arp_table *to_remove);
/* LANE2 functions */ /* LANE2 functions */
static void lane2_associate_ind (struct net_device *dev, u8 *mac_address, static void lane2_associate_ind (struct net_device *dev, u8 *mac_address,
...@@ -1468,7 +1468,7 @@ lec_arp_add(struct lec_priv *priv, struct lec_arp_table *to_add) ...@@ -1468,7 +1468,7 @@ lec_arp_add(struct lec_priv *priv, struct lec_arp_table *to_add)
/* /*
* Remove entry from lec_arp_table * Remove entry from lec_arp_table
*/ */
static inline int static int
lec_arp_remove(struct lec_priv *priv, lec_arp_remove(struct lec_priv *priv,
struct lec_arp_table *to_remove) struct lec_arp_table *to_remove)
{ {
...@@ -1755,7 +1755,7 @@ lec_arp_destroy(struct lec_priv *priv) ...@@ -1755,7 +1755,7 @@ lec_arp_destroy(struct lec_priv *priv)
/* /*
* Find entry by mac_address * Find entry by mac_address
*/ */
static inline struct lec_arp_table* static struct lec_arp_table*
lec_arp_find(struct lec_priv *priv, lec_arp_find(struct lec_priv *priv,
unsigned char *mac_addr) unsigned char *mac_addr)
{ {
......
net/bridge/br_stp.c
View file @ a8e0e8ab
...@@ -299,7 +299,7 @@ void br_topology_change_detection(struct net_bridge *br) ...@@ -299,7 +299,7 @@ void br_topology_change_detection(struct net_bridge *br)
int isroot = br_is_root_bridge(br); int isroot = br_is_root_bridge(br);
pr_info("%s: topology change detected, %s\n", br->dev->name, pr_info("%s: topology change detected, %s\n", br->dev->name,
isroot ? "propgating" : "sending tcn bpdu"); isroot ? "propagating" : "sending tcn bpdu");
if (isroot) { if (isroot) {
br->topology_change = 1; br->topology_change = 1;
......
net/core/dev.c
View file @ a8e0e8ab
...@@ -1320,6 +1320,13 @@ int __skb_linearize(struct sk_buff *skb, int gfp_mask) ...@@ -1320,6 +1320,13 @@ int __skb_linearize(struct sk_buff *skb, int gfp_mask)
} \ } \
} }
static inline void qdisc_run(struct net_device *dev)
{
while (!netif_queue_stopped(dev) &&
qdisc_restart(dev)<0)
/* NOTHING */;
}
/** /**
* dev_queue_xmit - transmit a buffer * dev_queue_xmit - transmit a buffer
* @skb: buffer to transmit * @skb: buffer to transmit
...@@ -3034,7 +3041,6 @@ static void netdev_wait_allrefs(struct net_device *dev) ...@@ -3034,7 +3041,6 @@ static void netdev_wait_allrefs(struct net_device *dev)
while (atomic_read(&dev->refcnt) != 0) { while (atomic_read(&dev->refcnt) != 0) {
if (time_after(jiffies, rebroadcast_time + 1 * HZ)) { if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
rtnl_shlock(); rtnl_shlock();
rtnl_exlock();
/* Rebroadcast unregister notification */ /* Rebroadcast unregister notification */
notifier_call_chain(&netdev_chain, notifier_call_chain(&netdev_chain,
...@@ -3051,7 +3057,6 @@ static void netdev_wait_allrefs(struct net_device *dev) ...@@ -3051,7 +3057,6 @@ static void netdev_wait_allrefs(struct net_device *dev)
linkwatch_run_queue(); linkwatch_run_queue();
} }
rtnl_exunlock();
rtnl_shunlock(); rtnl_shunlock();
rebroadcast_time = jiffies; rebroadcast_time = jiffies;
......
net/core/link_watch.c
View file @ a8e0e8ab
...@@ -93,9 +93,7 @@ static void linkwatch_event(void *dummy) ...@@ -93,9 +93,7 @@ static void linkwatch_event(void *dummy)
clear_bit(LW_RUNNING, &linkwatch_flags); clear_bit(LW_RUNNING, &linkwatch_flags);
rtnl_shlock(); rtnl_shlock();
rtnl_exlock();
linkwatch_run_queue(); linkwatch_run_queue();
rtnl_exunlock();
rtnl_shunlock(); rtnl_shunlock();
} }
......
net/core/pktgen.c
View file @ a8e0e8ab
...@@ -56,6 +56,7 @@ ...@@ -56,6 +56,7 @@
*/ */
#include <linux/module.h> #include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/types.h> #include <linux/types.h>
...@@ -1409,7 +1410,7 @@ module_exit(cleanup); ...@@ -1409,7 +1410,7 @@ module_exit(cleanup);
MODULE_AUTHOR("Robert Olsson <robert.olsson@its.uu.se"); MODULE_AUTHOR("Robert Olsson <robert.olsson@its.uu.se");
MODULE_DESCRIPTION("Packet Generator tool"); MODULE_DESCRIPTION("Packet Generator tool");
MODULE_LICENSE("GPL"); MODULE_LICENSE("GPL");
MODULE_PARM(count_d, "i"); module_param(count_d, int, 0);
MODULE_PARM(ipg_d, "i"); module_param(ipg_d, int, 0);
MODULE_PARM(cpu_speed, "i"); module_param(cpu_speed, int, 0);
MODULE_PARM(clone_skb_d, "i"); module_param(clone_skb_d, int, 0);
net/core/rtnetlink.c
View file @ a8e0e8ab
...@@ -56,12 +56,10 @@ DECLARE_MUTEX(rtnl_sem); ...@@ -56,12 +56,10 @@ DECLARE_MUTEX(rtnl_sem);
void rtnl_lock(void) void rtnl_lock(void)
{ {
rtnl_shlock(); rtnl_shlock();
rtnl_exlock();
} }
void rtnl_unlock(void) void rtnl_unlock(void)
{ {
rtnl_exunlock();
rtnl_shunlock(); rtnl_shunlock();
netdev_run_todo(); netdev_run_todo();
...@@ -337,7 +335,6 @@ rtnetlink_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, int *errp) ...@@ -337,7 +335,6 @@ rtnetlink_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, int *errp)
struct rtnetlink_link *link_tab; struct rtnetlink_link *link_tab;
struct rtattr *rta[RTATTR_MAX]; struct rtattr *rta[RTATTR_MAX];
int exclusive = 0;
int sz_idx, kind; int sz_idx, kind;
int min_len; int min_len;
int family; int family;
...@@ -404,14 +401,6 @@ rtnetlink_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, int *errp) ...@@ -404,14 +401,6 @@ rtnetlink_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, int *errp)
return -1; return -1;
} }
if (kind != 2) {
if (rtnl_exlock_nowait()) {
*errp = 0;
return -1;
}
exclusive = 1;
}
memset(&rta, 0, sizeof(rta)); memset(&rta, 0, sizeof(rta));
min_len = rtm_min[sz_idx]; min_len = rtm_min[sz_idx];
...@@ -439,14 +428,10 @@ rtnetlink_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, int *errp) ...@@ -439,14 +428,10 @@ rtnetlink_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, int *errp)
goto err_inval; goto err_inval;
err = link->doit(skb, nlh, (void *)&rta); err = link->doit(skb, nlh, (void *)&rta);
if (exclusive)
rtnl_exunlock();
*errp = err; *errp = err;
return err; return err;
err_inval: err_inval:
if (exclusive)
rtnl_exunlock();
*errp = -EINVAL; *errp = -EINVAL;
return -1; return -1;
} }
......
net/ipv6/af_inet6.c
View file @ a8e0e8ab
...@@ -560,8 +560,6 @@ static struct inet_protosw rawv6_protosw = { ...@@ -560,8 +560,6 @@ static struct inet_protosw rawv6_protosw = {
.flags = INET_PROTOSW_REUSE, .flags = INET_PROTOSW_REUSE,
}; };
#define INETSW6_ARRAY_LEN (sizeof(inetsw6_array) / sizeof(struct inet_protosw))
void void
inet6_register_protosw(struct inet_protosw *p) inet6_register_protosw(struct inet_protosw *p)
{ {
......
net/ipv6/ah6.c
View file @ a8e0e8ab
...@@ -246,9 +246,7 @@ int ah6_input(struct xfrm_state *x, struct xfrm_decap_state *decap, struct sk_bu ...@@ -246,9 +246,7 @@ int ah6_input(struct xfrm_state *x, struct xfrm_decap_state *decap, struct sk_bu
unsigned char *tmp_hdr = NULL; unsigned char *tmp_hdr = NULL;
u16 hdr_len; u16 hdr_len;
u16 ah_hlen; u16 ah_hlen;
u16 nh_offset = 0; int nexthdr;
u8 nexthdr = 0;
u8 *prevhdr;
if (!pskb_may_pull(skb, sizeof(struct ip_auth_hdr))) if (!pskb_may_pull(skb, sizeof(struct ip_auth_hdr)))
goto out; goto out;
...@@ -301,8 +299,6 @@ int ah6_input(struct xfrm_state *x, struct xfrm_decap_state *decap, struct sk_bu ...@@ -301,8 +299,6 @@ int ah6_input(struct xfrm_state *x, struct xfrm_decap_state *decap, struct sk_bu
skb->nh.raw = skb_pull(skb, ah_hlen); skb->nh.raw = skb_pull(skb, ah_hlen);
memcpy(skb->nh.raw, tmp_hdr, hdr_len); memcpy(skb->nh.raw, tmp_hdr, hdr_len);
prevhdr = (u8*)(skb->nh.raw + nh_offset);
*prevhdr = nexthdr;
skb->nh.ipv6h->payload_len = htons(skb->len - sizeof(struct ipv6hdr)); skb->nh.ipv6h->payload_len = htons(skb->len - sizeof(struct ipv6hdr));
skb_pull(skb, hdr_len); skb_pull(skb, hdr_len);
skb->h.raw = skb->data; skb->h.raw = skb->data;
......
net/ipv6/esp6.c
View file @ a8e0e8ab
...@@ -258,7 +258,6 @@ int esp6_input(struct xfrm_state *x, struct xfrm_decap_state *decap, struct sk_b ...@@ -258,7 +258,6 @@ int esp6_input(struct xfrm_state *x, struct xfrm_decap_state *decap, struct sk_b
u8 nexthdr[2]; u8 nexthdr[2];
struct scatterlist *sg = &esp->sgbuf[0]; struct scatterlist *sg = &esp->sgbuf[0];
u8 padlen; u8 padlen;
u8 *prevhdr;
if (unlikely(nfrags > ESP_NUM_FAST_SG)) { if (unlikely(nfrags > ESP_NUM_FAST_SG)) {
sg = kmalloc(sizeof(struct scatterlist)*nfrags, GFP_ATOMIC); sg = kmalloc(sizeof(struct scatterlist)*nfrags, GFP_ATOMIC);
...@@ -289,8 +288,7 @@ int esp6_input(struct xfrm_state *x, struct xfrm_decap_state *decap, struct sk_b ...@@ -289,8 +288,7 @@ int esp6_input(struct xfrm_state *x, struct xfrm_decap_state *decap, struct sk_b
skb->nh.raw += sizeof(struct ipv6_esp_hdr) + esp->conf.ivlen; skb->nh.raw += sizeof(struct ipv6_esp_hdr) + esp->conf.ivlen;
memcpy(skb->nh.raw, tmp_hdr, hdr_len); memcpy(skb->nh.raw, tmp_hdr, hdr_len);
skb->nh.ipv6h->payload_len = htons(skb->len - sizeof(struct ipv6hdr)); skb->nh.ipv6h->payload_len = htons(skb->len - sizeof(struct ipv6hdr));
ip6_find_1stfragopt(skb, &prevhdr); ret = nexthdr[1];
ret = *prevhdr = nexthdr[1];
} }
out: out:
......
net/ipv6/icmp.c
View file @ a8e0e8ab
...@@ -646,7 +646,13 @@ static int icmpv6_rcv(struct sk_buff **pskb, unsigned int *nhoffp) ...@@ -646,7 +646,13 @@ static int icmpv6_rcv(struct sk_buff **pskb, unsigned int *nhoffp)
break; break;
case ICMPV6_MGM_REDUCTION: case ICMPV6_MGM_REDUCTION:
case ICMPV6_NI_QUERY:
case ICMPV6_NI_REPLY:
case ICMPV6_MLD2_REPORT: case ICMPV6_MLD2_REPORT:
case ICMPV6_DHAAD_REQUEST:
case ICMPV6_DHAAD_REPLY:
case ICMPV6_MOBILE_PREFIX_SOL:
case ICMPV6_MOBILE_PREFIX_ADV:
break; break;
default: default:
......
net/ipv6/ipcomp6.c
View file @ a8e0e8ab
...@@ -49,7 +49,6 @@ static int ipcomp6_input(struct xfrm_state *x, struct xfrm_decap_state *decap, s ...@@ -49,7 +49,6 @@ static int ipcomp6_input(struct xfrm_state *x, struct xfrm_decap_state *decap, s
{ {
int err = 0; int err = 0;
u8 nexthdr = 0; u8 nexthdr = 0;
u8 *prevhdr;
int hdr_len = skb->h.raw - skb->nh.raw; int hdr_len = skb->h.raw - skb->nh.raw;
unsigned char *tmp_hdr = NULL; unsigned char *tmp_hdr = NULL;
struct ipv6hdr *iph; struct ipv6hdr *iph;
...@@ -106,8 +105,6 @@ static int ipcomp6_input(struct xfrm_state *x, struct xfrm_decap_state *decap, s ...@@ -106,8 +105,6 @@ static int ipcomp6_input(struct xfrm_state *x, struct xfrm_decap_state *decap, s
iph = skb->nh.ipv6h; iph = skb->nh.ipv6h;
iph->payload_len = htons(skb->len); iph->payload_len = htons(skb->len);
ip6_find_1stfragopt(skb, &prevhdr);
*prevhdr = nexthdr;
out: out:
if (tmp_hdr) if (tmp_hdr)
kfree(tmp_hdr); kfree(tmp_hdr);
......
net/ipv6/route.c
View file @ a8e0e8ab
...@@ -692,7 +692,7 @@ static int ipv6_get_mtu(struct net_device *dev) ...@@ -692,7 +692,7 @@ static int ipv6_get_mtu(struct net_device *dev)
return mtu; return mtu;
} }
static inline unsigned int ipv6_advmss(unsigned int mtu) static unsigned int ipv6_advmss(unsigned int mtu)
{ {
mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr); mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
......
net/ipv6/xfrm6_input.c
View file @ a8e0e8ab
...@@ -34,12 +34,11 @@ int xfrm6_rcv(struct sk_buff **pskb, unsigned int *nhoffp) ...@@ -34,12 +34,11 @@ int xfrm6_rcv(struct sk_buff **pskb, unsigned int *nhoffp)
struct xfrm_state *x; struct xfrm_state *x;
int xfrm_nr = 0; int xfrm_nr = 0;
int decaps = 0; int decaps = 0;
int nexthdr = 0; int nexthdr;
u8 *prevhdr = NULL; unsigned int nhoff;
ip6_find_1stfragopt(skb, &prevhdr); nhoff = *nhoffp;
nexthdr = *prevhdr; nexthdr = skb->nh.raw[nhoff];
*nhoffp = prevhdr - skb->nh.raw;
if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0)
goto drop; goto drop;
...@@ -67,6 +66,8 @@ int xfrm6_rcv(struct sk_buff **pskb, unsigned int *nhoffp) ...@@ -67,6 +66,8 @@ int xfrm6_rcv(struct sk_buff **pskb, unsigned int *nhoffp)
if (nexthdr <= 0) if (nexthdr <= 0)
goto drop_unlock; goto drop_unlock;
skb->nh.raw[nhoff] = nexthdr;
if (x->props.replay_window) if (x->props.replay_window)
xfrm_replay_advance(x, seq); xfrm_replay_advance(x, seq);
......
net/sched/cls_api.c
View file @ a8e0e8ab
...@@ -236,12 +236,12 @@ static int tc_ctl_tfilter(struct sk_buff *skb, struct nlmsghdr *n, void *arg) ...@@ -236,12 +236,12 @@ static int tc_ctl_tfilter(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
kfree(tp); kfree(tp);
goto errout; goto errout;
} }
write_lock(&qdisc_tree_lock);
spin_lock_bh(&dev->queue_lock); qdisc_lock_tree(dev);
tp->next = *back; tp->next = *back;
*back = tp; *back = tp;
spin_unlock_bh(&dev->queue_lock); qdisc_unlock_tree(dev);
write_unlock(&qdisc_tree_lock);
} else if (tca[TCA_KIND-1] && rtattr_strcmp(tca[TCA_KIND-1], tp->ops->kind)) } else if (tca[TCA_KIND-1] && rtattr_strcmp(tca[TCA_KIND-1], tp->ops->kind))
goto errout; goto errout;
...@@ -249,11 +249,10 @@ static int tc_ctl_tfilter(struct sk_buff *skb, struct nlmsghdr *n, void *arg) ...@@ -249,11 +249,10 @@ static int tc_ctl_tfilter(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
if (fh == 0) { if (fh == 0) {
if (n->nlmsg_type == RTM_DELTFILTER && t->tcm_handle == 0) { if (n->nlmsg_type == RTM_DELTFILTER && t->tcm_handle == 0) {
write_lock(&qdisc_tree_lock); qdisc_lock_tree(dev);
spin_lock_bh(&dev->queue_lock);
*back = tp->next; *back = tp->next;
spin_unlock_bh(&dev->queue_lock); qdisc_unlock_tree(dev);
write_unlock(&qdisc_tree_lock);
tfilter_notify(skb, n, tp, fh_s, RTM_DELTFILTER); tfilter_notify(skb, n, tp, fh_s, RTM_DELTFILTER);
tcf_destroy(tp); tcf_destroy(tp);
err = 0; err = 0;
...@@ -294,6 +293,19 @@ static int tc_ctl_tfilter(struct sk_buff *skb, struct nlmsghdr *n, void *arg) ...@@ -294,6 +293,19 @@ static int tc_ctl_tfilter(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
return err; return err;
} }
unsigned long tcf_set_class(struct tcf_proto *tp, unsigned long *clp,
unsigned long cl)
{
unsigned long old_cl;
tcf_tree_lock(tp);
old_cl = __cls_set_class(clp, cl);
tcf_tree_unlock(tp);
return old_cl;
}
static int static int
tcf_fill_node(struct sk_buff *skb, struct tcf_proto *tp, unsigned long fh, tcf_fill_node(struct sk_buff *skb, struct tcf_proto *tp, unsigned long fh,
u32 pid, u32 seq, unsigned flags, int event) u32 pid, u32 seq, unsigned flags, int event)
...@@ -459,3 +471,4 @@ subsys_initcall(tc_filter_init); ...@@ -459,3 +471,4 @@ subsys_initcall(tc_filter_init);
EXPORT_SYMBOL(register_tcf_proto_ops); EXPORT_SYMBOL(register_tcf_proto_ops);
EXPORT_SYMBOL(unregister_tcf_proto_ops); EXPORT_SYMBOL(unregister_tcf_proto_ops);
EXPORT_SYMBOL(tcf_set_class);
net/sched/sch_api.c
View file @ a8e0e8ab
...@@ -306,8 +306,7 @@ dev_graft_qdisc(struct net_device *dev, struct Qdisc *qdisc) ...@@ -306,8 +306,7 @@ dev_graft_qdisc(struct net_device *dev, struct Qdisc *qdisc)
if (dev->flags & IFF_UP) if (dev->flags & IFF_UP)
dev_deactivate(dev); dev_deactivate(dev);
write_lock(&qdisc_tree_lock); qdisc_lock_tree(dev);
spin_lock_bh(&dev->queue_lock);
if (qdisc && qdisc->flags&TCQ_F_INGRES) { if (qdisc && qdisc->flags&TCQ_F_INGRES) {
oqdisc = dev->qdisc_ingress; oqdisc = dev->qdisc_ingress;
/* Prune old scheduler */ /* Prune old scheduler */
...@@ -334,8 +333,7 @@ dev_graft_qdisc(struct net_device *dev, struct Qdisc *qdisc) ...@@ -334,8 +333,7 @@ dev_graft_qdisc(struct net_device *dev, struct Qdisc *qdisc)
dev->qdisc = &noop_qdisc; dev->qdisc = &noop_qdisc;
} }
spin_unlock_bh(&dev->queue_lock); qdisc_unlock_tree(dev);
write_unlock(&qdisc_tree_lock);
if (dev->flags & IFF_UP) if (dev->flags & IFF_UP)
dev_activate(dev); dev_activate(dev);
...@@ -454,10 +452,11 @@ qdisc_create(struct net_device *dev, u32 handle, struct rtattr **tca, int *errp) ...@@ -454,10 +452,11 @@ qdisc_create(struct net_device *dev, u32 handle, struct rtattr **tca, int *errp)
* before we set a netdevice's qdisc pointer to sch */ * before we set a netdevice's qdisc pointer to sch */
smp_wmb(); smp_wmb();
if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS-1])) == 0) { if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS-1])) == 0) {
write_lock(&qdisc_tree_lock); qdisc_lock_tree(dev);
sch->next = dev->qdisc_list; sch->next = dev->qdisc_list;
dev->qdisc_list = sch; dev->qdisc_list = sch;
write_unlock(&qdisc_tree_lock); qdisc_unlock_tree(dev);
#ifdef CONFIG_NET_ESTIMATOR #ifdef CONFIG_NET_ESTIMATOR
if (tca[TCA_RATE-1]) if (tca[TCA_RATE-1])
qdisc_new_estimator(&sch->stats, sch->stats_lock, qdisc_new_estimator(&sch->stats, sch->stats_lock,
......
net/sched/sch_generic.c
View file @ a8e0e8ab
...@@ -54,6 +54,18 @@ ...@@ -54,6 +54,18 @@
*/ */
rwlock_t qdisc_tree_lock = RW_LOCK_UNLOCKED; rwlock_t qdisc_tree_lock = RW_LOCK_UNLOCKED;
void qdisc_lock_tree(struct net_device *dev)
{
write_lock(&qdisc_tree_lock);
spin_lock_bh(&dev->queue_lock);
}
void qdisc_unlock_tree(struct net_device *dev)
{
spin_unlock_bh(&dev->queue_lock);
write_unlock(&qdisc_tree_lock);
}
/* /*
dev->queue_lock serializes queue accesses for this device dev->queue_lock serializes queue accesses for this device
AND dev->qdisc pointer itself. AND dev->qdisc pointer itself.
...@@ -513,13 +525,11 @@ void dev_deactivate(struct net_device *dev) ...@@ -513,13 +525,11 @@ void dev_deactivate(struct net_device *dev)
void dev_init_scheduler(struct net_device *dev) void dev_init_scheduler(struct net_device *dev)
{ {
write_lock(&qdisc_tree_lock); qdisc_lock_tree(dev);
spin_lock_bh(&dev->queue_lock);
dev->qdisc = &noop_qdisc; dev->qdisc = &noop_qdisc;
spin_unlock_bh(&dev->queue_lock);
dev->qdisc_sleeping = &noop_qdisc; dev->qdisc_sleeping = &noop_qdisc;
dev->qdisc_list = NULL; dev->qdisc_list = NULL;
write_unlock(&qdisc_tree_lock); qdisc_unlock_tree(dev);
dev_watchdog_init(dev); dev_watchdog_init(dev);
} }
...@@ -528,8 +538,7 @@ void dev_shutdown(struct net_device *dev) ...@@ -528,8 +538,7 @@ void dev_shutdown(struct net_device *dev)
{ {
struct Qdisc *qdisc; struct Qdisc *qdisc;
write_lock(&qdisc_tree_lock); qdisc_lock_tree(dev);
spin_lock_bh(&dev->queue_lock);
qdisc = dev->qdisc_sleeping; qdisc = dev->qdisc_sleeping;
dev->qdisc = &noop_qdisc; dev->qdisc = &noop_qdisc;
dev->qdisc_sleeping = &noop_qdisc; dev->qdisc_sleeping = &noop_qdisc;
...@@ -543,8 +552,7 @@ void dev_shutdown(struct net_device *dev) ...@@ -543,8 +552,7 @@ void dev_shutdown(struct net_device *dev)
BUG_TRAP(dev->qdisc_list == NULL); BUG_TRAP(dev->qdisc_list == NULL);
BUG_TRAP(!timer_pending(&dev->watchdog_timer)); BUG_TRAP(!timer_pending(&dev->watchdog_timer));
dev->qdisc_list = NULL; dev->qdisc_list = NULL;
spin_unlock_bh(&dev->queue_lock); qdisc_unlock_tree(dev);
write_unlock(&qdisc_tree_lock);
} }
EXPORT_SYMBOL(__netdev_watchdog_up); EXPORT_SYMBOL(__netdev_watchdog_up);
...@@ -554,4 +562,5 @@ EXPORT_SYMBOL(qdisc_create_dflt); ...@@ -554,4 +562,5 @@ EXPORT_SYMBOL(qdisc_create_dflt);
EXPORT_SYMBOL(qdisc_destroy); EXPORT_SYMBOL(qdisc_destroy);
EXPORT_SYMBOL(qdisc_reset); EXPORT_SYMBOL(qdisc_reset);
EXPORT_SYMBOL(qdisc_restart); EXPORT_SYMBOL(qdisc_restart);
EXPORT_SYMBOL(qdisc_tree_lock); EXPORT_SYMBOL(qdisc_lock_tree);
EXPORT_SYMBOL(qdisc_unlock_tree);
net/xfrm/xfrm_user.c
View file @ a8e0e8ab
...@@ -267,6 +267,8 @@ static int xfrm_add_sa(struct sk_buff *skb, struct nlmsghdr *nlh, void **xfrma) ...@@ -267,6 +267,8 @@ static int xfrm_add_sa(struct sk_buff *skb, struct nlmsghdr *nlh, void **xfrma)
if (err) if (err)
return err; return err;
xfrm_probe_algs();
x = xfrm_state_construct(p, (struct rtattr **) xfrma, &err); x = xfrm_state_construct(p, (struct rtattr **) xfrma, &err);
if (!x) if (!x)
return err; return err;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7