Commit 24594a2b authored by Andi Kleen's avatar Andi Kleen Committed by Linus Torvalds

[PATCH] x86-64 merge

 - Fix -funit-at-a-time compilation and enable it when possible
 - Remove -finline-limit as it should not be needed anymore.
 - Update defconfig
 - Fix Makefile bug that caused a recompilation of vsyscall.so on
   every make (thanks to Sam Ravnborg)
 - Add beginning of asm/dwarf2.h to support assembler CFI directives
   (not complete yet)
 - Remove old PDAREF macro from entry.S
 - Remove clever and buggy code in sys_ioperm/set_bitmap and replace it
   by simple and working code
 - Don't make acpi_disabled __initdata.  It is referenced after boot.
 - Fix TLB size reporting in /proc/cpuinfo
 - Cleanup oops printing a bit
 - Add "executive summary" at end of oopses
 - Reenable interrupts on oopses before calling do_exit
 - Remove some unneeded prefetches.  Just two are enough to kickstart
   the hardware prefetcher.
 - Add prefetch workaround (based on code from Richard Brunner)
 - Clean up signal checking in do_page_fault
 - Don't allow modify_ldt to set 64bit codesegments
 - Readd SIGEV_PAD_SIZE (Stephen Rothwell)
 - Add some likelys to uaccess.h (idea from Manfred Spraul)
parent 54735541
......@@ -37,6 +37,8 @@ LDFLAGS := -m elf_x86_64
OBJCOPYFLAGS := -O binary -R .note -R .comment -S
LDFLAGS_vmlinux := -e stext
check_gcc = $(shell if $(CC) $(1) -S -o /dev/null -xc /dev/null > /dev/null 2>&1 ; then echo "$(1)"; else echo "$(2)"; fi)
CFLAGS += -mno-red-zone
CFLAGS += -mcmodel=kernel
CFLAGS += -pipe
......@@ -44,13 +46,13 @@ CFLAGS += -pipe
# actually it makes the kernel smaller too.
CFLAGS += -fno-reorder-blocks
# should lower this a lot and see how much .text is saves
CFLAGS += -finline-limit=2000
# CFLAGS += -finline-limit=2000
CFLAGS += -Wno-sign-compare
# don't enable this when you use kgdb:
ifneq ($(CONFIG_X86_REMOTE_DEBUG),y)
CFLAGS += -fno-asynchronous-unwind-tables
endif
#CFLAGS += -funit-at-a-time
CFLAGS += $(call check_gcc,-funit-at-a-time,)
head-y := arch/x86_64/kernel/head.o arch/x86_64/kernel/head64.o arch/x86_64/kernel/init_task.o
......
......@@ -2,6 +2,7 @@
# Automatically generated make config: don't edit
#
CONFIG_X86_64=y
CONFIG_64BIT=y
CONFIG_X86=y
CONFIG_MMU=y
CONFIG_RWSEM_GENERIC_SPINLOCK=y
......@@ -14,7 +15,8 @@ CONFIG_GENERIC_ISA_DMA=y
# Code maturity level options
#
CONFIG_EXPERIMENTAL=y
# CONFIG_BROKEN is not set
CONFIG_CLEAN_COMPILE=y
CONFIG_STANDALONE=y
#
# General setup
......@@ -110,7 +112,6 @@ CONFIG_PCI_DIRECT=y
#
# Executable file formats / Emulations
#
CONFIG_KCORE_ELF=y
CONFIG_BINFMT_ELF=y
# CONFIG_BINFMT_MISC is not set
CONFIG_IA32_EMULATION=y
......@@ -215,6 +216,7 @@ CONFIG_IDEDMA_AUTO=y
# SCSI device support
#
CONFIG_SCSI=y
# CONFIG_SCSI_PROC_FS is not set
#
# SCSI support type (disk, tape, CD-ROM)
......@@ -243,7 +245,6 @@ CONFIG_BLK_DEV_SD=y
# CONFIG_SCSI_AIC7XXX_OLD is not set
# CONFIG_SCSI_AIC79XX is not set
# CONFIG_SCSI_ADVANSYS is not set
# CONFIG_SCSI_AM53C974 is not set
# CONFIG_SCSI_MEGARAID is not set
# CONFIG_SCSI_BUSLOGIC is not set
# CONFIG_SCSI_CPQFCTS is not set
......@@ -253,17 +254,12 @@ CONFIG_BLK_DEV_SD=y
# CONFIG_SCSI_FUTURE_DOMAIN is not set
# CONFIG_SCSI_GDTH is not set
CONFIG_SCSI_IPS=m
# CONFIG_SCSI_INITIO is not set
# CONFIG_SCSI_INIA100 is not set
# CONFIG_SCSI_SYM53C8XX_2 is not set
# CONFIG_SCSI_SYM53C8XX is not set
# CONFIG_SCSI_PCI2000 is not set
# CONFIG_SCSI_PCI2220I is not set
# CONFIG_SCSI_QLOGIC_ISP is not set
# CONFIG_SCSI_QLOGIC_FC is not set
# CONFIG_SCSI_QLOGIC_1280 is not set
# CONFIG_SCSI_DC395x is not set
# CONFIG_SCSI_DC390T is not set
# CONFIG_SCSI_NSP32 is not set
# CONFIG_SCSI_DEBUG is not set
......@@ -357,7 +353,6 @@ CONFIG_NETDEVICES=y
# CONFIG_BONDING is not set
# CONFIG_EQUALIZER is not set
# CONFIG_TUN is not set
# CONFIG_ETHERTAP is not set
#
# Ethernet (10 or 100Mbit)
......@@ -559,7 +554,6 @@ CONFIG_RTC=y
#
# Ftape, the floppy tape device driver
#
# CONFIG_FTAPE is not set
CONFIG_AGP=y
CONFIG_AGP_AMD64=y
# CONFIG_DRM is not set
......@@ -716,8 +710,6 @@ CONFIG_SOUND_ICH=y
# CONFIG_SOUND_RME96XX is not set
# CONFIG_SOUND_SONICVIBES is not set
# CONFIG_SOUND_TRIDENT is not set
# CONFIG_SOUND_MSNDCLAS is not set
# CONFIG_SOUND_MSNDPIN is not set
# CONFIG_SOUND_VIA82CXXX is not set
# CONFIG_SOUND_OSS is not set
# CONFIG_SOUND_ALI5455 is not set
......
......@@ -8,12 +8,16 @@ obj-$(CONFIG_IA32_EMULATION) := ia32entry.o sys_ia32.o ia32_ioctl.o \
$(obj)/syscall32.o: $(src)/syscall32.c $(obj)/vsyscall.so
# The DSO images are built using a special linker script.
$(obj)/vsyscall.so: $(src)/vsyscall.lds $(obj)/vsyscall.o
$(CC) -m32 -nostdlib -shared -s -Wl,-soname=linux-gate.so.1 \
-o $@ -Wl,-T,$^
# Teach kbuild about targets
targets := vsyscall.o vsyscall.so
clean-files += vsyscall.so
# The DSO images are built using a special linker script
quiet_cmd_vsyscall = SYSCALL $@
cmd_vsyscall = $(CC) -m32 -nostdlib -shared -s \
-Wl,-soname=linux-gate.so.1 -o $@ \
-Wl,-T,$(filter-out FORCE,$^)
$(obj)/vsyscall.so: $(src)/vsyscall.lds $(obj)/vsyscall.o FORCE
$(call if_changed,vsyscall)
AFLAGS_vsyscall.o = -m32
CFLAGS_ia32_ioctl.o += -Ifs/
......@@ -41,11 +41,10 @@
#include <asm/thread_info.h>
#include <asm/hw_irq.h>
#include <asm/errno.h>
#include <asm/dwarf2.h>
.code64
#define PDAREF(field) %gs:field
#ifdef CONFIG_PREEMPT
#define preempt_stop cli
#else
......@@ -63,7 +62,7 @@
/* %rsp:at FRAMEEND */
.macro FIXUP_TOP_OF_STACK tmp
movq PDAREF(pda_oldrsp),\tmp
movq %gs:pda_oldrsp,\tmp
movq \tmp,RSP(%rsp)
movq $__USER_DS,SS(%rsp)
movq $__USER_CS,CS(%rsp)
......@@ -74,7 +73,7 @@
.macro RESTORE_TOP_OF_STACK tmp,offset=0
movq RSP-\offset(%rsp),\tmp
movq \tmp,PDAREF(pda_oldrsp)
movq \tmp,%gs:pda_oldrsp
movq EFLAGS-\offset(%rsp),\tmp
movq \tmp,R11-\offset(%rsp)
.endm
......@@ -146,8 +145,8 @@ rff_trace:
ENTRY(system_call)
swapgs
movq %rsp,PDAREF(pda_oldrsp)
movq PDAREF(pda_kernelstack),%rsp
movq %rsp,%gs:pda_oldrsp
movq %gs:pda_kernelstack,%rsp
sti
SAVE_ARGS 8,1
movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
......@@ -176,7 +175,7 @@ sysret_check:
jnz sysret_careful
movq RIP-ARGOFFSET(%rsp),%rcx
RESTORE_ARGS 0,-ARG_SKIP,1
movq PDAREF(pda_oldrsp),%rsp
movq %gs:pda_oldrsp,%rsp
swapgs
sysretq
......@@ -369,8 +368,8 @@ ENTRY(stub_rt_sigreturn)
testl $3,CS(%rdi)
je 1f
swapgs
1: addl $1,PDAREF(pda_irqcount) # RED-PEN should check preempt count
movq PDAREF(pda_irqstackptr),%rax
1: addl $1,%gs:pda_irqcount # RED-PEN should check preempt count
movq %gs:pda_irqstackptr,%rax
cmoveq %rax,%rsp
pushq %rdi # save old stack
call \func
......@@ -382,7 +381,7 @@ ENTRY(common_interrupt)
ret_from_intr:
popq %rdi
cli
subl $1,PDAREF(pda_irqcount)
subl $1,%gs:pda_irqcount
leaq ARGOFFSET(%rdi),%rsp
exit_intr:
GET_THREAD_INFO(%rcx)
......
......@@ -19,35 +19,13 @@
/* Set EXTENT bits starting at BASE in BITMAP to value TURN_ON. */
static void set_bitmap(unsigned long *bitmap, unsigned int base, unsigned int extent, int new_value)
{
unsigned long mask;
unsigned long *bitmap_base = bitmap + (base / BITS_PER_LONG);
unsigned int low_index = base & (BITS_PER_LONG-1);
int length = low_index + extent;
if (low_index != 0) {
mask = (~0UL << low_index);
if (length < BITS_PER_LONG)
mask &= ~(~0UL << length);
if (new_value)
*bitmap_base++ |= mask;
else
*bitmap_base++ &= ~mask;
length -= BITS_PER_LONG;
}
mask = (new_value ? ~0UL : 0UL);
while (length >= BITS_PER_LONG) {
*bitmap_base++ = mask;
length -= BITS_PER_LONG;
}
if (length > 0) {
mask = ~(~0UL << length);
int i;
if (new_value)
*bitmap_base++ |= mask;
for (i = base; i < base + extent; i++)
__set_bit(i, bitmap);
else
*bitmap_base++ &= ~mask;
}
for (i = base; i < base + extent; i++)
clear_bit(i, bitmap);
}
......
......@@ -64,7 +64,7 @@ struct cpuinfo_x86 boot_cpu_data;
unsigned long mmu_cr4_features;
EXPORT_SYMBOL_GPL(mmu_cr4_features);
int acpi_disabled __initdata = 0;
int acpi_disabled = 0;
/* For PCI or other memory-mapped resources */
unsigned long pci_mem_start = 0x10000000;
......@@ -499,16 +499,16 @@ static void __init display_cacheinfo(struct cpuinfo_x86 *c)
cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
printk(KERN_INFO "CPU: L1 I Cache: %dK (%d bytes/line), D cache %dK (%d bytes/line)\n",
edx>>24, edx&0xFF, ecx>>24, ecx&0xFF);
c->x86_cache_size = (ecx>>24)+(edx>>24);
c->x86_cache_size=(ecx>>24)+(edx>>24);
/* DTLB and ITLB together, but only 4K */
c->x86_tlbsize = ((ebx >> 16) & 0xff) + (ebx & 0xff);
c->x86_tlbsize = ((ebx>>16)&0xff) + (ebx&0xff);
}
if (n >= 0x80000006) {
cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
ecx = cpuid_ecx(0x80000006);
c->x86_cache_size = ecx >> 16;
c->x86_tlbsize += ((ebx >> 16) & 0xff) + (ebx & 0xff);
c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
printk(KERN_INFO "CPU: L2 Cache: %dK (%d bytes/line)\n",
c->x86_cache_size, ecx & 0xFF);
......
......@@ -333,25 +333,34 @@ void oops_begin(void)
bust_spinlocks(1);
}
void oops_end(void)
{
die_owner = -1;
bust_spinlocks(0);
spin_unlock(&die_lock);
local_irq_enable(); /* make sure back scroll still works */
}
void __die(const char * str, struct pt_regs * regs, long err)
{
static int die_counter;
handle_BUG(regs);
printk(KERN_EMERG "%s: %04lx [%u]\n", str, err & 0xffff, ++die_counter);
printk(KERN_EMERG "%s: %04lx [%u]\n", str, err & 0xffff,++die_counter);
notify_die(DIE_OOPS, (char *)str, regs, err, 255, SIGSEGV);
show_registers(regs);
bust_spinlocks(0);
die_owner = -1;
spin_unlock_irq(&die_lock);
do_exit(SIGSEGV);
/* Execute summary in case the oops scrolled away */
printk(KERN_EMERG "RIP ");
printk_address(regs->rip);
printk(" RSP <%016lx>\n", regs->rsp);
}
void die(const char * str, struct pt_regs * regs, long err)
{
oops_begin();
handle_BUG(regs);
__die(str, regs, err);
oops_end();
do_exit(SIGSEGV);
}
static inline void die_if_kernel(const char * str, struct pt_regs * regs, long err)
{
if (!(regs->eflags & VM_MASK) && (regs->cs == __KERNEL_CS))
......
......@@ -10,14 +10,8 @@
copy_page:
prefetch (%rsi)
prefetch 1*64(%rsi)
prefetch 2*64(%rsi)
prefetch 3*64(%rsi)
prefetch 4*64(%rsi)
prefetchw (%rdi)
prefetchw 1*64(%rdi)
prefetchw 2*64(%rdi)
prefetchw 3*64(%rdi)
prefetchw 4*64(%rdi)
subq $3*8,%rsp
movq %rbx,(%rsp)
......
......@@ -64,21 +64,9 @@ csum_partial_copy_generic:
ignore
prefetch 1*64(%rdi)
ignore
prefetch 2*64(%rdi)
ignore
prefetch 3*64(%rdi)
ignore
prefetch 4*64(%rdi)
ignore
prefetchw (%rsi)
ignore
prefetchw 1*64(%rsi)
ignore
prefetchw 2*64(%rsi)
ignore
prefetchw 3*64(%rsi)
ignore
prefetchw 4*64(%rsi)
.Lignore:
subq $7*8,%rsp
......
......@@ -53,6 +53,87 @@ void bust_spinlocks(int yes)
}
}
/* Sometimes the CPU reports invalid exceptions on prefetch.
Check that here and ignore.
Opcode checker based on code by Richard Brunner */
static int is_prefetch(struct pt_regs *regs, unsigned long addr)
{
unsigned char *instr = (unsigned char *)(regs->rip);
int scan_more = 1;
int prefetch = 0;
unsigned char *max_instr = instr + 15;
/* Avoid recursive faults for this common case */
if (regs->rip == addr)
return 0;
/* Code segments in LDT could have a non zero base. Don't check
when that's possible */
if (regs->cs & (1<<2))
return 0;
while (scan_more && instr < max_instr) {
unsigned char opcode;
unsigned char instr_hi;
unsigned char instr_lo;
if (__get_user(opcode, instr))
break;
instr_hi = opcode & 0xf0;
instr_lo = opcode & 0x0f;
instr++;
switch (instr_hi) {
case 0x20:
case 0x30:
/* Values 0x26,0x2E,0x36,0x3E are valid x86
prefixes. In long mode, the CPU will signal
invalid opcode if some of these prefixes are
present so we will never get here anyway */
scan_more = ((instr_lo & 7) == 0x6);
break;
case 0x40:
/* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
Need to figure out under what instruction mode the
instruction was issued ... */
/* Could check the LDT for lm, but for now it's good
enough to assume that long mode only uses well known
segments or kernel. */
scan_more = ((regs->cs & 3) == 0) || (regs->cs == __USER_CS);
break;
case 0x60:
/* 0x64 thru 0x67 are valid prefixes in all modes. */
scan_more = (instr_lo & 0xC) == 0x4;
break;
case 0xF0:
/* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
scan_more = !instr_lo || (instr_lo>>1) == 1;
break;
case 0x00:
/* Prefetch instruction is 0x0F0D or 0x0F18 */
scan_more = 0;
if (__get_user(opcode, instr))
break;
prefetch = (instr_lo == 0xF) &&
(opcode == 0x0D || opcode == 0x18);
break;
default:
scan_more = 0;
break;
}
}
#if 1
if (prefetch)
printk("%s: prefetch caused page fault at %lx/%lx\n", current->comm,
regs->rip, addr);
#endif
return prefetch;
}
static int bad_address(void *p)
{
unsigned long dummy;
......@@ -90,6 +171,12 @@ void dump_pagetable(unsigned long address)
printk("BAD\n");
}
static inline int unhandled_signal(struct task_struct *tsk, int sig)
{
return (tsk->sighand->action[sig-1].sa.sa_handler == SIG_IGN) ||
(tsk->sighand->action[sig-1].sa.sa_handler == SIG_DFL);
}
int page_fault_trace;
int exception_trace = 1;
......@@ -151,7 +238,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code)
* context, we must not take the fault..
*/
if (unlikely(in_atomic() || !mm))
goto no_context;
goto bad_area_nosemaphore;
again:
down_read(&mm->mmap_sem);
......@@ -234,13 +321,16 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code)
/* User mode accesses just cause a SIGSEGV */
if (error_code & 4) {
if (is_prefetch(regs, address))
return;
if (exception_trace && !(tsk->ptrace & PT_PTRACED) &&
(tsk->sighand->action[SIGSEGV-1].sa.sa_handler == SIG_IGN ||
(tsk->sighand->action[SIGSEGV-1].sa.sa_handler == SIG_DFL)))
!unhandled_signal(tsk, SIGSEGV)) {
printk(KERN_INFO
"%s[%d]: segfault at %016lx rip %016lx rsp %016lx error %lx\n",
tsk->comm, tsk->pid, address, regs->rip,
regs->rsp, error_code);
}
tsk->thread.cr2 = address;
tsk->thread.error_code = error_code;
......@@ -262,6 +352,9 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code)
return;
}
if (is_prefetch(regs, address))
return;
/*
* Oops. The kernel tried to access some bad page. We'll have to
* terminate things with extreme prejudice.
......@@ -273,13 +366,13 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code)
printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
else
printk(KERN_ALERT "Unable to handle kernel paging request");
printk(" at virtual address %016lx\n",address);
printk(" printing rip:\n");
printk("%016lx\n", regs->rip);
printk(" at %016lx RIP: \n",address);
printk_address(regs->rip);
dump_pagetable(address);
__die("Oops", regs, error_code);
/* never reached */
/* Execute summary in case the body of the oops scrolled away */
printk(KERN_EMERG "CR2: %016lx\n", address);
oops_end();
do_exit(SIGKILL);
/*
......@@ -301,10 +394,10 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code)
do_sigbus:
up_read(&mm->mmap_sem);
/*
* Send a sigbus, regardless of whether we were in kernel
* or user mode.
*/
/* Kernel mode? Handle exceptions or die */
if (!(error_code & 4))
goto no_context;
tsk->thread.cr2 = address;
tsk->thread.error_code = error_code;
tsk->thread.trap_no = 14;
......@@ -313,13 +406,8 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code)
info.si_code = BUS_ADRERR;
info.si_addr = (void *)address;
force_sig_info(SIGBUS, &info, tsk);
/* Kernel mode? Handle exceptions or die */
if (!(error_code & 4))
goto no_context;
return;
vmalloc_fault:
{
pgd_t *pgd;
......
......@@ -149,6 +149,8 @@ static inline void set_seg_base(unsigned cpu, int entry, void *base)
#define LDT_entry_a(info) \
((((info)->base_addr & 0x0000ffff) << 16) | ((info)->limit & 0x0ffff))
/* Don't allow setting of the lm bit. It is useless anyways because
64bit system calls require __USER_CS. */
#define LDT_entry_b(info) \
(((info)->base_addr & 0xff000000) | \
(((info)->base_addr & 0x00ff0000) >> 16) | \
......@@ -159,7 +161,7 @@ static inline void set_seg_base(unsigned cpu, int entry, void *base)
((info)->seg_32bit << 22) | \
((info)->limit_in_pages << 23) | \
((info)->useable << 20) | \
((info)->lm << 21) | \
/* ((info)->lm << 21) | */ \
0x7000)
#define LDT_empty(info) (\
......
#ifndef _DWARF2_H
#define _DWARF2_H 1
#include <linux/config.h>
#ifndef __ASSEMBLY__
#warning "asm/dwarf2.h should be only included in pure assembly files"
#endif
/*
Macros for dwarf2 CFI unwind table entries.
See "as.info" for details on these pseudo ops. Unfortunately
they are only supported in very new binutils, so define them
away for older version.
*/
#ifdef CONFIG_CFI_BINUTILS
#define CFI_STARTPROC .cfi_startproc
#define CFI_ENDPROC .cfi_endproc
#define CFI_DEF_CFA .cfi_def_cfa
#define CFI_DEF_CFA_REGISTER .cfi_def_cfa_register
#define CFI_DEF_CFA_OFFSET .cfi_def_cfa_offset
#define CFI_ADJUST_CFA_OFFSET .cfi_adjust_cfa_offset
#define CFI_OFFSET .cfi_offset
#define CFI_REL_OFFSET .cfi_rel_offset
#else
#ifdef __ASSEMBLY__
.macro nothing
.endm
.macro nothing1 a
.endm
.macro nothing2 a,b
.endm
#endif
#define CFI_STARTPROC nothing
#define CFI_ENDPROC nothing
#define CFI_DEF_CFA nothing2
#define CFI_DEF_CFA_REGISTER nothing1
#define CFI_DEF_CFA_OFFSET nothing1
#define CFI_ADJUST_CFA_OFFSET nothing1
#define CFI_OFFSET nothing2
#define CFI_REL_OFFSET nothing2
#endif
#endif
......@@ -39,6 +39,12 @@ static inline int notify_die(enum die_val val,char *str,struct pt_regs *regs,lon
return notifier_call_chain(&die_chain, val, &args);
}
int printk_address(unsigned long address);
extern int printk_address(unsigned long address);
extern void die(const char *,struct pt_regs *,long);
extern void __die(const char *,struct pt_regs *,long);
extern void show_registers(struct pt_regs *regs);
extern void dump_pagetable(unsigned long);
extern void oops_begin(void);
extern void oops_end(void);
#endif
......@@ -5,6 +5,8 @@
#define __ARCH_SI_BAND_T long
#define SIGEV_PAD_SIZE ((SIGEV_MAX_SIZE/sizeof(int)) - 4)
#include <asm-generic/siginfo.h>
#endif
......@@ -147,7 +147,7 @@ extern void __put_user_bad(void);
({ \
int __pu_err = -EFAULT; \
__typeof__(*(ptr)) *__pu_addr = (ptr); \
if (access_ok(VERIFY_WRITE,__pu_addr,size)) \
if (likely(access_ok(VERIFY_WRITE,__pu_addr,size))) \
__put_user_size((x),__pu_addr,(size),__pu_err); \
__pu_err; \
})
......@@ -255,12 +255,12 @@ static inline int __copy_from_user(void *dst, const void *src, unsigned size)
return ret;
case 10:
__get_user_asm(*(u64*)dst,(u64*)src,ret,"q","","=r",16);
if (ret) return ret;
if (unlikely(ret)) return ret;
__get_user_asm(*(u16*)(8+(char*)dst),(u16*)(8+(char*)src),ret,"w","w","=r",2);
return ret;
case 16:
__get_user_asm(*(u64*)dst,(u64*)src,ret,"q","","=r",16);
if (ret) return ret;
if (unlikely(ret)) return ret;
__get_user_asm(*(u64*)(8+(char*)dst),(u64*)(8+(char*)src),ret,"q","","=r",8);
return ret;
default:
......@@ -284,13 +284,13 @@ static inline int __copy_to_user(void *dst, const void *src, unsigned size)
return ret;
case 10:
__put_user_asm(*(u64*)src,(u64*)dst,ret,"q","","ir",10);
if (ret) return ret;
if (unlikely(ret)) return ret;
asm("":::"memory");
__put_user_asm(4[(u16*)src],4+(u16*)dst,ret,"w","w","ir",2);
return ret;
case 16:
__put_user_asm(*(u64*)src,(u64*)dst,ret,"q","","ir",16);
if (ret) return ret;
if (unlikely(ret)) return ret;
asm("":::"memory");
__put_user_asm(1[(u64*)src],1+(u64*)dst,ret,"q","","ir",8);
return ret;
......@@ -309,14 +309,14 @@ static inline int __copy_in_user(void *dst, const void *src, unsigned size)
case 1: {
u8 tmp;
__get_user_asm(tmp,(u8 *)src,ret,"b","b","=q",1);
if (!ret)
if (likely(!ret))
__put_user_asm(tmp,(u8 *)dst,ret,"b","b","iq",1);
return ret;
}
case 2: {
u16 tmp;
__get_user_asm(tmp,(u16 *)src,ret,"w","w","=r",2);
if (!ret)
if (likely(!ret))
__put_user_asm(tmp,(u16 *)dst,ret,"w","w","ir",2);
return ret;
}
......@@ -324,14 +324,14 @@ static inline int __copy_in_user(void *dst, const void *src, unsigned size)
case 4: {
u32 tmp;
__get_user_asm(tmp,(u32 *)src,ret,"l","k","=r",4);
if (!ret)
if (likely(!ret))
__put_user_asm(tmp,(u32 *)dst,ret,"l","k","ir",4);
return ret;
}
case 8: {
u64 tmp;
__get_user_asm(tmp,(u64 *)src,ret,"q","","=r",8);
if (!ret)
if (likely(!ret))
__put_user_asm(tmp,(u64 *)dst,ret,"q","","ir",8);
return ret;
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment