Commit 450b4497 authored by Linus Torvalds's avatar Linus Torvalds

Move "used FPU status" into new non-atomic thread_info->status field.

This allows us to avoid having to use atomic updates for the lazy FP
status setting, since we don't have to worry about other CPU's racing
on the fields.

Also, fix x86 FP state after fork() by making sure the FP is unlazied
_before_ we copy the state information. Otherwise, if a process did a
fork() while holding the FP state lazily in the registers, the child
would incorrectly unlazy bogus state.
parent 36f06199
...@@ -480,7 +480,7 @@ void __init cpu_init (void) ...@@ -480,7 +480,7 @@ void __init cpu_init (void)
*/ */
atomic_inc(&init_mm.mm_count); atomic_inc(&init_mm.mm_count);
current->active_mm = &init_mm; current->active_mm = &init_mm;
if(current->mm) if (current->mm)
BUG(); BUG();
enter_lazy_tlb(&init_mm, current, cpu); enter_lazy_tlb(&init_mm, current, cpu);
...@@ -508,7 +508,7 @@ void __init cpu_init (void) ...@@ -508,7 +508,7 @@ void __init cpu_init (void)
/* /*
* Force FPU initialization: * Force FPU initialization:
*/ */
clear_thread_flag(TIF_USEDFPU); current_thread_info()->status = 0;
current->used_math = 0; current->used_math = 0;
stts(); stts();
} }
...@@ -54,9 +54,11 @@ void init_fpu(struct task_struct *tsk) ...@@ -54,9 +54,11 @@ void init_fpu(struct task_struct *tsk)
void kernel_fpu_begin(void) void kernel_fpu_begin(void)
{ {
struct thread_info *thread = current_thread_info();
preempt_disable(); preempt_disable();
if (test_thread_flag(TIF_USEDFPU)) { if (thread->status & TS_USEDFPU) {
__save_init_fpu(current); __save_init_fpu(thread->task);
return; return;
} }
clts(); clts();
......
...@@ -275,6 +275,15 @@ void release_thread(struct task_struct *dead_task) ...@@ -275,6 +275,15 @@ void release_thread(struct task_struct *dead_task)
release_x86_irqs(dead_task); release_x86_irqs(dead_task);
} }
/*
* This gets called before we allocate a new thread and copy
* the current task into it.
*/
void prepare_to_copy(struct task_struct *tsk)
{
unlazy_fpu(tsk);
}
int copy_thread(int nr, unsigned long clone_flags, unsigned long esp, int copy_thread(int nr, unsigned long clone_flags, unsigned long esp,
unsigned long unused, unsigned long unused,
struct task_struct * p, struct pt_regs * regs) struct task_struct * p, struct pt_regs * regs)
...@@ -297,9 +306,6 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long esp, ...@@ -297,9 +306,6 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long esp,
savesegment(gs,p->thread.gs); savesegment(gs,p->thread.gs);
tsk = current; tsk = current;
unlazy_fpu(tsk);
struct_cpy(&p->thread.i387, &tsk->thread.i387);
if (unlikely(NULL != tsk->thread.ts_io_bitmap)) { if (unlikely(NULL != tsk->thread.ts_io_bitmap)) {
p->thread.ts_io_bitmap = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL); p->thread.ts_io_bitmap = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL);
if (!p->thread.ts_io_bitmap) if (!p->thread.ts_io_bitmap)
......
...@@ -737,13 +737,14 @@ asmlinkage void do_spurious_interrupt_bug(struct pt_regs * regs, ...@@ -737,13 +737,14 @@ asmlinkage void do_spurious_interrupt_bug(struct pt_regs * regs,
*/ */
asmlinkage void math_state_restore(struct pt_regs regs) asmlinkage void math_state_restore(struct pt_regs regs)
{ {
struct task_struct *tsk = current; struct thread_info *thread = current_thread_info();
clts(); /* Allow maths ops (or we recurse) */ struct task_struct *tsk = thread->task;
clts(); /* Allow maths ops (or we recurse) */
if (!tsk->used_math) if (!tsk->used_math)
init_fpu(tsk); init_fpu(tsk);
restore_fpu(tsk); restore_fpu(tsk);
set_thread_flag(TIF_USEDFPU); /* So we fnsave on switch_to() */ thread->status |= TS_USEDFPU; /* So we fnsave on switch_to() */
} }
#ifndef CONFIG_MATH_EMULATION #ifndef CONFIG_MATH_EMULATION
......
...@@ -51,6 +51,9 @@ extern void start_thread(struct pt_regs *, unsigned long, unsigned long); ...@@ -51,6 +51,9 @@ extern void start_thread(struct pt_regs *, unsigned long, unsigned long);
/* Free all resources held by a thread. */ /* Free all resources held by a thread. */
extern void release_thread(struct task_struct *); extern void release_thread(struct task_struct *);
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
/* Create a kernel thread without removing it from tasklists. */ /* Create a kernel thread without removing it from tasklists. */
extern long kernel_thread(int (*fn)(void *), void *arg, unsigned long flags); extern long kernel_thread(int (*fn)(void *), void *arg, unsigned long flags);
......
...@@ -62,6 +62,9 @@ struct task_struct; ...@@ -62,6 +62,9 @@ struct task_struct;
/* Free all resources held by a thread. */ /* Free all resources held by a thread. */
extern void release_thread(struct task_struct *); extern void release_thread(struct task_struct *);
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
unsigned long get_wchan(struct task_struct *p); unsigned long get_wchan(struct task_struct *p);
#define cpu_relax() barrier() #define cpu_relax() barrier()
......
...@@ -123,6 +123,9 @@ static inline void release_thread(struct task_struct *dead_task) ...@@ -123,6 +123,9 @@ static inline void release_thread(struct task_struct *dead_task)
/* Nothing needs to be done. */ /* Nothing needs to be done. */
} }
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
/* /*
* Return saved PC of a blocked thread. * Return saved PC of a blocked thread.
*/ */
......
...@@ -36,7 +36,7 @@ static inline void __save_init_fpu( struct task_struct *tsk ) ...@@ -36,7 +36,7 @@ static inline void __save_init_fpu( struct task_struct *tsk )
asm volatile( "fnsave %0 ; fwait" asm volatile( "fnsave %0 ; fwait"
: "=m" (tsk->thread.i387.fsave) ); : "=m" (tsk->thread.i387.fsave) );
} }
tsk->thread_info->flags &= ~_TIF_USEDFPU; tsk->thread_info->status &= ~TS_USEDFPU;
} }
static inline void save_init_fpu( struct task_struct *tsk ) static inline void save_init_fpu( struct task_struct *tsk )
...@@ -47,15 +47,15 @@ static inline void save_init_fpu( struct task_struct *tsk ) ...@@ -47,15 +47,15 @@ static inline void save_init_fpu( struct task_struct *tsk )
#define unlazy_fpu( tsk ) do { \ #define unlazy_fpu( tsk ) do { \
if ((tsk)->thread_info->flags & _TIF_USEDFPU) \ if ((tsk)->thread_info->status & TS_USEDFPU) \
save_init_fpu( tsk ); \ save_init_fpu( tsk ); \
} while (0) } while (0)
#define clear_fpu( tsk ) \ #define clear_fpu( tsk ) \
do { \ do { \
if ((tsk)->thread_info->flags & _TIF_USEDFPU) { \ if ((tsk)->thread_info->status & TS_USEDFPU) { \
asm volatile("fwait"); \ asm volatile("fwait"); \
(tsk)->thread_info->flags &= ~_TIF_USEDFPU; \ (tsk)->thread_info->status &= ~TS_USEDFPU; \
stts(); \ stts(); \
} \ } \
} while (0) } while (0)
......
...@@ -446,6 +446,10 @@ struct mm_struct; ...@@ -446,6 +446,10 @@ struct mm_struct;
/* Free all resources held by a thread. */ /* Free all resources held by a thread. */
extern void release_thread(struct task_struct *); extern void release_thread(struct task_struct *);
/* Prepare to copy thread state - unlazy all lazy status */
extern void prepare_to_copy(struct task_struct *tsk);
/* /*
* create a kernel thread without removing it from tasklists * create a kernel thread without removing it from tasklists
*/ */
......
...@@ -25,6 +25,7 @@ struct thread_info { ...@@ -25,6 +25,7 @@ struct thread_info {
struct task_struct *task; /* main task structure */ struct task_struct *task; /* main task structure */
struct exec_domain *exec_domain; /* execution domain */ struct exec_domain *exec_domain; /* execution domain */
unsigned long flags; /* low level flags */ unsigned long flags; /* low level flags */
unsigned long status; /* thread-synchronous flags */
__u32 cpu; /* current CPU */ __u32 cpu; /* current CPU */
__s32 preempt_count; /* 0 => preemptable, <0 => BUG */ __s32 preempt_count; /* 0 => preemptable, <0 => BUG */
...@@ -43,10 +44,11 @@ struct thread_info { ...@@ -43,10 +44,11 @@ struct thread_info {
#define TI_TASK 0x00000000 #define TI_TASK 0x00000000
#define TI_EXEC_DOMAIN 0x00000004 #define TI_EXEC_DOMAIN 0x00000004
#define TI_FLAGS 0x00000008 #define TI_FLAGS 0x00000008
#define TI_CPU 0x0000000C #define TI_STATUS 0x0000000C
#define TI_PRE_COUNT 0x00000010 #define TI_CPU 0x00000010
#define TI_ADDR_LIMIT 0x00000014 #define TI_PRE_COUNT 0x00000014
#define TI_RESTART_BLOCK 0x0000018 #define TI_ADDR_LIMIT 0x00000018
#define TI_RESTART_BLOCK 0x000001C
#endif #endif
...@@ -111,8 +113,7 @@ static inline struct thread_info *current_thread_info(void) ...@@ -111,8 +113,7 @@ static inline struct thread_info *current_thread_info(void)
#define TIF_NEED_RESCHED 3 /* rescheduling necessary */ #define TIF_NEED_RESCHED 3 /* rescheduling necessary */
#define TIF_SINGLESTEP 4 /* restore singlestep on return to user mode */ #define TIF_SINGLESTEP 4 /* restore singlestep on return to user mode */
#define TIF_IRET 5 /* return with iret */ #define TIF_IRET 5 /* return with iret */
#define TIF_USEDFPU 16 /* FPU was used by this task this quantum (SMP) */ #define TIF_POLLING_NRFLAG 16 /* true if poll_idle() is polling TIF_NEED_RESCHED */
#define TIF_POLLING_NRFLAG 17 /* true if poll_idle() is polling TIF_NEED_RESCHED */
#define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE) #define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE)
#define _TIF_NOTIFY_RESUME (1<<TIF_NOTIFY_RESUME) #define _TIF_NOTIFY_RESUME (1<<TIF_NOTIFY_RESUME)
...@@ -120,12 +121,20 @@ static inline struct thread_info *current_thread_info(void) ...@@ -120,12 +121,20 @@ static inline struct thread_info *current_thread_info(void)
#define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED) #define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED)
#define _TIF_SINGLESTEP (1<<TIF_SINGLESTEP) #define _TIF_SINGLESTEP (1<<TIF_SINGLESTEP)
#define _TIF_IRET (1<<TIF_IRET) #define _TIF_IRET (1<<TIF_IRET)
#define _TIF_USEDFPU (1<<TIF_USEDFPU)
#define _TIF_POLLING_NRFLAG (1<<TIF_POLLING_NRFLAG) #define _TIF_POLLING_NRFLAG (1<<TIF_POLLING_NRFLAG)
#define _TIF_WORK_MASK 0x0000FFFE /* work to do on interrupt/exception return */ #define _TIF_WORK_MASK 0x0000FFFE /* work to do on interrupt/exception return */
#define _TIF_ALLWORK_MASK 0x0000FFFF /* work to do on any return to u-space */ #define _TIF_ALLWORK_MASK 0x0000FFFF /* work to do on any return to u-space */
/*
* Thread-synchronous status.
*
* This is different from the flags in that nobody else
* ever touches our thread-synchronous status, so we don't
* have to worry about atomic accesses.
*/
#define TS_USEDFPU 0x0001 /* FPU was used by this task this quantum (SMP) */
#endif /* __KERNEL__ */ #endif /* __KERNEL__ */
#endif /* _ASM_THREAD_INFO_H */ #endif /* _ASM_THREAD_INFO_H */
...@@ -344,6 +344,9 @@ struct task_struct; ...@@ -344,6 +344,9 @@ struct task_struct;
# define release_thread(dead_task) # define release_thread(dead_task)
#endif #endif
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
/* /*
* This is the mechanism for creating a new kernel thread. * This is the mechanism for creating a new kernel thread.
* *
......
...@@ -112,6 +112,9 @@ static inline void release_thread(struct task_struct *dead_task) ...@@ -112,6 +112,9 @@ static inline void release_thread(struct task_struct *dead_task)
{ {
} }
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags); extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
/* /*
......
...@@ -108,6 +108,9 @@ static inline void release_thread(struct task_struct *dead_task) ...@@ -108,6 +108,9 @@ static inline void release_thread(struct task_struct *dead_task)
{ {
} }
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags); extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
/* /*
......
...@@ -208,6 +208,9 @@ struct thread_struct { ...@@ -208,6 +208,9 @@ struct thread_struct {
/* Free all resources held by a thread. */ /* Free all resources held by a thread. */
#define release_thread(thread) do { } while(0) #define release_thread(thread) do { } while(0)
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags); extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
/* /*
......
...@@ -231,6 +231,9 @@ struct thread_struct { ...@@ -231,6 +231,9 @@ struct thread_struct {
/* Free all resources held by a thread. */ /* Free all resources held by a thread. */
#define release_thread(thread) do { } while(0) #define release_thread(thread) do { } while(0)
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags); extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
/* /*
......
...@@ -290,6 +290,9 @@ struct mm_struct; ...@@ -290,6 +290,9 @@ struct mm_struct;
extern void release_thread(struct task_struct *); extern void release_thread(struct task_struct *);
extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags); extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
extern void map_hpux_gateway_page(struct task_struct *tsk, struct mm_struct *mm); extern void map_hpux_gateway_page(struct task_struct *tsk, struct mm_struct *mm);
static inline unsigned long get_wchan(struct task_struct *p) static inline unsigned long get_wchan(struct task_struct *p)
......
...@@ -608,6 +608,9 @@ struct task_struct; ...@@ -608,6 +608,9 @@ struct task_struct;
void start_thread(struct pt_regs *regs, unsigned long nip, unsigned long sp); void start_thread(struct pt_regs *regs, unsigned long nip, unsigned long sp);
void release_thread(struct task_struct *); void release_thread(struct task_struct *);
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
/* /*
* Create a new kernel thread. * Create a new kernel thread.
*/ */
......
...@@ -601,6 +601,9 @@ struct task_struct; ...@@ -601,6 +601,9 @@ struct task_struct;
void start_thread(struct pt_regs *regs, unsigned long fdptr, unsigned long sp); void start_thread(struct pt_regs *regs, unsigned long fdptr, unsigned long sp);
void release_thread(struct task_struct *); void release_thread(struct task_struct *);
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
/* /*
* Create a new kernel thread. * Create a new kernel thread.
*/ */
......
...@@ -114,6 +114,9 @@ struct mm_struct; ...@@ -114,6 +114,9 @@ struct mm_struct;
extern void release_thread(struct task_struct *); extern void release_thread(struct task_struct *);
extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags); extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
/* /*
* Return saved PC of a blocked thread. * Return saved PC of a blocked thread.
*/ */
......
...@@ -129,6 +129,9 @@ struct mm_struct; ...@@ -129,6 +129,9 @@ struct mm_struct;
extern void release_thread(struct task_struct *); extern void release_thread(struct task_struct *);
extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags); extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
/* /*
* Return saved PC of a blocked thread. * Return saved PC of a blocked thread.
*/ */
......
...@@ -134,6 +134,10 @@ struct mm_struct; ...@@ -134,6 +134,10 @@ struct mm_struct;
/* Free all resources held by a thread. */ /* Free all resources held by a thread. */
extern void release_thread(struct task_struct *); extern void release_thread(struct task_struct *);
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
/* /*
* create a kernel thread without removing it from tasklists * create a kernel thread without removing it from tasklists
*/ */
......
...@@ -139,6 +139,9 @@ extern __inline__ void start_thread(struct pt_regs * regs, unsigned long pc, ...@@ -139,6 +139,9 @@ extern __inline__ void start_thread(struct pt_regs * regs, unsigned long pc,
#define release_thread(tsk) do { } while(0) #define release_thread(tsk) do { } while(0)
extern pid_t kernel_thread(int (*fn)(void *), void * arg, unsigned long flags); extern pid_t kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
extern unsigned long get_wchan(struct task_struct *); extern unsigned long get_wchan(struct task_struct *);
#define KSTK_EIP(tsk) ((tsk)->thread.kregs->pc) #define KSTK_EIP(tsk) ((tsk)->thread.kregs->pc)
......
...@@ -186,6 +186,9 @@ do { \ ...@@ -186,6 +186,9 @@ do { \
/* Free all resources held by a thread. */ /* Free all resources held by a thread. */
#define release_thread(tsk) do { } while (0) #define release_thread(tsk) do { } while (0)
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
extern pid_t kernel_thread(int (*fn)(void *), void * arg, unsigned long flags); extern pid_t kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
extern unsigned long get_wchan(struct task_struct *task); extern unsigned long get_wchan(struct task_struct *task);
......
...@@ -81,6 +81,9 @@ extern inline void release_thread (struct task_struct *dead_task) ...@@ -81,6 +81,9 @@ extern inline void release_thread (struct task_struct *dead_task)
{ {
} }
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
extern int kernel_thread (int (*fn)(void *), void * arg, unsigned long flags); extern int kernel_thread (int (*fn)(void *), void * arg, unsigned long flags);
/* Free current thread data structures etc. */ /* Free current thread data structures etc. */
......
...@@ -267,6 +267,10 @@ struct mm_struct; ...@@ -267,6 +267,10 @@ struct mm_struct;
/* Free all resources held by a thread. */ /* Free all resources held by a thread. */
extern void release_thread(struct task_struct *); extern void release_thread(struct task_struct *);
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
/* /*
* create a kernel thread without removing it from tasklists * create a kernel thread without removing it from tasklists
*/ */
......
...@@ -214,6 +214,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) ...@@ -214,6 +214,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig)
struct thread_info *ti; struct thread_info *ti;
int cpu = get_cpu(); int cpu = get_cpu();
prepare_to_copy(orig);
tsk = task_cache[cpu]; tsk = task_cache[cpu];
task_cache[cpu] = NULL; task_cache[cpu] = NULL;
put_cpu(); put_cpu();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment