Commit 2d724ffd authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'x86-fpu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 fpu updates from Ingo Molnar:
 "The main x86 FPU changes in this cycle were:

   - a large series of cleanups, fixes and enhancements to re-enable the
     XSAVES instruction on Intel CPUs - which is the most advanced
     instruction to do FPU context switches (Yu-cheng Yu, Fenghua Yu)

   - Add FPU tracepoints for the FPU state machine (Dave Hansen)"

* 'x86-fpu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/fpu: Do not BUG_ON() in early FPU code
  x86/fpu/xstate: Re-enable XSAVES
  x86/fpu/xstate: Fix fpstate_init() for XRSTORS
  x86/fpu/xstate: Return NULL for disabled xstate component address
  x86/fpu/xstate: Fix __fpu_restore_sig() for XSAVES
  x86/fpu/xstate: Fix xstate_offsets, xstate_sizes for non-extended xstates
  x86/fpu/xstate: Fix XSTATE component offset print out
  x86/fpu/xstate: Fix PTRACE frames for XSAVES
  x86/fpu/xstate: Fix supervisor xstate component offset
  x86/fpu/xstate: Align xstate components according to CPUID
  x86/fpu/xstate: Copy xstate registers directly to the signal frame when compacted format is in use
  x86/fpu/xstate: Keep init_fpstate.xsave.header.xfeatures as zero for init optimization
  x86/fpu/xstate: Rename 'xstate_size' to 'fpu_kernel_xstate_size', to distinguish it from 'fpu_user_xstate_size'
  x86/fpu/xstate: Define and use 'fpu_user_xstate_size'
  x86/fpu: Add tracepoints to dump FPU state at key points
parents 36e635cb ec3ed4a2
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include <asm/fpu/api.h> #include <asm/fpu/api.h>
#include <asm/fpu/xstate.h> #include <asm/fpu/xstate.h>
#include <asm/cpufeature.h> #include <asm/cpufeature.h>
#include <asm/trace/fpu.h>
/* /*
* High level FPU state handling functions: * High level FPU state handling functions:
...@@ -524,6 +525,7 @@ static inline void __fpregs_deactivate(struct fpu *fpu) ...@@ -524,6 +525,7 @@ static inline void __fpregs_deactivate(struct fpu *fpu)
fpu->fpregs_active = 0; fpu->fpregs_active = 0;
this_cpu_write(fpu_fpregs_owner_ctx, NULL); this_cpu_write(fpu_fpregs_owner_ctx, NULL);
trace_x86_fpu_regs_deactivated(fpu);
} }
/* Must be paired with a 'clts' (fpregs_activate_hw()) before! */ /* Must be paired with a 'clts' (fpregs_activate_hw()) before! */
...@@ -533,6 +535,7 @@ static inline void __fpregs_activate(struct fpu *fpu) ...@@ -533,6 +535,7 @@ static inline void __fpregs_activate(struct fpu *fpu)
fpu->fpregs_active = 1; fpu->fpregs_active = 1;
this_cpu_write(fpu_fpregs_owner_ctx, fpu); this_cpu_write(fpu_fpregs_owner_ctx, fpu);
trace_x86_fpu_regs_activated(fpu);
} }
/* /*
...@@ -604,11 +607,13 @@ switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu) ...@@ -604,11 +607,13 @@ switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu)
/* But leave fpu_fpregs_owner_ctx! */ /* But leave fpu_fpregs_owner_ctx! */
old_fpu->fpregs_active = 0; old_fpu->fpregs_active = 0;
trace_x86_fpu_regs_deactivated(old_fpu);
/* Don't change CR0.TS if we just switch! */ /* Don't change CR0.TS if we just switch! */
if (fpu.preload) { if (fpu.preload) {
new_fpu->counter++; new_fpu->counter++;
__fpregs_activate(new_fpu); __fpregs_activate(new_fpu);
trace_x86_fpu_regs_activated(new_fpu);
prefetch(&new_fpu->state); prefetch(&new_fpu->state);
} else { } else {
__fpregs_deactivate_hw(); __fpregs_deactivate_hw();
......
...@@ -122,6 +122,7 @@ enum xfeature { ...@@ -122,6 +122,7 @@ enum xfeature {
#define XFEATURE_MASK_OPMASK (1 << XFEATURE_OPMASK) #define XFEATURE_MASK_OPMASK (1 << XFEATURE_OPMASK)
#define XFEATURE_MASK_ZMM_Hi256 (1 << XFEATURE_ZMM_Hi256) #define XFEATURE_MASK_ZMM_Hi256 (1 << XFEATURE_ZMM_Hi256)
#define XFEATURE_MASK_Hi16_ZMM (1 << XFEATURE_Hi16_ZMM) #define XFEATURE_MASK_Hi16_ZMM (1 << XFEATURE_Hi16_ZMM)
#define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR)
#define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU) #define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU)
#define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE) #define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE)
...@@ -230,6 +231,12 @@ struct xstate_header { ...@@ -230,6 +231,12 @@ struct xstate_header {
u64 reserved[6]; u64 reserved[6];
} __attribute__((packed)); } __attribute__((packed));
/*
* xstate_header.xcomp_bv[63] indicates that the extended_state_area
* is in compacted format.
*/
#define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63)
/* /*
* This is our most modern FPU state format, as saved by the XSAVE * This is our most modern FPU state format, as saved by the XSAVE
* and restored by the XRSTOR instructions. * and restored by the XRSTOR instructions.
......
...@@ -18,6 +18,9 @@ ...@@ -18,6 +18,9 @@
#define XSAVE_YMM_SIZE 256 #define XSAVE_YMM_SIZE 256
#define XSAVE_YMM_OFFSET (XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET) #define XSAVE_YMM_OFFSET (XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET)
/* Supervisor features */
#define XFEATURE_MASK_SUPERVISOR (XFEATURE_MASK_PT)
/* Supported features which support lazy state saving */ /* Supported features which support lazy state saving */
#define XFEATURE_MASK_LAZY (XFEATURE_MASK_FP | \ #define XFEATURE_MASK_LAZY (XFEATURE_MASK_FP | \
XFEATURE_MASK_SSE | \ XFEATURE_MASK_SSE | \
...@@ -39,7 +42,6 @@ ...@@ -39,7 +42,6 @@
#define REX_PREFIX #define REX_PREFIX
#endif #endif
extern unsigned int xstate_size;
extern u64 xfeatures_mask; extern u64 xfeatures_mask;
extern u64 xstate_fx_sw_bytes[USER_XSTATE_FX_SW_WORDS]; extern u64 xstate_fx_sw_bytes[USER_XSTATE_FX_SW_WORDS];
...@@ -48,5 +50,9 @@ extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask); ...@@ -48,5 +50,9 @@ extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask);
void fpu__xstate_clear_all_cpu_caps(void); void fpu__xstate_clear_all_cpu_caps(void);
void *get_xsave_addr(struct xregs_state *xsave, int xstate); void *get_xsave_addr(struct xregs_state *xsave, int xstate);
const void *get_xsave_field_ptr(int xstate_field); const void *get_xsave_field_ptr(int xstate_field);
int using_compacted_format(void);
int copyout_from_xsaves(unsigned int pos, unsigned int count, void *kbuf,
void __user *ubuf, struct xregs_state *xsave);
int copyin_to_xsaves(const void *kbuf, const void __user *ubuf,
struct xregs_state *xsave);
#endif #endif
...@@ -367,7 +367,8 @@ DECLARE_PER_CPU(struct irq_stack *, hardirq_stack); ...@@ -367,7 +367,8 @@ DECLARE_PER_CPU(struct irq_stack *, hardirq_stack);
DECLARE_PER_CPU(struct irq_stack *, softirq_stack); DECLARE_PER_CPU(struct irq_stack *, softirq_stack);
#endif /* X86_64 */ #endif /* X86_64 */
extern unsigned int xstate_size; extern unsigned int fpu_kernel_xstate_size;
extern unsigned int fpu_user_xstate_size;
struct perf_event; struct perf_event;
......
#undef TRACE_SYSTEM
#define TRACE_SYSTEM x86_fpu
#if !defined(_TRACE_FPU_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_FPU_H
#include <linux/tracepoint.h>
DECLARE_EVENT_CLASS(x86_fpu,
TP_PROTO(struct fpu *fpu),
TP_ARGS(fpu),
TP_STRUCT__entry(
__field(struct fpu *, fpu)
__field(bool, fpregs_active)
__field(bool, fpstate_active)
__field(int, counter)
__field(u64, xfeatures)
__field(u64, xcomp_bv)
),
TP_fast_assign(
__entry->fpu = fpu;
__entry->fpregs_active = fpu->fpregs_active;
__entry->fpstate_active = fpu->fpstate_active;
__entry->counter = fpu->counter;
if (boot_cpu_has(X86_FEATURE_OSXSAVE)) {
__entry->xfeatures = fpu->state.xsave.header.xfeatures;
__entry->xcomp_bv = fpu->state.xsave.header.xcomp_bv;
}
),
TP_printk("x86/fpu: %p fpregs_active: %d fpstate_active: %d counter: %d xfeatures: %llx xcomp_bv: %llx",
__entry->fpu,
__entry->fpregs_active,
__entry->fpstate_active,
__entry->counter,
__entry->xfeatures,
__entry->xcomp_bv
)
);
DEFINE_EVENT(x86_fpu, x86_fpu_state,
TP_PROTO(struct fpu *fpu),
TP_ARGS(fpu)
);
DEFINE_EVENT(x86_fpu, x86_fpu_before_save,
TP_PROTO(struct fpu *fpu),
TP_ARGS(fpu)
);
DEFINE_EVENT(x86_fpu, x86_fpu_after_save,
TP_PROTO(struct fpu *fpu),
TP_ARGS(fpu)
);
DEFINE_EVENT(x86_fpu, x86_fpu_before_restore,
TP_PROTO(struct fpu *fpu),
TP_ARGS(fpu)
);
DEFINE_EVENT(x86_fpu, x86_fpu_after_restore,
TP_PROTO(struct fpu *fpu),
TP_ARGS(fpu)
);
DEFINE_EVENT(x86_fpu, x86_fpu_regs_activated,
TP_PROTO(struct fpu *fpu),
TP_ARGS(fpu)
);
DEFINE_EVENT(x86_fpu, x86_fpu_regs_deactivated,
TP_PROTO(struct fpu *fpu),
TP_ARGS(fpu)
);
DEFINE_EVENT(x86_fpu, x86_fpu_activate_state,
TP_PROTO(struct fpu *fpu),
TP_ARGS(fpu)
);
DEFINE_EVENT(x86_fpu, x86_fpu_deactivate_state,
TP_PROTO(struct fpu *fpu),
TP_ARGS(fpu)
);
DEFINE_EVENT(x86_fpu, x86_fpu_init_state,
TP_PROTO(struct fpu *fpu),
TP_ARGS(fpu)
);
DEFINE_EVENT(x86_fpu, x86_fpu_dropped,
TP_PROTO(struct fpu *fpu),
TP_ARGS(fpu)
);
DEFINE_EVENT(x86_fpu, x86_fpu_copy_src,
TP_PROTO(struct fpu *fpu),
TP_ARGS(fpu)
);
DEFINE_EVENT(x86_fpu, x86_fpu_copy_dst,
TP_PROTO(struct fpu *fpu),
TP_ARGS(fpu)
);
DEFINE_EVENT(x86_fpu, x86_fpu_xstate_check_failed,
TP_PROTO(struct fpu *fpu),
TP_ARGS(fpu)
);
#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH asm/trace/
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE fpu
#endif /* _TRACE_FPU_H */
/* This part must be outside protection */
#include <trace/define_trace.h>
...@@ -8,10 +8,14 @@ ...@@ -8,10 +8,14 @@
#include <asm/fpu/internal.h> #include <asm/fpu/internal.h>
#include <asm/fpu/regset.h> #include <asm/fpu/regset.h>
#include <asm/fpu/signal.h> #include <asm/fpu/signal.h>
#include <asm/fpu/types.h>
#include <asm/traps.h> #include <asm/traps.h>
#include <linux/hardirq.h> #include <linux/hardirq.h>
#define CREATE_TRACE_POINTS
#include <asm/trace/fpu.h>
/* /*
* Represents the initial FPU state. It's mostly (but not completely) zeroes, * Represents the initial FPU state. It's mostly (but not completely) zeroes,
* depending on the FPU hardware format: * depending on the FPU hardware format:
...@@ -192,6 +196,7 @@ void fpu__save(struct fpu *fpu) ...@@ -192,6 +196,7 @@ void fpu__save(struct fpu *fpu)
WARN_ON_FPU(fpu != &current->thread.fpu); WARN_ON_FPU(fpu != &current->thread.fpu);
preempt_disable(); preempt_disable();
trace_x86_fpu_before_save(fpu);
if (fpu->fpregs_active) { if (fpu->fpregs_active) {
if (!copy_fpregs_to_fpstate(fpu)) { if (!copy_fpregs_to_fpstate(fpu)) {
if (use_eager_fpu()) if (use_eager_fpu())
...@@ -200,6 +205,7 @@ void fpu__save(struct fpu *fpu) ...@@ -200,6 +205,7 @@ void fpu__save(struct fpu *fpu)
fpregs_deactivate(fpu); fpregs_deactivate(fpu);
} }
} }
trace_x86_fpu_after_save(fpu);
preempt_enable(); preempt_enable();
} }
EXPORT_SYMBOL_GPL(fpu__save); EXPORT_SYMBOL_GPL(fpu__save);
...@@ -222,7 +228,14 @@ void fpstate_init(union fpregs_state *state) ...@@ -222,7 +228,14 @@ void fpstate_init(union fpregs_state *state)
return; return;
} }
memset(state, 0, xstate_size); memset(state, 0, fpu_kernel_xstate_size);
/*
* XRSTORS requires that this bit is set in xcomp_bv, or
* it will #GP. Make sure it is replaced after the memset().
*/
if (static_cpu_has(X86_FEATURE_XSAVES))
state->xsave.header.xcomp_bv = XCOMP_BV_COMPACTED_FORMAT;
if (static_cpu_has(X86_FEATURE_FXSR)) if (static_cpu_has(X86_FEATURE_FXSR))
fpstate_init_fxstate(&state->fxsave); fpstate_init_fxstate(&state->fxsave);
...@@ -247,7 +260,7 @@ int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu) ...@@ -247,7 +260,7 @@ int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu)
* leak into the child task: * leak into the child task:
*/ */
if (use_eager_fpu()) if (use_eager_fpu())
memset(&dst_fpu->state.xsave, 0, xstate_size); memset(&dst_fpu->state.xsave, 0, fpu_kernel_xstate_size);
/* /*
* Save current FPU registers directly into the child * Save current FPU registers directly into the child
...@@ -266,7 +279,8 @@ int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu) ...@@ -266,7 +279,8 @@ int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu)
*/ */
preempt_disable(); preempt_disable();
if (!copy_fpregs_to_fpstate(dst_fpu)) { if (!copy_fpregs_to_fpstate(dst_fpu)) {
memcpy(&src_fpu->state, &dst_fpu->state, xstate_size); memcpy(&src_fpu->state, &dst_fpu->state,
fpu_kernel_xstate_size);
if (use_eager_fpu()) if (use_eager_fpu())
copy_kernel_to_fpregs(&src_fpu->state); copy_kernel_to_fpregs(&src_fpu->state);
...@@ -275,6 +289,9 @@ int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu) ...@@ -275,6 +289,9 @@ int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu)
} }
preempt_enable(); preempt_enable();
trace_x86_fpu_copy_src(src_fpu);
trace_x86_fpu_copy_dst(dst_fpu);
return 0; return 0;
} }
...@@ -288,7 +305,9 @@ void fpu__activate_curr(struct fpu *fpu) ...@@ -288,7 +305,9 @@ void fpu__activate_curr(struct fpu *fpu)
if (!fpu->fpstate_active) { if (!fpu->fpstate_active) {
fpstate_init(&fpu->state); fpstate_init(&fpu->state);
trace_x86_fpu_init_state(fpu);
trace_x86_fpu_activate_state(fpu);
/* Safe to do for the current task: */ /* Safe to do for the current task: */
fpu->fpstate_active = 1; fpu->fpstate_active = 1;
} }
...@@ -314,7 +333,9 @@ void fpu__activate_fpstate_read(struct fpu *fpu) ...@@ -314,7 +333,9 @@ void fpu__activate_fpstate_read(struct fpu *fpu)
} else { } else {
if (!fpu->fpstate_active) { if (!fpu->fpstate_active) {
fpstate_init(&fpu->state); fpstate_init(&fpu->state);
trace_x86_fpu_init_state(fpu);
trace_x86_fpu_activate_state(fpu);
/* Safe to do for current and for stopped child tasks: */ /* Safe to do for current and for stopped child tasks: */
fpu->fpstate_active = 1; fpu->fpstate_active = 1;
} }
...@@ -347,7 +368,9 @@ void fpu__activate_fpstate_write(struct fpu *fpu) ...@@ -347,7 +368,9 @@ void fpu__activate_fpstate_write(struct fpu *fpu)
fpu->last_cpu = -1; fpu->last_cpu = -1;
} else { } else {
fpstate_init(&fpu->state); fpstate_init(&fpu->state);
trace_x86_fpu_init_state(fpu);
trace_x86_fpu_activate_state(fpu);
/* Safe to do for stopped child tasks: */ /* Safe to do for stopped child tasks: */
fpu->fpstate_active = 1; fpu->fpstate_active = 1;
} }
...@@ -432,9 +455,11 @@ void fpu__restore(struct fpu *fpu) ...@@ -432,9 +455,11 @@ void fpu__restore(struct fpu *fpu)
/* Avoid __kernel_fpu_begin() right after fpregs_activate() */ /* Avoid __kernel_fpu_begin() right after fpregs_activate() */
kernel_fpu_disable(); kernel_fpu_disable();
trace_x86_fpu_before_restore(fpu);
fpregs_activate(fpu); fpregs_activate(fpu);
copy_kernel_to_fpregs(&fpu->state); copy_kernel_to_fpregs(&fpu->state);
fpu->counter++; fpu->counter++;
trace_x86_fpu_after_restore(fpu);
kernel_fpu_enable(); kernel_fpu_enable();
} }
EXPORT_SYMBOL_GPL(fpu__restore); EXPORT_SYMBOL_GPL(fpu__restore);
...@@ -463,6 +488,8 @@ void fpu__drop(struct fpu *fpu) ...@@ -463,6 +488,8 @@ void fpu__drop(struct fpu *fpu)
fpu->fpstate_active = 0; fpu->fpstate_active = 0;
trace_x86_fpu_dropped(fpu);
preempt_enable(); preempt_enable();
} }
......
...@@ -145,8 +145,8 @@ static void __init fpu__init_system_generic(void) ...@@ -145,8 +145,8 @@ static void __init fpu__init_system_generic(void)
* This is inherent to the XSAVE architecture which puts all state * This is inherent to the XSAVE architecture which puts all state
* components into a single, continuous memory block: * components into a single, continuous memory block:
*/ */
unsigned int xstate_size; unsigned int fpu_kernel_xstate_size;
EXPORT_SYMBOL_GPL(xstate_size); EXPORT_SYMBOL_GPL(fpu_kernel_xstate_size);
/* Get alignment of the TYPE. */ /* Get alignment of the TYPE. */
#define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test) #define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test)
...@@ -178,7 +178,7 @@ static void __init fpu__init_task_struct_size(void) ...@@ -178,7 +178,7 @@ static void __init fpu__init_task_struct_size(void)
* Add back the dynamically-calculated register state * Add back the dynamically-calculated register state
* size. * size.
*/ */
task_size += xstate_size; task_size += fpu_kernel_xstate_size;
/* /*
* We dynamically size 'struct fpu', so we require that * We dynamically size 'struct fpu', so we require that
...@@ -195,7 +195,7 @@ static void __init fpu__init_task_struct_size(void) ...@@ -195,7 +195,7 @@ static void __init fpu__init_task_struct_size(void)
} }
/* /*
* Set up the xstate_size based on the legacy FPU context size. * Set up the user and kernel xstate sizes based on the legacy FPU context size.
* *
* We set this up first, and later it will be overwritten by * We set this up first, and later it will be overwritten by
* fpu__init_system_xstate() if the CPU knows about xstates. * fpu__init_system_xstate() if the CPU knows about xstates.
...@@ -208,7 +208,7 @@ static void __init fpu__init_system_xstate_size_legacy(void) ...@@ -208,7 +208,7 @@ static void __init fpu__init_system_xstate_size_legacy(void)
on_boot_cpu = 0; on_boot_cpu = 0;
/* /*
* Note that xstate_size might be overwriten later during * Note that xstate sizes might be overwritten later during
* fpu__init_system_xstate(). * fpu__init_system_xstate().
*/ */
...@@ -219,27 +219,17 @@ static void __init fpu__init_system_xstate_size_legacy(void) ...@@ -219,27 +219,17 @@ static void __init fpu__init_system_xstate_size_legacy(void)
*/ */
setup_clear_cpu_cap(X86_FEATURE_XSAVE); setup_clear_cpu_cap(X86_FEATURE_XSAVE);
setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT); setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
xstate_size = sizeof(struct swregs_state); fpu_kernel_xstate_size = sizeof(struct swregs_state);
} else { } else {
if (boot_cpu_has(X86_FEATURE_FXSR)) if (boot_cpu_has(X86_FEATURE_FXSR))
xstate_size = sizeof(struct fxregs_state); fpu_kernel_xstate_size =
sizeof(struct fxregs_state);
else else
xstate_size = sizeof(struct fregs_state); fpu_kernel_xstate_size =
sizeof(struct fregs_state);
} }
/*
* Quirk: we don't yet handle the XSAVES* instructions fpu_user_xstate_size = fpu_kernel_xstate_size;
* correctly, as we don't correctly convert between
* standard and compacted format when interfacing
* with user-space - so disable it for now.
*
* The difference is small: with recent CPUs the
* compacted format is only marginally smaller than
* the standard FPU state format.
*
* ( This is easy to backport while we are fixing
* XSAVES* support. )
*/
setup_clear_cpu_cap(X86_FEATURE_XSAVES);
} }
/* /*
......
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
#include <asm/fpu/internal.h> #include <asm/fpu/internal.h>
#include <asm/fpu/signal.h> #include <asm/fpu/signal.h>
#include <asm/fpu/regset.h> #include <asm/fpu/regset.h>
#include <asm/fpu/xstate.h>
/* /*
* The xstateregs_active() routine is the same as the regset_fpregs_active() routine, * The xstateregs_active() routine is the same as the regset_fpregs_active() routine,
...@@ -85,21 +86,26 @@ int xstateregs_get(struct task_struct *target, const struct user_regset *regset, ...@@ -85,21 +86,26 @@ int xstateregs_get(struct task_struct *target, const struct user_regset *regset,
if (!boot_cpu_has(X86_FEATURE_XSAVE)) if (!boot_cpu_has(X86_FEATURE_XSAVE))
return -ENODEV; return -ENODEV;
fpu__activate_fpstate_read(fpu);
xsave = &fpu->state.xsave; xsave = &fpu->state.xsave;
fpu__activate_fpstate_read(fpu);
if (using_compacted_format()) {
ret = copyout_from_xsaves(pos, count, kbuf, ubuf, xsave);
} else {
fpstate_sanitize_xstate(fpu);
/* /*
* Copy the 48bytes defined by the software first into the xstate * Copy the 48 bytes defined by the software into the xsave
* memory layout in the thread struct, so that we can copy the entire * area in the thread struct, so that we can copy the whole
* xstateregs to the user using one user_regset_copyout(). * area to user using one user_regset_copyout().
*/ */
memcpy(&xsave->i387.sw_reserved, memcpy(&xsave->i387.sw_reserved, xstate_fx_sw_bytes, sizeof(xstate_fx_sw_bytes));
xstate_fx_sw_bytes, sizeof(xstate_fx_sw_bytes));
/* /*
* Copy the xstate memory layout. * Copy the xstate memory layout.
*/ */
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, xsave, 0, -1); ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, xsave, 0, -1);
}
return ret; return ret;
} }
...@@ -114,11 +120,27 @@ int xstateregs_set(struct task_struct *target, const struct user_regset *regset, ...@@ -114,11 +120,27 @@ int xstateregs_set(struct task_struct *target, const struct user_regset *regset,
if (!boot_cpu_has(X86_FEATURE_XSAVE)) if (!boot_cpu_has(X86_FEATURE_XSAVE))
return -ENODEV; return -ENODEV;
fpu__activate_fpstate_write(fpu); /*
* A whole standard-format XSAVE buffer is needed:
*/
if ((pos != 0) || (count < fpu_user_xstate_size))
return -EFAULT;
xsave = &fpu->state.xsave; xsave = &fpu->state.xsave;
fpu__activate_fpstate_write(fpu);
if (boot_cpu_has(X86_FEATURE_XSAVES))
ret = copyin_to_xsaves(kbuf, ubuf, xsave);
else
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, xsave, 0, -1); ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, xsave, 0, -1);
/*
* In case of failure, mark all states as init:
*/
if (ret)
fpstate_init(&fpu->state);
/* /*
* mxcsr reserved bits must be masked to zero for security reasons. * mxcsr reserved bits must be masked to zero for security reasons.
*/ */
......
...@@ -8,8 +8,10 @@ ...@@ -8,8 +8,10 @@
#include <asm/fpu/internal.h> #include <asm/fpu/internal.h>
#include <asm/fpu/signal.h> #include <asm/fpu/signal.h>
#include <asm/fpu/regset.h> #include <asm/fpu/regset.h>
#include <asm/fpu/xstate.h>
#include <asm/sigframe.h> #include <asm/sigframe.h>
#include <asm/trace/fpu.h>
static struct _fpx_sw_bytes fx_sw_reserved, fx_sw_reserved_ia32; static struct _fpx_sw_bytes fx_sw_reserved, fx_sw_reserved_ia32;
...@@ -31,7 +33,7 @@ static inline int check_for_xstate(struct fxregs_state __user *buf, ...@@ -31,7 +33,7 @@ static inline int check_for_xstate(struct fxregs_state __user *buf,
/* Check for the first magic field and other error scenarios. */ /* Check for the first magic field and other error scenarios. */
if (fx_sw->magic1 != FP_XSTATE_MAGIC1 || if (fx_sw->magic1 != FP_XSTATE_MAGIC1 ||
fx_sw->xstate_size < min_xstate_size || fx_sw->xstate_size < min_xstate_size ||
fx_sw->xstate_size > xstate_size || fx_sw->xstate_size > fpu_user_xstate_size ||
fx_sw->xstate_size > fx_sw->extended_size) fx_sw->xstate_size > fx_sw->extended_size)
return -1; return -1;
...@@ -88,7 +90,8 @@ static inline int save_xstate_epilog(void __user *buf, int ia32_frame) ...@@ -88,7 +90,8 @@ static inline int save_xstate_epilog(void __user *buf, int ia32_frame)
if (!use_xsave()) if (!use_xsave())
return err; return err;
err |= __put_user(FP_XSTATE_MAGIC2, (__u32 *)(buf + xstate_size)); err |= __put_user(FP_XSTATE_MAGIC2,
(__u32 *)(buf + fpu_user_xstate_size));
/* /*
* Read the xfeatures which we copied (directly from the cpu or * Read the xfeatures which we copied (directly from the cpu or
...@@ -125,7 +128,7 @@ static inline int copy_fpregs_to_sigframe(struct xregs_state __user *buf) ...@@ -125,7 +128,7 @@ static inline int copy_fpregs_to_sigframe(struct xregs_state __user *buf)
else else
err = copy_fregs_to_user((struct fregs_state __user *) buf); err = copy_fregs_to_user((struct fregs_state __user *) buf);
if (unlikely(err) && __clear_user(buf, xstate_size)) if (unlikely(err) && __clear_user(buf, fpu_user_xstate_size))
err = -EFAULT; err = -EFAULT;
return err; return err;
} }
...@@ -167,7 +170,7 @@ int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size) ...@@ -167,7 +170,7 @@ int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size)
sizeof(struct user_i387_ia32_struct), NULL, sizeof(struct user_i387_ia32_struct), NULL,
(struct _fpstate_32 __user *) buf) ? -1 : 1; (struct _fpstate_32 __user *) buf) ? -1 : 1;
if (fpregs_active()) { if (fpregs_active() || using_compacted_format()) {
/* Save the live register state to the user directly. */ /* Save the live register state to the user directly. */
if (copy_fpregs_to_sigframe(buf_fx)) if (copy_fpregs_to_sigframe(buf_fx))
return -1; return -1;
...@@ -175,8 +178,19 @@ int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size) ...@@ -175,8 +178,19 @@ int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size)
if (ia32_fxstate) if (ia32_fxstate)
copy_fxregs_to_kernel(&tsk->thread.fpu); copy_fxregs_to_kernel(&tsk->thread.fpu);
} else { } else {
/*
* It is a *bug* if kernel uses compacted-format for xsave
* area and we copy it out directly to a signal frame. It
* should have been handled above by saving the registers
* directly.
*/
if (boot_cpu_has(X86_FEATURE_XSAVES)) {
WARN_ONCE(1, "x86/fpu: saving compacted-format xsave area to a signal frame!\n");
return -1;
}
fpstate_sanitize_xstate(&tsk->thread.fpu); fpstate_sanitize_xstate(&tsk->thread.fpu);
if (__copy_to_user(buf_fx, xsave, xstate_size)) if (__copy_to_user(buf_fx, xsave, fpu_user_xstate_size))
return -1; return -1;
} }
...@@ -250,7 +264,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size) ...@@ -250,7 +264,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
int ia32_fxstate = (buf != buf_fx); int ia32_fxstate = (buf != buf_fx);
struct task_struct *tsk = current; struct task_struct *tsk = current;
struct fpu *fpu = &tsk->thread.fpu; struct fpu *fpu = &tsk->thread.fpu;
int state_size = xstate_size; int state_size = fpu_kernel_xstate_size;
u64 xfeatures = 0; u64 xfeatures = 0;
int fx_only = 0; int fx_only = 0;
...@@ -282,6 +296,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size) ...@@ -282,6 +296,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
*/ */
state_size = sizeof(struct fxregs_state); state_size = sizeof(struct fxregs_state);
fx_only = 1; fx_only = 1;
trace_x86_fpu_xstate_check_failed(fpu);
} else { } else {
state_size = fx_sw_user.xstate_size; state_size = fx_sw_user.xstate_size;
xfeatures = fx_sw_user.xfeatures; xfeatures = fx_sw_user.xfeatures;
...@@ -308,9 +323,17 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size) ...@@ -308,9 +323,17 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
*/ */
fpu__drop(fpu); fpu__drop(fpu);
if (__copy_from_user(&fpu->state.xsave, buf_fx, state_size) || if (using_compacted_format()) {
__copy_from_user(&env, buf, sizeof(env))) { err = copyin_to_xsaves(NULL, buf_fx,
&fpu->state.xsave);
} else {
err = __copy_from_user(&fpu->state.xsave,
buf_fx, state_size);
}
if (err || __copy_from_user(&env, buf, sizeof(env))) {
fpstate_init(&fpu->state); fpstate_init(&fpu->state);
trace_x86_fpu_init_state(fpu);
err = -1; err = -1;
} else { } else {
sanitize_restored_xstate(tsk, &env, xfeatures, fx_only); sanitize_restored_xstate(tsk, &env, xfeatures, fx_only);
...@@ -341,7 +364,8 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size) ...@@ -341,7 +364,8 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
static inline int xstate_sigframe_size(void) static inline int xstate_sigframe_size(void)
{ {
return use_xsave() ? xstate_size + FP_XSTATE_MAGIC2_SIZE : xstate_size; return use_xsave() ? fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE :
fpu_user_xstate_size;
} }
/* /*
...@@ -385,12 +409,12 @@ fpu__alloc_mathframe(unsigned long sp, int ia32_frame, ...@@ -385,12 +409,12 @@ fpu__alloc_mathframe(unsigned long sp, int ia32_frame,
*/ */
void fpu__init_prepare_fx_sw_frame(void) void fpu__init_prepare_fx_sw_frame(void)
{ {
int size = xstate_size + FP_XSTATE_MAGIC2_SIZE; int size = fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE;
fx_sw_reserved.magic1 = FP_XSTATE_MAGIC1; fx_sw_reserved.magic1 = FP_XSTATE_MAGIC1;
fx_sw_reserved.extended_size = size; fx_sw_reserved.extended_size = size;
fx_sw_reserved.xfeatures = xfeatures_mask; fx_sw_reserved.xfeatures = xfeatures_mask;
fx_sw_reserved.xstate_size = xstate_size; fx_sw_reserved.xstate_size = fpu_user_xstate_size;
if (config_enabled(CONFIG_IA32_EMULATION) || if (config_enabled(CONFIG_IA32_EMULATION) ||
config_enabled(CONFIG_X86_32)) { config_enabled(CONFIG_X86_32)) {
......
This diff is collapsed.
...@@ -55,9 +55,6 @@ ...@@ -55,9 +55,6 @@
#include <linux/irqbypass.h> #include <linux/irqbypass.h>
#include <trace/events/kvm.h> #include <trace/events/kvm.h>
#define CREATE_TRACE_POINTS
#include "trace.h"
#include <asm/debugreg.h> #include <asm/debugreg.h>
#include <asm/msr.h> #include <asm/msr.h>
#include <asm/desc.h> #include <asm/desc.h>
...@@ -68,6 +65,9 @@ ...@@ -68,6 +65,9 @@
#include <asm/div64.h> #include <asm/div64.h>
#include <asm/irq_remapping.h> #include <asm/irq_remapping.h>
#define CREATE_TRACE_POINTS
#include "trace.h"
#define MAX_IO_MSRS 256 #define MAX_IO_MSRS 256
#define KVM_MAX_MCE_BANKS 32 #define KVM_MAX_MCE_BANKS 32
#define KVM_MCE_CAP_SUPPORTED (MCG_CTL_P | MCG_SER_P) #define KVM_MCE_CAP_SUPPORTED (MCG_CTL_P | MCG_SER_P)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment