Commit 5529acf4 authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Borislav Petkov

x86/fpu: Add sanity checks for XFD

Add debug functionality to ensure that the XFD MSR is up to date for XSAVE*
and XRSTOR* operations.

 [ tglx: Improve comment. ]
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarChang S. Bae <chang.seok.bae@intel.com>
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Link: https://lkml.kernel.org/r/20211021225527.10184-16-chang.seok.bae@intel.com
parent 8bf26758
...@@ -166,7 +166,7 @@ void restore_fpregs_from_fpstate(struct fpstate *fpstate, u64 mask) ...@@ -166,7 +166,7 @@ void restore_fpregs_from_fpstate(struct fpstate *fpstate, u64 mask)
*/ */
mask = fpu_kernel_cfg.max_features & mask; mask = fpu_kernel_cfg.max_features & mask;
os_xrstor(&fpstate->regs.xsave, mask); os_xrstor(fpstate, mask);
} else { } else {
if (use_fxsr()) if (use_fxsr())
fxrstor(&fpstate->regs.fxsave); fxrstor(&fpstate->regs.fxsave);
...@@ -534,7 +534,7 @@ void fpu__drop(struct fpu *fpu) ...@@ -534,7 +534,7 @@ void fpu__drop(struct fpu *fpu)
static inline void restore_fpregs_from_init_fpstate(u64 features_mask) static inline void restore_fpregs_from_init_fpstate(u64 features_mask)
{ {
if (use_xsave()) if (use_xsave())
os_xrstor(&init_fpstate.regs.xsave, features_mask); os_xrstor(&init_fpstate, features_mask);
else if (use_fxsr()) else if (use_fxsr())
fxrstor(&init_fpstate.regs.fxsave); fxrstor(&init_fpstate.regs.fxsave);
else else
...@@ -591,9 +591,8 @@ void fpu__clear_user_states(struct fpu *fpu) ...@@ -591,9 +591,8 @@ void fpu__clear_user_states(struct fpu *fpu)
* corresponding registers. * corresponding registers.
*/ */
if (xfeatures_mask_supervisor() && if (xfeatures_mask_supervisor() &&
!fpregs_state_valid(fpu, smp_processor_id())) { !fpregs_state_valid(fpu, smp_processor_id()))
os_xrstor(&fpu->fpstate->regs.xsave, xfeatures_mask_supervisor()); os_xrstor_supervisor(fpu->fpstate);
}
/* Reset user states in registers. */ /* Reset user states in registers. */
restore_fpregs_from_init_fpstate(XFEATURE_MASK_USER_RESTORE); restore_fpregs_from_init_fpstate(XFEATURE_MASK_USER_RESTORE);
......
...@@ -261,7 +261,7 @@ static int __restore_fpregs_from_user(void __user *buf, u64 ufeatures, ...@@ -261,7 +261,7 @@ static int __restore_fpregs_from_user(void __user *buf, u64 ufeatures,
ret = fxrstor_from_user_sigframe(buf); ret = fxrstor_from_user_sigframe(buf);
if (!ret && unlikely(init_bv)) if (!ret && unlikely(init_bv))
os_xrstor(&init_fpstate.regs.xsave, init_bv); os_xrstor(&init_fpstate, init_bv);
return ret; return ret;
} else if (use_fxsr()) { } else if (use_fxsr()) {
return fxrstor_from_user_sigframe(buf); return fxrstor_from_user_sigframe(buf);
...@@ -322,7 +322,7 @@ static bool restore_fpregs_from_user(void __user *buf, u64 xrestore, ...@@ -322,7 +322,7 @@ static bool restore_fpregs_from_user(void __user *buf, u64 xrestore,
* been restored from a user buffer directly. * been restored from a user buffer directly.
*/ */
if (test_thread_flag(TIF_NEED_FPU_LOAD) && xfeatures_mask_supervisor()) if (test_thread_flag(TIF_NEED_FPU_LOAD) && xfeatures_mask_supervisor())
os_xrstor(&fpu->fpstate->regs.xsave, xfeatures_mask_supervisor()); os_xrstor_supervisor(fpu->fpstate);
fpregs_mark_activate(); fpregs_mark_activate();
fpregs_unlock(); fpregs_unlock();
...@@ -432,7 +432,7 @@ static bool __fpu_restore_sig(void __user *buf, void __user *buf_fx, ...@@ -432,7 +432,7 @@ static bool __fpu_restore_sig(void __user *buf, void __user *buf_fx,
u64 mask = user_xfeatures | xfeatures_mask_supervisor(); u64 mask = user_xfeatures | xfeatures_mask_supervisor();
fpregs->xsave.header.xfeatures &= mask; fpregs->xsave.header.xfeatures &= mask;
success = !os_xrstor_safe(&fpregs->xsave, success = !os_xrstor_safe(fpu->fpstate,
fpu_kernel_cfg.max_features); fpu_kernel_cfg.max_features);
} else { } else {
success = !fxrstor_safe(&fpregs->fxsave); success = !fxrstor_safe(&fpregs->fxsave);
......
...@@ -1301,6 +1301,64 @@ EXPORT_SYMBOL_GPL(fpstate_clear_xstate_component); ...@@ -1301,6 +1301,64 @@ EXPORT_SYMBOL_GPL(fpstate_clear_xstate_component);
#endif #endif
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
#ifdef CONFIG_X86_DEBUG_FPU
/*
* Ensure that a subsequent XSAVE* or XRSTOR* instruction with RFBM=@mask
* can safely operate on the @fpstate buffer.
*/
static bool xstate_op_valid(struct fpstate *fpstate, u64 mask, bool rstor)
{
u64 xfd = __this_cpu_read(xfd_state);
if (fpstate->xfd == xfd)
return true;
/*
* The XFD MSR does not match fpstate->xfd. That's invalid when
* the passed in fpstate is current's fpstate.
*/
if (fpstate->xfd == current->thread.fpu.fpstate->xfd)
return false;
/*
* XRSTOR(S) from init_fpstate are always correct as it will just
* bring all components into init state and not read from the
* buffer. XSAVE(S) raises #PF after init.
*/
if (fpstate == &init_fpstate)
return rstor;
/*
* XSAVE(S): clone(), fpu_swap_kvm_fpu()
* XRSTORS(S): fpu_swap_kvm_fpu()
*/
/*
* No XSAVE/XRSTOR instructions (except XSAVE itself) touch
* the buffer area for XFD-disabled state components.
*/
mask &= ~xfd;
/*
* Remove features which are valid in fpstate. They
* have space allocated in fpstate.
*/
mask &= ~fpstate->xfeatures;
/*
* Any remaining state components in 'mask' might be written
* by XSAVE/XRSTOR. Fail validation it found.
*/
return !mask;
}
void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rstor)
{
WARN_ON_ONCE(!xstate_op_valid(fpstate, mask, rstor));
}
#endif /* CONFIG_X86_DEBUG_FPU */
static int validate_sigaltstack(unsigned int usize) static int validate_sigaltstack(unsigned int usize)
{ {
struct task_struct *thread, *leader = current->group_leader; struct task_struct *thread, *leader = current->group_leader;
......
...@@ -130,6 +130,12 @@ static inline u64 xfeatures_mask_independent(void) ...@@ -130,6 +130,12 @@ static inline u64 xfeatures_mask_independent(void)
: "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \
: "memory") : "memory")
#if defined(CONFIG_X86_64) && defined(CONFIG_X86_DEBUG_FPU)
extern void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rstor);
#else
static inline void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rstor) { }
#endif
/* /*
* Save processor xstate to xsave area. * Save processor xstate to xsave area.
* *
...@@ -144,6 +150,7 @@ static inline void os_xsave(struct fpstate *fpstate) ...@@ -144,6 +150,7 @@ static inline void os_xsave(struct fpstate *fpstate)
int err; int err;
WARN_ON_FPU(!alternatives_patched); WARN_ON_FPU(!alternatives_patched);
xfd_validate_state(fpstate, mask, false);
XSTATE_XSAVE(&fpstate->regs.xsave, lmask, hmask, err); XSTATE_XSAVE(&fpstate->regs.xsave, lmask, hmask, err);
...@@ -156,12 +163,23 @@ static inline void os_xsave(struct fpstate *fpstate) ...@@ -156,12 +163,23 @@ static inline void os_xsave(struct fpstate *fpstate)
* *
* Uses XRSTORS when XSAVES is used, XRSTOR otherwise. * Uses XRSTORS when XSAVES is used, XRSTOR otherwise.
*/ */
static inline void os_xrstor(struct xregs_state *xstate, u64 mask) static inline void os_xrstor(struct fpstate *fpstate, u64 mask)
{
u32 lmask = mask;
u32 hmask = mask >> 32;
xfd_validate_state(fpstate, mask, true);
XSTATE_XRESTORE(&fpstate->regs.xsave, lmask, hmask);
}
/* Restore of supervisor state. Does not require XFD */
static inline void os_xrstor_supervisor(struct fpstate *fpstate)
{ {
u64 mask = xfeatures_mask_supervisor();
u32 lmask = mask; u32 lmask = mask;
u32 hmask = mask >> 32; u32 hmask = mask >> 32;
XSTATE_XRESTORE(xstate, lmask, hmask); XSTATE_XRESTORE(&fpstate->regs.xsave, lmask, hmask);
} }
/* /*
...@@ -184,11 +202,14 @@ static inline int xsave_to_user_sigframe(struct xregs_state __user *buf) ...@@ -184,11 +202,14 @@ static inline int xsave_to_user_sigframe(struct xregs_state __user *buf)
* internally, e.g. PKRU. That's user space ABI and also required * internally, e.g. PKRU. That's user space ABI and also required
* to allow the signal handler to modify PKRU. * to allow the signal handler to modify PKRU.
*/ */
u64 mask = current->thread.fpu.fpstate->user_xfeatures; struct fpstate *fpstate = current->thread.fpu.fpstate;
u64 mask = fpstate->user_xfeatures;
u32 lmask = mask; u32 lmask = mask;
u32 hmask = mask >> 32; u32 hmask = mask >> 32;
int err; int err;
xfd_validate_state(fpstate, mask, false);
stac(); stac();
XSTATE_OP(XSAVE, buf, lmask, hmask, err); XSTATE_OP(XSAVE, buf, lmask, hmask, err);
clac(); clac();
...@@ -206,6 +227,8 @@ static inline int xrstor_from_user_sigframe(struct xregs_state __user *buf, u64 ...@@ -206,6 +227,8 @@ static inline int xrstor_from_user_sigframe(struct xregs_state __user *buf, u64
u32 hmask = mask >> 32; u32 hmask = mask >> 32;
int err; int err;
xfd_validate_state(current->thread.fpu.fpstate, mask, true);
stac(); stac();
XSTATE_OP(XRSTOR, xstate, lmask, hmask, err); XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
clac(); clac();
...@@ -217,12 +240,15 @@ static inline int xrstor_from_user_sigframe(struct xregs_state __user *buf, u64 ...@@ -217,12 +240,15 @@ static inline int xrstor_from_user_sigframe(struct xregs_state __user *buf, u64
* Restore xstate from kernel space xsave area, return an error code instead of * Restore xstate from kernel space xsave area, return an error code instead of
* an exception. * an exception.
*/ */
static inline int os_xrstor_safe(struct xregs_state *xstate, u64 mask) static inline int os_xrstor_safe(struct fpstate *fpstate, u64 mask)
{ {
struct xregs_state *xstate = &fpstate->regs.xsave;
u32 lmask = mask; u32 lmask = mask;
u32 hmask = mask >> 32; u32 hmask = mask >> 32;
int err; int err;
/* Must enforce XFD update here */
if (cpu_feature_enabled(X86_FEATURE_XSAVES)) if (cpu_feature_enabled(X86_FEATURE_XSAVES))
XSTATE_OP(XRSTORS, xstate, lmask, hmask, err); XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
else else
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment