s390/fpu: provide and use ld and std inline assemblies

Deduplicate the 64 ld and std inline assemblies. Provide an fpu inline
assembly for both instructions, and use them in the new save_fp_regs()
and load_fp_regs() helper functions.
Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>

arch/s390/include/asm/fpu-insn.h

@@ -36,6 +36,15 @@ asm(".include \"asm/fpu-insn-asm.h\"\n");
  * barrier.
  */
 
+static __always_inline void fpu_ld(unsigned short fpr, freg_t *reg)
+{
+	instrument_read(reg, sizeof(*reg));
+	asm volatile("ld	 %[fpr],%[reg]\n"
+		     :
+		     : [fpr] "I" (fpr), [reg] "Q" (reg->ui)
+		     : "memory");
+}
+
 /**
  * fpu_lfpc_safe - Load floating point control register safely.
  * @fpc: new value for floating point control register
@@ -64,5 +73,14 @@ static inline void fpu_lfpc_safe(unsigned int *fpc)
 		: "memory");
 }
 
+static __always_inline void fpu_std(unsigned short fpr, freg_t *reg)
+{
+	instrument_write(reg, sizeof(*reg));
+	asm volatile("std	 %[fpr],%[reg]\n"
+		     : [reg] "=Q" (reg->ui)
+		     : [fpr] "I" (fpr)
+		     : "memory");
+}
+
 #endif /* __ASSEMBLY__ */
 #endif	/* __ASM_S390_FPU_INSN_H */

arch/s390/include/asm/fpu.h

@@ -84,6 +84,46 @@ void __load_fpu_regs(void);
 void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags);
 void __kernel_fpu_end(struct kernel_fpu *state, u32 flags);
 
+static __always_inline void save_fp_regs(freg_t *fprs)
+{
+	fpu_std(0, &fprs[0]);
+	fpu_std(1, &fprs[1]);
+	fpu_std(2, &fprs[2]);
+	fpu_std(3, &fprs[3]);
+	fpu_std(4, &fprs[4]);
+	fpu_std(5, &fprs[5]);
+	fpu_std(6, &fprs[6]);
+	fpu_std(7, &fprs[7]);
+	fpu_std(8, &fprs[8]);
+	fpu_std(9, &fprs[9]);
+	fpu_std(10, &fprs[10]);
+	fpu_std(11, &fprs[11]);
+	fpu_std(12, &fprs[12]);
+	fpu_std(13, &fprs[13]);
+	fpu_std(14, &fprs[14]);
+	fpu_std(15, &fprs[15]);
+}
+
+static __always_inline void load_fp_regs(freg_t *fprs)
+{
+	fpu_ld(0, &fprs[0]);
+	fpu_ld(1, &fprs[1]);
+	fpu_ld(2, &fprs[2]);
+	fpu_ld(3, &fprs[3]);
+	fpu_ld(4, &fprs[4]);
+	fpu_ld(5, &fprs[5]);
+	fpu_ld(6, &fprs[6]);
+	fpu_ld(7, &fprs[7]);
+	fpu_ld(8, &fprs[8]);
+	fpu_ld(9, &fprs[9]);
+	fpu_ld(10, &fprs[10]);
+	fpu_ld(11, &fprs[11]);
+	fpu_ld(12, &fprs[12]);
+	fpu_ld(13, &fprs[13]);
+	fpu_ld(14, &fprs[14]);
+	fpu_ld(15, &fprs[15]);
+}
+
 static inline void kernel_fpu_begin(struct kernel_fpu *state, u32 flags)
 {
 	preempt_disable();

arch/s390/kernel/fpu.c

@@ -22,25 +22,8 @@ void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags)
 		asm volatile("stfpc %0" : "=Q" (state->fpc));
 	}
 	if (!cpu_has_vx()) {
-		if (flags & KERNEL_VXR_LOW) {
-			/* Save floating-point registers */
-			asm volatile("std 0,%0" : "=Q" (state->fprs[0]));
-			asm volatile("std 1,%0" : "=Q" (state->fprs[1]));
-			asm volatile("std 2,%0" : "=Q" (state->fprs[2]));
-			asm volatile("std 3,%0" : "=Q" (state->fprs[3]));
-			asm volatile("std 4,%0" : "=Q" (state->fprs[4]));
-			asm volatile("std 5,%0" : "=Q" (state->fprs[5]));
-			asm volatile("std 6,%0" : "=Q" (state->fprs[6]));
-			asm volatile("std 7,%0" : "=Q" (state->fprs[7]));
-			asm volatile("std 8,%0" : "=Q" (state->fprs[8]));
-			asm volatile("std 9,%0" : "=Q" (state->fprs[9]));
-			asm volatile("std 10,%0" : "=Q" (state->fprs[10]));
-			asm volatile("std 11,%0" : "=Q" (state->fprs[11]));
-			asm volatile("std 12,%0" : "=Q" (state->fprs[12]));
-			asm volatile("std 13,%0" : "=Q" (state->fprs[13]));
-			asm volatile("std 14,%0" : "=Q" (state->fprs[14]));
-			asm volatile("std 15,%0" : "=Q" (state->fprs[15]));
-		}
+		if (flags & KERNEL_VXR_LOW)
+			save_fp_regs(state->fprs);
 		return;
 	}
 	/* Test and save vector registers */
@@ -102,25 +85,8 @@ void __kernel_fpu_end(struct kernel_fpu *state, u32 flags)
 		asm volatile("lfpc %0" : : "Q" (state->fpc));
 	}
 	if (!cpu_has_vx()) {
-		if (flags & KERNEL_VXR_LOW) {
-			/* Restore floating-point registers */
-			asm volatile("ld 0,%0" : : "Q" (state->fprs[0]));
-			asm volatile("ld 1,%0" : : "Q" (state->fprs[1]));
-			asm volatile("ld 2,%0" : : "Q" (state->fprs[2]));
-			asm volatile("ld 3,%0" : : "Q" (state->fprs[3]));
-			asm volatile("ld 4,%0" : : "Q" (state->fprs[4]));
-			asm volatile("ld 5,%0" : : "Q" (state->fprs[5]));
-			asm volatile("ld 6,%0" : : "Q" (state->fprs[6]));
-			asm volatile("ld 7,%0" : : "Q" (state->fprs[7]));
-			asm volatile("ld 8,%0" : : "Q" (state->fprs[8]));
-			asm volatile("ld 9,%0" : : "Q" (state->fprs[9]));
-			asm volatile("ld 10,%0" : : "Q" (state->fprs[10]));
-			asm volatile("ld 11,%0" : : "Q" (state->fprs[11]));
-			asm volatile("ld 12,%0" : : "Q" (state->fprs[12]));
-			asm volatile("ld 13,%0" : : "Q" (state->fprs[13]));
-			asm volatile("ld 14,%0" : : "Q" (state->fprs[14]));
-			asm volatile("ld 15,%0" : : "Q" (state->fprs[15]));
-		}
+		if (flags & KERNEL_VXR_LOW)
+			load_fp_regs(state->fprs);
 		return;
 	}
 	/* Test and restore (load) vector registers */
@@ -171,8 +137,8 @@ EXPORT_SYMBOL(__kernel_fpu_end);
 
 void __load_fpu_regs(void)
 {
-	unsigned long *regs = current->thread.fpu.regs;
 	struct fpu *state = &current->thread.fpu;
+	void *regs = current->thread.fpu.regs;
 
 	fpu_lfpc_safe(&state->fpc);
 	if (likely(cpu_has_vx())) {
@@ -183,22 +149,7 @@ void __load_fpu_regs(void)
 			     : "d" (regs)
 			     : "1", "cc", "memory");
 	} else {
-		asm volatile("ld 0,%0" : : "Q" (regs[0]));
-		asm volatile("ld 1,%0" : : "Q" (regs[1]));
-		asm volatile("ld 2,%0" : : "Q" (regs[2]));
-		asm volatile("ld 3,%0" : : "Q" (regs[3]));
-		asm volatile("ld 4,%0" : : "Q" (regs[4]));
-		asm volatile("ld 5,%0" : : "Q" (regs[5]));
-		asm volatile("ld 6,%0" : : "Q" (regs[6]));
-		asm volatile("ld 7,%0" : : "Q" (regs[7]));
-		asm volatile("ld 8,%0" : : "Q" (regs[8]));
-		asm volatile("ld 9,%0" : : "Q" (regs[9]));
-		asm volatile("ld 10,%0" : : "Q" (regs[10]));
-		asm volatile("ld 11,%0" : : "Q" (regs[11]));
-		asm volatile("ld 12,%0" : : "Q" (regs[12]));
-		asm volatile("ld 13,%0" : : "Q" (regs[13]));
-		asm volatile("ld 14,%0" : : "Q" (regs[14]));
-		asm volatile("ld 15,%0" : : "Q" (regs[15]));
+		load_fp_regs(regs);
 	}
 	clear_cpu_flag(CIF_FPU);
 }
@@ -213,8 +164,9 @@ EXPORT_SYMBOL(load_fpu_regs);
 
 void save_fpu_regs(void)
 {
-	unsigned long flags, *regs;
+	unsigned long flags;
 	struct fpu *state;
+	void *regs;
 
 	local_irq_save(flags);
 
@@ -233,22 +185,7 @@ void save_fpu_regs(void)
 			     : "d" (regs)
 			     : "1", "cc", "memory");
 	} else {
-		asm volatile("std 0,%0" : "=Q" (regs[0]));
-		asm volatile("std 1,%0" : "=Q" (regs[1]));
-		asm volatile("std 2,%0" : "=Q" (regs[2]));
-		asm volatile("std 3,%0" : "=Q" (regs[3]));
-		asm volatile("std 4,%0" : "=Q" (regs[4]));
-		asm volatile("std 5,%0" : "=Q" (regs[5]));
-		asm volatile("std 6,%0" : "=Q" (regs[6]));
-		asm volatile("std 7,%0" : "=Q" (regs[7]));
-		asm volatile("std 8,%0" : "=Q" (regs[8]));
-		asm volatile("std 9,%0" : "=Q" (regs[9]));
-		asm volatile("std 10,%0" : "=Q" (regs[10]));
-		asm volatile("std 11,%0" : "=Q" (regs[11]));
-		asm volatile("std 12,%0" : "=Q" (regs[12]));
-		asm volatile("std 13,%0" : "=Q" (regs[13]));
-		asm volatile("std 14,%0" : "=Q" (regs[14]));
-		asm volatile("std 15,%0" : "=Q" (regs[15]));
+		save_fp_regs(regs);
 	}
 	set_cpu_flag(CIF_FPU);
 out: