Skip to content

L
linux
Commit 8b9ed79c authored by Linus Torvalds's avatar Linus Torvalds
Browse files
Options

Merge tag 'x86_asm_for_v6.2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 asm updates from Borislav Petkov:

 - Move the 32-bit memmove() asm implementation out-of-line in order to
   fix a 32-bit full LTO build failure with clang where it would fail at
   register allocation.

   Move it to an asm file and clean it up while at it, similar to what
   has been already done on 64-bit

* tag 'x86_asm_for_v6.2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/mem: Move memmove to out of line assembler
parents fc4c9f45 bce5a1e8
Hide whitespace changes
Inline Side-by-side
Showing with 223 additions and 187 deletions
arch/x86/lib/Makefile
View file @ 8b9ed79c
...@@ -60,6 +60,7 @@ ifeq ($(CONFIG_X86_32),y) ...@@ -60,6 +60,7 @@ ifeq ($(CONFIG_X86_32),y)
lib-y += checksum_32.o lib-y += checksum_32.o
lib-y += strstr_32.o lib-y += strstr_32.o
lib-y += string_32.o lib-y += string_32.o
lib-y += memmove_32.o
ifneq ($(CONFIG_X86_CMPXCHG64),y) ifneq ($(CONFIG_X86_CMPXCHG64),y)
lib-y += cmpxchg8b_emu.o atomic64_386_32.o lib-y += cmpxchg8b_emu.o atomic64_386_32.o
endif endif
......
arch/x86/lib/memcpy_32.c
View file @ 8b9ed79c
...@@ -17,190 +17,3 @@ __visible void *memset(void *s, int c, size_t count) ...@@ -17,190 +17,3 @@ __visible void *memset(void *s, int c, size_t count)
return __memset(s, c, count); return __memset(s, c, count);
} }
EXPORT_SYMBOL(memset); EXPORT_SYMBOL(memset);
__visible void *memmove(void *dest, const void *src, size_t n)
{
int d0,d1,d2,d3,d4,d5;
char *ret = dest;
__asm__ __volatile__(
/* Handle more 16 bytes in loop */
"cmp $0x10, %0\n\t"
"jb 1f\n\t"
/* Decide forward/backward copy mode */
"cmp %2, %1\n\t"
"jb 2f\n\t"
/*
* movs instruction have many startup latency
* so we handle small size by general register.
*/
"cmp $680, %0\n\t"
"jb 3f\n\t"
/*
* movs instruction is only good for aligned case.
*/
"mov %1, %3\n\t"
"xor %2, %3\n\t"
"and $0xff, %3\n\t"
"jz 4f\n\t"
"3:\n\t"
"sub $0x10, %0\n\t"
/*
* We gobble 16 bytes forward in each loop.
*/
"3:\n\t"
"sub $0x10, %0\n\t"
"mov 0*4(%1), %3\n\t"
"mov 1*4(%1), %4\n\t"
"mov %3, 0*4(%2)\n\t"
"mov %4, 1*4(%2)\n\t"
"mov 2*4(%1), %3\n\t"
"mov 3*4(%1), %4\n\t"
"mov %3, 2*4(%2)\n\t"
"mov %4, 3*4(%2)\n\t"
"lea 0x10(%1), %1\n\t"
"lea 0x10(%2), %2\n\t"
"jae 3b\n\t"
"add $0x10, %0\n\t"
"jmp 1f\n\t"
/*
* Handle data forward by movs.
*/
".p2align 4\n\t"
"4:\n\t"
"mov -4(%1, %0), %3\n\t"
"lea -4(%2, %0), %4\n\t"
"shr $2, %0\n\t"
"rep movsl\n\t"
"mov %3, (%4)\n\t"
"jmp 11f\n\t"
/*
* Handle data backward by movs.
*/
".p2align 4\n\t"
"6:\n\t"
"mov (%1), %3\n\t"
"mov %2, %4\n\t"
"lea -4(%1, %0), %1\n\t"
"lea -4(%2, %0), %2\n\t"
"shr $2, %0\n\t"
"std\n\t"
"rep movsl\n\t"
"mov %3,(%4)\n\t"
"cld\n\t"
"jmp 11f\n\t"
/*
* Start to prepare for backward copy.
*/
".p2align 4\n\t"
"2:\n\t"
"cmp $680, %0\n\t"
"jb 5f\n\t"
"mov %1, %3\n\t"
"xor %2, %3\n\t"
"and $0xff, %3\n\t"
"jz 6b\n\t"
/*
* Calculate copy position to tail.
*/
"5:\n\t"
"add %0, %1\n\t"
"add %0, %2\n\t"
"sub $0x10, %0\n\t"
/*
* We gobble 16 bytes backward in each loop.
*/
"7:\n\t"
"sub $0x10, %0\n\t"
"mov -1*4(%1), %3\n\t"
"mov -2*4(%1), %4\n\t"
"mov %3, -1*4(%2)\n\t"
"mov %4, -2*4(%2)\n\t"
"mov -3*4(%1), %3\n\t"
"mov -4*4(%1), %4\n\t"
"mov %3, -3*4(%2)\n\t"
"mov %4, -4*4(%2)\n\t"
"lea -0x10(%1), %1\n\t"
"lea -0x10(%2), %2\n\t"
"jae 7b\n\t"
/*
* Calculate copy position to head.
*/
"add $0x10, %0\n\t"
"sub %0, %1\n\t"
"sub %0, %2\n\t"
/*
* Move data from 8 bytes to 15 bytes.
*/
".p2align 4\n\t"
"1:\n\t"
"cmp $8, %0\n\t"
"jb 8f\n\t"
"mov 0*4(%1), %3\n\t"
"mov 1*4(%1), %4\n\t"
"mov -2*4(%1, %0), %5\n\t"
"mov -1*4(%1, %0), %1\n\t"
"mov %3, 0*4(%2)\n\t"
"mov %4, 1*4(%2)\n\t"
"mov %5, -2*4(%2, %0)\n\t"
"mov %1, -1*4(%2, %0)\n\t"
"jmp 11f\n\t"
/*
* Move data from 4 bytes to 7 bytes.
*/
".p2align 4\n\t"
"8:\n\t"
"cmp $4, %0\n\t"
"jb 9f\n\t"
"mov 0*4(%1), %3\n\t"
"mov -1*4(%1, %0), %4\n\t"
"mov %3, 0*4(%2)\n\t"
"mov %4, -1*4(%2, %0)\n\t"
"jmp 11f\n\t"
/*
* Move data from 2 bytes to 3 bytes.
*/
".p2align 4\n\t"
"9:\n\t"
"cmp $2, %0\n\t"
"jb 10f\n\t"
"movw 0*2(%1), %%dx\n\t"
"movw -1*2(%1, %0), %%bx\n\t"
"movw %%dx, 0*2(%2)\n\t"
"movw %%bx, -1*2(%2, %0)\n\t"
"jmp 11f\n\t"
/*
* Move data for 1 byte.
*/
".p2align 4\n\t"
"10:\n\t"
"cmp $1, %0\n\t"
"jb 11f\n\t"
"movb (%1), %%cl\n\t"
"movb %%cl, (%2)\n\t"
".p2align 4\n\t"
"11:"
: "=&c" (d0), "=&S" (d1), "=&D" (d2),
"=r" (d3),"=r" (d4), "=r"(d5)
:"0" (n),
"1" (src),
"2" (dest)
:"memory");
return ret;
}
EXPORT_SYMBOL(memmove);
arch/x86/lib/memmove_32.S 0 → 100644
View file @ 8b9ed79c
/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/linkage.h>
#include <asm/export.h>
SYM_FUNC_START(memmove)
/*
* void *memmove(void *dest_in, const void *src_in, size_t n)
* -mregparm=3 passes these in registers:
* dest_in: %eax
* src_in: %edx
* n: %ecx
* See also: arch/x86/entry/calling.h for description of the calling convention.
*
* n can remain in %ecx, but for `rep movsl`, we'll need dest in %edi and src
* in %esi.
*/
.set dest_in, %eax
.set dest, %edi
.set src_in, %edx
.set src, %esi
.set n, %ecx
.set tmp0, %edx
.set tmp0w, %dx
.set tmp1, %ebx
.set tmp1w, %bx
.set tmp2, %eax
.set tmp3b, %cl
/*
* Save all callee-saved registers, because this function is going to clobber
* all of them:
*/
pushl %ebp
movl %esp, %ebp // set standard frame pointer
pushl %ebx
pushl %edi
pushl %esi
pushl %eax // save 'dest_in' parameter [eax] as the return value
movl src_in, src
movl dest_in, dest
/* Handle more 16 bytes in loop */
cmpl $0x10, n
jb .Lmove_16B
/* Decide forward/backward copy mode */
cmpl dest, src
jb .Lbackwards_header
/*
* movs instruction have many startup latency
* so we handle small size by general register.
*/
cmpl $680, n
jb .Ltoo_small_forwards
/* movs instruction is only good for aligned case. */
movl src, tmp0
xorl dest, tmp0
andl $0xff, tmp0
jz .Lforward_movs
.Ltoo_small_forwards:
subl $0x10, n
/* We gobble 16 bytes forward in each loop. */
.Lmove_16B_forwards_loop:
subl $0x10, n
movl 0*4(src), tmp0
movl 1*4(src), tmp1
movl tmp0, 0*4(dest)
movl tmp1, 1*4(dest)
movl 2*4(src), tmp0
movl 3*4(src), tmp1
movl tmp0, 2*4(dest)
movl tmp1, 3*4(dest)
leal 0x10(src), src
leal 0x10(dest), dest
jae .Lmove_16B_forwards_loop
addl $0x10, n
jmp .Lmove_16B
/* Handle data forward by movs. */
.p2align 4
.Lforward_movs:
movl -4(src, n), tmp0
leal -4(dest, n), tmp1
shrl $2, n
rep movsl
movl tmp0, (tmp1)
jmp .Ldone
/* Handle data backward by movs. */
.p2align 4
.Lbackwards_movs:
movl (src), tmp0
movl dest, tmp1
leal -4(src, n), src
leal -4(dest, n), dest
shrl $2, n
std
rep movsl
movl tmp0,(tmp1)
cld
jmp .Ldone
/* Start to prepare for backward copy. */
.p2align 4
.Lbackwards_header:
cmpl $680, n
jb .Ltoo_small_backwards
movl src, tmp0
xorl dest, tmp0
andl $0xff, tmp0
jz .Lbackwards_movs
/* Calculate copy position to tail. */
.Ltoo_small_backwards:
addl n, src
addl n, dest
subl $0x10, n
/* We gobble 16 bytes backward in each loop. */
.Lmove_16B_backwards_loop:
subl $0x10, n
movl -1*4(src), tmp0
movl -2*4(src), tmp1
movl tmp0, -1*4(dest)
movl tmp1, -2*4(dest)
movl -3*4(src), tmp0
movl -4*4(src), tmp1
movl tmp0, -3*4(dest)
movl tmp1, -4*4(dest)
leal -0x10(src), src
leal -0x10(dest), dest
jae .Lmove_16B_backwards_loop
/* Calculate copy position to head. */
addl $0x10, n
subl n, src
subl n, dest
/* Move data from 8 bytes to 15 bytes. */
.p2align 4
.Lmove_16B:
cmpl $8, n
jb .Lmove_8B
movl 0*4(src), tmp0
movl 1*4(src), tmp1
movl -2*4(src, n), tmp2
movl -1*4(src, n), src
movl tmp0, 0*4(dest)
movl tmp1, 1*4(dest)
movl tmp2, -2*4(dest, n)
movl src, -1*4(dest, n)
jmp .Ldone
/* Move data from 4 bytes to 7 bytes. */
.p2align 4
.Lmove_8B:
cmpl $4, n
jb .Lmove_4B
movl 0*4(src), tmp0
movl -1*4(src, n), tmp1
movl tmp0, 0*4(dest)
movl tmp1, -1*4(dest, n)
jmp .Ldone
/* Move data from 2 bytes to 3 bytes. */
.p2align 4
.Lmove_4B:
cmpl $2, n
jb .Lmove_1B
movw 0*2(src), tmp0w
movw -1*2(src, n), tmp1w
movw tmp0w, 0*2(dest)
movw tmp1w, -1*2(dest, n)
jmp .Ldone
/* Move data for 1 byte. */
.p2align 4
.Lmove_1B:
cmpl $1, n
jb .Ldone
movb (src), tmp3b
movb tmp3b, (dest)
.p2align 4
.Ldone:
popl dest_in // restore 'dest_in' [eax] as the return value
/* Restore all callee-saved registers: */
popl %esi
popl %edi
popl %ebx
popl %ebp
RET
SYM_FUNC_END(memmove)
EXPORT_SYMBOL(memmove)
lib/memcpy_kunit.c
View file @ 8b9ed79c
...@@ -105,6 +105,8 @@ static void memcpy_test(struct kunit *test) ...@@ -105,6 +105,8 @@ static void memcpy_test(struct kunit *test)
#undef TEST_OP #undef TEST_OP
} }
static unsigned char larger_array [2048];
static void memmove_test(struct kunit *test) static void memmove_test(struct kunit *test)
{ {
#define TEST_OP "memmove" #define TEST_OP "memmove"
...@@ -179,6 +181,26 @@ static void memmove_test(struct kunit *test) ...@@ -179,6 +181,26 @@ static void memmove_test(struct kunit *test)
ptr = &overlap.data[2]; ptr = &overlap.data[2];
memmove(ptr, overlap.data, 5); memmove(ptr, overlap.data, 5);
compare("overlapping write", overlap, overlap_expected); compare("overlapping write", overlap, overlap_expected);
/* Verify larger overlapping moves. */
larger_array[256] = 0xAAu;
/*
* Test a backwards overlapping memmove first. 256 and 1024 are
* important for i386 to use rep movsl.
*/
memmove(larger_array, larger_array + 256, 1024);
KUNIT_ASSERT_EQ(test, larger_array[0], 0xAAu);
KUNIT_ASSERT_EQ(test, larger_array[256], 0x00);
KUNIT_ASSERT_NULL(test,
memchr(larger_array + 1, 0xaa, ARRAY_SIZE(larger_array) - 1));
/* Test a forwards overlapping memmove. */
larger_array[0] = 0xBBu;
memmove(larger_array + 256, larger_array, 1024);
KUNIT_ASSERT_EQ(test, larger_array[0], 0xBBu);
KUNIT_ASSERT_EQ(test, larger_array[256], 0xBBu);
KUNIT_ASSERT_NULL(test, memchr(larger_array + 1, 0xBBu, 256 - 1));
KUNIT_ASSERT_NULL(test,
memchr(larger_array + 257, 0xBBu, ARRAY_SIZE(larger_array) - 257));
#undef TEST_OP #undef TEST_OP
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7