Commit 0d4ddce3 authored by Maciej Fijalkowski's avatar Maciej Fijalkowski Committed by Alexei Starovoitov

bpf, x64: use %rcx instead of %rax for tail call retpolines

Currently, %rax is used to store the jump target when BPF program is
emitting the retpoline instructions that are handling the indirect
tailcall.

There is a plan to use %rax for different purpose, which is storing the
tail call counter. In order to preserve this value across the tailcalls,
adjust the BPF indirect tailcalls so that the target program will reside
in %rcx and teach the retpoline instructions about new location of jump
target.
Signed-off-by: default avatarMaciej Fijalkowski <maciej.fijalkowski@intel.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent c64779e2
...@@ -314,19 +314,19 @@ static inline void mds_idle_clear_cpu_buffers(void) ...@@ -314,19 +314,19 @@ static inline void mds_idle_clear_cpu_buffers(void)
* lfence * lfence
* jmp spec_trap * jmp spec_trap
* do_rop: * do_rop:
* mov %rax,(%rsp) for x86_64 * mov %rcx,(%rsp) for x86_64
* mov %edx,(%esp) for x86_32 * mov %edx,(%esp) for x86_32
* retq * retq
* *
* Without retpolines configured: * Without retpolines configured:
* *
* jmp *%rax for x86_64 * jmp *%rcx for x86_64
* jmp *%edx for x86_32 * jmp *%edx for x86_32
*/ */
#ifdef CONFIG_RETPOLINE #ifdef CONFIG_RETPOLINE
# ifdef CONFIG_X86_64 # ifdef CONFIG_X86_64
# define RETPOLINE_RAX_BPF_JIT_SIZE 17 # define RETPOLINE_RCX_BPF_JIT_SIZE 17
# define RETPOLINE_RAX_BPF_JIT() \ # define RETPOLINE_RCX_BPF_JIT() \
do { \ do { \
EMIT1_off32(0xE8, 7); /* callq do_rop */ \ EMIT1_off32(0xE8, 7); /* callq do_rop */ \
/* spec_trap: */ \ /* spec_trap: */ \
...@@ -334,7 +334,7 @@ do { \ ...@@ -334,7 +334,7 @@ do { \
EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \
EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \
/* do_rop: */ \ /* do_rop: */ \
EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \ EMIT4(0x48, 0x89, 0x0C, 0x24); /* mov %rcx,(%rsp) */ \
EMIT1(0xC3); /* retq */ \ EMIT1(0xC3); /* retq */ \
} while (0) } while (0)
# else /* !CONFIG_X86_64 */ # else /* !CONFIG_X86_64 */
...@@ -352,9 +352,9 @@ do { \ ...@@ -352,9 +352,9 @@ do { \
# endif # endif
#else /* !CONFIG_RETPOLINE */ #else /* !CONFIG_RETPOLINE */
# ifdef CONFIG_X86_64 # ifdef CONFIG_X86_64
# define RETPOLINE_RAX_BPF_JIT_SIZE 2 # define RETPOLINE_RCX_BPF_JIT_SIZE 2
# define RETPOLINE_RAX_BPF_JIT() \ # define RETPOLINE_RCX_BPF_JIT() \
EMIT2(0xFF, 0xE0); /* jmp *%rax */ EMIT2(0xFF, 0xE1); /* jmp *%rcx */
# else /* !CONFIG_X86_64 */ # else /* !CONFIG_X86_64 */
# define RETPOLINE_EDX_BPF_JIT() \ # define RETPOLINE_EDX_BPF_JIT() \
EMIT2(0xFF, 0xE2) /* jmp *%edx */ EMIT2(0xFF, 0xE2) /* jmp *%edx */
......
...@@ -370,7 +370,7 @@ static void emit_bpf_tail_call_indirect(u8 **pprog) ...@@ -370,7 +370,7 @@ static void emit_bpf_tail_call_indirect(u8 **pprog)
EMIT2(0x89, 0xD2); /* mov edx, edx */ EMIT2(0x89, 0xD2); /* mov edx, edx */
EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */ EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */
offsetof(struct bpf_array, map.max_entries)); offsetof(struct bpf_array, map.max_entries));
#define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* Number of bytes to jump */ #define OFFSET1 (41 + RETPOLINE_RCX_BPF_JIT_SIZE) /* Number of bytes to jump */
EMIT2(X86_JBE, OFFSET1); /* jbe out */ EMIT2(X86_JBE, OFFSET1); /* jbe out */
label1 = cnt; label1 = cnt;
...@@ -380,36 +380,36 @@ static void emit_bpf_tail_call_indirect(u8 **pprog) ...@@ -380,36 +380,36 @@ static void emit_bpf_tail_call_indirect(u8 **pprog)
*/ */
EMIT2_off32(0x8B, 0x85, -36 - MAX_BPF_STACK); /* mov eax, dword ptr [rbp - 548] */ EMIT2_off32(0x8B, 0x85, -36 - MAX_BPF_STACK); /* mov eax, dword ptr [rbp - 548] */
EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */
#define OFFSET2 (30 + RETPOLINE_RAX_BPF_JIT_SIZE) #define OFFSET2 (30 + RETPOLINE_RCX_BPF_JIT_SIZE)
EMIT2(X86_JA, OFFSET2); /* ja out */ EMIT2(X86_JA, OFFSET2); /* ja out */
label2 = cnt; label2 = cnt;
EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */ EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */
EMIT2_off32(0x89, 0x85, -36 - MAX_BPF_STACK); /* mov dword ptr [rbp -548], eax */ EMIT2_off32(0x89, 0x85, -36 - MAX_BPF_STACK); /* mov dword ptr [rbp -548], eax */
/* prog = array->ptrs[index]; */ /* prog = array->ptrs[index]; */
EMIT4_off32(0x48, 0x8B, 0x84, 0xD6, /* mov rax, [rsi + rdx * 8 + offsetof(...)] */ EMIT4_off32(0x48, 0x8B, 0x8C, 0xD6, /* mov rcx, [rsi + rdx * 8 + offsetof(...)] */
offsetof(struct bpf_array, ptrs)); offsetof(struct bpf_array, ptrs));
/* /*
* if (prog == NULL) * if (prog == NULL)
* goto out; * goto out;
*/ */
EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */ EMIT3(0x48, 0x85, 0xC9); /* test rcx,rcx */
#define OFFSET3 (8 + RETPOLINE_RAX_BPF_JIT_SIZE) #define OFFSET3 (8 + RETPOLINE_RCX_BPF_JIT_SIZE)
EMIT2(X86_JE, OFFSET3); /* je out */ EMIT2(X86_JE, OFFSET3); /* je out */
label3 = cnt; label3 = cnt;
/* goto *(prog->bpf_func + prologue_size); */ /* goto *(prog->bpf_func + prologue_size); */
EMIT4(0x48, 0x8B, 0x40, /* mov rax, qword ptr [rax + 32] */ EMIT4(0x48, 0x8B, 0x49, /* mov rcx, qword ptr [rcx + 32] */
offsetof(struct bpf_prog, bpf_func)); offsetof(struct bpf_prog, bpf_func));
EMIT4(0x48, 0x83, 0xC0, PROLOGUE_SIZE); /* add rax, prologue_size */ EMIT4(0x48, 0x83, 0xC1, PROLOGUE_SIZE); /* add rcx, prologue_size */
/* /*
* Wow we're ready to jump into next BPF program * Now we're ready to jump into next BPF program
* rdi == ctx (1st arg) * rdi == ctx (1st arg)
* rax == prog->bpf_func + prologue_size * rcx == prog->bpf_func + prologue_size
*/ */
RETPOLINE_RAX_BPF_JIT(); RETPOLINE_RCX_BPF_JIT();
/* out: */ /* out: */
BUILD_BUG_ON(cnt - label1 != OFFSET1); BUILD_BUG_ON(cnt - label1 != OFFSET1);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment