Skip to content

L
linux
Commit f7e0beaf authored by Kui-Feng Lee's avatar Kui-Feng Lee Committed by Andrii Nakryiko
Browse files
Options

bpf, x86: Generate trampolines from bpf_tramp_links 

Replace struct bpf_tramp_progs with struct bpf_tramp_links to collect
struct bpf_tramp_link(s) for a trampoline.  struct bpf_tramp_link
extends bpf_link to act as a linked list node.

arch_prepare_bpf_trampoline() accepts a struct bpf_tramp_links to
collects all bpf_tramp_link(s) that a trampoline should call.

Change BPF trampoline and bpf_struct_ops to pass bpf_tramp_links
instead of bpf_tramp_progs.
Signed-off-by: default avatarKui-Feng Lee <kuifeng@fb.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
Acked-by: default avatarAndrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20220510205923.3206889-2-kuifeng@fb.com
parent cb411545
Show whitespace changes
Inline Side-by-side
Showing with 164 additions and 103 deletions
arch/x86/net/bpf_jit_comp.c
View file @ f7e0beaf
...@@ -1762,10 +1762,12 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args, ...@@ -1762,10 +1762,12 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
} }
static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
struct bpf_prog *p, int stack_size, bool save_ret) struct bpf_tramp_link *l, int stack_size,
bool save_ret)
{ {
u8 *prog = *pprog; u8 *prog = *pprog;
u8 *jmp_insn; u8 *jmp_insn;
struct bpf_prog *p = l->link.prog;
/* arg1: mov rdi, progs[i] */ /* arg1: mov rdi, progs[i] */
emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p); emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
...@@ -1850,14 +1852,14 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) ...@@ -1850,14 +1852,14 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)
} }
static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
struct bpf_tramp_progs *tp, int stack_size, struct bpf_tramp_links *tl, int stack_size,
bool save_ret) bool save_ret)
{ {
int i; int i;
u8 *prog = *pprog; u8 *prog = *pprog;
for (i = 0; i < tp->nr_progs; i++) { for (i = 0; i < tl->nr_links; i++) {
if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size,
save_ret)) save_ret))
return -EINVAL; return -EINVAL;
} }
...@@ -1866,7 +1868,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, ...@@ -1866,7 +1868,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
} }
static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
struct bpf_tramp_progs *tp, int stack_size, struct bpf_tramp_links *tl, int stack_size,
u8 **branches) u8 **branches)
{ {
u8 *prog = *pprog; u8 *prog = *pprog;
...@@ -1877,8 +1879,8 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, ...@@ -1877,8 +1879,8 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
*/ */
emit_mov_imm32(&prog, false, BPF_REG_0, 0); emit_mov_imm32(&prog, false, BPF_REG_0, 0);
emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
for (i = 0; i < tp->nr_progs; i++) { for (i = 0; i < tl->nr_links; i++) {
if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, true)) if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, true))
return -EINVAL; return -EINVAL;
/* mod_ret prog stored return value into [rbp - 8]. Emit: /* mod_ret prog stored return value into [rbp - 8]. Emit:
...@@ -1980,14 +1982,14 @@ static bool is_valid_bpf_tramp_flags(unsigned int flags) ...@@ -1980,14 +1982,14 @@ static bool is_valid_bpf_tramp_flags(unsigned int flags)
*/ */
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
const struct btf_func_model *m, u32 flags, const struct btf_func_model *m, u32 flags,
struct bpf_tramp_progs *tprogs, struct bpf_tramp_links *tlinks,
void *orig_call) void *orig_call)
{ {
int ret, i, nr_args = m->nr_args; int ret, i, nr_args = m->nr_args;
int regs_off, ip_off, args_off, stack_size = nr_args * 8; int regs_off, ip_off, args_off, stack_size = nr_args * 8;
struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY]; struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT]; struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN]; struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
u8 **branches = NULL; u8 **branches = NULL;
u8 *prog; u8 *prog;
bool save_ret; bool save_ret;
...@@ -2078,13 +2080,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i ...@@ -2078,13 +2080,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
} }
} }
if (fentry->nr_progs) if (fentry->nr_links)
if (invoke_bpf(m, &prog, fentry, regs_off, if (invoke_bpf(m, &prog, fentry, regs_off,
flags & BPF_TRAMP_F_RET_FENTRY_RET)) flags & BPF_TRAMP_F_RET_FENTRY_RET))
return -EINVAL; return -EINVAL;
if (fmod_ret->nr_progs) { if (fmod_ret->nr_links) {
branches = kcalloc(fmod_ret->nr_progs, sizeof(u8 *), branches = kcalloc(fmod_ret->nr_links, sizeof(u8 *),
GFP_KERNEL); GFP_KERNEL);
if (!branches) if (!branches)
return -ENOMEM; return -ENOMEM;
...@@ -2111,7 +2113,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i ...@@ -2111,7 +2113,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
prog += X86_PATCH_SIZE; prog += X86_PATCH_SIZE;
} }
if (fmod_ret->nr_progs) { if (fmod_ret->nr_links) {
/* From Intel 64 and IA-32 Architectures Optimization /* From Intel 64 and IA-32 Architectures Optimization
* Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler
* Coding Rule 11: All branch targets should be 16-byte * Coding Rule 11: All branch targets should be 16-byte
...@@ -2121,12 +2123,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i ...@@ -2121,12 +2123,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
/* Update the branches saved in invoke_bpf_mod_ret with the /* Update the branches saved in invoke_bpf_mod_ret with the
* aligned address of do_fexit. * aligned address of do_fexit.
*/ */
for (i = 0; i < fmod_ret->nr_progs; i++) for (i = 0; i < fmod_ret->nr_links; i++)
emit_cond_near_jump(&branches[i], prog, branches[i], emit_cond_near_jump(&branches[i], prog, branches[i],
X86_JNE); X86_JNE);
} }
if (fexit->nr_progs) if (fexit->nr_links)
if (invoke_bpf(m, &prog, fexit, regs_off, false)) { if (invoke_bpf(m, &prog, fexit, regs_off, false)) {
ret = -EINVAL; ret = -EINVAL;
goto cleanup; goto cleanup;
......
include/linux/bpf.h
View file @ f7e0beaf
...@@ -723,11 +723,11 @@ struct btf_func_model { ...@@ -723,11 +723,11 @@ struct btf_func_model {
/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
* bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2 * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2
*/ */
#define BPF_MAX_TRAMP_PROGS 38 #define BPF_MAX_TRAMP_LINKS 38
struct bpf_tramp_progs { struct bpf_tramp_links {
struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS]; struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS];
int nr_progs; int nr_links;
}; };
/* Different use cases for BPF trampoline: /* Different use cases for BPF trampoline:
...@@ -753,7 +753,7 @@ struct bpf_tramp_progs { ...@@ -753,7 +753,7 @@ struct bpf_tramp_progs {
struct bpf_tramp_image; struct bpf_tramp_image;
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end, int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end,
const struct btf_func_model *m, u32 flags, const struct btf_func_model *m, u32 flags,
struct bpf_tramp_progs *tprogs, struct bpf_tramp_links *tlinks,
void *orig_call); void *orig_call);
/* these two functions are called from generated trampoline */ /* these two functions are called from generated trampoline */
u64 notrace __bpf_prog_enter(struct bpf_prog *prog); u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
...@@ -852,9 +852,10 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( ...@@ -852,9 +852,10 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
{ {
return bpf_func(ctx, insnsi); return bpf_func(ctx, insnsi);
} }
#ifdef CONFIG_BPF_JIT #ifdef CONFIG_BPF_JIT
int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr); int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr); int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
struct bpf_trampoline *bpf_trampoline_get(u64 key, struct bpf_trampoline *bpf_trampoline_get(u64 key,
struct bpf_attach_target_info *tgt_info); struct bpf_attach_target_info *tgt_info);
void bpf_trampoline_put(struct bpf_trampoline *tr); void bpf_trampoline_put(struct bpf_trampoline *tr);
...@@ -905,12 +906,12 @@ int bpf_jit_charge_modmem(u32 size); ...@@ -905,12 +906,12 @@ int bpf_jit_charge_modmem(u32 size);
void bpf_jit_uncharge_modmem(u32 size); void bpf_jit_uncharge_modmem(u32 size);
bool bpf_prog_has_trampoline(const struct bpf_prog *prog); bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
#else #else
static inline int bpf_trampoline_link_prog(struct bpf_prog *prog, static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
struct bpf_trampoline *tr) struct bpf_trampoline *tr)
{ {
return -ENOTSUPP; return -ENOTSUPP;
} }
static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog, static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
struct bpf_trampoline *tr) struct bpf_trampoline *tr)
{ {
return -ENOTSUPP; return -ENOTSUPP;
...@@ -1009,7 +1010,6 @@ struct bpf_prog_aux { ...@@ -1009,7 +1010,6 @@ struct bpf_prog_aux {
bool tail_call_reachable; bool tail_call_reachable;
bool xdp_has_frags; bool xdp_has_frags;
bool use_bpf_prog_pack; bool use_bpf_prog_pack;
struct hlist_node tramp_hlist;
/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
const struct btf_type *attach_func_proto; const struct btf_type *attach_func_proto;
/* function name for valid attach_btf_id */ /* function name for valid attach_btf_id */
...@@ -1096,6 +1096,18 @@ struct bpf_link_ops { ...@@ -1096,6 +1096,18 @@ struct bpf_link_ops {
struct bpf_link_info *info); struct bpf_link_info *info);
}; };
struct bpf_tramp_link {
struct bpf_link link;
struct hlist_node tramp_hlist;
};
struct bpf_tracing_link {
struct bpf_tramp_link link;
enum bpf_attach_type attach_type;
struct bpf_trampoline *trampoline;
struct bpf_prog *tgt_prog;
};
struct bpf_link_primer { struct bpf_link_primer {
struct bpf_link *link; struct bpf_link *link;
struct file *file; struct file *file;
...@@ -1133,8 +1145,8 @@ bool bpf_struct_ops_get(const void *kdata); ...@@ -1133,8 +1145,8 @@ bool bpf_struct_ops_get(const void *kdata);
void bpf_struct_ops_put(const void *kdata); void bpf_struct_ops_put(const void *kdata);
int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
void *value); void *value);
int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs *tprogs, int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
struct bpf_prog *prog, struct bpf_tramp_link *link,
const struct btf_func_model *model, const struct btf_func_model *model,
void *image, void *image_end); void *image, void *image_end);
static inline bool bpf_try_module_get(const void *data, struct module *owner) static inline bool bpf_try_module_get(const void *data, struct module *owner)
......
include/linux/bpf_types.h
View file @ f7e0beaf
...@@ -141,3 +141,4 @@ BPF_LINK_TYPE(BPF_LINK_TYPE_XDP, xdp) ...@@ -141,3 +141,4 @@ BPF_LINK_TYPE(BPF_LINK_TYPE_XDP, xdp)
BPF_LINK_TYPE(BPF_LINK_TYPE_PERF_EVENT, perf) BPF_LINK_TYPE(BPF_LINK_TYPE_PERF_EVENT, perf)
#endif #endif
BPF_LINK_TYPE(BPF_LINK_TYPE_KPROBE_MULTI, kprobe_multi) BPF_LINK_TYPE(BPF_LINK_TYPE_KPROBE_MULTI, kprobe_multi)
BPF_LINK_TYPE(BPF_LINK_TYPE_STRUCT_OPS, struct_ops)
include/uapi/linux/bpf.h
View file @ f7e0beaf
...@@ -1013,6 +1013,7 @@ enum bpf_link_type { ...@@ -1013,6 +1013,7 @@ enum bpf_link_type {
BPF_LINK_TYPE_XDP = 6, BPF_LINK_TYPE_XDP = 6,
BPF_LINK_TYPE_PERF_EVENT = 7, BPF_LINK_TYPE_PERF_EVENT = 7,
BPF_LINK_TYPE_KPROBE_MULTI = 8, BPF_LINK_TYPE_KPROBE_MULTI = 8,
BPF_LINK_TYPE_STRUCT_OPS = 9,
MAX_BPF_LINK_TYPE, MAX_BPF_LINK_TYPE,
}; };
......
kernel/bpf/bpf_struct_ops.c
View file @ f7e0beaf
...@@ -33,15 +33,15 @@ struct bpf_struct_ops_map { ...@@ -33,15 +33,15 @@ struct bpf_struct_ops_map {
const struct bpf_struct_ops *st_ops; const struct bpf_struct_ops *st_ops;
/* protect map_update */ /* protect map_update */
struct mutex lock; struct mutex lock;
/* progs has all the bpf_prog that is populated /* link has all the bpf_links that is populated
* to the func ptr of the kernel's struct * to the func ptr of the kernel's struct
* (in kvalue.data). * (in kvalue.data).
*/ */
struct bpf_prog **progs; struct bpf_link **links;
/* image is a page that has all the trampolines /* image is a page that has all the trampolines
* that stores the func args before calling the bpf_prog. * that stores the func args before calling the bpf_prog.
* A PAGE_SIZE "image" is enough to store all trampoline for * A PAGE_SIZE "image" is enough to store all trampoline for
* "progs[]". * "links[]".
*/ */
void *image; void *image;
/* uvalue->data stores the kernel struct /* uvalue->data stores the kernel struct
...@@ -283,9 +283,9 @@ static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map) ...@@ -283,9 +283,9 @@ static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)
u32 i; u32 i;
for (i = 0; i < btf_type_vlen(t); i++) { for (i = 0; i < btf_type_vlen(t); i++) {
if (st_map->progs[i]) { if (st_map->links[i]) {
bpf_prog_put(st_map->progs[i]); bpf_link_put(st_map->links[i]);
st_map->progs[i] = NULL; st_map->links[i] = NULL;
} }
} }
} }
...@@ -316,18 +316,34 @@ static int check_zero_holes(const struct btf_type *t, void *data) ...@@ -316,18 +316,34 @@ static int check_zero_holes(const struct btf_type *t, void *data)
return 0; return 0;
} }
int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs *tprogs, static void bpf_struct_ops_link_release(struct bpf_link *link)
struct bpf_prog *prog, {
}
static void bpf_struct_ops_link_dealloc(struct bpf_link *link)
{
struct bpf_tramp_link *tlink = container_of(link, struct bpf_tramp_link, link);
kfree(tlink);
}
const struct bpf_link_ops bpf_struct_ops_link_lops = {
.release = bpf_struct_ops_link_release,
.dealloc = bpf_struct_ops_link_dealloc,
};
int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
struct bpf_tramp_link *link,
const struct btf_func_model *model, const struct btf_func_model *model,
void *image, void *image_end) void *image, void *image_end)
{ {
u32 flags; u32 flags;
tprogs[BPF_TRAMP_FENTRY].progs[0] = prog; tlinks[BPF_TRAMP_FENTRY].links[0] = link;
tprogs[BPF_TRAMP_FENTRY].nr_progs = 1; tlinks[BPF_TRAMP_FENTRY].nr_links = 1;
flags = model->ret_size > 0 ? BPF_TRAMP_F_RET_FENTRY_RET : 0; flags = model->ret_size > 0 ? BPF_TRAMP_F_RET_FENTRY_RET : 0;
return arch_prepare_bpf_trampoline(NULL, image, image_end, return arch_prepare_bpf_trampoline(NULL, image, image_end,
model, flags, tprogs, NULL); model, flags, tlinks, NULL);
} }
static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
...@@ -338,7 +354,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, ...@@ -338,7 +354,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
struct bpf_struct_ops_value *uvalue, *kvalue; struct bpf_struct_ops_value *uvalue, *kvalue;
const struct btf_member *member; const struct btf_member *member;
const struct btf_type *t = st_ops->type; const struct btf_type *t = st_ops->type;
struct bpf_tramp_progs *tprogs = NULL; struct bpf_tramp_links *tlinks = NULL;
void *udata, *kdata; void *udata, *kdata;
int prog_fd, err = 0; int prog_fd, err = 0;
void *image, *image_end; void *image, *image_end;
...@@ -362,8 +378,8 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, ...@@ -362,8 +378,8 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
if (uvalue->state || refcount_read(&uvalue->refcnt)) if (uvalue->state || refcount_read(&uvalue->refcnt))
return -EINVAL; return -EINVAL;
tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL); tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
if (!tprogs) if (!tlinks)
return -ENOMEM; return -ENOMEM;
uvalue = (struct bpf_struct_ops_value *)st_map->uvalue; uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;
...@@ -386,6 +402,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, ...@@ -386,6 +402,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
for_each_member(i, t, member) { for_each_member(i, t, member) {
const struct btf_type *mtype, *ptype; const struct btf_type *mtype, *ptype;
struct bpf_prog *prog; struct bpf_prog *prog;
struct bpf_tramp_link *link;
u32 moff; u32 moff;
moff = __btf_member_bit_offset(t, member) / 8; moff = __btf_member_bit_offset(t, member) / 8;
...@@ -439,16 +456,26 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, ...@@ -439,16 +456,26 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
err = PTR_ERR(prog); err = PTR_ERR(prog);
goto reset_unlock; goto reset_unlock;
} }
st_map->progs[i] = prog;
if (prog->type != BPF_PROG_TYPE_STRUCT_OPS || if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
prog->aux->attach_btf_id != st_ops->type_id || prog->aux->attach_btf_id != st_ops->type_id ||
prog->expected_attach_type != i) { prog->expected_attach_type != i) {
bpf_prog_put(prog);
err = -EINVAL; err = -EINVAL;
goto reset_unlock; goto reset_unlock;
} }
err = bpf_struct_ops_prepare_trampoline(tprogs, prog, link = kzalloc(sizeof(*link), GFP_USER);
if (!link) {
bpf_prog_put(prog);
err = -ENOMEM;
goto reset_unlock;
}
bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS,
&bpf_struct_ops_link_lops, prog);
st_map->links[i] = &link->link;
err = bpf_struct_ops_prepare_trampoline(tlinks, link,
&st_ops->func_models[i], &st_ops->func_models[i],
image, image_end); image, image_end);
if (err < 0) if (err < 0)
...@@ -491,7 +518,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, ...@@ -491,7 +518,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
memset(uvalue, 0, map->value_size); memset(uvalue, 0, map->value_size);
memset(kvalue, 0, map->value_size); memset(kvalue, 0, map->value_size);
unlock: unlock:
kfree(tprogs); kfree(tlinks);
mutex_unlock(&st_map->lock); mutex_unlock(&st_map->lock);
return err; return err;
} }
...@@ -546,9 +573,9 @@ static void bpf_struct_ops_map_free(struct bpf_map *map) ...@@ -546,9 +573,9 @@ static void bpf_struct_ops_map_free(struct bpf_map *map)
{ {
struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
if (st_map->progs) if (st_map->links)
bpf_struct_ops_map_put_progs(st_map); bpf_struct_ops_map_put_progs(st_map);
bpf_map_area_free(st_map->progs); bpf_map_area_free(st_map->links);
bpf_jit_free_exec(st_map->image); bpf_jit_free_exec(st_map->image);
bpf_map_area_free(st_map->uvalue); bpf_map_area_free(st_map->uvalue);
bpf_map_area_free(st_map); bpf_map_area_free(st_map);
...@@ -597,11 +624,11 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) ...@@ -597,11 +624,11 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
map = &st_map->map; map = &st_map->map;
st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE); st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);
st_map->progs = st_map->links =
bpf_map_area_alloc(btf_type_vlen(t) * sizeof(struct bpf_prog *), bpf_map_area_alloc(btf_type_vlen(t) * sizeof(struct bpf_links *),
NUMA_NO_NODE); NUMA_NO_NODE);
st_map->image = bpf_jit_alloc_exec(PAGE_SIZE); st_map->image = bpf_jit_alloc_exec(PAGE_SIZE);
if (!st_map->uvalue || !st_map->progs || !st_map->image) { if (!st_map->uvalue || !st_map->links || !st_map->image) {
bpf_struct_ops_map_free(map); bpf_struct_ops_map_free(map);
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
} }
......
kernel/bpf/syscall.c
View file @ f7e0beaf
...@@ -2864,19 +2864,12 @@ struct bpf_link *bpf_link_get_from_fd(u32 ufd) ...@@ -2864,19 +2864,12 @@ struct bpf_link *bpf_link_get_from_fd(u32 ufd)
} }
EXPORT_SYMBOL(bpf_link_get_from_fd); EXPORT_SYMBOL(bpf_link_get_from_fd);
struct bpf_tracing_link {
struct bpf_link link;
enum bpf_attach_type attach_type;
struct bpf_trampoline *trampoline;
struct bpf_prog *tgt_prog;
};
static void bpf_tracing_link_release(struct bpf_link *link) static void bpf_tracing_link_release(struct bpf_link *link)
{ {
struct bpf_tracing_link *tr_link = struct bpf_tracing_link *tr_link =
container_of(link, struct bpf_tracing_link, link); container_of(link, struct bpf_tracing_link, link.link);
WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog, WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link,
tr_link->trampoline)); tr_link->trampoline));
bpf_trampoline_put(tr_link->trampoline); bpf_trampoline_put(tr_link->trampoline);
...@@ -2889,7 +2882,7 @@ static void bpf_tracing_link_release(struct bpf_link *link) ...@@ -2889,7 +2882,7 @@ static void bpf_tracing_link_release(struct bpf_link *link)
static void bpf_tracing_link_dealloc(struct bpf_link *link) static void bpf_tracing_link_dealloc(struct bpf_link *link)
{ {
struct bpf_tracing_link *tr_link = struct bpf_tracing_link *tr_link =
container_of(link, struct bpf_tracing_link, link); container_of(link, struct bpf_tracing_link, link.link);
kfree(tr_link); kfree(tr_link);
} }
...@@ -2898,7 +2891,7 @@ static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link, ...@@ -2898,7 +2891,7 @@ static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link,
struct seq_file *seq) struct seq_file *seq)
{ {
struct bpf_tracing_link *tr_link = struct bpf_tracing_link *tr_link =
container_of(link, struct bpf_tracing_link, link); container_of(link, struct bpf_tracing_link, link.link);
seq_printf(seq, seq_printf(seq,
"attach_type:\t%d\n", "attach_type:\t%d\n",
...@@ -2909,7 +2902,7 @@ static int bpf_tracing_link_fill_link_info(const struct bpf_link *link, ...@@ -2909,7 +2902,7 @@ static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
struct bpf_link_info *info) struct bpf_link_info *info)
{ {
struct bpf_tracing_link *tr_link = struct bpf_tracing_link *tr_link =
container_of(link, struct bpf_tracing_link, link); container_of(link, struct bpf_tracing_link, link.link);
info->tracing.attach_type = tr_link->attach_type; info->tracing.attach_type = tr_link->attach_type;
bpf_trampoline_unpack_key(tr_link->trampoline->key, bpf_trampoline_unpack_key(tr_link->trampoline->key,
...@@ -2990,7 +2983,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, ...@@ -2990,7 +2983,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
err = -ENOMEM; err = -ENOMEM;
goto out_put_prog; goto out_put_prog;
} }
bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING, bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING,
&bpf_tracing_link_lops, prog); &bpf_tracing_link_lops, prog);
link->attach_type = prog->expected_attach_type; link->attach_type = prog->expected_attach_type;
...@@ -3060,11 +3053,11 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, ...@@ -3060,11 +3053,11 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
tgt_prog = prog->aux->dst_prog; tgt_prog = prog->aux->dst_prog;
} }
err = bpf_link_prime(&link->link, &link_primer); err = bpf_link_prime(&link->link.link, &link_primer);
if (err) if (err)
goto out_unlock; goto out_unlock;
err = bpf_trampoline_link_prog(prog, tr); err = bpf_trampoline_link_prog(&link->link, tr);
if (err) { if (err) {
bpf_link_cleanup(&link_primer); bpf_link_cleanup(&link_primer);
link = NULL; link = NULL;
......
kernel/bpf/trampoline.c
View file @ f7e0beaf
...@@ -168,30 +168,30 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr) ...@@ -168,30 +168,30 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
return ret; return ret;
} }
static struct bpf_tramp_progs * static struct bpf_tramp_links *
bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg) bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg)
{ {
const struct bpf_prog_aux *aux; struct bpf_tramp_link *link;
struct bpf_tramp_progs *tprogs; struct bpf_tramp_links *tlinks;
struct bpf_prog **progs; struct bpf_tramp_link **links;
int kind; int kind;
*total = 0; *total = 0;
tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL); tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
if (!tprogs) if (!tlinks)
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
for (kind = 0; kind < BPF_TRAMP_MAX; kind++) { for (kind = 0; kind < BPF_TRAMP_MAX; kind++) {
tprogs[kind].nr_progs = tr->progs_cnt[kind]; tlinks[kind].nr_links = tr->progs_cnt[kind];
*total += tr->progs_cnt[kind]; *total += tr->progs_cnt[kind];
progs = tprogs[kind].progs; links = tlinks[kind].links;
hlist_for_each_entry(aux, &tr->progs_hlist[kind], tramp_hlist) { hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) {
*ip_arg |= aux->prog->call_get_func_ip; *ip_arg |= link->link.prog->call_get_func_ip;
*progs++ = aux->prog; *links++ = link;
} }
} }
return tprogs; return tlinks;
} }
static void __bpf_tramp_image_put_deferred(struct work_struct *work) static void __bpf_tramp_image_put_deferred(struct work_struct *work)
...@@ -330,14 +330,14 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx) ...@@ -330,14 +330,14 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx)
static int bpf_trampoline_update(struct bpf_trampoline *tr) static int bpf_trampoline_update(struct bpf_trampoline *tr)
{ {
struct bpf_tramp_image *im; struct bpf_tramp_image *im;
struct bpf_tramp_progs *tprogs; struct bpf_tramp_links *tlinks;
u32 flags = BPF_TRAMP_F_RESTORE_REGS; u32 flags = BPF_TRAMP_F_RESTORE_REGS;
bool ip_arg = false; bool ip_arg = false;
int err, total; int err, total;
tprogs = bpf_trampoline_get_progs(tr, &total, &ip_arg); tlinks = bpf_trampoline_get_progs(tr, &total, &ip_arg);
if (IS_ERR(tprogs)) if (IS_ERR(tlinks))
return PTR_ERR(tprogs); return PTR_ERR(tlinks);
if (total == 0) { if (total == 0) {
err = unregister_fentry(tr, tr->cur_image->image); err = unregister_fentry(tr, tr->cur_image->image);
...@@ -353,15 +353,15 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) ...@@ -353,15 +353,15 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr)
goto out; goto out;
} }
if (tprogs[BPF_TRAMP_FEXIT].nr_progs || if (tlinks[BPF_TRAMP_FEXIT].nr_links ||
tprogs[BPF_TRAMP_MODIFY_RETURN].nr_progs) tlinks[BPF_TRAMP_MODIFY_RETURN].nr_links)
flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME; flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;
if (ip_arg) if (ip_arg)
flags |= BPF_TRAMP_F_IP_ARG; flags |= BPF_TRAMP_F_IP_ARG;
err = arch_prepare_bpf_trampoline(im, im->image, im->image + PAGE_SIZE, err = arch_prepare_bpf_trampoline(im, im->image, im->image + PAGE_SIZE,
&tr->func.model, flags, tprogs, &tr->func.model, flags, tlinks,
tr->func.addr); tr->func.addr);
if (err < 0) if (err < 0)
goto out; goto out;
...@@ -381,7 +381,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) ...@@ -381,7 +381,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr)
tr->cur_image = im; tr->cur_image = im;
tr->selector++; tr->selector++;
out: out:
kfree(tprogs); kfree(tlinks);
return err; return err;
} }
...@@ -407,13 +407,14 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) ...@@ -407,13 +407,14 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
} }
} }
int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr) int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
{ {
enum bpf_tramp_prog_type kind; enum bpf_tramp_prog_type kind;
struct bpf_tramp_link *link_exiting;
int err = 0; int err = 0;
int cnt; int cnt;
kind = bpf_attach_type_to_tramp(prog); kind = bpf_attach_type_to_tramp(link->link.prog);
mutex_lock(&tr->mutex); mutex_lock(&tr->mutex);
if (tr->extension_prog) { if (tr->extension_prog) {
/* cannot attach fentry/fexit if extension prog is attached. /* cannot attach fentry/fexit if extension prog is attached.
...@@ -429,25 +430,33 @@ int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr) ...@@ -429,25 +430,33 @@ int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
err = -EBUSY; err = -EBUSY;
goto out; goto out;
} }
tr->extension_prog = prog; tr->extension_prog = link->link.prog;
err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL, err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL,
prog->bpf_func); link->link.prog->bpf_func);
goto out; goto out;
} }
if (cnt >= BPF_MAX_TRAMP_PROGS) { if (cnt >= BPF_MAX_TRAMP_LINKS) {
err = -E2BIG; err = -E2BIG;
goto out; goto out;
} }
if (!hlist_unhashed(&prog->aux->tramp_hlist)) { if (!hlist_unhashed(&link->tramp_hlist)) {
/* prog already linked */ /* prog already linked */
err = -EBUSY; err = -EBUSY;
goto out; goto out;
} }
hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]); hlist_for_each_entry(link_exiting, &tr->progs_hlist[kind], tramp_hlist) {
if (link_exiting->link.prog != link->link.prog)
continue;
/* prog already linked */
err = -EBUSY;
goto out;
}
hlist_add_head(&link->tramp_hlist, &tr->progs_hlist[kind]);
tr->progs_cnt[kind]++; tr->progs_cnt[kind]++;
err = bpf_trampoline_update(tr); err = bpf_trampoline_update(tr);
if (err) { if (err) {
hlist_del_init(&prog->aux->tramp_hlist); hlist_del_init(&link->tramp_hlist);
tr->progs_cnt[kind]--; tr->progs_cnt[kind]--;
} }
out: out:
...@@ -456,12 +465,12 @@ int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr) ...@@ -456,12 +465,12 @@ int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
} }
/* bpf_trampoline_unlink_prog() should never fail. */ /* bpf_trampoline_unlink_prog() should never fail. */
int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr) int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
{ {
enum bpf_tramp_prog_type kind; enum bpf_tramp_prog_type kind;
int err; int err;
kind = bpf_attach_type_to_tramp(prog); kind = bpf_attach_type_to_tramp(link->link.prog);
mutex_lock(&tr->mutex); mutex_lock(&tr->mutex);
if (kind == BPF_TRAMP_REPLACE) { if (kind == BPF_TRAMP_REPLACE) {
WARN_ON_ONCE(!tr->extension_prog); WARN_ON_ONCE(!tr->extension_prog);
...@@ -470,7 +479,7 @@ int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr) ...@@ -470,7 +479,7 @@ int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
tr->extension_prog = NULL; tr->extension_prog = NULL;
goto out; goto out;
} }
hlist_del_init(&prog->aux->tramp_hlist); hlist_del_init(&link->tramp_hlist);
tr->progs_cnt[kind]--; tr->progs_cnt[kind]--;
err = bpf_trampoline_update(tr); err = bpf_trampoline_update(tr);
out: out:
...@@ -635,7 +644,7 @@ void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr) ...@@ -635,7 +644,7 @@ void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr)
int __weak int __weak
arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end, arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end,
const struct btf_func_model *m, u32 flags, const struct btf_func_model *m, u32 flags,
struct bpf_tramp_progs *tprogs, struct bpf_tramp_links *tlinks,
void *orig_call) void *orig_call)
{ {
return -ENOTSUPP; return -ENOTSUPP;
......
net/bpf/bpf_dummy_struct_ops.c
View file @ f7e0beaf
...@@ -72,13 +72,16 @@ static int dummy_ops_call_op(void *image, struct bpf_dummy_ops_test_args *args) ...@@ -72,13 +72,16 @@ static int dummy_ops_call_op(void *image, struct bpf_dummy_ops_test_args *args)
args->args[3], args->args[4]); args->args[3], args->args[4]);
} }
extern const struct bpf_link_ops bpf_struct_ops_link_lops;
int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
union bpf_attr __user *uattr) union bpf_attr __user *uattr)
{ {
const struct bpf_struct_ops *st_ops = &bpf_bpf_dummy_ops; const struct bpf_struct_ops *st_ops = &bpf_bpf_dummy_ops;
const struct btf_type *func_proto; const struct btf_type *func_proto;
struct bpf_dummy_ops_test_args *args; struct bpf_dummy_ops_test_args *args;
struct bpf_tramp_progs *tprogs; struct bpf_tramp_links *tlinks;
struct bpf_tramp_link *link = NULL;
void *image = NULL; void *image = NULL;
unsigned int op_idx; unsigned int op_idx;
int prog_ret; int prog_ret;
...@@ -92,8 +95,8 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, ...@@ -92,8 +95,8 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
if (IS_ERR(args)) if (IS_ERR(args))
return PTR_ERR(args); return PTR_ERR(args);
tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL); tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
if (!tprogs) { if (!tlinks) {
err = -ENOMEM; err = -ENOMEM;
goto out; goto out;
} }
...@@ -105,8 +108,17 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, ...@@ -105,8 +108,17 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
} }
set_vm_flush_reset_perms(image); set_vm_flush_reset_perms(image);
link = kzalloc(sizeof(*link), GFP_USER);
if (!link) {
err = -ENOMEM;
goto out;
}
/* prog doesn't take the ownership of the reference from caller */
bpf_prog_inc(prog);
bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, &bpf_struct_ops_link_lops, prog);
op_idx = prog->expected_attach_type; op_idx = prog->expected_attach_type;
err = bpf_struct_ops_prepare_trampoline(tprogs, prog, err = bpf_struct_ops_prepare_trampoline(tlinks, link,
&st_ops->func_models[op_idx], &st_ops->func_models[op_idx],
image, image + PAGE_SIZE); image, image + PAGE_SIZE);
if (err < 0) if (err < 0)
...@@ -124,7 +136,9 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, ...@@ -124,7 +136,9 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
out: out:
kfree(args); kfree(args);
bpf_jit_free_exec(image); bpf_jit_free_exec(image);
kfree(tprogs); if (link)
bpf_link_put(&link->link);
kfree(tlinks);
return err; return err;
} }
......
tools/bpf/bpftool/link.c
View file @ f7e0beaf
...@@ -23,6 +23,7 @@ static const char * const link_type_name[] = { ...@@ -23,6 +23,7 @@ static const char * const link_type_name[] = {
[BPF_LINK_TYPE_XDP] = "xdp", [BPF_LINK_TYPE_XDP] = "xdp",
[BPF_LINK_TYPE_PERF_EVENT] = "perf_event", [BPF_LINK_TYPE_PERF_EVENT] = "perf_event",
[BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi", [BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi",
[BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops",
}; };
static struct hashmap *link_table; static struct hashmap *link_table;
......
tools/include/uapi/linux/bpf.h
View file @ f7e0beaf
...@@ -1013,6 +1013,7 @@ enum bpf_link_type { ...@@ -1013,6 +1013,7 @@ enum bpf_link_type {
BPF_LINK_TYPE_XDP = 6, BPF_LINK_TYPE_XDP = 6,
BPF_LINK_TYPE_PERF_EVENT = 7, BPF_LINK_TYPE_PERF_EVENT = 7,
BPF_LINK_TYPE_KPROBE_MULTI = 8, BPF_LINK_TYPE_KPROBE_MULTI = 8,
BPF_LINK_TYPE_STRUCT_OPS = 9,
MAX_BPF_LINK_TYPE, MAX_BPF_LINK_TYPE,
}; };
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7