Commit 638f5b90 authored by Alexei Starovoitov's avatar Alexei Starovoitov Committed by David S. Miller

bpf: reduce verifier memory consumption

the verifier got progressively smarter over time and size of its internal
state grew as well. Time to reduce the memory consumption.

Before:
sizeof(struct bpf_verifier_state) = 6520
After:
sizeof(struct bpf_verifier_state) = 896

It's done by observing that majority of BPF programs use little to
no stack whereas verifier kept all of 512 stack slots ready always.
Instead dynamically reallocate struct verifier state when stack
access is detected.
Runtime difference before vs after is within a noise.
The number of processed instructions stays the same.
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 3179698d
...@@ -76,9 +76,9 @@ nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, ...@@ -76,9 +76,9 @@ nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
static int static int
nfp_bpf_check_exit(struct nfp_prog *nfp_prog, nfp_bpf_check_exit(struct nfp_prog *nfp_prog,
const struct bpf_verifier_env *env) struct bpf_verifier_env *env)
{ {
const struct bpf_reg_state *reg0 = &env->cur_state.regs[0]; const struct bpf_reg_state *reg0 = cur_regs(env) + BPF_REG_0;
u64 imm; u64 imm;
if (nfp_prog->act == NN_ACT_XDP) if (nfp_prog->act == NN_ACT_XDP)
...@@ -144,9 +144,9 @@ nfp_bpf_check_stack_access(struct nfp_prog *nfp_prog, ...@@ -144,9 +144,9 @@ nfp_bpf_check_stack_access(struct nfp_prog *nfp_prog,
static int static int
nfp_bpf_check_ptr(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_bpf_check_ptr(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
const struct bpf_verifier_env *env, u8 reg_no) struct bpf_verifier_env *env, u8 reg_no)
{ {
const struct bpf_reg_state *reg = &env->cur_state.regs[reg_no]; const struct bpf_reg_state *reg = cur_regs(env) + reg_no;
int err; int err;
if (reg->type != PTR_TO_CTX && if (reg->type != PTR_TO_CTX &&
......
...@@ -88,14 +88,19 @@ enum bpf_stack_slot_type { ...@@ -88,14 +88,19 @@ enum bpf_stack_slot_type {
#define BPF_REG_SIZE 8 /* size of eBPF register in bytes */ #define BPF_REG_SIZE 8 /* size of eBPF register in bytes */
struct bpf_stack_state {
struct bpf_reg_state spilled_ptr;
u8 slot_type[BPF_REG_SIZE];
};
/* state of the program: /* state of the program:
* type of all registers and stack info * type of all registers and stack info
*/ */
struct bpf_verifier_state { struct bpf_verifier_state {
struct bpf_reg_state regs[MAX_BPF_REG]; struct bpf_reg_state regs[MAX_BPF_REG];
u8 stack_slot_type[MAX_BPF_STACK];
struct bpf_reg_state spilled_regs[MAX_BPF_STACK / BPF_REG_SIZE];
struct bpf_verifier_state *parent; struct bpf_verifier_state *parent;
int allocated_stack;
struct bpf_stack_state *stack;
}; };
/* linked list of verifier states used to prune search */ /* linked list of verifier states used to prune search */
...@@ -145,7 +150,7 @@ struct bpf_verifier_env { ...@@ -145,7 +150,7 @@ struct bpf_verifier_env {
struct bpf_verifier_stack_elem *head; /* stack of verifier states to be processed */ struct bpf_verifier_stack_elem *head; /* stack of verifier states to be processed */
int stack_size; /* number of states to be processed */ int stack_size; /* number of states to be processed */
bool strict_alignment; /* perform strict pointer alignment checks */ bool strict_alignment; /* perform strict pointer alignment checks */
struct bpf_verifier_state cur_state; /* current verifier state */ struct bpf_verifier_state *cur_state; /* current verifier state */
struct bpf_verifier_state_list **explored_states; /* search pruning optimization */ struct bpf_verifier_state_list **explored_states; /* search pruning optimization */
const struct bpf_ext_analyzer_ops *analyzer_ops; /* external analyzer ops */ const struct bpf_ext_analyzer_ops *analyzer_ops; /* external analyzer ops */
void *analyzer_priv; /* pointer to external analyzer's private data */ void *analyzer_priv; /* pointer to external analyzer's private data */
...@@ -159,6 +164,11 @@ struct bpf_verifier_env { ...@@ -159,6 +164,11 @@ struct bpf_verifier_env {
struct bpf_verifer_log log; struct bpf_verifer_log log;
}; };
static inline struct bpf_reg_state *cur_regs(struct bpf_verifier_env *env)
{
return env->cur_state->regs;
}
int bpf_analyzer(struct bpf_prog *prog, const struct bpf_ext_analyzer_ops *ops, int bpf_analyzer(struct bpf_prog *prog, const struct bpf_ext_analyzer_ops *ops,
void *priv); void *priv);
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment