Commit 0a4c58f5 authored by Roman Gushchin's avatar Roman Gushchin Committed by Daniel Borkmann

bpf: add ability to charge bpf maps memory dynamically

This commits extends existing bpf maps memory charging API
to support dynamic charging/uncharging.

This is required to account memory used by maps,
if all entries are created dynamically after
the map initialization.
Signed-off-by: default avatarRoman Gushchin <guro@fb.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: default avatarMartin KaFai Lau <kafai@fb.com>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parent fbeb1603
...@@ -435,6 +435,8 @@ struct bpf_map * __must_check bpf_map_inc(struct bpf_map *map, bool uref); ...@@ -435,6 +435,8 @@ struct bpf_map * __must_check bpf_map_inc(struct bpf_map *map, bool uref);
void bpf_map_put_with_uref(struct bpf_map *map); void bpf_map_put_with_uref(struct bpf_map *map);
void bpf_map_put(struct bpf_map *map); void bpf_map_put(struct bpf_map *map);
int bpf_map_precharge_memlock(u32 pages); int bpf_map_precharge_memlock(u32 pages);
int bpf_map_charge_memlock(struct bpf_map *map, u32 pages);
void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages);
void *bpf_map_area_alloc(size_t size, int numa_node); void *bpf_map_area_alloc(size_t size, int numa_node);
void bpf_map_area_free(void *base); void bpf_map_area_free(void *base);
void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr); void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
......
...@@ -181,32 +181,60 @@ int bpf_map_precharge_memlock(u32 pages) ...@@ -181,32 +181,60 @@ int bpf_map_precharge_memlock(u32 pages)
return 0; return 0;
} }
static int bpf_map_charge_memlock(struct bpf_map *map) static int bpf_charge_memlock(struct user_struct *user, u32 pages)
{ {
struct user_struct *user = get_current_user(); unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
unsigned long memlock_limit;
memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) {
atomic_long_sub(pages, &user->locked_vm);
return -EPERM;
}
return 0;
}
atomic_long_add(map->pages, &user->locked_vm); static void bpf_uncharge_memlock(struct user_struct *user, u32 pages)
{
atomic_long_sub(pages, &user->locked_vm);
}
static int bpf_map_init_memlock(struct bpf_map *map)
{
struct user_struct *user = get_current_user();
int ret;
if (atomic_long_read(&user->locked_vm) > memlock_limit) { ret = bpf_charge_memlock(user, map->pages);
atomic_long_sub(map->pages, &user->locked_vm); if (ret) {
free_uid(user); free_uid(user);
return -EPERM; return ret;
} }
map->user = user; map->user = user;
return 0; return ret;
} }
static void bpf_map_uncharge_memlock(struct bpf_map *map) static void bpf_map_release_memlock(struct bpf_map *map)
{ {
struct user_struct *user = map->user; struct user_struct *user = map->user;
bpf_uncharge_memlock(user, map->pages);
atomic_long_sub(map->pages, &user->locked_vm);
free_uid(user); free_uid(user);
} }
int bpf_map_charge_memlock(struct bpf_map *map, u32 pages)
{
int ret;
ret = bpf_charge_memlock(map->user, pages);
if (ret)
return ret;
map->pages += pages;
return ret;
}
void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages)
{
bpf_uncharge_memlock(map->user, pages);
map->pages -= pages;
}
static int bpf_map_alloc_id(struct bpf_map *map) static int bpf_map_alloc_id(struct bpf_map *map)
{ {
int id; int id;
...@@ -256,7 +284,7 @@ static void bpf_map_free_deferred(struct work_struct *work) ...@@ -256,7 +284,7 @@ static void bpf_map_free_deferred(struct work_struct *work)
{ {
struct bpf_map *map = container_of(work, struct bpf_map, work); struct bpf_map *map = container_of(work, struct bpf_map, work);
bpf_map_uncharge_memlock(map); bpf_map_release_memlock(map);
security_bpf_map_free(map); security_bpf_map_free(map);
/* implementation dependent freeing */ /* implementation dependent freeing */
map->ops->map_free(map); map->ops->map_free(map);
...@@ -492,7 +520,7 @@ static int map_create(union bpf_attr *attr) ...@@ -492,7 +520,7 @@ static int map_create(union bpf_attr *attr)
if (err) if (err)
goto free_map_nouncharge; goto free_map_nouncharge;
err = bpf_map_charge_memlock(map); err = bpf_map_init_memlock(map);
if (err) if (err)
goto free_map_sec; goto free_map_sec;
...@@ -515,7 +543,7 @@ static int map_create(union bpf_attr *attr) ...@@ -515,7 +543,7 @@ static int map_create(union bpf_attr *attr)
return err; return err;
free_map: free_map:
bpf_map_uncharge_memlock(map); bpf_map_release_memlock(map);
free_map_sec: free_map_sec:
security_bpf_map_free(map); security_bpf_map_free(map);
free_map_nouncharge: free_map_nouncharge:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment