Commit 4d44175a authored by Florian Westphal's avatar Florian Westphal Committed by Pablo Neira Ayuso

netfilter: nf_tables: handle nft_object lookups via rhltable

Instead of linear search, use rhlist interface to look up the objects.
This fixes rulesets with thousands of named objects (quota, counters and
the like).

We only use a single table for this and consider the address of the
table we're doing the lookup in as a part of the key.

This reduces restore time of a sample ruleset with ~20k named counters
from 37 seconds to 0.8 seconds.
Signed-off-by: default avatarFlorian Westphal <fw@strlen.de>
Signed-off-by: default avatarPablo Neira Ayuso <pablo@netfilter.org>
parent d152159b
...@@ -1027,6 +1027,7 @@ struct nft_object_hash_key { ...@@ -1027,6 +1027,7 @@ struct nft_object_hash_key {
* *
* @list: table stateful object list node * @list: table stateful object list node
* @key: keys that identify this object * @key: keys that identify this object
* @rhlhead: nft_objname_ht node
* @genmask: generation mask * @genmask: generation mask
* @use: number of references to this stateful object * @use: number of references to this stateful object
* @handle: unique object handle * @handle: unique object handle
...@@ -1035,6 +1036,7 @@ struct nft_object_hash_key { ...@@ -1035,6 +1036,7 @@ struct nft_object_hash_key {
*/ */
struct nft_object { struct nft_object {
struct list_head list; struct list_head list;
struct rhlist_head rhlhead;
struct nft_object_hash_key key; struct nft_object_hash_key key;
u32 genmask:2, u32 genmask:2,
use:30; use:30;
...@@ -1052,7 +1054,8 @@ static inline void *nft_obj_data(const struct nft_object *obj) ...@@ -1052,7 +1054,8 @@ static inline void *nft_obj_data(const struct nft_object *obj)
#define nft_expr_obj(expr) *((struct nft_object **)nft_expr_priv(expr)) #define nft_expr_obj(expr) *((struct nft_object **)nft_expr_priv(expr))
struct nft_object *nft_obj_lookup(const struct nft_table *table, struct nft_object *nft_obj_lookup(const struct net *net,
const struct nft_table *table,
const struct nlattr *nla, u32 objtype, const struct nlattr *nla, u32 objtype,
u8 genmask); u8 genmask);
......
...@@ -37,10 +37,16 @@ enum { ...@@ -37,10 +37,16 @@ enum {
NFT_VALIDATE_DO, NFT_VALIDATE_DO,
}; };
static struct rhltable nft_objname_ht;
static u32 nft_chain_hash(const void *data, u32 len, u32 seed); static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed); static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *); static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
static const struct rhashtable_params nft_chain_ht_params = { static const struct rhashtable_params nft_chain_ht_params = {
.head_offset = offsetof(struct nft_chain, rhlhead), .head_offset = offsetof(struct nft_chain, rhlhead),
.key_offset = offsetof(struct nft_chain, name), .key_offset = offsetof(struct nft_chain, name),
...@@ -51,6 +57,15 @@ static const struct rhashtable_params nft_chain_ht_params = { ...@@ -51,6 +57,15 @@ static const struct rhashtable_params nft_chain_ht_params = {
.automatic_shrinking = true, .automatic_shrinking = true,
}; };
static const struct rhashtable_params nft_objname_ht_params = {
.head_offset = offsetof(struct nft_object, rhlhead),
.key_offset = offsetof(struct nft_object, key),
.hashfn = nft_objname_hash,
.obj_hashfn = nft_objname_hash_obj,
.obj_cmpfn = nft_objname_hash_cmp,
.automatic_shrinking = true,
};
static void nft_validate_state_update(struct net *net, u8 new_validate_state) static void nft_validate_state_update(struct net *net, u8 new_validate_state)
{ {
switch (net->nft.validate_state) { switch (net->nft.validate_state) {
...@@ -814,6 +829,34 @@ static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg, ...@@ -814,6 +829,34 @@ static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
return strcmp(chain->name, name); return strcmp(chain->name, name);
} }
static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
{
const struct nft_object_hash_key *k = data;
seed ^= hash_ptr(k->table, 32);
return jhash(k->name, strlen(k->name), seed);
}
static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
{
const struct nft_object *obj = data;
return nft_objname_hash(&obj->key, 0, seed);
}
static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
const void *ptr)
{
const struct nft_object_hash_key *k = arg->key;
const struct nft_object *obj = ptr;
if (obj->key.table != k->table)
return -1;
return strcmp(obj->key.name, k->name);
}
static int nf_tables_newtable(struct net *net, struct sock *nlsk, static int nf_tables_newtable(struct net *net, struct sock *nlsk,
struct sk_buff *skb, const struct nlmsghdr *nlh, struct sk_buff *skb, const struct nlmsghdr *nlh,
const struct nlattr * const nla[], const struct nlattr * const nla[],
...@@ -1070,7 +1113,7 @@ nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask) ...@@ -1070,7 +1113,7 @@ nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
return ERR_PTR(-ENOENT); return ERR_PTR(-ENOENT);
} }
static bool lockdep_commit_lock_is_held(struct net *net) static bool lockdep_commit_lock_is_held(const struct net *net)
{ {
#ifdef CONFIG_PROVE_LOCKING #ifdef CONFIG_PROVE_LOCKING
return lockdep_is_held(&net->nft.commit_mutex); return lockdep_is_held(&net->nft.commit_mutex);
...@@ -4386,7 +4429,8 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, ...@@ -4386,7 +4429,8 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
err = -EINVAL; err = -EINVAL;
goto err2; goto err2;
} }
obj = nft_obj_lookup(ctx->table, nla[NFTA_SET_ELEM_OBJREF], obj = nft_obj_lookup(ctx->net, ctx->table,
nla[NFTA_SET_ELEM_OBJREF],
set->objtype, genmask); set->objtype, genmask);
if (IS_ERR(obj)) { if (IS_ERR(obj)) {
err = PTR_ERR(obj); err = PTR_ERR(obj);
...@@ -4819,18 +4863,36 @@ void nft_unregister_obj(struct nft_object_type *obj_type) ...@@ -4819,18 +4863,36 @@ void nft_unregister_obj(struct nft_object_type *obj_type)
} }
EXPORT_SYMBOL_GPL(nft_unregister_obj); EXPORT_SYMBOL_GPL(nft_unregister_obj);
struct nft_object *nft_obj_lookup(const struct nft_table *table, struct nft_object *nft_obj_lookup(const struct net *net,
const struct nft_table *table,
const struct nlattr *nla, u32 objtype, const struct nlattr *nla, u32 objtype,
u8 genmask) u8 genmask)
{ {
struct nft_object_hash_key k = { .table = table };
char search[NFT_OBJ_MAXNAMELEN];
struct rhlist_head *tmp, *list;
struct nft_object *obj; struct nft_object *obj;
list_for_each_entry_rcu(obj, &table->objects, list) { nla_strlcpy(search, nla, sizeof(search));
if (!nla_strcmp(nla, obj->key.name) && k.name = search;
objtype == obj->ops->type->type &&
nft_active_genmask(obj, genmask)) WARN_ON_ONCE(!rcu_read_lock_held() &&
!lockdep_commit_lock_is_held(net));
rcu_read_lock();
list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
if (!list)
goto out;
rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
if (objtype == obj->ops->type->type &&
nft_active_genmask(obj, genmask)) {
rcu_read_unlock();
return obj; return obj;
} }
}
out:
rcu_read_unlock();
return ERR_PTR(-ENOENT); return ERR_PTR(-ENOENT);
} }
EXPORT_SYMBOL_GPL(nft_obj_lookup); EXPORT_SYMBOL_GPL(nft_obj_lookup);
...@@ -4988,7 +5050,7 @@ static int nf_tables_newobj(struct net *net, struct sock *nlsk, ...@@ -4988,7 +5050,7 @@ static int nf_tables_newobj(struct net *net, struct sock *nlsk,
} }
objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE])); objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
obj = nft_obj_lookup(table, nla[NFTA_OBJ_NAME], objtype, genmask); obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
if (IS_ERR(obj)) { if (IS_ERR(obj)) {
err = PTR_ERR(obj); err = PTR_ERR(obj);
if (err != -ENOENT) { if (err != -ENOENT) {
...@@ -5027,9 +5089,18 @@ static int nf_tables_newobj(struct net *net, struct sock *nlsk, ...@@ -5027,9 +5089,18 @@ static int nf_tables_newobj(struct net *net, struct sock *nlsk,
if (err < 0) if (err < 0)
goto err3; goto err3;
err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
nft_objname_ht_params);
if (err < 0)
goto err4;
list_add_tail_rcu(&obj->list, &table->objects); list_add_tail_rcu(&obj->list, &table->objects);
table->use++; table->use++;
return 0; return 0;
err4:
/* queued in transaction log */
INIT_LIST_HEAD(&obj->list);
return err;
err3: err3:
kfree(obj->key.name); kfree(obj->key.name);
err2: err2:
...@@ -5215,7 +5286,7 @@ static int nf_tables_getobj(struct net *net, struct sock *nlsk, ...@@ -5215,7 +5286,7 @@ static int nf_tables_getobj(struct net *net, struct sock *nlsk,
} }
objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE])); objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
obj = nft_obj_lookup(table, nla[NFTA_OBJ_NAME], objtype, genmask); obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
if (IS_ERR(obj)) { if (IS_ERR(obj)) {
NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]); NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
return PTR_ERR(obj); return PTR_ERR(obj);
...@@ -5280,7 +5351,7 @@ static int nf_tables_delobj(struct net *net, struct sock *nlsk, ...@@ -5280,7 +5351,7 @@ static int nf_tables_delobj(struct net *net, struct sock *nlsk,
obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask); obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
} else { } else {
attr = nla[NFTA_OBJ_NAME]; attr = nla[NFTA_OBJ_NAME];
obj = nft_obj_lookup(table, attr, objtype, genmask); obj = nft_obj_lookup(net, table, attr, objtype, genmask);
} }
if (IS_ERR(obj)) { if (IS_ERR(obj)) {
...@@ -6406,6 +6477,7 @@ static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain) ...@@ -6406,6 +6477,7 @@ static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
static void nft_obj_del(struct nft_object *obj) static void nft_obj_del(struct nft_object *obj)
{ {
rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
list_del_rcu(&obj->list); list_del_rcu(&obj->list);
} }
...@@ -6721,7 +6793,7 @@ static int __nf_tables_abort(struct net *net) ...@@ -6721,7 +6793,7 @@ static int __nf_tables_abort(struct net *net)
break; break;
case NFT_MSG_NEWOBJ: case NFT_MSG_NEWOBJ:
trans->ctx.table->use--; trans->ctx.table->use--;
list_del_rcu(&nft_trans_obj(trans)->list); nft_obj_del(nft_trans_obj(trans));
break; break;
case NFT_MSG_DELOBJ: case NFT_MSG_DELOBJ:
trans->ctx.table->use++; trans->ctx.table->use++;
...@@ -7397,12 +7469,18 @@ static int __init nf_tables_module_init(void) ...@@ -7397,12 +7469,18 @@ static int __init nf_tables_module_init(void)
if (err < 0) if (err < 0)
goto err3; goto err3;
err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
if (err < 0)
goto err4;
/* must be last */ /* must be last */
err = nfnetlink_subsys_register(&nf_tables_subsys); err = nfnetlink_subsys_register(&nf_tables_subsys);
if (err < 0) if (err < 0)
goto err4; goto err5;
return err; return err;
err5:
rhltable_destroy(&nft_objname_ht);
err4: err4:
unregister_netdevice_notifier(&nf_tables_flowtable_notifier); unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
err3: err3:
...@@ -7422,6 +7500,7 @@ static void __exit nf_tables_module_exit(void) ...@@ -7422,6 +7500,7 @@ static void __exit nf_tables_module_exit(void)
unregister_pernet_subsys(&nf_tables_net_ops); unregister_pernet_subsys(&nf_tables_net_ops);
cancel_work_sync(&trans_destroy_work); cancel_work_sync(&trans_destroy_work);
rcu_barrier(); rcu_barrier();
rhltable_destroy(&nft_objname_ht);
nf_tables_core_module_exit(); nf_tables_core_module_exit();
} }
......
...@@ -38,7 +38,8 @@ static int nft_objref_init(const struct nft_ctx *ctx, ...@@ -38,7 +38,8 @@ static int nft_objref_init(const struct nft_ctx *ctx,
return -EINVAL; return -EINVAL;
objtype = ntohl(nla_get_be32(tb[NFTA_OBJREF_IMM_TYPE])); objtype = ntohl(nla_get_be32(tb[NFTA_OBJREF_IMM_TYPE]));
obj = nft_obj_lookup(ctx->table, tb[NFTA_OBJREF_IMM_NAME], objtype, obj = nft_obj_lookup(ctx->net, ctx->table,
tb[NFTA_OBJREF_IMM_NAME], objtype,
genmask); genmask);
if (IS_ERR(obj)) if (IS_ERR(obj))
return -ENOENT; return -ENOENT;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment