Commit b0f38338 authored by Florian Westphal's avatar Florian Westphal Committed by Pablo Neira Ayuso

netfilter: reduce size of hook entry point locations

struct net contains:

struct nf_hook_entries __rcu *hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS];

which store the hook entry point locations for the various protocol
families and the hooks.

Using array results in compact c code when doing accesses, i.e.
  x = rcu_dereference(net->nf.hooks[pf][hook]);

but its also wasting a lot of memory, as most families are
not used.

So split the array into those families that are used, which
are only 5 (instead of 13).  In most cases, the 'pf' argument is
constant, i.e. gcc removes switch statement.

struct net before:
 /* size: 5184, cachelines: 81, members: 46 */
after:
 /* size: 4672, cachelines: 73, members: 46 */
Signed-off-by: default avatarFlorian Westphal <fw@strlen.de>
Signed-off-by: default avatarPablo Neira Ayuso <pablo@netfilter.org>
parent 8c873e21
...@@ -195,7 +195,7 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net, ...@@ -195,7 +195,7 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net,
struct net_device *indev, struct net_device *outdev, struct net_device *indev, struct net_device *outdev,
int (*okfn)(struct net *, struct sock *, struct sk_buff *)) int (*okfn)(struct net *, struct sock *, struct sk_buff *))
{ {
struct nf_hook_entries *hook_head; struct nf_hook_entries *hook_head = NULL;
int ret = 1; int ret = 1;
#ifdef HAVE_JUMP_LABEL #ifdef HAVE_JUMP_LABEL
...@@ -206,7 +206,27 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net, ...@@ -206,7 +206,27 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net,
#endif #endif
rcu_read_lock(); rcu_read_lock();
hook_head = rcu_dereference(net->nf.hooks[pf][hook]); switch (pf) {
case NFPROTO_IPV4:
hook_head = rcu_dereference(net->nf.hooks_ipv4[hook]);
break;
case NFPROTO_IPV6:
hook_head = rcu_dereference(net->nf.hooks_ipv6[hook]);
break;
case NFPROTO_ARP:
hook_head = rcu_dereference(net->nf.hooks_arp[hook]);
break;
case NFPROTO_BRIDGE:
hook_head = rcu_dereference(net->nf.hooks_bridge[hook]);
break;
case NFPROTO_DECNET:
hook_head = rcu_dereference(net->nf.hooks_decnet[hook]);
break;
default:
WARN_ON_ONCE(1);
break;
}
if (hook_head) { if (hook_head) {
struct nf_hook_state state; struct nf_hook_state state;
......
...@@ -17,7 +17,11 @@ struct netns_nf { ...@@ -17,7 +17,11 @@ struct netns_nf {
#ifdef CONFIG_SYSCTL #ifdef CONFIG_SYSCTL
struct ctl_table_header *nf_log_dir_header; struct ctl_table_header *nf_log_dir_header;
#endif #endif
struct nf_hook_entries __rcu *hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; struct nf_hook_entries __rcu *hooks_ipv4[NF_MAX_HOOKS];
struct nf_hook_entries __rcu *hooks_ipv6[NF_MAX_HOOKS];
struct nf_hook_entries __rcu *hooks_arp[NF_MAX_HOOKS];
struct nf_hook_entries __rcu *hooks_bridge[NF_MAX_HOOKS];
struct nf_hook_entries __rcu *hooks_decnet[NF_MAX_HOOKS];
#if IS_ENABLED(CONFIG_NF_DEFRAG_IPV4) #if IS_ENABLED(CONFIG_NF_DEFRAG_IPV4)
bool defrag_ipv4; bool defrag_ipv4;
#endif #endif
......
...@@ -991,7 +991,7 @@ int br_nf_hook_thresh(unsigned int hook, struct net *net, ...@@ -991,7 +991,7 @@ int br_nf_hook_thresh(unsigned int hook, struct net *net,
unsigned int i; unsigned int i;
int ret; int ret;
e = rcu_dereference(net->nf.hooks[NFPROTO_BRIDGE][hook]); e = rcu_dereference(net->nf.hooks_bridge[hook]);
if (!e) if (!e)
return okfn(net, sk, skb); return okfn(net, sk, skb);
......
...@@ -264,8 +264,23 @@ static void *__nf_hook_entries_try_shrink(struct nf_hook_entries __rcu **pp) ...@@ -264,8 +264,23 @@ static void *__nf_hook_entries_try_shrink(struct nf_hook_entries __rcu **pp)
static struct nf_hook_entries __rcu **nf_hook_entry_head(struct net *net, const struct nf_hook_ops *reg) static struct nf_hook_entries __rcu **nf_hook_entry_head(struct net *net, const struct nf_hook_ops *reg)
{ {
if (reg->pf != NFPROTO_NETDEV) switch (reg->pf) {
return net->nf.hooks[reg->pf]+reg->hooknum; case NFPROTO_NETDEV:
break;
case NFPROTO_ARP:
return net->nf.hooks_arp + reg->hooknum;
case NFPROTO_BRIDGE:
return net->nf.hooks_bridge + reg->hooknum;
case NFPROTO_IPV4:
return net->nf.hooks_ipv4 + reg->hooknum;
case NFPROTO_IPV6:
return net->nf.hooks_ipv6 + reg->hooknum;
case NFPROTO_DECNET:
return net->nf.hooks_decnet + reg->hooknum;
default:
WARN_ON_ONCE(1);
return NULL;
}
#ifdef CONFIG_NETFILTER_INGRESS #ifdef CONFIG_NETFILTER_INGRESS
if (reg->hooknum == NF_NETDEV_INGRESS) { if (reg->hooknum == NF_NETDEV_INGRESS) {
...@@ -534,14 +549,21 @@ void (*nf_nat_decode_session_hook)(struct sk_buff *, struct flowi *); ...@@ -534,14 +549,21 @@ void (*nf_nat_decode_session_hook)(struct sk_buff *, struct flowi *);
EXPORT_SYMBOL(nf_nat_decode_session_hook); EXPORT_SYMBOL(nf_nat_decode_session_hook);
#endif #endif
static int __net_init netfilter_net_init(struct net *net) static void __net_init __netfilter_net_init(struct nf_hook_entries *e[NF_MAX_HOOKS])
{ {
int i, h; int h;
for (i = 0; i < ARRAY_SIZE(net->nf.hooks); i++) { for (h = 0; h < NF_MAX_HOOKS; h++)
for (h = 0; h < NF_MAX_HOOKS; h++) RCU_INIT_POINTER(e[h], NULL);
RCU_INIT_POINTER(net->nf.hooks[i][h], NULL); }
}
static int __net_init netfilter_net_init(struct net *net)
{
__netfilter_net_init(net->nf.hooks_ipv4);
__netfilter_net_init(net->nf.hooks_ipv6);
__netfilter_net_init(net->nf.hooks_arp);
__netfilter_net_init(net->nf.hooks_bridge);
__netfilter_net_init(net->nf.hooks_decnet);
#ifdef CONFIG_PROC_FS #ifdef CONFIG_PROC_FS
net->nf.proc_netfilter = proc_net_mkdir(net, "netfilter", net->nf.proc_netfilter = proc_net_mkdir(net, "netfilter",
......
...@@ -201,6 +201,23 @@ static unsigned int nf_iterate(struct sk_buff *skb, ...@@ -201,6 +201,23 @@ static unsigned int nf_iterate(struct sk_buff *skb,
return NF_ACCEPT; return NF_ACCEPT;
} }
static struct nf_hook_entries *nf_hook_entries_head(const struct net *net, u8 pf, u8 hooknum)
{
switch (pf) {
case NFPROTO_BRIDGE:
return rcu_dereference(net->nf.hooks_bridge[hooknum]);
case NFPROTO_IPV4:
return rcu_dereference(net->nf.hooks_ipv4[hooknum]);
case NFPROTO_IPV6:
return rcu_dereference(net->nf.hooks_ipv6[hooknum]);
default:
WARN_ON_ONCE(1);
return NULL;
}
return NULL;
}
/* Caller must hold rcu read-side lock */ /* Caller must hold rcu read-side lock */
void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict) void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict)
{ {
...@@ -216,12 +233,12 @@ void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict) ...@@ -216,12 +233,12 @@ void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict)
net = entry->state.net; net = entry->state.net;
pf = entry->state.pf; pf = entry->state.pf;
hooks = rcu_dereference(net->nf.hooks[pf][entry->state.hook]); hooks = nf_hook_entries_head(net, pf, entry->state.hook);
nf_queue_entry_release_refs(entry); nf_queue_entry_release_refs(entry);
i = entry->hook_index; i = entry->hook_index;
if (WARN_ON_ONCE(i >= hooks->num_hook_entries)) { if (WARN_ON_ONCE(!hooks || i >= hooks->num_hook_entries)) {
kfree_skb(skb); kfree_skb(skb);
kfree(entry); kfree(entry);
return; return;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment