Commit 42aecaa9 authored by Tom Herbert's avatar Tom Herbert Committed by David S. Miller

net: Get skb hash over flow_keys structure

This patch changes flow hashing to use jhash2 over the flow_keys
structure instead just doing jhash_3words over src, dst, and ports.
This method will allow us take more input into the hashing function
so that we can include full IPv6 addresses, VLAN, flow labels etc.
without needing to resort to xor'ing which makes for a poor hash.
Acked-by: default avatarJiri Pirko <jiri@resnulli.us>
Signed-off-by: default avatarTom Herbert <tom@herbertland.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent c468efe2
...@@ -1943,7 +1943,7 @@ static inline void skb_probe_transport_header(struct sk_buff *skb, ...@@ -1943,7 +1943,7 @@ static inline void skb_probe_transport_header(struct sk_buff *skb,
if (skb_transport_header_was_set(skb)) if (skb_transport_header_was_set(skb))
return; return;
else if (skb_flow_dissect_flow_keys(skb, &keys)) else if (skb_flow_dissect_flow_keys(skb, &keys))
skb_set_transport_header(skb, keys.basic.thoff); skb_set_transport_header(skb, keys.control.thoff);
else else
skb_set_transport_header(skb, offset_hint); skb_set_transport_header(skb, offset_hint);
} }
......
...@@ -6,6 +6,15 @@ ...@@ -6,6 +6,15 @@
#include <linux/in6.h> #include <linux/in6.h>
#include <uapi/linux/if_ether.h> #include <uapi/linux/if_ether.h>
/**
* struct flow_dissector_key_control:
* @thoff: Transport header offset
*/
struct flow_dissector_key_control {
u16 thoff;
u16 padding;
};
/** /**
* struct flow_dissector_key_basic: * struct flow_dissector_key_basic:
* @thoff: Transport header offset * @thoff: Transport header offset
...@@ -13,9 +22,9 @@ ...@@ -13,9 +22,9 @@
* @ip_proto: Transport header protocol (eg. TCP/UDP) * @ip_proto: Transport header protocol (eg. TCP/UDP)
*/ */
struct flow_dissector_key_basic { struct flow_dissector_key_basic {
u16 thoff;
__be16 n_proto; __be16 n_proto;
u8 ip_proto; u8 ip_proto;
u8 padding;
}; };
/** /**
...@@ -70,6 +79,7 @@ struct flow_dissector_key_eth_addrs { ...@@ -70,6 +79,7 @@ struct flow_dissector_key_eth_addrs {
}; };
enum flow_dissector_key_id { enum flow_dissector_key_id {
FLOW_DISSECTOR_KEY_CONTROL, /* struct flow_dissector_key_control */
FLOW_DISSECTOR_KEY_BASIC, /* struct flow_dissector_key_basic */ FLOW_DISSECTOR_KEY_BASIC, /* struct flow_dissector_key_basic */
FLOW_DISSECTOR_KEY_IPV4_ADDRS, /* struct flow_dissector_key_addrs */ FLOW_DISSECTOR_KEY_IPV4_ADDRS, /* struct flow_dissector_key_addrs */
FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS, /* struct flow_dissector_key_addrs */ FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS, /* struct flow_dissector_key_addrs */
...@@ -109,11 +119,16 @@ static inline bool skb_flow_dissect(const struct sk_buff *skb, ...@@ -109,11 +119,16 @@ static inline bool skb_flow_dissect(const struct sk_buff *skb,
} }
struct flow_keys { struct flow_keys {
struct flow_dissector_key_addrs addrs; struct flow_dissector_key_control control;
struct flow_dissector_key_ports ports; #define FLOW_KEYS_HASH_START_FIELD basic
struct flow_dissector_key_basic basic; struct flow_dissector_key_basic basic;
struct flow_dissector_key_ports ports;
struct flow_dissector_key_addrs addrs;
}; };
#define FLOW_KEYS_HASH_OFFSET \
offsetof(struct flow_keys, FLOW_KEYS_HASH_START_FIELD)
extern struct flow_dissector flow_keys_dissector; extern struct flow_dissector flow_keys_dissector;
extern struct flow_dissector flow_keys_buf_dissector; extern struct flow_dissector flow_keys_buf_dissector;
......
...@@ -360,6 +360,8 @@ static inline void inet_set_txhash(struct sock *sk) ...@@ -360,6 +360,8 @@ static inline void inet_set_txhash(struct sock *sk)
struct inet_sock *inet = inet_sk(sk); struct inet_sock *inet = inet_sk(sk);
struct flow_keys keys; struct flow_keys keys;
memset(&keys, 0, sizeof(keys));
keys.addrs.src = inet->inet_saddr; keys.addrs.src = inet->inet_saddr;
keys.addrs.dst = inet->inet_daddr; keys.addrs.dst = inet->inet_daddr;
keys.ports.src = inet->inet_sport; keys.ports.src = inet->inet_sport;
......
...@@ -699,6 +699,8 @@ static inline void ip6_set_txhash(struct sock *sk) ...@@ -699,6 +699,8 @@ static inline void ip6_set_txhash(struct sock *sk)
struct ipv6_pinfo *np = inet6_sk(sk); struct ipv6_pinfo *np = inet6_sk(sk);
struct flow_keys keys; struct flow_keys keys;
memset(&keys, 0, sizeof(keys));
keys.addrs.src = (__force __be32)ipv6_addr_hash(&np->saddr); keys.addrs.src = (__force __be32)ipv6_addr_hash(&np->saddr);
keys.addrs.dst = (__force __be32)ipv6_addr_hash(&sk->sk_v6_daddr); keys.addrs.dst = (__force __be32)ipv6_addr_hash(&sk->sk_v6_daddr);
keys.ports.src = inet->inet_sport; keys.ports.src = inet->inet_sport;
......
...@@ -57,9 +57,11 @@ void skb_flow_dissector_init(struct flow_dissector *flow_dissector, ...@@ -57,9 +57,11 @@ void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
flow_dissector->offset[key->key_id] = key->offset; flow_dissector->offset[key->key_id] = key->offset;
} }
/* Ensure that the dissector always includes basic key. That way /* Ensure that the dissector always includes control and basic key.
* we are able to avoid handling lack of it in fast path. * That way we are able to avoid handling lack of these in fast path.
*/ */
BUG_ON(!skb_flow_dissector_uses_key(flow_dissector,
FLOW_DISSECTOR_KEY_CONTROL));
BUG_ON(!skb_flow_dissector_uses_key(flow_dissector, BUG_ON(!skb_flow_dissector_uses_key(flow_dissector,
FLOW_DISSECTOR_KEY_BASIC)); FLOW_DISSECTOR_KEY_BASIC));
} }
...@@ -120,6 +122,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb, ...@@ -120,6 +122,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb,
void *target_container, void *target_container,
void *data, __be16 proto, int nhoff, int hlen) void *data, __be16 proto, int nhoff, int hlen)
{ {
struct flow_dissector_key_control *key_control;
struct flow_dissector_key_basic *key_basic; struct flow_dissector_key_basic *key_basic;
struct flow_dissector_key_addrs *key_addrs; struct flow_dissector_key_addrs *key_addrs;
struct flow_dissector_key_ports *key_ports; struct flow_dissector_key_ports *key_ports;
...@@ -132,6 +135,13 @@ bool __skb_flow_dissect(const struct sk_buff *skb, ...@@ -132,6 +135,13 @@ bool __skb_flow_dissect(const struct sk_buff *skb,
hlen = skb_headlen(skb); hlen = skb_headlen(skb);
} }
/* It is ensured by skb_flow_dissector_init() that control key will
* be always present.
*/
key_control = skb_flow_dissector_target(flow_dissector,
FLOW_DISSECTOR_KEY_CONTROL,
target_container);
/* It is ensured by skb_flow_dissector_init() that basic key will /* It is ensured by skb_flow_dissector_init() that basic key will
* be always present. * be always present.
*/ */
...@@ -219,7 +229,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb, ...@@ -219,7 +229,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb,
key_basic->n_proto = proto; key_basic->n_proto = proto;
key_basic->ip_proto = ip_proto; key_basic->ip_proto = ip_proto;
key_basic->thoff = (u16)nhoff; key_control->thoff = (u16)nhoff;
if (skb_flow_dissector_uses_key(flow_dissector, if (skb_flow_dissector_uses_key(flow_dissector,
FLOW_DISSECTOR_KEY_PORTS)) { FLOW_DISSECTOR_KEY_PORTS)) {
...@@ -275,7 +285,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb, ...@@ -275,7 +285,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb,
if (!hdr) if (!hdr)
return false; return false;
key_basic->n_proto = proto; key_basic->n_proto = proto;
key_basic->thoff = (u16)nhoff; key_control->thoff = (u16)nhoff;
if (skb_flow_dissector_uses_key(flow_dissector, if (skb_flow_dissector_uses_key(flow_dissector,
FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) { FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
...@@ -288,7 +298,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb, ...@@ -288,7 +298,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb,
return true; return true;
} }
case htons(ETH_P_FCOE): case htons(ETH_P_FCOE):
key_basic->thoff = (u16)(nhoff + FCOE_HEADER_LEN); key_control->thoff = (u16)(nhoff + FCOE_HEADER_LEN);
/* fall through */ /* fall through */
default: default:
return false; return false;
...@@ -345,7 +355,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb, ...@@ -345,7 +355,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb,
key_basic->n_proto = proto; key_basic->n_proto = proto;
key_basic->ip_proto = ip_proto; key_basic->ip_proto = ip_proto;
key_basic->thoff = (u16) nhoff; key_control->thoff = (u16)nhoff;
if (skb_flow_dissector_uses_key(flow_dissector, if (skb_flow_dissector_uses_key(flow_dissector,
FLOW_DISSECTOR_KEY_PORTS)) { FLOW_DISSECTOR_KEY_PORTS)) {
...@@ -366,9 +376,21 @@ static __always_inline void __flow_hash_secret_init(void) ...@@ -366,9 +376,21 @@ static __always_inline void __flow_hash_secret_init(void)
net_get_random_once(&hashrnd, sizeof(hashrnd)); net_get_random_once(&hashrnd, sizeof(hashrnd));
} }
static __always_inline u32 __flow_hash_3words(u32 a, u32 b, u32 c, u32 keyval) static __always_inline u32 __flow_hash_words(u32 *words, u32 length, u32 keyval)
{
return jhash2(words, length, keyval);
}
static inline void *flow_keys_hash_start(struct flow_keys *flow)
{ {
return jhash_3words(a, b, c, keyval); BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % sizeof(u32));
return (void *)flow + FLOW_KEYS_HASH_OFFSET;
}
static inline size_t flow_keys_hash_length(struct flow_keys *flow)
{
BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32));
return (sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) / sizeof(u32);
} }
static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval) static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval)
...@@ -383,10 +405,8 @@ static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval) ...@@ -383,10 +405,8 @@ static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval)
swap(keys->ports.src, keys->ports.dst); swap(keys->ports.src, keys->ports.dst);
} }
hash = __flow_hash_3words((__force u32)keys->addrs.dst, hash = __flow_hash_words((u32 *)flow_keys_hash_start(keys),
(__force u32)keys->addrs.src, flow_keys_hash_length(keys), keyval);
(__force u32)keys->ports.ports,
keyval);
if (!hash) if (!hash)
hash = 1; hash = 1;
...@@ -473,7 +493,7 @@ EXPORT_SYMBOL(skb_get_hash_perturb); ...@@ -473,7 +493,7 @@ EXPORT_SYMBOL(skb_get_hash_perturb);
u32 __skb_get_poff(const struct sk_buff *skb, void *data, u32 __skb_get_poff(const struct sk_buff *skb, void *data,
const struct flow_keys *keys, int hlen) const struct flow_keys *keys, int hlen)
{ {
u32 poff = keys->basic.thoff; u32 poff = keys->control.thoff;
switch (keys->basic.ip_proto) { switch (keys->basic.ip_proto) {
case IPPROTO_TCP: { case IPPROTO_TCP: {
...@@ -536,6 +556,10 @@ u32 skb_get_poff(const struct sk_buff *skb) ...@@ -536,6 +556,10 @@ u32 skb_get_poff(const struct sk_buff *skb)
} }
static const struct flow_dissector_key flow_keys_dissector_keys[] = { static const struct flow_dissector_key flow_keys_dissector_keys[] = {
{
.key_id = FLOW_DISSECTOR_KEY_CONTROL,
.offset = offsetof(struct flow_keys, control),
},
{ {
.key_id = FLOW_DISSECTOR_KEY_BASIC, .key_id = FLOW_DISSECTOR_KEY_BASIC,
.offset = offsetof(struct flow_keys, basic), .offset = offsetof(struct flow_keys, basic),
...@@ -555,6 +579,10 @@ static const struct flow_dissector_key flow_keys_dissector_keys[] = { ...@@ -555,6 +579,10 @@ static const struct flow_dissector_key flow_keys_dissector_keys[] = {
}; };
static const struct flow_dissector_key flow_keys_buf_dissector_keys[] = { static const struct flow_dissector_key flow_keys_buf_dissector_keys[] = {
{
.key_id = FLOW_DISSECTOR_KEY_CONTROL,
.offset = offsetof(struct flow_keys, control),
},
{ {
.key_id = FLOW_DISSECTOR_KEY_BASIC, .key_id = FLOW_DISSECTOR_KEY_BASIC,
.offset = offsetof(struct flow_keys, basic), .offset = offsetof(struct flow_keys, basic),
......
...@@ -25,6 +25,7 @@ ...@@ -25,6 +25,7 @@
struct fl_flow_key { struct fl_flow_key {
int indev_ifindex; int indev_ifindex;
struct flow_dissector_key_control control;
struct flow_dissector_key_basic basic; struct flow_dissector_key_basic basic;
struct flow_dissector_key_eth_addrs eth; struct flow_dissector_key_eth_addrs eth;
union { union {
...@@ -347,6 +348,7 @@ static void fl_init_dissector(struct cls_fl_head *head, ...@@ -347,6 +348,7 @@ static void fl_init_dissector(struct cls_fl_head *head,
struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX]; struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX];
size_t cnt = 0; size_t cnt = 0;
FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control);
FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic); FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic);
FL_KEY_SET_IF_IN_RANGE(mask, keys, cnt, FL_KEY_SET_IF_IN_RANGE(mask, keys, cnt,
FLOW_DISSECTOR_KEY_ETH_ADDRS, eth); FLOW_DISSECTOR_KEY_ETH_ADDRS, eth);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment