Commit c617f398 authored by David S. Miller's avatar David S. Miller

Merge branch 'soreuseport'

Tom Herbert says:

====================
This series implements so_reuseport (SO_REUSEPORT socket option) for
TCP and UDP.  For TCP, so_reuseport allows multiple listener sockets
to be bound to the same port.  In the case of UDP, so_reuseport allows
multiple sockets to bind to the same port.  To prevent port hijacking
all sockets bound to the same port using so_reuseport must have the
same uid.  Received packets are distributed to multiple sockets bound
to the same port using a 4-tuple hash.

The motivating case for so_resuseport in TCP would be something like
a web server binding to port 80 running with multiple threads, where
each thread might have it's own listener socket.  This could be done
as an alternative to other models: 1) have one listener thread which
dispatches completed connections to workers. 2) accept on a single
listener socket from multiple threads.  In case #1 the listener thread
can easily become the bottleneck with high connection turn-over rate.
In case #2, the proportion of connections accepted per thread tends
to be uneven under high connection load (assuming simple event loop:
while (1) { accept(); process() }, wakeup does not promote fairness
among the sockets.  We have seen the  disproportion to be as high
as 3:1 ratio between thread accepting most connections and the one
accepting the fewest.  With so_reusport the distribution is
uniform.

The TCP implementation has a problem in that the request sockets for a
listener are attached to a listener socket.  If a SYN is received, a
listener socket is chosen and request structure is created (SYN-RECV
state).  If the subsequent ack in 3WHS does not match the same port
by so_reusport, the connection state is not found (reset) and the
request structure is orphaned.  This scenario would occur when the
number of listener sockets bound to a port changes (new ones are
added, or old ones closed).  We are looking for a solution to this,
maybe allow multiple sockets to share the same request table...

The motivating case for so_reuseport in UDP would be something like a
DNS server.  An alternative would be to recv on the same socket from
multiple threads.  As in the case of TCP, the load across these threads
tends to be disproportionate and we also see a lot of contection on
the socket lock.  Note that SO_REUSEADDR already allows multiple UDP
sockets to bind to the same port, however there is no provision to
prevent hijacking and nothing to distribute packets across all the
sockets sharing the same bound port.  This patch does not change the
semantics of SO_REUSEADDR, but provides usable functionality of it
for unicast.
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 4a633a60 72289b96
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
#define SO_BROADCAST 0x0020 #define SO_BROADCAST 0x0020
#define SO_LINGER 0x0080 #define SO_LINGER 0x0080
#define SO_OOBINLINE 0x0100 #define SO_OOBINLINE 0x0100
/* To add :#define SO_REUSEPORT 0x0200 */ #define SO_REUSEPORT 0x0200
#define SO_TYPE 0x1008 #define SO_TYPE 0x1008
#define SO_ERROR 0x1007 #define SO_ERROR 0x1007
......
...@@ -22,7 +22,7 @@ ...@@ -22,7 +22,7 @@
#define SO_PRIORITY 12 #define SO_PRIORITY 12
#define SO_LINGER 13 #define SO_LINGER 13
#define SO_BSDCOMPAT 14 #define SO_BSDCOMPAT 14
/* To add :#define SO_REUSEPORT 15 */ #define SO_REUSEPORT 15
#define SO_PASSCRED 16 #define SO_PASSCRED 16
#define SO_PEERCRED 17 #define SO_PEERCRED 17
#define SO_RCVLOWAT 18 #define SO_RCVLOWAT 18
......
...@@ -24,7 +24,7 @@ ...@@ -24,7 +24,7 @@
#define SO_PRIORITY 12 #define SO_PRIORITY 12
#define SO_LINGER 13 #define SO_LINGER 13
#define SO_BSDCOMPAT 14 #define SO_BSDCOMPAT 14
/* To add :#define SO_REUSEPORT 15 */ #define SO_REUSEPORT 15
#define SO_PASSCRED 16 #define SO_PASSCRED 16
#define SO_PEERCRED 17 #define SO_PEERCRED 17
#define SO_RCVLOWAT 18 #define SO_RCVLOWAT 18
......
...@@ -22,7 +22,7 @@ ...@@ -22,7 +22,7 @@
#define SO_PRIORITY 12 #define SO_PRIORITY 12
#define SO_LINGER 13 #define SO_LINGER 13
#define SO_BSDCOMPAT 14 #define SO_BSDCOMPAT 14
/* To add :#define SO_REUSEPORT 15 */ #define SO_REUSEPORT 15
#define SO_PASSCRED 16 #define SO_PASSCRED 16
#define SO_PEERCRED 17 #define SO_PEERCRED 17
#define SO_RCVLOWAT 18 #define SO_RCVLOWAT 18
......
...@@ -22,7 +22,7 @@ ...@@ -22,7 +22,7 @@
#define SO_PRIORITY 12 #define SO_PRIORITY 12
#define SO_LINGER 13 #define SO_LINGER 13
#define SO_BSDCOMPAT 14 #define SO_BSDCOMPAT 14
/* To add :#define SO_REUSEPORT 15 */ #define SO_REUSEPORT 15
#define SO_PASSCRED 16 #define SO_PASSCRED 16
#define SO_PEERCRED 17 #define SO_PEERCRED 17
#define SO_RCVLOWAT 18 #define SO_RCVLOWAT 18
......
...@@ -31,7 +31,7 @@ ...@@ -31,7 +31,7 @@
#define SO_PRIORITY 12 #define SO_PRIORITY 12
#define SO_LINGER 13 #define SO_LINGER 13
#define SO_BSDCOMPAT 14 #define SO_BSDCOMPAT 14
/* To add :#define SO_REUSEPORT 15 */ #define SO_REUSEPORT 15
#define SO_PASSCRED 16 #define SO_PASSCRED 16
#define SO_PEERCRED 17 #define SO_PEERCRED 17
#define SO_RCVLOWAT 18 #define SO_RCVLOWAT 18
......
...@@ -22,7 +22,7 @@ ...@@ -22,7 +22,7 @@
#define SO_PRIORITY 12 #define SO_PRIORITY 12
#define SO_LINGER 13 #define SO_LINGER 13
#define SO_BSDCOMPAT 14 #define SO_BSDCOMPAT 14
/* To add :#define SO_REUSEPORT 15 */ #define SO_REUSEPORT 15
#define SO_PASSCRED 16 #define SO_PASSCRED 16
#define SO_PEERCRED 17 #define SO_PEERCRED 17
#define SO_RCVLOWAT 18 #define SO_RCVLOWAT 18
......
...@@ -28,8 +28,7 @@ ...@@ -28,8 +28,7 @@
#define SO_LINGER 0x0080 /* Block on close of a reliable #define SO_LINGER 0x0080 /* Block on close of a reliable
socket to transmit pending data. */ socket to transmit pending data. */
#define SO_OOBINLINE 0x0100 /* Receive out-of-band data in-band. */ #define SO_OOBINLINE 0x0100 /* Receive out-of-band data in-band. */
#if 0 #define SO_REUSEPORT 0x0200 /* Allow local address and port reuse. */
To add: #define SO_REUSEPORT 0x0200 /* Allow local address and port reuse. */
#endif #endif
#define SO_TYPE 0x1008 /* Compatible name for SO_STYLE. */ #define SO_TYPE 0x1008 /* Compatible name for SO_STYLE. */
......
...@@ -22,7 +22,7 @@ ...@@ -22,7 +22,7 @@
#define SO_PRIORITY 12 #define SO_PRIORITY 12
#define SO_LINGER 13 #define SO_LINGER 13
#define SO_BSDCOMPAT 14 #define SO_BSDCOMPAT 14
/* To add :#define SO_REUSEPORT 15 */ #define SO_REUSEPORT 15
#define SO_PASSCRED 16 #define SO_PASSCRED 16
#define SO_PEERCRED 17 #define SO_PEERCRED 17
#define SO_RCVLOWAT 18 #define SO_RCVLOWAT 18
......
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
#define SO_BROADCAST 0x0020 #define SO_BROADCAST 0x0020
#define SO_LINGER 0x0080 #define SO_LINGER 0x0080
#define SO_OOBINLINE 0x0100 #define SO_OOBINLINE 0x0100
/* To add :#define SO_REUSEPORT 0x0200 */ #define SO_REUSEPORT 0x0200
#define SO_SNDBUF 0x1001 #define SO_SNDBUF 0x1001
#define SO_RCVBUF 0x1002 #define SO_RCVBUF 0x1002
#define SO_SNDBUFFORCE 0x100a #define SO_SNDBUFFORCE 0x100a
......
...@@ -29,7 +29,7 @@ ...@@ -29,7 +29,7 @@
#define SO_PRIORITY 12 #define SO_PRIORITY 12
#define SO_LINGER 13 #define SO_LINGER 13
#define SO_BSDCOMPAT 14 #define SO_BSDCOMPAT 14
/* To add :#define SO_REUSEPORT 15 */ #define SO_REUSEPORT 15
#define SO_RCVLOWAT 16 #define SO_RCVLOWAT 16
#define SO_SNDLOWAT 17 #define SO_SNDLOWAT 17
#define SO_RCVTIMEO 18 #define SO_RCVTIMEO 18
......
...@@ -28,7 +28,7 @@ ...@@ -28,7 +28,7 @@
#define SO_PRIORITY 12 #define SO_PRIORITY 12
#define SO_LINGER 13 #define SO_LINGER 13
#define SO_BSDCOMPAT 14 #define SO_BSDCOMPAT 14
/* To add :#define SO_REUSEPORT 15 */ #define SO_REUSEPORT 15
#define SO_PASSCRED 16 #define SO_PASSCRED 16
#define SO_PEERCRED 17 #define SO_PEERCRED 17
#define SO_RCVLOWAT 18 #define SO_RCVLOWAT 18
......
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
#define SO_PEERCRED 0x0040 #define SO_PEERCRED 0x0040
#define SO_LINGER 0x0080 #define SO_LINGER 0x0080
#define SO_OOBINLINE 0x0100 #define SO_OOBINLINE 0x0100
/* To add :#define SO_REUSEPORT 0x0200 */ #define SO_REUSEPORT 0x0200
#define SO_BSDCOMPAT 0x0400 #define SO_BSDCOMPAT 0x0400
#define SO_RCVLOWAT 0x0800 #define SO_RCVLOWAT 0x0800
#define SO_SNDLOWAT 0x1000 #define SO_SNDLOWAT 0x1000
......
...@@ -32,7 +32,7 @@ ...@@ -32,7 +32,7 @@
#define SO_PRIORITY 12 #define SO_PRIORITY 12
#define SO_LINGER 13 #define SO_LINGER 13
#define SO_BSDCOMPAT 14 #define SO_BSDCOMPAT 14
/* To add :#define SO_REUSEPORT 15 */ #define SO_REUSEPORT 15
#define SO_PASSCRED 16 #define SO_PASSCRED 16
#define SO_PEERCRED 17 #define SO_PEERCRED 17
#define SO_RCVLOWAT 18 #define SO_RCVLOWAT 18
......
...@@ -74,4 +74,10 @@ static inline int arch_get_random_int(unsigned int *v) ...@@ -74,4 +74,10 @@ static inline int arch_get_random_int(unsigned int *v)
} }
#endif #endif
/* Pseudo random number generator from numerical recipes. */
static inline u32 next_pseudo_random32(u32 seed)
{
return seed * 1664525 + 1013904223;
}
#endif /* _LINUX_RANDOM_H */ #endif /* _LINUX_RANDOM_H */
...@@ -71,6 +71,8 @@ extern struct sock *__inet6_lookup_established(struct net *net, ...@@ -71,6 +71,8 @@ extern struct sock *__inet6_lookup_established(struct net *net,
extern struct sock *inet6_lookup_listener(struct net *net, extern struct sock *inet6_lookup_listener(struct net *net,
struct inet_hashinfo *hashinfo, struct inet_hashinfo *hashinfo,
const struct in6_addr *saddr,
const __be16 sport,
const struct in6_addr *daddr, const struct in6_addr *daddr,
const unsigned short hnum, const unsigned short hnum,
const int dif); const int dif);
...@@ -88,7 +90,8 @@ static inline struct sock *__inet6_lookup(struct net *net, ...@@ -88,7 +90,8 @@ static inline struct sock *__inet6_lookup(struct net *net,
if (sk) if (sk)
return sk; return sk;
return inet6_lookup_listener(net, hashinfo, daddr, hnum, dif); return inet6_lookup_listener(net, hashinfo, saddr, sport,
daddr, hnum, dif);
} }
static inline struct sock *__inet6_lookup_skb(struct inet_hashinfo *hashinfo, static inline struct sock *__inet6_lookup_skb(struct inet_hashinfo *hashinfo,
......
...@@ -81,7 +81,9 @@ struct inet_bind_bucket { ...@@ -81,7 +81,9 @@ struct inet_bind_bucket {
struct net *ib_net; struct net *ib_net;
#endif #endif
unsigned short port; unsigned short port;
signed short fastreuse; signed char fastreuse;
signed char fastreuseport;
kuid_t fastuid;
int num_owners; int num_owners;
struct hlist_node node; struct hlist_node node;
struct hlist_head owners; struct hlist_head owners;
...@@ -257,15 +259,19 @@ extern void inet_unhash(struct sock *sk); ...@@ -257,15 +259,19 @@ extern void inet_unhash(struct sock *sk);
extern struct sock *__inet_lookup_listener(struct net *net, extern struct sock *__inet_lookup_listener(struct net *net,
struct inet_hashinfo *hashinfo, struct inet_hashinfo *hashinfo,
const __be32 saddr,
const __be16 sport,
const __be32 daddr, const __be32 daddr,
const unsigned short hnum, const unsigned short hnum,
const int dif); const int dif);
static inline struct sock *inet_lookup_listener(struct net *net, static inline struct sock *inet_lookup_listener(struct net *net,
struct inet_hashinfo *hashinfo, struct inet_hashinfo *hashinfo,
__be32 saddr, __be16 sport,
__be32 daddr, __be16 dport, int dif) __be32 daddr, __be16 dport, int dif)
{ {
return __inet_lookup_listener(net, hashinfo, daddr, ntohs(dport), dif); return __inet_lookup_listener(net, hashinfo, saddr, sport,
daddr, ntohs(dport), dif);
} }
/* Socket demux engine toys. */ /* Socket demux engine toys. */
...@@ -358,7 +364,8 @@ static inline struct sock *__inet_lookup(struct net *net, ...@@ -358,7 +364,8 @@ static inline struct sock *__inet_lookup(struct net *net,
struct sock *sk = __inet_lookup_established(net, hashinfo, struct sock *sk = __inet_lookup_established(net, hashinfo,
saddr, sport, daddr, hnum, dif); saddr, sport, daddr, hnum, dif);
return sk ? : __inet_lookup_listener(net, hashinfo, daddr, hnum, dif); return sk ? : __inet_lookup_listener(net, hashinfo, saddr, sport,
daddr, hnum, dif);
} }
static inline struct sock *inet_lookup(struct net *net, static inline struct sock *inet_lookup(struct net *net,
......
...@@ -82,6 +82,7 @@ nf_tproxy_get_sock_v4(struct net *net, const u8 protocol, ...@@ -82,6 +82,7 @@ nf_tproxy_get_sock_v4(struct net *net, const u8 protocol,
break; break;
case NFT_LOOKUP_LISTENER: case NFT_LOOKUP_LISTENER:
sk = inet_lookup_listener(net, &tcp_hashinfo, sk = inet_lookup_listener(net, &tcp_hashinfo,
saddr, sport,
daddr, dport, daddr, dport,
in->ifindex); in->ifindex);
...@@ -151,6 +152,7 @@ nf_tproxy_get_sock_v6(struct net *net, const u8 protocol, ...@@ -151,6 +152,7 @@ nf_tproxy_get_sock_v6(struct net *net, const u8 protocol,
break; break;
case NFT_LOOKUP_LISTENER: case NFT_LOOKUP_LISTENER:
sk = inet6_lookup_listener(net, &tcp_hashinfo, sk = inet6_lookup_listener(net, &tcp_hashinfo,
saddr, sport,
daddr, ntohs(dport), daddr, ntohs(dport),
in->ifindex); in->ifindex);
......
...@@ -140,6 +140,7 @@ typedef __u64 __bitwise __addrpair; ...@@ -140,6 +140,7 @@ typedef __u64 __bitwise __addrpair;
* @skc_family: network address family * @skc_family: network address family
* @skc_state: Connection state * @skc_state: Connection state
* @skc_reuse: %SO_REUSEADDR setting * @skc_reuse: %SO_REUSEADDR setting
* @skc_reuseport: %SO_REUSEPORT setting
* @skc_bound_dev_if: bound device index if != 0 * @skc_bound_dev_if: bound device index if != 0
* @skc_bind_node: bind hash linkage for various protocol lookup tables * @skc_bind_node: bind hash linkage for various protocol lookup tables
* @skc_portaddr_node: second hash linkage for UDP/UDP-Lite protocol * @skc_portaddr_node: second hash linkage for UDP/UDP-Lite protocol
...@@ -179,7 +180,8 @@ struct sock_common { ...@@ -179,7 +180,8 @@ struct sock_common {
unsigned short skc_family; unsigned short skc_family;
volatile unsigned char skc_state; volatile unsigned char skc_state;
unsigned char skc_reuse; unsigned char skc_reuse:4;
unsigned char skc_reuseport:4;
int skc_bound_dev_if; int skc_bound_dev_if;
union { union {
struct hlist_node skc_bind_node; struct hlist_node skc_bind_node;
...@@ -297,6 +299,7 @@ struct sock { ...@@ -297,6 +299,7 @@ struct sock {
#define sk_family __sk_common.skc_family #define sk_family __sk_common.skc_family
#define sk_state __sk_common.skc_state #define sk_state __sk_common.skc_state
#define sk_reuse __sk_common.skc_reuse #define sk_reuse __sk_common.skc_reuse
#define sk_reuseport __sk_common.skc_reuseport
#define sk_bound_dev_if __sk_common.skc_bound_dev_if #define sk_bound_dev_if __sk_common.skc_bound_dev_if
#define sk_bind_node __sk_common.skc_bind_node #define sk_bind_node __sk_common.skc_bind_node
#define sk_prot __sk_common.skc_prot #define sk_prot __sk_common.skc_prot
......
...@@ -22,8 +22,7 @@ ...@@ -22,8 +22,7 @@
#define SO_PRIORITY 12 #define SO_PRIORITY 12
#define SO_LINGER 13 #define SO_LINGER 13
#define SO_BSDCOMPAT 14 #define SO_BSDCOMPAT 14
/* To add :#define SO_REUSEPORT 15 */ #define SO_REUSEPORT 15
#ifndef SO_PASSCRED /* powerpc only differs in these */ #ifndef SO_PASSCRED /* powerpc only differs in these */
#define SO_PASSCRED 16 #define SO_PASSCRED 16
#define SO_PEERCRED 17 #define SO_PEERCRED 17
......
...@@ -665,6 +665,9 @@ int sock_setsockopt(struct socket *sock, int level, int optname, ...@@ -665,6 +665,9 @@ int sock_setsockopt(struct socket *sock, int level, int optname,
case SO_REUSEADDR: case SO_REUSEADDR:
sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE); sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE);
break; break;
case SO_REUSEPORT:
sk->sk_reuseport = valbool;
break;
case SO_TYPE: case SO_TYPE:
case SO_PROTOCOL: case SO_PROTOCOL:
case SO_DOMAIN: case SO_DOMAIN:
...@@ -972,6 +975,10 @@ int sock_getsockopt(struct socket *sock, int level, int optname, ...@@ -972,6 +975,10 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
v.val = sk->sk_reuse; v.val = sk->sk_reuse;
break; break;
case SO_REUSEPORT:
v.val = sk->sk_reuseport;
break;
case SO_KEEPALIVE: case SO_KEEPALIVE:
v.val = sock_flag(sk, SOCK_KEEPOPEN); v.val = sock_flag(sk, SOCK_KEEPOPEN);
break; break;
......
...@@ -59,6 +59,8 @@ int inet_csk_bind_conflict(const struct sock *sk, ...@@ -59,6 +59,8 @@ int inet_csk_bind_conflict(const struct sock *sk,
struct sock *sk2; struct sock *sk2;
struct hlist_node *node; struct hlist_node *node;
int reuse = sk->sk_reuse; int reuse = sk->sk_reuse;
int reuseport = sk->sk_reuseport;
kuid_t uid = sock_i_uid((struct sock *)sk);
/* /*
* Unlike other sk lookup places we do not check * Unlike other sk lookup places we do not check
...@@ -73,8 +75,11 @@ int inet_csk_bind_conflict(const struct sock *sk, ...@@ -73,8 +75,11 @@ int inet_csk_bind_conflict(const struct sock *sk,
(!sk->sk_bound_dev_if || (!sk->sk_bound_dev_if ||
!sk2->sk_bound_dev_if || !sk2->sk_bound_dev_if ||
sk->sk_bound_dev_if == sk2->sk_bound_dev_if)) { sk->sk_bound_dev_if == sk2->sk_bound_dev_if)) {
if (!reuse || !sk2->sk_reuse || if ((!reuse || !sk2->sk_reuse ||
sk2->sk_state == TCP_LISTEN) { sk2->sk_state == TCP_LISTEN) &&
(!reuseport || !sk2->sk_reuseport ||
(sk2->sk_state != TCP_TIME_WAIT &&
!uid_eq(uid, sock_i_uid(sk2))))) {
const __be32 sk2_rcv_saddr = sk_rcv_saddr(sk2); const __be32 sk2_rcv_saddr = sk_rcv_saddr(sk2);
if (!sk2_rcv_saddr || !sk_rcv_saddr(sk) || if (!sk2_rcv_saddr || !sk_rcv_saddr(sk) ||
sk2_rcv_saddr == sk_rcv_saddr(sk)) sk2_rcv_saddr == sk_rcv_saddr(sk))
...@@ -106,6 +111,7 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum) ...@@ -106,6 +111,7 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum)
int ret, attempts = 5; int ret, attempts = 5;
struct net *net = sock_net(sk); struct net *net = sock_net(sk);
int smallest_size = -1, smallest_rover; int smallest_size = -1, smallest_rover;
kuid_t uid = sock_i_uid(sk);
local_bh_disable(); local_bh_disable();
if (!snum) { if (!snum) {
...@@ -125,9 +131,12 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum) ...@@ -125,9 +131,12 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum)
spin_lock(&head->lock); spin_lock(&head->lock);
inet_bind_bucket_for_each(tb, node, &head->chain) inet_bind_bucket_for_each(tb, node, &head->chain)
if (net_eq(ib_net(tb), net) && tb->port == rover) { if (net_eq(ib_net(tb), net) && tb->port == rover) {
if (tb->fastreuse > 0 && if (((tb->fastreuse > 0 &&
sk->sk_reuse && sk->sk_reuse &&
sk->sk_state != TCP_LISTEN && sk->sk_state != TCP_LISTEN) ||
(tb->fastreuseport > 0 &&
sk->sk_reuseport &&
uid_eq(tb->fastuid, uid))) &&
(tb->num_owners < smallest_size || smallest_size == -1)) { (tb->num_owners < smallest_size || smallest_size == -1)) {
smallest_size = tb->num_owners; smallest_size = tb->num_owners;
smallest_rover = rover; smallest_rover = rover;
...@@ -185,14 +194,17 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum) ...@@ -185,14 +194,17 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum)
if (sk->sk_reuse == SK_FORCE_REUSE) if (sk->sk_reuse == SK_FORCE_REUSE)
goto success; goto success;
if (tb->fastreuse > 0 && if (((tb->fastreuse > 0 &&
sk->sk_reuse && sk->sk_state != TCP_LISTEN && sk->sk_reuse && sk->sk_state != TCP_LISTEN) ||
(tb->fastreuseport > 0 &&
sk->sk_reuseport && uid_eq(tb->fastuid, uid))) &&
smallest_size == -1) { smallest_size == -1) {
goto success; goto success;
} else { } else {
ret = 1; ret = 1;
if (inet_csk(sk)->icsk_af_ops->bind_conflict(sk, tb, true)) { if (inet_csk(sk)->icsk_af_ops->bind_conflict(sk, tb, true)) {
if (sk->sk_reuse && sk->sk_state != TCP_LISTEN && if (((sk->sk_reuse && sk->sk_state != TCP_LISTEN) ||
(sk->sk_reuseport && uid_eq(tb->fastuid, uid))) &&
smallest_size != -1 && --attempts >= 0) { smallest_size != -1 && --attempts >= 0) {
spin_unlock(&head->lock); spin_unlock(&head->lock);
goto again; goto again;
...@@ -212,9 +224,23 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum) ...@@ -212,9 +224,23 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum)
tb->fastreuse = 1; tb->fastreuse = 1;
else else
tb->fastreuse = 0; tb->fastreuse = 0;
} else if (tb->fastreuse && if (sk->sk_reuseport) {
(!sk->sk_reuse || sk->sk_state == TCP_LISTEN)) tb->fastreuseport = 1;
tb->fastreuse = 0; tb->fastuid = uid;
} else {
tb->fastreuseport = 0;
tb->fastuid = 0;
}
} else {
if (tb->fastreuse &&
(!sk->sk_reuse || sk->sk_state == TCP_LISTEN))
tb->fastreuse = 0;
if (tb->fastreuseport &&
(!sk->sk_reuseport || !uid_eq(tb->fastuid, uid))) {
tb->fastreuseport = 0;
tb->fastuid = 0;
}
}
success: success:
if (!inet_csk(sk)->icsk_bind_hash) if (!inet_csk(sk)->icsk_bind_hash)
inet_bind_hash(sk, tb, snum); inet_bind_hash(sk, tb, snum);
......
...@@ -39,6 +39,7 @@ struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep, ...@@ -39,6 +39,7 @@ struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep,
write_pnet(&tb->ib_net, hold_net(net)); write_pnet(&tb->ib_net, hold_net(net));
tb->port = snum; tb->port = snum;
tb->fastreuse = 0; tb->fastreuse = 0;
tb->fastreuseport = 0;
tb->num_owners = 0; tb->num_owners = 0;
INIT_HLIST_HEAD(&tb->owners); INIT_HLIST_HEAD(&tb->owners);
hlist_add_head(&tb->node, &head->chain); hlist_add_head(&tb->node, &head->chain);
...@@ -151,16 +152,16 @@ static inline int compute_score(struct sock *sk, struct net *net, ...@@ -151,16 +152,16 @@ static inline int compute_score(struct sock *sk, struct net *net,
if (net_eq(sock_net(sk), net) && inet->inet_num == hnum && if (net_eq(sock_net(sk), net) && inet->inet_num == hnum &&
!ipv6_only_sock(sk)) { !ipv6_only_sock(sk)) {
__be32 rcv_saddr = inet->inet_rcv_saddr; __be32 rcv_saddr = inet->inet_rcv_saddr;
score = sk->sk_family == PF_INET ? 1 : 0; score = sk->sk_family == PF_INET ? 2 : 1;
if (rcv_saddr) { if (rcv_saddr) {
if (rcv_saddr != daddr) if (rcv_saddr != daddr)
return -1; return -1;
score += 2; score += 4;
} }
if (sk->sk_bound_dev_if) { if (sk->sk_bound_dev_if) {
if (sk->sk_bound_dev_if != dif) if (sk->sk_bound_dev_if != dif)
return -1; return -1;
score += 2; score += 4;
} }
} }
return score; return score;
...@@ -176,6 +177,7 @@ static inline int compute_score(struct sock *sk, struct net *net, ...@@ -176,6 +177,7 @@ static inline int compute_score(struct sock *sk, struct net *net,
struct sock *__inet_lookup_listener(struct net *net, struct sock *__inet_lookup_listener(struct net *net,
struct inet_hashinfo *hashinfo, struct inet_hashinfo *hashinfo,
const __be32 saddr, __be16 sport,
const __be32 daddr, const unsigned short hnum, const __be32 daddr, const unsigned short hnum,
const int dif) const int dif)
{ {
...@@ -183,17 +185,29 @@ struct sock *__inet_lookup_listener(struct net *net, ...@@ -183,17 +185,29 @@ struct sock *__inet_lookup_listener(struct net *net,
struct hlist_nulls_node *node; struct hlist_nulls_node *node;
unsigned int hash = inet_lhashfn(net, hnum); unsigned int hash = inet_lhashfn(net, hnum);
struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash]; struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];
int score, hiscore; int score, hiscore, matches = 0, reuseport = 0;
u32 phash = 0;
rcu_read_lock(); rcu_read_lock();
begin: begin:
result = NULL; result = NULL;
hiscore = -1; hiscore = 0;
sk_nulls_for_each_rcu(sk, node, &ilb->head) { sk_nulls_for_each_rcu(sk, node, &ilb->head) {
score = compute_score(sk, net, hnum, daddr, dif); score = compute_score(sk, net, hnum, daddr, dif);
if (score > hiscore) { if (score > hiscore) {
result = sk; result = sk;
hiscore = score; hiscore = score;
reuseport = sk->sk_reuseport;
if (reuseport) {
phash = inet_ehashfn(net, daddr, hnum,
saddr, sport);
matches = 1;
}
} else if (score == hiscore && reuseport) {
matches++;
if (((u64)phash * matches) >> 32 == 0)
result = sk;
phash = next_pseudo_random32(phash);
} }
} }
/* /*
...@@ -501,7 +515,8 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, ...@@ -501,7 +515,8 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
inet_bind_bucket_for_each(tb, node, &head->chain) { inet_bind_bucket_for_each(tb, node, &head->chain) {
if (net_eq(ib_net(tb), net) && if (net_eq(ib_net(tb), net) &&
tb->port == port) { tb->port == port) {
if (tb->fastreuse >= 0) if (tb->fastreuse >= 0 ||
tb->fastreuseport >= 0)
goto next_port; goto next_port;
WARN_ON(hlist_empty(&tb->owners)); WARN_ON(hlist_empty(&tb->owners));
if (!check_established(death_row, sk, if (!check_established(death_row, sk,
...@@ -518,6 +533,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, ...@@ -518,6 +533,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
break; break;
} }
tb->fastreuse = -1; tb->fastreuse = -1;
tb->fastreuseport = -1;
goto ok; goto ok;
next_port: next_port:
......
...@@ -657,7 +657,8 @@ static void tcp_v4_send_reset(struct sock *sk, struct sk_buff *skb) ...@@ -657,7 +657,8 @@ static void tcp_v4_send_reset(struct sock *sk, struct sk_buff *skb)
* no RST generated if md5 hash doesn't match. * no RST generated if md5 hash doesn't match.
*/ */
sk1 = __inet_lookup_listener(dev_net(skb_dst(skb)->dev), sk1 = __inet_lookup_listener(dev_net(skb_dst(skb)->dev),
&tcp_hashinfo, ip_hdr(skb)->daddr, &tcp_hashinfo, ip_hdr(skb)->saddr,
th->source, ip_hdr(skb)->daddr,
ntohs(th->source), inet_iif(skb)); ntohs(th->source), inet_iif(skb));
/* don't send rst if it can't find key */ /* don't send rst if it can't find key */
if (!sk1) if (!sk1)
...@@ -2074,6 +2075,7 @@ int tcp_v4_rcv(struct sk_buff *skb) ...@@ -2074,6 +2075,7 @@ int tcp_v4_rcv(struct sk_buff *skb)
case TCP_TW_SYN: { case TCP_TW_SYN: {
struct sock *sk2 = inet_lookup_listener(dev_net(skb->dev), struct sock *sk2 = inet_lookup_listener(dev_net(skb->dev),
&tcp_hashinfo, &tcp_hashinfo,
iph->saddr, th->source,
iph->daddr, th->dest, iph->daddr, th->dest,
inet_iif(skb)); inet_iif(skb));
if (sk2) { if (sk2) {
......
...@@ -139,6 +139,7 @@ static int udp_lib_lport_inuse(struct net *net, __u16 num, ...@@ -139,6 +139,7 @@ static int udp_lib_lport_inuse(struct net *net, __u16 num,
{ {
struct sock *sk2; struct sock *sk2;
struct hlist_nulls_node *node; struct hlist_nulls_node *node;
kuid_t uid = sock_i_uid(sk);
sk_nulls_for_each(sk2, node, &hslot->head) sk_nulls_for_each(sk2, node, &hslot->head)
if (net_eq(sock_net(sk2), net) && if (net_eq(sock_net(sk2), net) &&
...@@ -147,6 +148,8 @@ static int udp_lib_lport_inuse(struct net *net, __u16 num, ...@@ -147,6 +148,8 @@ static int udp_lib_lport_inuse(struct net *net, __u16 num,
(!sk2->sk_reuse || !sk->sk_reuse) && (!sk2->sk_reuse || !sk->sk_reuse) &&
(!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if || (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
(!sk2->sk_reuseport || !sk->sk_reuseport ||
!uid_eq(uid, sock_i_uid(sk2))) &&
(*saddr_comp)(sk, sk2)) { (*saddr_comp)(sk, sk2)) {
if (bitmap) if (bitmap)
__set_bit(udp_sk(sk2)->udp_port_hash >> log, __set_bit(udp_sk(sk2)->udp_port_hash >> log,
...@@ -169,6 +172,7 @@ static int udp_lib_lport_inuse2(struct net *net, __u16 num, ...@@ -169,6 +172,7 @@ static int udp_lib_lport_inuse2(struct net *net, __u16 num,
{ {
struct sock *sk2; struct sock *sk2;
struct hlist_nulls_node *node; struct hlist_nulls_node *node;
kuid_t uid = sock_i_uid(sk);
int res = 0; int res = 0;
spin_lock(&hslot2->lock); spin_lock(&hslot2->lock);
...@@ -179,6 +183,8 @@ static int udp_lib_lport_inuse2(struct net *net, __u16 num, ...@@ -179,6 +183,8 @@ static int udp_lib_lport_inuse2(struct net *net, __u16 num,
(!sk2->sk_reuse || !sk->sk_reuse) && (!sk2->sk_reuse || !sk->sk_reuse) &&
(!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if || (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
(!sk2->sk_reuseport || !sk->sk_reuseport ||
!uid_eq(uid, sock_i_uid(sk2))) &&
(*saddr_comp)(sk, sk2)) { (*saddr_comp)(sk, sk2)) {
res = 1; res = 1;
break; break;
...@@ -337,26 +343,26 @@ static inline int compute_score(struct sock *sk, struct net *net, __be32 saddr, ...@@ -337,26 +343,26 @@ static inline int compute_score(struct sock *sk, struct net *net, __be32 saddr,
!ipv6_only_sock(sk)) { !ipv6_only_sock(sk)) {
struct inet_sock *inet = inet_sk(sk); struct inet_sock *inet = inet_sk(sk);
score = (sk->sk_family == PF_INET ? 1 : 0); score = (sk->sk_family == PF_INET ? 2 : 1);
if (inet->inet_rcv_saddr) { if (inet->inet_rcv_saddr) {
if (inet->inet_rcv_saddr != daddr) if (inet->inet_rcv_saddr != daddr)
return -1; return -1;
score += 2; score += 4;
} }
if (inet->inet_daddr) { if (inet->inet_daddr) {
if (inet->inet_daddr != saddr) if (inet->inet_daddr != saddr)
return -1; return -1;
score += 2; score += 4;
} }
if (inet->inet_dport) { if (inet->inet_dport) {
if (inet->inet_dport != sport) if (inet->inet_dport != sport)
return -1; return -1;
score += 2; score += 4;
} }
if (sk->sk_bound_dev_if) { if (sk->sk_bound_dev_if) {
if (sk->sk_bound_dev_if != dif) if (sk->sk_bound_dev_if != dif)
return -1; return -1;
score += 2; score += 4;
} }
} }
return score; return score;
...@@ -365,7 +371,6 @@ static inline int compute_score(struct sock *sk, struct net *net, __be32 saddr, ...@@ -365,7 +371,6 @@ static inline int compute_score(struct sock *sk, struct net *net, __be32 saddr,
/* /*
* In this second variant, we check (daddr, dport) matches (inet_rcv_sadd, inet_num) * In this second variant, we check (daddr, dport) matches (inet_rcv_sadd, inet_num)
*/ */
#define SCORE2_MAX (1 + 2 + 2 + 2)
static inline int compute_score2(struct sock *sk, struct net *net, static inline int compute_score2(struct sock *sk, struct net *net,
__be32 saddr, __be16 sport, __be32 saddr, __be16 sport,
__be32 daddr, unsigned int hnum, int dif) __be32 daddr, unsigned int hnum, int dif)
...@@ -380,21 +385,21 @@ static inline int compute_score2(struct sock *sk, struct net *net, ...@@ -380,21 +385,21 @@ static inline int compute_score2(struct sock *sk, struct net *net,
if (inet->inet_num != hnum) if (inet->inet_num != hnum)
return -1; return -1;
score = (sk->sk_family == PF_INET ? 1 : 0); score = (sk->sk_family == PF_INET ? 2 : 1);
if (inet->inet_daddr) { if (inet->inet_daddr) {
if (inet->inet_daddr != saddr) if (inet->inet_daddr != saddr)
return -1; return -1;
score += 2; score += 4;
} }
if (inet->inet_dport) { if (inet->inet_dport) {
if (inet->inet_dport != sport) if (inet->inet_dport != sport)
return -1; return -1;
score += 2; score += 4;
} }
if (sk->sk_bound_dev_if) { if (sk->sk_bound_dev_if) {
if (sk->sk_bound_dev_if != dif) if (sk->sk_bound_dev_if != dif)
return -1; return -1;
score += 2; score += 4;
} }
} }
return score; return score;
...@@ -409,19 +414,29 @@ static struct sock *udp4_lib_lookup2(struct net *net, ...@@ -409,19 +414,29 @@ static struct sock *udp4_lib_lookup2(struct net *net,
{ {
struct sock *sk, *result; struct sock *sk, *result;
struct hlist_nulls_node *node; struct hlist_nulls_node *node;
int score, badness; int score, badness, matches = 0, reuseport = 0;
u32 hash = 0;
begin: begin:
result = NULL; result = NULL;
badness = -1; badness = 0;
udp_portaddr_for_each_entry_rcu(sk, node, &hslot2->head) { udp_portaddr_for_each_entry_rcu(sk, node, &hslot2->head) {
score = compute_score2(sk, net, saddr, sport, score = compute_score2(sk, net, saddr, sport,
daddr, hnum, dif); daddr, hnum, dif);
if (score > badness) { if (score > badness) {
result = sk; result = sk;
badness = score; badness = score;
if (score == SCORE2_MAX) reuseport = sk->sk_reuseport;
goto exact_match; if (reuseport) {
hash = inet_ehashfn(net, daddr, hnum,
saddr, htons(sport));
matches = 1;
}
} else if (score == badness && reuseport) {
matches++;
if (((u64)hash * matches) >> 32 == 0)
result = sk;
hash = next_pseudo_random32(hash);
} }
} }
/* /*
...@@ -431,9 +446,7 @@ static struct sock *udp4_lib_lookup2(struct net *net, ...@@ -431,9 +446,7 @@ static struct sock *udp4_lib_lookup2(struct net *net,
*/ */
if (get_nulls_value(node) != slot2) if (get_nulls_value(node) != slot2)
goto begin; goto begin;
if (result) { if (result) {
exact_match:
if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2))) if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
result = NULL; result = NULL;
else if (unlikely(compute_score2(result, net, saddr, sport, else if (unlikely(compute_score2(result, net, saddr, sport,
...@@ -457,7 +470,8 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, ...@@ -457,7 +470,8 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
unsigned short hnum = ntohs(dport); unsigned short hnum = ntohs(dport);
unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask); unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask);
struct udp_hslot *hslot2, *hslot = &udptable->hash[slot]; struct udp_hslot *hslot2, *hslot = &udptable->hash[slot];
int score, badness; int score, badness, matches = 0, reuseport = 0;
u32 hash = 0;
rcu_read_lock(); rcu_read_lock();
if (hslot->count > 10) { if (hslot->count > 10) {
...@@ -486,13 +500,24 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, ...@@ -486,13 +500,24 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
} }
begin: begin:
result = NULL; result = NULL;
badness = -1; badness = 0;
sk_nulls_for_each_rcu(sk, node, &hslot->head) { sk_nulls_for_each_rcu(sk, node, &hslot->head) {
score = compute_score(sk, net, saddr, hnum, sport, score = compute_score(sk, net, saddr, hnum, sport,
daddr, dport, dif); daddr, dport, dif);
if (score > badness) { if (score > badness) {
result = sk; result = sk;
badness = score; badness = score;
reuseport = sk->sk_reuseport;
if (reuseport) {
hash = inet_ehashfn(net, daddr, hnum,
saddr, htons(sport));
matches = 1;
}
} else if (score == badness && reuseport) {
matches++;
if (((u64)hash * matches) >> 32 == 0)
result = sk;
hash = next_pseudo_random32(hash);
} }
} }
/* /*
......
...@@ -32,6 +32,9 @@ int inet6_csk_bind_conflict(const struct sock *sk, ...@@ -32,6 +32,9 @@ int inet6_csk_bind_conflict(const struct sock *sk,
{ {
const struct sock *sk2; const struct sock *sk2;
const struct hlist_node *node; const struct hlist_node *node;
int reuse = sk->sk_reuse;
int reuseport = sk->sk_reuseport;
int uid = sock_i_uid((struct sock *)sk);
/* We must walk the whole port owner list in this case. -DaveM */ /* We must walk the whole port owner list in this case. -DaveM */
/* /*
...@@ -42,11 +45,17 @@ int inet6_csk_bind_conflict(const struct sock *sk, ...@@ -42,11 +45,17 @@ int inet6_csk_bind_conflict(const struct sock *sk,
if (sk != sk2 && if (sk != sk2 &&
(!sk->sk_bound_dev_if || (!sk->sk_bound_dev_if ||
!sk2->sk_bound_dev_if || !sk2->sk_bound_dev_if ||
sk->sk_bound_dev_if == sk2->sk_bound_dev_if) && sk->sk_bound_dev_if == sk2->sk_bound_dev_if)) {
(!sk->sk_reuse || !sk2->sk_reuse || if ((!reuse || !sk2->sk_reuse ||
sk2->sk_state == TCP_LISTEN) && sk2->sk_state == TCP_LISTEN) &&
ipv6_rcv_saddr_equal(sk, sk2)) (!reuseport || !sk2->sk_reuseport ||
break; (sk2->sk_state != TCP_TIME_WAIT &&
!uid_eq(uid,
sock_i_uid((struct sock *)sk2))))) {
if (ipv6_rcv_saddr_equal(sk, sk2))
break;
}
}
} }
return node != NULL; return node != NULL;
......
...@@ -158,25 +158,38 @@ static inline int compute_score(struct sock *sk, struct net *net, ...@@ -158,25 +158,38 @@ static inline int compute_score(struct sock *sk, struct net *net,
} }
struct sock *inet6_lookup_listener(struct net *net, struct sock *inet6_lookup_listener(struct net *net,
struct inet_hashinfo *hashinfo, const struct in6_addr *daddr, struct inet_hashinfo *hashinfo, const struct in6_addr *saddr,
const __be16 sport, const struct in6_addr *daddr,
const unsigned short hnum, const int dif) const unsigned short hnum, const int dif)
{ {
struct sock *sk; struct sock *sk;
const struct hlist_nulls_node *node; const struct hlist_nulls_node *node;
struct sock *result; struct sock *result;
int score, hiscore; int score, hiscore, matches = 0, reuseport = 0;
u32 phash = 0;
unsigned int hash = inet_lhashfn(net, hnum); unsigned int hash = inet_lhashfn(net, hnum);
struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash]; struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];
rcu_read_lock(); rcu_read_lock();
begin: begin:
result = NULL; result = NULL;
hiscore = -1; hiscore = 0;
sk_nulls_for_each(sk, node, &ilb->head) { sk_nulls_for_each(sk, node, &ilb->head) {
score = compute_score(sk, net, hnum, daddr, dif); score = compute_score(sk, net, hnum, daddr, dif);
if (score > hiscore) { if (score > hiscore) {
hiscore = score; hiscore = score;
result = sk; result = sk;
reuseport = sk->sk_reuseport;
if (reuseport) {
phash = inet6_ehashfn(net, daddr, hnum,
saddr, sport);
matches = 1;
}
} else if (score == hiscore && reuseport) {
matches++;
if (((u64)phash * matches) >> 32 == 0)
result = sk;
phash = next_pseudo_random32(phash);
} }
} }
/* /*
......
...@@ -834,7 +834,8 @@ static void tcp_v6_send_reset(struct sock *sk, struct sk_buff *skb) ...@@ -834,7 +834,8 @@ static void tcp_v6_send_reset(struct sock *sk, struct sk_buff *skb)
* no RST generated if md5 hash doesn't match. * no RST generated if md5 hash doesn't match.
*/ */
sk1 = inet6_lookup_listener(dev_net(skb_dst(skb)->dev), sk1 = inet6_lookup_listener(dev_net(skb_dst(skb)->dev),
&tcp_hashinfo, &ipv6h->daddr, &tcp_hashinfo, &ipv6h->saddr,
th->source, &ipv6h->daddr,
ntohs(th->source), inet6_iif(skb)); ntohs(th->source), inet6_iif(skb));
if (!sk1) if (!sk1)
return; return;
...@@ -1598,6 +1599,7 @@ static int tcp_v6_rcv(struct sk_buff *skb) ...@@ -1598,6 +1599,7 @@ static int tcp_v6_rcv(struct sk_buff *skb)
struct sock *sk2; struct sock *sk2;
sk2 = inet6_lookup_listener(dev_net(skb->dev), &tcp_hashinfo, sk2 = inet6_lookup_listener(dev_net(skb->dev), &tcp_hashinfo,
&ipv6_hdr(skb)->saddr, th->source,
&ipv6_hdr(skb)->daddr, &ipv6_hdr(skb)->daddr,
ntohs(th->dest), inet6_iif(skb)); ntohs(th->dest), inet6_iif(skb));
if (sk2 != NULL) { if (sk2 != NULL) {
......
...@@ -45,6 +45,7 @@ ...@@ -45,6 +45,7 @@
#include <net/tcp_states.h> #include <net/tcp_states.h>
#include <net/ip6_checksum.h> #include <net/ip6_checksum.h>
#include <net/xfrm.h> #include <net/xfrm.h>
#include <net/inet6_hashtables.h>
#include <linux/proc_fs.h> #include <linux/proc_fs.h>
#include <linux/seq_file.h> #include <linux/seq_file.h>
...@@ -203,7 +204,8 @@ static struct sock *udp6_lib_lookup2(struct net *net, ...@@ -203,7 +204,8 @@ static struct sock *udp6_lib_lookup2(struct net *net,
{ {
struct sock *sk, *result; struct sock *sk, *result;
struct hlist_nulls_node *node; struct hlist_nulls_node *node;
int score, badness; int score, badness, matches = 0, reuseport = 0;
u32 hash = 0;
begin: begin:
result = NULL; result = NULL;
...@@ -214,8 +216,18 @@ static struct sock *udp6_lib_lookup2(struct net *net, ...@@ -214,8 +216,18 @@ static struct sock *udp6_lib_lookup2(struct net *net,
if (score > badness) { if (score > badness) {
result = sk; result = sk;
badness = score; badness = score;
if (score == SCORE2_MAX) reuseport = sk->sk_reuseport;
if (reuseport) {
hash = inet6_ehashfn(net, daddr, hnum,
saddr, sport);
matches = 1;
} else if (score == SCORE2_MAX)
goto exact_match; goto exact_match;
} else if (score == badness && reuseport) {
matches++;
if (((u64)hash * matches) >> 32 == 0)
result = sk;
hash = next_pseudo_random32(hash);
} }
} }
/* /*
...@@ -249,7 +261,8 @@ struct sock *__udp6_lib_lookup(struct net *net, ...@@ -249,7 +261,8 @@ struct sock *__udp6_lib_lookup(struct net *net,
unsigned short hnum = ntohs(dport); unsigned short hnum = ntohs(dport);
unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask); unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask);
struct udp_hslot *hslot2, *hslot = &udptable->hash[slot]; struct udp_hslot *hslot2, *hslot = &udptable->hash[slot];
int score, badness; int score, badness, matches = 0, reuseport = 0;
u32 hash = 0;
rcu_read_lock(); rcu_read_lock();
if (hslot->count > 10) { if (hslot->count > 10) {
...@@ -284,6 +297,17 @@ struct sock *__udp6_lib_lookup(struct net *net, ...@@ -284,6 +297,17 @@ struct sock *__udp6_lib_lookup(struct net *net,
if (score > badness) { if (score > badness) {
result = sk; result = sk;
badness = score; badness = score;
reuseport = sk->sk_reuseport;
if (reuseport) {
hash = inet6_ehashfn(net, daddr, hnum,
saddr, sport);
matches = 1;
}
} else if (score == badness && reuseport) {
matches++;
if (((u64)hash * matches) >> 32 == 0)
result = sk;
hash = next_pseudo_random32(hash);
} }
} }
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment