Commit d983ea6f authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

tcp: add rcu protection around tp->fastopen_rsk

Both tcp_v4_err() and tcp_v6_err() do the following operations
while they do not own the socket lock :

	fastopen = tp->fastopen_rsk;
 	snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;

The problem is that without appropriate barrier, the compiler
might reload tp->fastopen_rsk and trigger a NULL deref.

request sockets are protected by RCU, we can simply add
the missing annotations and barriers to solve the issue.

Fixes: 168a8f58 ("tcp: TCP Fast Open Server - main code path")
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 8caf8a91
...@@ -393,7 +393,7 @@ struct tcp_sock { ...@@ -393,7 +393,7 @@ struct tcp_sock {
/* fastopen_rsk points to request_sock that resulted in this big /* fastopen_rsk points to request_sock that resulted in this big
* socket. Used to retransmit SYNACKs etc. * socket. Used to retransmit SYNACKs etc.
*/ */
struct request_sock *fastopen_rsk; struct request_sock __rcu *fastopen_rsk;
u32 *saved_syn; u32 *saved_syn;
}; };
...@@ -447,8 +447,8 @@ static inline struct tcp_timewait_sock *tcp_twsk(const struct sock *sk) ...@@ -447,8 +447,8 @@ static inline struct tcp_timewait_sock *tcp_twsk(const struct sock *sk)
static inline bool tcp_passive_fastopen(const struct sock *sk) static inline bool tcp_passive_fastopen(const struct sock *sk)
{ {
return (sk->sk_state == TCP_SYN_RECV && return sk->sk_state == TCP_SYN_RECV &&
tcp_sk(sk)->fastopen_rsk != NULL); rcu_access_pointer(tcp_sk(sk)->fastopen_rsk) != NULL;
} }
static inline void fastopen_queue_tune(struct sock *sk, int backlog) static inline void fastopen_queue_tune(struct sock *sk, int backlog)
......
...@@ -96,7 +96,7 @@ void reqsk_fastopen_remove(struct sock *sk, struct request_sock *req, ...@@ -96,7 +96,7 @@ void reqsk_fastopen_remove(struct sock *sk, struct request_sock *req,
fastopenq = &inet_csk(lsk)->icsk_accept_queue.fastopenq; fastopenq = &inet_csk(lsk)->icsk_accept_queue.fastopenq;
tcp_sk(sk)->fastopen_rsk = NULL; RCU_INIT_POINTER(tcp_sk(sk)->fastopen_rsk, NULL);
spin_lock_bh(&fastopenq->lock); spin_lock_bh(&fastopenq->lock);
fastopenq->qlen--; fastopenq->qlen--;
tcp_rsk(req)->tfo_listener = false; tcp_rsk(req)->tfo_listener = false;
......
...@@ -906,7 +906,7 @@ static void inet_child_forget(struct sock *sk, struct request_sock *req, ...@@ -906,7 +906,7 @@ static void inet_child_forget(struct sock *sk, struct request_sock *req,
percpu_counter_inc(sk->sk_prot->orphan_count); percpu_counter_inc(sk->sk_prot->orphan_count);
if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(req)->tfo_listener) { if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(req)->tfo_listener) {
BUG_ON(tcp_sk(child)->fastopen_rsk != req); BUG_ON(rcu_access_pointer(tcp_sk(child)->fastopen_rsk) != req);
BUG_ON(sk != req->rsk_listener); BUG_ON(sk != req->rsk_listener);
/* Paranoid, to prevent race condition if /* Paranoid, to prevent race condition if
...@@ -915,7 +915,7 @@ static void inet_child_forget(struct sock *sk, struct request_sock *req, ...@@ -915,7 +915,7 @@ static void inet_child_forget(struct sock *sk, struct request_sock *req,
* Also to satisfy an assertion in * Also to satisfy an assertion in
* tcp_v4_destroy_sock(). * tcp_v4_destroy_sock().
*/ */
tcp_sk(child)->fastopen_rsk = NULL; RCU_INIT_POINTER(tcp_sk(child)->fastopen_rsk, NULL);
} }
inet_csk_destroy_sock(child); inet_csk_destroy_sock(child);
} }
......
...@@ -543,7 +543,7 @@ __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait) ...@@ -543,7 +543,7 @@ __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
/* Connected or passive Fast Open socket? */ /* Connected or passive Fast Open socket? */
if (state != TCP_SYN_SENT && if (state != TCP_SYN_SENT &&
(state != TCP_SYN_RECV || tp->fastopen_rsk)) { (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) {
int target = sock_rcvlowat(sk, 0, INT_MAX); int target = sock_rcvlowat(sk, 0, INT_MAX);
if (tp->urg_seq == tp->copied_seq && if (tp->urg_seq == tp->copied_seq &&
...@@ -2487,7 +2487,10 @@ void tcp_close(struct sock *sk, long timeout) ...@@ -2487,7 +2487,10 @@ void tcp_close(struct sock *sk, long timeout)
} }
if (sk->sk_state == TCP_CLOSE) { if (sk->sk_state == TCP_CLOSE) {
struct request_sock *req = tcp_sk(sk)->fastopen_rsk; struct request_sock *req;
req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk,
lockdep_sock_is_held(sk));
/* We could get here with a non-NULL req if the socket is /* We could get here with a non-NULL req if the socket is
* aborted (e.g., closed with unread data) before 3WHS * aborted (e.g., closed with unread data) before 3WHS
* finishes. * finishes.
...@@ -3831,8 +3834,10 @@ EXPORT_SYMBOL(tcp_md5_hash_key); ...@@ -3831,8 +3834,10 @@ EXPORT_SYMBOL(tcp_md5_hash_key);
void tcp_done(struct sock *sk) void tcp_done(struct sock *sk)
{ {
struct request_sock *req = tcp_sk(sk)->fastopen_rsk; struct request_sock *req;
req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk,
lockdep_sock_is_held(sk));
if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV) if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS); TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
......
...@@ -253,7 +253,7 @@ static struct sock *tcp_fastopen_create_child(struct sock *sk, ...@@ -253,7 +253,7 @@ static struct sock *tcp_fastopen_create_child(struct sock *sk,
*/ */
tp = tcp_sk(child); tp = tcp_sk(child);
tp->fastopen_rsk = req; rcu_assign_pointer(tp->fastopen_rsk, req);
tcp_rsk(req)->tfo_listener = true; tcp_rsk(req)->tfo_listener = true;
/* RFC1323: The window in SYN & SYN/ACK segments is never /* RFC1323: The window in SYN & SYN/ACK segments is never
......
...@@ -2666,7 +2666,7 @@ static void tcp_process_loss(struct sock *sk, int flag, int num_dupack, ...@@ -2666,7 +2666,7 @@ static void tcp_process_loss(struct sock *sk, int flag, int num_dupack,
struct tcp_sock *tp = tcp_sk(sk); struct tcp_sock *tp = tcp_sk(sk);
bool recovered = !before(tp->snd_una, tp->high_seq); bool recovered = !before(tp->snd_una, tp->high_seq);
if ((flag & FLAG_SND_UNA_ADVANCED || tp->fastopen_rsk) && if ((flag & FLAG_SND_UNA_ADVANCED || rcu_access_pointer(tp->fastopen_rsk)) &&
tcp_try_undo_loss(sk, false)) tcp_try_undo_loss(sk, false))
return; return;
...@@ -2990,7 +2990,7 @@ void tcp_rearm_rto(struct sock *sk) ...@@ -2990,7 +2990,7 @@ void tcp_rearm_rto(struct sock *sk)
/* If the retrans timer is currently being used by Fast Open /* If the retrans timer is currently being used by Fast Open
* for SYN-ACK retrans purpose, stay put. * for SYN-ACK retrans purpose, stay put.
*/ */
if (tp->fastopen_rsk) if (rcu_access_pointer(tp->fastopen_rsk))
return; return;
if (!tp->packets_out) { if (!tp->packets_out) {
...@@ -6087,6 +6087,8 @@ static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb, ...@@ -6087,6 +6087,8 @@ static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb,
static void tcp_rcv_synrecv_state_fastopen(struct sock *sk) static void tcp_rcv_synrecv_state_fastopen(struct sock *sk)
{ {
struct request_sock *req;
tcp_try_undo_loss(sk, false); tcp_try_undo_loss(sk, false);
/* Reset rtx states to prevent spurious retransmits_timed_out() */ /* Reset rtx states to prevent spurious retransmits_timed_out() */
...@@ -6096,7 +6098,9 @@ static void tcp_rcv_synrecv_state_fastopen(struct sock *sk) ...@@ -6096,7 +6098,9 @@ static void tcp_rcv_synrecv_state_fastopen(struct sock *sk)
/* Once we leave TCP_SYN_RECV or TCP_FIN_WAIT_1, /* Once we leave TCP_SYN_RECV or TCP_FIN_WAIT_1,
* we no longer need req so release it. * we no longer need req so release it.
*/ */
reqsk_fastopen_remove(sk, tcp_sk(sk)->fastopen_rsk, false); req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk,
lockdep_sock_is_held(sk));
reqsk_fastopen_remove(sk, req, false);
/* Re-arm the timer because data may have been sent out. /* Re-arm the timer because data may have been sent out.
* This is similar to the regular data transmission case * This is similar to the regular data transmission case
...@@ -6171,7 +6175,8 @@ int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb) ...@@ -6171,7 +6175,8 @@ int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb)
tcp_mstamp_refresh(tp); tcp_mstamp_refresh(tp);
tp->rx_opt.saw_tstamp = 0; tp->rx_opt.saw_tstamp = 0;
req = tp->fastopen_rsk; req = rcu_dereference_protected(tp->fastopen_rsk,
lockdep_sock_is_held(sk));
if (req) { if (req) {
bool req_stolen; bool req_stolen;
......
...@@ -478,7 +478,7 @@ int tcp_v4_err(struct sk_buff *icmp_skb, u32 info) ...@@ -478,7 +478,7 @@ int tcp_v4_err(struct sk_buff *icmp_skb, u32 info)
icsk = inet_csk(sk); icsk = inet_csk(sk);
tp = tcp_sk(sk); tp = tcp_sk(sk);
/* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */ /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
fastopen = tp->fastopen_rsk; fastopen = rcu_dereference(tp->fastopen_rsk);
snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una; snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
if (sk->sk_state != TCP_LISTEN && if (sk->sk_state != TCP_LISTEN &&
!between(seq, snd_una, tp->snd_nxt)) { !between(seq, snd_una, tp->snd_nxt)) {
...@@ -2121,7 +2121,7 @@ void tcp_v4_destroy_sock(struct sock *sk) ...@@ -2121,7 +2121,7 @@ void tcp_v4_destroy_sock(struct sock *sk)
if (inet_csk(sk)->icsk_bind_hash) if (inet_csk(sk)->icsk_bind_hash)
inet_put_port(sk); inet_put_port(sk);
BUG_ON(tp->fastopen_rsk); BUG_ON(rcu_access_pointer(tp->fastopen_rsk));
/* If socket is aborted during connect operation */ /* If socket is aborted during connect operation */
tcp_free_fastopen_req(tp); tcp_free_fastopen_req(tp);
......
...@@ -541,7 +541,7 @@ struct sock *tcp_create_openreq_child(const struct sock *sk, ...@@ -541,7 +541,7 @@ struct sock *tcp_create_openreq_child(const struct sock *sk,
newtp->rx_opt.mss_clamp = req->mss; newtp->rx_opt.mss_clamp = req->mss;
tcp_ecn_openreq_child(newtp, req); tcp_ecn_openreq_child(newtp, req);
newtp->fastopen_req = NULL; newtp->fastopen_req = NULL;
newtp->fastopen_rsk = NULL; RCU_INIT_POINTER(newtp->fastopen_rsk, NULL);
__TCP_INC_STATS(sock_net(sk), TCP_MIB_PASSIVEOPENS); __TCP_INC_STATS(sock_net(sk), TCP_MIB_PASSIVEOPENS);
......
...@@ -2482,7 +2482,7 @@ bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto) ...@@ -2482,7 +2482,7 @@ bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto)
/* Don't do any loss probe on a Fast Open connection before 3WHS /* Don't do any loss probe on a Fast Open connection before 3WHS
* finishes. * finishes.
*/ */
if (tp->fastopen_rsk) if (rcu_access_pointer(tp->fastopen_rsk))
return false; return false;
early_retrans = sock_net(sk)->ipv4.sysctl_tcp_early_retrans; early_retrans = sock_net(sk)->ipv4.sysctl_tcp_early_retrans;
......
...@@ -386,15 +386,13 @@ abort: tcp_write_err(sk); ...@@ -386,15 +386,13 @@ abort: tcp_write_err(sk);
* Timer for Fast Open socket to retransmit SYNACK. Note that the * Timer for Fast Open socket to retransmit SYNACK. Note that the
* sk here is the child socket, not the parent (listener) socket. * sk here is the child socket, not the parent (listener) socket.
*/ */
static void tcp_fastopen_synack_timer(struct sock *sk) static void tcp_fastopen_synack_timer(struct sock *sk, struct request_sock *req)
{ {
struct inet_connection_sock *icsk = inet_csk(sk); struct inet_connection_sock *icsk = inet_csk(sk);
int max_retries = icsk->icsk_syn_retries ? : int max_retries = icsk->icsk_syn_retries ? :
sock_net(sk)->ipv4.sysctl_tcp_synack_retries + 1; /* add one more retry for fastopen */ sock_net(sk)->ipv4.sysctl_tcp_synack_retries + 1; /* add one more retry for fastopen */
struct tcp_sock *tp = tcp_sk(sk); struct tcp_sock *tp = tcp_sk(sk);
struct request_sock *req;
req = tcp_sk(sk)->fastopen_rsk;
req->rsk_ops->syn_ack_timeout(req); req->rsk_ops->syn_ack_timeout(req);
if (req->num_timeout >= max_retries) { if (req->num_timeout >= max_retries) {
...@@ -435,11 +433,14 @@ void tcp_retransmit_timer(struct sock *sk) ...@@ -435,11 +433,14 @@ void tcp_retransmit_timer(struct sock *sk)
struct tcp_sock *tp = tcp_sk(sk); struct tcp_sock *tp = tcp_sk(sk);
struct net *net = sock_net(sk); struct net *net = sock_net(sk);
struct inet_connection_sock *icsk = inet_csk(sk); struct inet_connection_sock *icsk = inet_csk(sk);
struct request_sock *req;
if (tp->fastopen_rsk) { req = rcu_dereference_protected(tp->fastopen_rsk,
lockdep_sock_is_held(sk));
if (req) {
WARN_ON_ONCE(sk->sk_state != TCP_SYN_RECV && WARN_ON_ONCE(sk->sk_state != TCP_SYN_RECV &&
sk->sk_state != TCP_FIN_WAIT1); sk->sk_state != TCP_FIN_WAIT1);
tcp_fastopen_synack_timer(sk); tcp_fastopen_synack_timer(sk, req);
/* Before we receive ACK to our SYN-ACK don't retransmit /* Before we receive ACK to our SYN-ACK don't retransmit
* anything else (e.g., data or FIN segments). * anything else (e.g., data or FIN segments).
*/ */
......
...@@ -406,7 +406,7 @@ static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, ...@@ -406,7 +406,7 @@ static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
tp = tcp_sk(sk); tp = tcp_sk(sk);
/* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */ /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
fastopen = tp->fastopen_rsk; fastopen = rcu_dereference(tp->fastopen_rsk);
snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una; snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
if (sk->sk_state != TCP_LISTEN && if (sk->sk_state != TCP_LISTEN &&
!between(seq, snd_una, tp->snd_nxt)) { !between(seq, snd_una, tp->snd_nxt)) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment