Commit 10e361e1 authored by David S. Miller's avatar David S. Miller

Merge branch 'tcp-default-RACK-loss-recovery'

Yuchung Cheng says:

====================
tcp: default RACK loss recovery

This patch set implements the features correspond to the
draft-ietf-tcpm-rack-03 version of the RACK draft.
https://datatracker.ietf.org/meeting/101/materials/slides-101-tcpm-update-on-tcp-rack-00

1. SACK: implement equivalent DUPACK threshold heuristic in RACK to
   replace existing RFC6675 recovery (tcp_mark_head_lost).

2. Non-SACK: simplify RFC6582 NewReno implementation

3. RTO: apply RACK's time-based approach to avoid spuriouly
   marking very recently sent packets lost.

4. with (1)(2)(3), make RACK the exclusive fast recovery mechanism to
   mark losses based on time on S/ACK. Tail loss probe and F-RTO remain
   enabled by default as complementary mechanisms to send probes in
   CA_Open and CA_Loss states. The probes would solicit S/ACKs to trigger
   RACK time-based loss detection.

All Google web and internal servers have been running RACK-only mode
(4) for a while now. a/b experiments indicate RACK/TLP on average
reduces recovery latency by 10% compared to RFC6675. RFC6675
is default-off now but can be enabled by disabling RACK (sysctl
net.ipv4.tcp_recovery=0) for unseen issues.
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 9611d6d6 56f8c5d7
...@@ -449,8 +449,10 @@ tcp_recovery - INTEGER ...@@ -449,8 +449,10 @@ tcp_recovery - INTEGER
features. features.
RACK: 0x1 enables the RACK loss detection for fast detection of lost RACK: 0x1 enables the RACK loss detection for fast detection of lost
retransmissions and tail drops. retransmissions and tail drops. It also subsumes and disables
RFC6675 recovery for SACK connections.
RACK: 0x2 makes RACK's reordering window static (min_rtt/4). RACK: 0x2 makes RACK's reordering window static (min_rtt/4).
RACK: 0x4 disables RACK's DUPACK threshold heuristic
Default: 0x1 Default: 0x1
......
...@@ -245,6 +245,7 @@ extern long sysctl_tcp_mem[3]; ...@@ -245,6 +245,7 @@ extern long sysctl_tcp_mem[3];
#define TCP_RACK_LOSS_DETECTION 0x1 /* Use RACK to detect losses */ #define TCP_RACK_LOSS_DETECTION 0x1 /* Use RACK to detect losses */
#define TCP_RACK_STATIC_REO_WND 0x2 /* Use static RACK reo wnd */ #define TCP_RACK_STATIC_REO_WND 0x2 /* Use static RACK reo wnd */
#define TCP_RACK_NO_DUPTHRESH 0x4 /* Do not use DUPACK threshold in RACK */
extern atomic_long_t tcp_memory_allocated; extern atomic_long_t tcp_memory_allocated;
extern struct percpu_counter tcp_sockets_allocated; extern struct percpu_counter tcp_sockets_allocated;
...@@ -1876,6 +1877,10 @@ void tcp_v4_init(void); ...@@ -1876,6 +1877,10 @@ void tcp_v4_init(void);
void tcp_init(void); void tcp_init(void);
/* tcp_recovery.c */ /* tcp_recovery.c */
void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb);
void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced);
extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb,
u32 reo_wnd);
extern void tcp_rack_mark_lost(struct sock *sk); extern void tcp_rack_mark_lost(struct sock *sk);
extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq, extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq,
u64 xmit_time); u64 xmit_time);
......
...@@ -1917,19 +1917,54 @@ static inline void tcp_init_undo(struct tcp_sock *tp) ...@@ -1917,19 +1917,54 @@ static inline void tcp_init_undo(struct tcp_sock *tp)
tp->undo_retrans = tp->retrans_out ? : -1; tp->undo_retrans = tp->retrans_out ? : -1;
} }
/* Enter Loss state. If we detect SACK reneging, forget all SACK information static bool tcp_is_rack(const struct sock *sk)
{
return sock_net(sk)->ipv4.sysctl_tcp_recovery & TCP_RACK_LOSS_DETECTION;
}
/* If we detect SACK reneging, forget all SACK information
* and reset tags completely, otherwise preserve SACKs. If receiver * and reset tags completely, otherwise preserve SACKs. If receiver
* dropped its ofo queue, we will know this due to reneging detection. * dropped its ofo queue, we will know this due to reneging detection.
*/ */
static void tcp_timeout_mark_lost(struct sock *sk)
{
struct tcp_sock *tp = tcp_sk(sk);
struct sk_buff *skb, *head;
bool is_reneg; /* is receiver reneging on SACKs? */
head = tcp_rtx_queue_head(sk);
is_reneg = head && (TCP_SKB_CB(head)->sacked & TCPCB_SACKED_ACKED);
if (is_reneg) {
NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSACKRENEGING);
tp->sacked_out = 0;
/* Mark SACK reneging until we recover from this loss event. */
tp->is_sack_reneg = 1;
} else if (tcp_is_reno(tp)) {
tcp_reset_reno_sack(tp);
}
skb = head;
skb_rbtree_walk_from(skb) {
if (is_reneg)
TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_ACKED;
else if (tcp_is_rack(sk) && skb != head &&
tcp_rack_skb_timeout(tp, skb, 0) > 0)
continue; /* Don't mark recently sent ones lost yet */
tcp_mark_skb_lost(sk, skb);
}
tcp_verify_left_out(tp);
tcp_clear_all_retrans_hints(tp);
}
/* Enter Loss state. */
void tcp_enter_loss(struct sock *sk) void tcp_enter_loss(struct sock *sk)
{ {
const struct inet_connection_sock *icsk = inet_csk(sk); const struct inet_connection_sock *icsk = inet_csk(sk);
struct tcp_sock *tp = tcp_sk(sk); struct tcp_sock *tp = tcp_sk(sk);
struct net *net = sock_net(sk); struct net *net = sock_net(sk);
struct sk_buff *skb;
bool new_recovery = icsk->icsk_ca_state < TCP_CA_Recovery; bool new_recovery = icsk->icsk_ca_state < TCP_CA_Recovery;
bool is_reneg; /* is receiver reneging on SACKs? */
bool mark_lost; tcp_timeout_mark_lost(sk);
/* Reduce ssthresh if it has not yet been made inside this window. */ /* Reduce ssthresh if it has not yet been made inside this window. */
if (icsk->icsk_ca_state <= TCP_CA_Disorder || if (icsk->icsk_ca_state <= TCP_CA_Disorder ||
...@@ -1941,40 +1976,10 @@ void tcp_enter_loss(struct sock *sk) ...@@ -1941,40 +1976,10 @@ void tcp_enter_loss(struct sock *sk)
tcp_ca_event(sk, CA_EVENT_LOSS); tcp_ca_event(sk, CA_EVENT_LOSS);
tcp_init_undo(tp); tcp_init_undo(tp);
} }
tp->snd_cwnd = 1; tp->snd_cwnd = tcp_packets_in_flight(tp) + 1;
tp->snd_cwnd_cnt = 0; tp->snd_cwnd_cnt = 0;
tp->snd_cwnd_stamp = tcp_jiffies32; tp->snd_cwnd_stamp = tcp_jiffies32;
tp->retrans_out = 0;
tp->lost_out = 0;
if (tcp_is_reno(tp))
tcp_reset_reno_sack(tp);
skb = tcp_rtx_queue_head(sk);
is_reneg = skb && (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED);
if (is_reneg) {
NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSACKRENEGING);
tp->sacked_out = 0;
/* Mark SACK reneging until we recover from this loss event. */
tp->is_sack_reneg = 1;
}
tcp_clear_all_retrans_hints(tp);
skb_rbtree_walk_from(skb) {
mark_lost = (!(TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED) ||
is_reneg);
if (mark_lost)
tcp_sum_lost(tp, skb);
TCP_SKB_CB(skb)->sacked &= (~TCPCB_TAGBITS)|TCPCB_SACKED_ACKED;
if (mark_lost) {
TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_ACKED;
TCP_SKB_CB(skb)->sacked |= TCPCB_LOST;
tp->lost_out += tcp_skb_pcount(skb);
}
}
tcp_verify_left_out(tp);
/* Timeout in disordered state after receiving substantial DUPACKs /* Timeout in disordered state after receiving substantial DUPACKs
* suggests that the degree of reordering is over-estimated. * suggests that the degree of reordering is over-estimated.
*/ */
...@@ -2141,7 +2146,7 @@ static bool tcp_time_to_recover(struct sock *sk, int flag) ...@@ -2141,7 +2146,7 @@ static bool tcp_time_to_recover(struct sock *sk, int flag)
return true; return true;
/* Not-A-Trick#2 : Classic rule... */ /* Not-A-Trick#2 : Classic rule... */
if (tcp_dupack_heuristics(tp) > tp->reordering) if (!tcp_is_rack(sk) && tcp_dupack_heuristics(tp) > tp->reordering)
return true; return true;
return false; return false;
...@@ -2218,9 +2223,7 @@ static void tcp_update_scoreboard(struct sock *sk, int fast_rexmit) ...@@ -2218,9 +2223,7 @@ static void tcp_update_scoreboard(struct sock *sk, int fast_rexmit)
{ {
struct tcp_sock *tp = tcp_sk(sk); struct tcp_sock *tp = tcp_sk(sk);
if (tcp_is_reno(tp)) { if (tcp_is_sack(tp)) {
tcp_mark_head_lost(sk, 1, 1);
} else {
int sacked_upto = tp->sacked_out - tp->reordering; int sacked_upto = tp->sacked_out - tp->reordering;
if (sacked_upto >= 0) if (sacked_upto >= 0)
tcp_mark_head_lost(sk, sacked_upto, 0); tcp_mark_head_lost(sk, sacked_upto, 0);
...@@ -2718,12 +2721,16 @@ static bool tcp_try_undo_partial(struct sock *sk, u32 prior_snd_una) ...@@ -2718,12 +2721,16 @@ static bool tcp_try_undo_partial(struct sock *sk, u32 prior_snd_una)
return false; return false;
} }
static void tcp_rack_identify_loss(struct sock *sk, int *ack_flag) static void tcp_identify_packet_loss(struct sock *sk, int *ack_flag)
{ {
struct tcp_sock *tp = tcp_sk(sk); struct tcp_sock *tp = tcp_sk(sk);
/* Use RACK to detect loss */ if (tcp_rtx_queue_empty(sk))
if (sock_net(sk)->ipv4.sysctl_tcp_recovery & TCP_RACK_LOSS_DETECTION) { return;
if (unlikely(tcp_is_reno(tp))) {
tcp_newreno_mark_lost(sk, *ack_flag & FLAG_SND_UNA_ADVANCED);
} else if (tcp_is_rack(sk)) {
u32 prior_retrans = tp->retrans_out; u32 prior_retrans = tp->retrans_out;
tcp_rack_mark_lost(sk); tcp_rack_mark_lost(sk);
...@@ -2819,11 +2826,11 @@ static void tcp_fastretrans_alert(struct sock *sk, const u32 prior_snd_una, ...@@ -2819,11 +2826,11 @@ static void tcp_fastretrans_alert(struct sock *sk, const u32 prior_snd_una,
tcp_try_keep_open(sk); tcp_try_keep_open(sk);
return; return;
} }
tcp_rack_identify_loss(sk, ack_flag); tcp_identify_packet_loss(sk, ack_flag);
break; break;
case TCP_CA_Loss: case TCP_CA_Loss:
tcp_process_loss(sk, flag, is_dupack, rexmit); tcp_process_loss(sk, flag, is_dupack, rexmit);
tcp_rack_identify_loss(sk, ack_flag); tcp_identify_packet_loss(sk, ack_flag);
if (!(icsk->icsk_ca_state == TCP_CA_Open || if (!(icsk->icsk_ca_state == TCP_CA_Open ||
(*ack_flag & FLAG_LOST_RETRANS))) (*ack_flag & FLAG_LOST_RETRANS)))
return; return;
...@@ -2840,7 +2847,7 @@ static void tcp_fastretrans_alert(struct sock *sk, const u32 prior_snd_una, ...@@ -2840,7 +2847,7 @@ static void tcp_fastretrans_alert(struct sock *sk, const u32 prior_snd_una,
if (icsk->icsk_ca_state <= TCP_CA_Disorder) if (icsk->icsk_ca_state <= TCP_CA_Disorder)
tcp_try_undo_dsack(sk); tcp_try_undo_dsack(sk);
tcp_rack_identify_loss(sk, ack_flag); tcp_identify_packet_loss(sk, ack_flag);
if (!tcp_time_to_recover(sk, flag)) { if (!tcp_time_to_recover(sk, flag)) {
tcp_try_to_open(sk, flag); tcp_try_to_open(sk, flag);
return; return;
...@@ -2862,7 +2869,7 @@ static void tcp_fastretrans_alert(struct sock *sk, const u32 prior_snd_una, ...@@ -2862,7 +2869,7 @@ static void tcp_fastretrans_alert(struct sock *sk, const u32 prior_snd_una,
fast_rexmit = 1; fast_rexmit = 1;
} }
if (do_lost) if (!tcp_is_rack(sk) && do_lost)
tcp_update_scoreboard(sk, fast_rexmit); tcp_update_scoreboard(sk, fast_rexmit);
*rexmit = REXMIT_LOST; *rexmit = REXMIT_LOST;
} }
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
#include <linux/tcp.h> #include <linux/tcp.h>
#include <net/tcp.h> #include <net/tcp.h>
static void tcp_rack_mark_skb_lost(struct sock *sk, struct sk_buff *skb) void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb)
{ {
struct tcp_sock *tp = tcp_sk(sk); struct tcp_sock *tp = tcp_sk(sk);
...@@ -21,6 +21,38 @@ static bool tcp_rack_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2) ...@@ -21,6 +21,38 @@ static bool tcp_rack_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2)
return t1 > t2 || (t1 == t2 && after(seq1, seq2)); return t1 > t2 || (t1 == t2 && after(seq1, seq2));
} }
u32 tcp_rack_reo_wnd(const struct sock *sk)
{
struct tcp_sock *tp = tcp_sk(sk);
if (!tp->rack.reord) {
/* If reordering has not been observed, be aggressive during
* the recovery or starting the recovery by DUPACK threshold.
*/
if (inet_csk(sk)->icsk_ca_state >= TCP_CA_Recovery)
return 0;
if (tp->sacked_out >= tp->reordering &&
!(sock_net(sk)->ipv4.sysctl_tcp_recovery & TCP_RACK_NO_DUPTHRESH))
return 0;
}
/* To be more reordering resilient, allow min_rtt/4 settling delay.
* Use min_rtt instead of the smoothed RTT because reordering is
* often a path property and less related to queuing or delayed ACKs.
* Upon receiving DSACKs, linearly increase the window up to the
* smoothed RTT.
*/
return min((tcp_min_rtt(tp) >> 2) * tp->rack.reo_wnd_steps,
tp->srtt_us >> 3);
}
s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb, u32 reo_wnd)
{
return tp->rack.rtt_us + reo_wnd -
tcp_stamp_us_delta(tp->tcp_mstamp, skb->skb_mstamp);
}
/* RACK loss detection (IETF draft draft-ietf-tcpm-rack-01): /* RACK loss detection (IETF draft draft-ietf-tcpm-rack-01):
* *
* Marks a packet lost, if some packet sent later has been (s)acked. * Marks a packet lost, if some packet sent later has been (s)acked.
...@@ -44,23 +76,11 @@ static bool tcp_rack_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2) ...@@ -44,23 +76,11 @@ static bool tcp_rack_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2)
static void tcp_rack_detect_loss(struct sock *sk, u32 *reo_timeout) static void tcp_rack_detect_loss(struct sock *sk, u32 *reo_timeout)
{ {
struct tcp_sock *tp = tcp_sk(sk); struct tcp_sock *tp = tcp_sk(sk);
u32 min_rtt = tcp_min_rtt(tp);
struct sk_buff *skb, *n; struct sk_buff *skb, *n;
u32 reo_wnd; u32 reo_wnd;
*reo_timeout = 0; *reo_timeout = 0;
/* To be more reordering resilient, allow min_rtt/4 settling delay reo_wnd = tcp_rack_reo_wnd(sk);
* (lower-bounded to 1000uS). We use min_rtt instead of the smoothed
* RTT because reordering is often a path property and less related
* to queuing or delayed ACKs.
*/
reo_wnd = 1000;
if ((tp->rack.reord || inet_csk(sk)->icsk_ca_state < TCP_CA_Recovery) &&
min_rtt != ~0U) {
reo_wnd = max((min_rtt >> 2) * tp->rack.reo_wnd_steps, reo_wnd);
reo_wnd = min(reo_wnd, tp->srtt_us >> 3);
}
list_for_each_entry_safe(skb, n, &tp->tsorted_sent_queue, list_for_each_entry_safe(skb, n, &tp->tsorted_sent_queue,
tcp_tsorted_anchor) { tcp_tsorted_anchor) {
struct tcp_skb_cb *scb = TCP_SKB_CB(skb); struct tcp_skb_cb *scb = TCP_SKB_CB(skb);
...@@ -78,10 +98,9 @@ static void tcp_rack_detect_loss(struct sock *sk, u32 *reo_timeout) ...@@ -78,10 +98,9 @@ static void tcp_rack_detect_loss(struct sock *sk, u32 *reo_timeout)
/* A packet is lost if it has not been s/acked beyond /* A packet is lost if it has not been s/acked beyond
* the recent RTT plus the reordering window. * the recent RTT plus the reordering window.
*/ */
remaining = tp->rack.rtt_us + reo_wnd - remaining = tcp_rack_skb_timeout(tp, skb, reo_wnd);
tcp_stamp_us_delta(tp->tcp_mstamp, skb->skb_mstamp);
if (remaining <= 0) { if (remaining <= 0) {
tcp_rack_mark_skb_lost(sk, skb); tcp_mark_skb_lost(sk, skb);
list_del_init(&skb->tcp_tsorted_anchor); list_del_init(&skb->tcp_tsorted_anchor);
} else { } else {
/* Record maximum wait time */ /* Record maximum wait time */
...@@ -202,3 +221,30 @@ void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs) ...@@ -202,3 +221,30 @@ void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs)
tp->rack.reo_wnd_steps = 1; tp->rack.reo_wnd_steps = 1;
} }
} }
/* RFC6582 NewReno recovery for non-SACK connection. It simply retransmits
* the next unacked packet upon receiving
* a) three or more DUPACKs to start the fast recovery
* b) an ACK acknowledging new data during the fast recovery.
*/
void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced)
{
const u8 state = inet_csk(sk)->icsk_ca_state;
struct tcp_sock *tp = tcp_sk(sk);
if ((state < TCP_CA_Recovery && tp->sacked_out >= tp->reordering) ||
(state == TCP_CA_Recovery && snd_una_advanced)) {
struct sk_buff *skb = tcp_rtx_queue_head(sk);
u32 mss;
if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST)
return;
mss = tcp_skb_mss(skb);
if (tcp_skb_pcount(skb) > 1 && skb->len > mss)
tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb,
mss, mss, GFP_ATOMIC);
tcp_skb_mark_lost_uncond_verify(tp, skb);
}
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment