rxrpc: Save last ACK's SACK table rather than marking txbufs

Improve the tracking of which packets need to be transmitted by saving the
last ACK packet that we receive that has a populated soft-ACK table rather
than marking packets.  Then we can step through the soft-ACK table and look
at the packets we've transmitted beyond that to determine which packets we
might want to retransmit.

We also look at the highest serial number that has been acked to try and
guess which packets we've transmitted the peer is likely to have seen.  If
necessary, we send a ping to retrieve that number.

One downside that might be a problem is that we can't then compare the
previous acked/unacked state so easily in rxrpc_input_soft_acks() - which
is a potential problem for the slow-start algorithm.
Signed-off-by: David Howells <dhowells@redhat.com>
cc: Marc Dionne <marc.dionne@auristor.com>
cc: linux-afs@lists.infradead.org

include/trace/events/rxrpc.h

@@ -17,6 +17,7 @@
  * Declare tracing information enums and their string mappings for display.
  */
 #define rxrpc_skb_traces \
+	EM(rxrpc_skb_ack,			"ACK") \
 	EM(rxrpc_skb_cleaned,			"CLN") \
 	EM(rxrpc_skb_cloned_jumbo,		"CLJ") \
 	EM(rxrpc_skb_freed,			"FRE") \
@@ -1257,7 +1258,7 @@ TRACE_EVENT(rxrpc_congest,
 		    memcpy(&__entry->sum, summary, sizeof(__entry->sum));
 			   ),
 
-	    TP_printk("c=%08x r=%08x %s q=%08x %s cw=%u ss=%u nA=%u,%u+%u,%u r=%u b=%u u=%u d=%u l=%x%s%s%s",
+	    TP_printk("c=%08x r=%08x %s q=%08x %s cw=%u ss=%u nA=%u,%u+%u r=%u b=%u u=%u d=%u l=%x%s%s%s",
 		      __entry->call,
 		      __entry->ack_serial,
 		      __print_symbolic(__entry->sum.ack_reason, rxrpc_ack_names),
@@ -1265,8 +1266,8 @@ TRACE_EVENT(rxrpc_congest,
 		      __print_symbolic(__entry->sum.mode, rxrpc_congest_modes),
 		      __entry->sum.cwnd,
 		      __entry->sum.ssthresh,
-		      __entry->sum.nr_acks, __entry->sum.nr_nacks,
-		      __entry->sum.nr_new_acks, __entry->sum.nr_new_nacks,
+		      __entry->sum.nr_acks, __entry->sum.saw_nacks,
+		      __entry->sum.nr_new_acks,
 		      __entry->sum.nr_rot_new_acks,
 		      __entry->top - __entry->hard_ack,
 		      __entry->sum.cumulative_acks,

net/core/skbuff.c

@@ -6431,6 +6431,7 @@ void skb_condense(struct sk_buff *skb)
 	 */
 	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
 }
+EXPORT_SYMBOL(skb_condense);
 
 #ifdef CONFIG_SKB_EXTENSIONS
 static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id)

net/rxrpc/ar-internal.h

@@ -694,6 +694,8 @@ struct rxrpc_call {
 	rxrpc_seq_t		acks_lost_top;	/* tx_top at the time lost-ack ping sent */
 	rxrpc_serial_t		acks_lost_ping;	/* Serial number of probe ACK */
 	rxrpc_serial_t		acks_highest_serial; /* Highest serial number ACK'd */
+	struct sk_buff		*acks_soft_tbl;	/* The last ACK packet with NAKs in it */
+	spinlock_t		acks_ack_lock;	/* Access to ->acks_last_ack */
 };
 
 /*
@@ -702,10 +704,9 @@ struct rxrpc_call {
 struct rxrpc_ack_summary {
 	u8			ack_reason;
 	u8			nr_acks;		/* Number of ACKs in packet */
-	u8			nr_nacks;		/* Number of NACKs in packet */
 	u8			nr_new_acks;		/* Number of new ACKs in packet */
-	u8			nr_new_nacks;		/* Number of new NACKs in packet */
 	u8			nr_rot_new_acks;	/* Number of rotated new ACKs */
+	bool			saw_nacks;		/* Saw NACKs in packet */
 	bool			new_low_nack;		/* T if new low NACK found */
 	bool			retrans_timeo;		/* T if reTx due to timeout happened */
 	u8			flight_size;		/* Number of unreceived transmissions */
@@ -765,11 +766,8 @@ struct rxrpc_txbuf {
 	unsigned int		space;		/* Remaining data space */
 	unsigned int		offset;		/* Offset of fill point */
 	unsigned long		flags;
-#define RXRPC_TXBUF_ACKED	0		/* Set if ACK'd */
-#define RXRPC_TXBUF_NACKED	1		/* Set if NAK'd */
-#define RXRPC_TXBUF_LAST	2		/* Set if last packet in Tx phase */
-#define RXRPC_TXBUF_RESENT	3		/* Set if has been resent */
-#define RXRPC_TXBUF_RETRANS	4		/* Set if should be retransmitted */
+#define RXRPC_TXBUF_LAST	0		/* Set if last packet in Tx phase */
+#define RXRPC_TXBUF_RESENT	1		/* Set if has been resent */
 	u8 /*enum rxrpc_propose_ack_trace*/ ack_why;	/* If ack, why */
 	struct {
 		/* The packet for encrypting and DMA'ing.  We align it such

net/rxrpc/call_event.c

@@ -132,48 +132,127 @@ static void rxrpc_congestion_timeout(struct rxrpc_call *call)
  */
 static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j)
 {
+	struct rxrpc_ackpacket *ack = NULL;
 	struct rxrpc_txbuf *txb;
+	struct sk_buff *ack_skb = NULL;
 	unsigned long resend_at;
 	rxrpc_seq_t transmitted = READ_ONCE(call->tx_transmitted);
 	ktime_t now, max_age, oldest, ack_ts;
 	bool unacked = false;
+	unsigned int i;
 	LIST_HEAD(retrans_queue);
 
 	_enter("{%d,%d}", call->acks_hard_ack, call->tx_top);
 
 	now = ktime_get_real();
 	max_age = ktime_sub_us(now, jiffies_to_usecs(call->peer->rto_j));
+	oldest = now;
+
+	/* See if there's an ACK saved with a soft-ACK table in it. */
+	if (call->acks_soft_tbl) {
+		spin_lock_bh(&call->acks_ack_lock);
+		ack_skb = call->acks_soft_tbl;
+		if (ack_skb) {
+			rxrpc_get_skb(ack_skb, rxrpc_skb_ack);
+			ack = (void *)ack_skb->data + sizeof(struct rxrpc_wire_header);
+		}
+		spin_unlock_bh(&call->acks_ack_lock);
+	}
+
+	if (list_empty(&call->tx_buffer))
+		goto no_resend;
 
 	spin_lock(&call->tx_lock);
 
-	/* Scan the packet list without dropping the lock and decide which of
-	 * the packets in the Tx buffer we're going to resend and what the new
-	 * resend timeout will be.
-	 */
+	if (list_empty(&call->tx_buffer))
+		goto no_further_resend;
+
 	trace_rxrpc_resend(call);
-	oldest = now;
-	list_for_each_entry(txb, &call->tx_buffer, call_link) {
-		if (test_bit(RXRPC_TXBUF_ACKED, &txb->flags))
-			continue;
-		if (after(txb->seq, transmitted))
-			break;
+	txb = list_first_entry(&call->tx_buffer, struct rxrpc_txbuf, call_link);
 
-		rxrpc_see_txbuf(txb, rxrpc_txbuf_see_unacked);
+	/* Scan the soft ACK table without dropping the lock and resend any
+	 * explicitly NAK'd packets.
+	 */
+	if (ack) {
+		for (i = 0; i < ack->nAcks; i++) {
+			rxrpc_seq_t seq;
 
-		if (test_bit(RXRPC_TXBUF_RESENT, &txb->flags)) {
-			if (ktime_after(txb->last_sent, max_age)) {
-				if (ktime_before(txb->last_sent, oldest))
-					oldest = txb->last_sent;
+			if (ack->acks[i] & 1)
 				continue;
+			seq = ntohl(ack->firstPacket) + i;
+			if (after(txb->seq, transmitted))
+				break;
+			if (after(txb->seq, seq))
+				continue; /* A new hard ACK probably came in */
+			list_for_each_entry_from(txb, &call->tx_buffer, call_link) {
+				if (txb->seq == seq)
+					goto found_txb;
+			}
+			goto no_further_resend;
+
+		found_txb:
+			if (after(ntohl(txb->wire.serial), call->acks_highest_serial))
+				continue; /* Ack point not yet reached */
+
+			rxrpc_see_txbuf(txb, rxrpc_txbuf_see_unacked);
+
+			if (list_empty(&txb->tx_link)) {
+				rxrpc_get_txbuf(txb, rxrpc_txbuf_get_retrans);
+				rxrpc_get_call(call, rxrpc_call_got_tx);
+				list_add_tail(&txb->tx_link, &retrans_queue);
+				set_bit(RXRPC_TXBUF_RESENT, &txb->flags);
 			}
-			unacked = true;
+
+			trace_rxrpc_retransmit(call, txb->seq,
+					       ktime_to_ns(ktime_sub(txb->last_sent,
+								     max_age)));
+
+			if (list_is_last(&txb->call_link, &call->tx_buffer))
+				goto no_further_resend;
+			txb = list_next_entry(txb, call_link);
 		}
+	}
 
-		rxrpc_get_txbuf(txb, rxrpc_txbuf_get_retrans);
-		list_move_tail(&txb->tx_link, &retrans_queue);
+	/* Fast-forward through the Tx queue to the point the peer says it has
+	 * seen.  Anything between the soft-ACK table and that point will get
+	 * ACK'd or NACK'd in due course, so don't worry about it here; here we
+	 * need to consider retransmitting anything beyond that point.
+	 *
+	 * Note that ACK for a packet can beat the update of tx_transmitted.
+	 */
+	if (after_eq(READ_ONCE(call->acks_prev_seq), READ_ONCE(call->tx_transmitted)))
+		goto no_further_resend;
+
+	list_for_each_entry_from(txb, &call->tx_buffer, call_link) {
+		if (before_eq(txb->seq, READ_ONCE(call->acks_prev_seq)))
+			continue;
+		if (after(txb->seq, READ_ONCE(call->tx_transmitted)))
+			break; /* Not transmitted yet */
+
+		if (ack && ack->reason == RXRPC_ACK_PING_RESPONSE &&
+		    before(ntohl(txb->wire.serial), ntohl(ack->serial)))
+			goto do_resend; /* Wasn't accounted for by a more recent ping. */
+
+		if (ktime_after(txb->last_sent, max_age)) {
+			if (ktime_before(txb->last_sent, oldest))
+				oldest = txb->last_sent;
+			continue;
+		}
+
+	do_resend:
+		unacked = true;
+		if (list_empty(&txb->tx_link)) {
+			rxrpc_get_txbuf(txb, rxrpc_txbuf_get_retrans);
+			list_add_tail(&txb->tx_link, &retrans_queue);
+			set_bit(RXRPC_TXBUF_RESENT, &txb->flags);
+			rxrpc_inc_stat(call->rxnet, stat_tx_data_retrans);
+		}
 	}
 
+no_further_resend:
 	spin_unlock(&call->tx_lock);
+no_resend:
+	rxrpc_free_skb(ack_skb, rxrpc_skb_freed);
 
 	resend_at = nsecs_to_jiffies(ktime_to_ns(ktime_sub(now, oldest)));
 	resend_at += jiffies + rxrpc_get_rto_backoff(call->peer,
@@ -201,8 +280,6 @@ static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j)
 	while ((txb = list_first_entry_or_null(&retrans_queue,
 					       struct rxrpc_txbuf, tx_link))) {
 		list_del_init(&txb->tx_link);
-		set_bit(RXRPC_TXBUF_RESENT, &txb->flags);
-		rxrpc_inc_stat(call->rxnet, stat_tx_data_retrans);
 		rxrpc_send_data_packet(call, txb);
 		rxrpc_put_txbuf(txb, rxrpc_txbuf_put_trans);
 

net/rxrpc/call_object.c

@@ -162,6 +162,7 @@ struct rxrpc_call *rxrpc_alloc_call(struct rxrpc_sock *rx, gfp_t gfp,
 	spin_lock_init(&call->notify_lock);
 	spin_lock_init(&call->tx_lock);
 	spin_lock_init(&call->input_lock);
+	spin_lock_init(&call->acks_ack_lock);
 	rwlock_init(&call->state_lock);
 	refcount_set(&call->ref, 1);
 	call->debug_id = debug_id;
@@ -701,6 +702,7 @@ void rxrpc_cleanup_call(struct rxrpc_call *call)
 		rxrpc_put_txbuf(txb, rxrpc_txbuf_put_cleaned);
 	}
 	rxrpc_put_txbuf(call->tx_pending, rxrpc_txbuf_put_cleaned);
+	rxrpc_free_skb(call->acks_soft_tbl, rxrpc_skb_cleaned);
 
 	call_rcu(&call->rcu, rxrpc_rcu_destroy_call);
 }

net/rxrpc/sendmsg.c

@@ -600,7 +600,6 @@ rxrpc_new_client_call_for_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg,
  */
 int rxrpc_do_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, size_t len)
 	__releases(&rx->sk.sk_lock.slock)
-	__releases(&call->user_mutex)
 {
 	enum rxrpc_call_state state;
 	struct rxrpc_call *call;