Commit ea1627c2 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

tcp: minor optimizations around tcp_hdr() usage

tcp_hdr() is slightly more expensive than using skb->data in contexts
where we know they point to the same byte.

In receive path, tcp_v4_rcv() and tcp_v6_rcv() are in this situation,
as tcp header has not been pulled yet.

In output path, the same can be said when we just pushed the tcp header
in the skb, in tcp_transmit_skb() and tcp_make_synack()

Also factorize the two checks for tcb->tcp_flags & TCPHDR_SYN in
tcp_transmit_skb() and pass tcp header pointer to tcp_ecn_send(),
so that compiler can further optimize and avoid a reload.
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 50225243
...@@ -1556,9 +1556,9 @@ int tcp_v4_rcv(struct sk_buff *skb) ...@@ -1556,9 +1556,9 @@ int tcp_v4_rcv(struct sk_buff *skb)
if (!pskb_may_pull(skb, sizeof(struct tcphdr))) if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
goto discard_it; goto discard_it;
th = tcp_hdr(skb); th = (const struct tcphdr *)skb->data;
if (th->doff < sizeof(struct tcphdr) / 4) if (unlikely(th->doff < sizeof(struct tcphdr) / 4))
goto bad_packet; goto bad_packet;
if (!pskb_may_pull(skb, th->doff * 4)) if (!pskb_may_pull(skb, th->doff * 4))
goto discard_it; goto discard_it;
...@@ -1571,7 +1571,7 @@ int tcp_v4_rcv(struct sk_buff *skb) ...@@ -1571,7 +1571,7 @@ int tcp_v4_rcv(struct sk_buff *skb)
if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo)) if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo))
goto csum_error; goto csum_error;
th = tcp_hdr(skb); th = (const struct tcphdr *)skb->data;
iph = ip_hdr(skb); iph = ip_hdr(skb);
/* This is tricky : We move IPCB at its correct location into TCP_SKB_CB() /* This is tricky : We move IPCB at its correct location into TCP_SKB_CB()
* barrier() makes sure compiler wont play fool^Waliasing games. * barrier() makes sure compiler wont play fool^Waliasing games.
......
...@@ -364,7 +364,7 @@ tcp_ecn_make_synack(const struct request_sock *req, struct tcphdr *th) ...@@ -364,7 +364,7 @@ tcp_ecn_make_synack(const struct request_sock *req, struct tcphdr *th)
* be sent. * be sent.
*/ */
static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb,
int tcp_header_len) struct tcphdr *th, int tcp_header_len)
{ {
struct tcp_sock *tp = tcp_sk(sk); struct tcp_sock *tp = tcp_sk(sk);
...@@ -375,7 +375,7 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, ...@@ -375,7 +375,7 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb,
INET_ECN_xmit(sk); INET_ECN_xmit(sk);
if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) { if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) {
tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR; tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
tcp_hdr(skb)->cwr = 1; th->cwr = 1;
skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
} }
} else if (!tcp_ca_needs_ecn(sk)) { } else if (!tcp_ca_needs_ecn(sk)) {
...@@ -383,7 +383,7 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, ...@@ -383,7 +383,7 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb,
INET_ECN_dontxmit(sk); INET_ECN_dontxmit(sk);
} }
if (tp->ecn_flags & TCP_ECN_DEMAND_CWR) if (tp->ecn_flags & TCP_ECN_DEMAND_CWR)
tcp_hdr(skb)->ece = 1; th->ece = 1;
} }
} }
...@@ -954,7 +954,7 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it, ...@@ -954,7 +954,7 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
atomic_add(skb->truesize, &sk->sk_wmem_alloc); atomic_add(skb->truesize, &sk->sk_wmem_alloc);
/* Build TCP header and checksum it. */ /* Build TCP header and checksum it. */
th = tcp_hdr(skb); th = (struct tcphdr *)skb->data;
th->source = inet->inet_sport; th->source = inet->inet_sport;
th->dest = inet->inet_dport; th->dest = inet->inet_dport;
th->seq = htonl(tcb->seq); th->seq = htonl(tcb->seq);
...@@ -962,14 +962,6 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it, ...@@ -962,14 +962,6 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
*(((__be16 *)th) + 6) = htons(((tcp_header_size >> 2) << 12) | *(((__be16 *)th) + 6) = htons(((tcp_header_size >> 2) << 12) |
tcb->tcp_flags); tcb->tcp_flags);
if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) {
/* RFC1323: The window in SYN & SYN/ACK segments
* is never scaled.
*/
th->window = htons(min(tp->rcv_wnd, 65535U));
} else {
th->window = htons(tcp_select_window(sk));
}
th->check = 0; th->check = 0;
th->urg_ptr = 0; th->urg_ptr = 0;
...@@ -986,9 +978,15 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it, ...@@ -986,9 +978,15 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
tcp_options_write((__be32 *)(th + 1), tp, &opts); tcp_options_write((__be32 *)(th + 1), tp, &opts);
skb_shinfo(skb)->gso_type = sk->sk_gso_type; skb_shinfo(skb)->gso_type = sk->sk_gso_type;
if (likely((tcb->tcp_flags & TCPHDR_SYN) == 0)) if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) {
tcp_ecn_send(sk, skb, tcp_header_size); th->window = htons(tcp_select_window(sk));
tcp_ecn_send(sk, skb, th, tcp_header_size);
} else {
/* RFC1323: The window in SYN & SYN/ACK segments
* is never scaled.
*/
th->window = htons(min(tp->rcv_wnd, 65535U));
}
#ifdef CONFIG_TCP_MD5SIG #ifdef CONFIG_TCP_MD5SIG
/* Calculate the MD5 hash, as we have all we need now */ /* Calculate the MD5 hash, as we have all we need now */
if (md5) { if (md5) {
...@@ -3040,7 +3038,7 @@ struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, ...@@ -3040,7 +3038,7 @@ struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
skb_push(skb, tcp_header_size); skb_push(skb, tcp_header_size);
skb_reset_transport_header(skb); skb_reset_transport_header(skb);
th = tcp_hdr(skb); th = (struct tcphdr *)skb->data;
memset(th, 0, sizeof(struct tcphdr)); memset(th, 0, sizeof(struct tcphdr));
th->syn = 1; th->syn = 1;
th->ack = 1; th->ack = 1;
......
...@@ -1369,9 +1369,9 @@ static int tcp_v6_rcv(struct sk_buff *skb) ...@@ -1369,9 +1369,9 @@ static int tcp_v6_rcv(struct sk_buff *skb)
if (!pskb_may_pull(skb, sizeof(struct tcphdr))) if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
goto discard_it; goto discard_it;
th = tcp_hdr(skb); th = (const struct tcphdr *)skb->data;
if (th->doff < sizeof(struct tcphdr)/4) if (unlikely(th->doff < sizeof(struct tcphdr)/4))
goto bad_packet; goto bad_packet;
if (!pskb_may_pull(skb, th->doff*4)) if (!pskb_may_pull(skb, th->doff*4))
goto discard_it; goto discard_it;
...@@ -1379,7 +1379,7 @@ static int tcp_v6_rcv(struct sk_buff *skb) ...@@ -1379,7 +1379,7 @@ static int tcp_v6_rcv(struct sk_buff *skb)
if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo)) if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo))
goto csum_error; goto csum_error;
th = tcp_hdr(skb); th = (const struct tcphdr *)skb->data;
hdr = ipv6_hdr(skb); hdr = ipv6_hdr(skb);
lookup: lookup:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment