Commit 2d37490b authored by Alexander Duyck's avatar Alexander Duyck Committed by Jeff Kirsher

i40e/i40evf: Rewrite logic for 8 descriptor per packet check

This patch is meant to rewrite the logic for how we determine if we can
transmit the frame or if it needs to be linearized.

The previous code for this function was using a mix of division and modulus
division as a part of computing if we need to take the slow path.  Instead
I have replaced this by simply working with a sliding window which will
tell us if the frame would be capable of causing a single packet to span
several descriptors.

The logic for the scan is fairly simple.  If any given group of 6 fragments
is less than gso_size - 1 then it is possible for us to have one byte
coming out of the first fragment, 6 fragments, and one or more bytes coming
out of the last fragment.  This gives us a total of 8 fragments
which exceeds what we can allow so we send such frames to be linearized.

Arguably the use of modulus might be more exact as the approach I propose
may generate some false positives.  However the likelihood of us taking much
of a hit for those false positives is fairly low, and I would rather not
add more overhead in the case where we are receiving a frame composed of 4K
pages.
Signed-off-by: default avatarAlexander Duyck <aduyck@mirantis.com>
Tested-by: default avatarAndrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: default avatarJeff Kirsher <jeffrey.t.kirsher@intel.com>
parent 4ec441df
...@@ -1368,6 +1368,12 @@ static netdev_tx_t i40e_fcoe_xmit_frame(struct sk_buff *skb, ...@@ -1368,6 +1368,12 @@ static netdev_tx_t i40e_fcoe_xmit_frame(struct sk_buff *skb,
goto out_drop; goto out_drop;
count = i40e_xmit_descriptor_count(skb); count = i40e_xmit_descriptor_count(skb);
if (i40e_chk_linearize(skb, count)) {
if (__skb_linearize(skb))
goto out_drop;
count = TXD_USE_COUNT(skb->len);
tx_ring->tx_stats.tx_linearize++;
}
/* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD, /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
* + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD, * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
......
...@@ -2593,59 +2593,71 @@ int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size) ...@@ -2593,59 +2593,71 @@ int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
} }
/** /**
* i40e_chk_linearize - Check if there are more than 8 fragments per packet * __i40e_chk_linearize - Check if there are more than 8 fragments per packet
* @skb: send buffer * @skb: send buffer
* @tx_flags: collected send information
* *
* Note: Our HW can't scatter-gather more than 8 fragments to build * Note: Our HW can't scatter-gather more than 8 fragments to build
* a packet on the wire and so we need to figure out the cases where we * a packet on the wire and so we need to figure out the cases where we
* need to linearize the skb. * need to linearize the skb.
**/ **/
static bool i40e_chk_linearize(struct sk_buff *skb, u32 tx_flags) bool __i40e_chk_linearize(struct sk_buff *skb)
{ {
struct skb_frag_struct *frag; const struct skb_frag_struct *frag, *stale;
bool linearize = false; int gso_size, nr_frags, sum;
unsigned int size = 0;
u16 num_frags;
u16 gso_segs;
num_frags = skb_shinfo(skb)->nr_frags; /* check to see if TSO is enabled, if so we may get a repreive */
gso_segs = skb_shinfo(skb)->gso_segs; gso_size = skb_shinfo(skb)->gso_size;
if (unlikely(!gso_size))
return true;
if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO)) { /* no need to check if number of frags is less than 8 */
u16 j = 0; nr_frags = skb_shinfo(skb)->nr_frags;
if (nr_frags < I40E_MAX_BUFFER_TXD)
return false;
if (num_frags < (I40E_MAX_BUFFER_TXD)) /* We need to walk through the list and validate that each group
goto linearize_chk_done; * of 6 fragments totals at least gso_size. However we don't need
/* try the simple math, if we have too many frags per segment */ * to perform such validation on the first or last 6 since the first
if (DIV_ROUND_UP((num_frags + gso_segs), gso_segs) > * 6 cannot inherit any data from a descriptor before them, and the
I40E_MAX_BUFFER_TXD) { * last 6 cannot inherit any data from a descriptor after them.
linearize = true; */
goto linearize_chk_done; nr_frags -= I40E_MAX_BUFFER_TXD - 1;
} frag = &skb_shinfo(skb)->frags[0];
frag = &skb_shinfo(skb)->frags[0];
/* we might still have more fragments per segment */ /* Initialize size to the negative value of gso_size minus 1. We
do { * use this as the worst case scenerio in which the frag ahead
size += skb_frag_size(frag); * of us only provides one byte which is why we are limited to 6
frag++; j++; * descriptors for a single transmit as the header and previous
if ((size >= skb_shinfo(skb)->gso_size) && * fragment are already consuming 2 descriptors.
(j < I40E_MAX_BUFFER_TXD)) { */
size = (size % skb_shinfo(skb)->gso_size); sum = 1 - gso_size;
j = (size) ? 1 : 0;
} /* Add size of frags 1 through 5 to create our initial sum */
if (j == I40E_MAX_BUFFER_TXD) { sum += skb_frag_size(++frag);
linearize = true; sum += skb_frag_size(++frag);
break; sum += skb_frag_size(++frag);
} sum += skb_frag_size(++frag);
num_frags--; sum += skb_frag_size(++frag);
} while (num_frags);
} else { /* Walk through fragments adding latest fragment, testing it, and
if (num_frags >= I40E_MAX_BUFFER_TXD) * then removing stale fragments from the sum.
linearize = true; */
stale = &skb_shinfo(skb)->frags[0];
for (;;) {
sum += skb_frag_size(++frag);
/* if sum is negative we failed to make sufficient progress */
if (sum < 0)
return true;
/* use pre-decrement to avoid processing last fragment */
if (!--nr_frags)
break;
sum -= skb_frag_size(++stale);
} }
linearize_chk_done: return false;
return linearize;
} }
/** /**
...@@ -2876,6 +2888,12 @@ static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb, ...@@ -2876,6 +2888,12 @@ static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
prefetch(skb->data); prefetch(skb->data);
count = i40e_xmit_descriptor_count(skb); count = i40e_xmit_descriptor_count(skb);
if (i40e_chk_linearize(skb, count)) {
if (__skb_linearize(skb))
goto out_drop;
count = TXD_USE_COUNT(skb->len);
tx_ring->tx_stats.tx_linearize++;
}
/* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD, /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
* + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD, * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
...@@ -2916,11 +2934,6 @@ static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb, ...@@ -2916,11 +2934,6 @@ static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
if (tsyn) if (tsyn)
tx_flags |= I40E_TX_FLAGS_TSYN; tx_flags |= I40E_TX_FLAGS_TSYN;
if (i40e_chk_linearize(skb, tx_flags)) {
if (skb_linearize(skb))
goto out_drop;
tx_ring->tx_stats.tx_linearize++;
}
skb_tx_timestamp(skb); skb_tx_timestamp(skb);
/* always enable CRC insertion offload */ /* always enable CRC insertion offload */
......
...@@ -337,6 +337,7 @@ int i40e_tx_prepare_vlan_flags(struct sk_buff *skb, ...@@ -337,6 +337,7 @@ int i40e_tx_prepare_vlan_flags(struct sk_buff *skb,
void i40e_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector); void i40e_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector);
u32 i40e_get_tx_pending(struct i40e_ring *ring, bool in_sw); u32 i40e_get_tx_pending(struct i40e_ring *ring, bool in_sw);
int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size); int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size);
bool __i40e_chk_linearize(struct sk_buff *skb);
/** /**
* i40e_get_head - Retrieve head from head writeback * i40e_get_head - Retrieve head from head writeback
...@@ -392,4 +393,22 @@ static inline int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size) ...@@ -392,4 +393,22 @@ static inline int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
return 0; return 0;
return __i40e_maybe_stop_tx(tx_ring, size); return __i40e_maybe_stop_tx(tx_ring, size);
} }
/**
* i40e_chk_linearize - Check if there are more than 8 fragments per packet
* @skb: send buffer
* @count: number of buffers used
*
* Note: Our HW can't scatter-gather more than 8 fragments to build
* a packet on the wire and so we need to figure out the cases where we
* need to linearize the skb.
**/
static inline bool i40e_chk_linearize(struct sk_buff *skb, int count)
{
/* we can only support up to 8 data buffers for a single send */
if (likely(count <= I40E_MAX_BUFFER_TXD))
return false;
return __i40e_chk_linearize(skb);
}
#endif /* _I40E_TXRX_H_ */ #endif /* _I40E_TXRX_H_ */
...@@ -1796,59 +1796,71 @@ static void i40e_create_tx_ctx(struct i40e_ring *tx_ring, ...@@ -1796,59 +1796,71 @@ static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
} }
/** /**
* i40e_chk_linearize - Check if there are more than 8 fragments per packet * __i40evf_chk_linearize - Check if there are more than 8 fragments per packet
* @skb: send buffer * @skb: send buffer
* @tx_flags: collected send information
* *
* Note: Our HW can't scatter-gather more than 8 fragments to build * Note: Our HW can't scatter-gather more than 8 fragments to build
* a packet on the wire and so we need to figure out the cases where we * a packet on the wire and so we need to figure out the cases where we
* need to linearize the skb. * need to linearize the skb.
**/ **/
static bool i40e_chk_linearize(struct sk_buff *skb, u32 tx_flags) bool __i40evf_chk_linearize(struct sk_buff *skb)
{ {
struct skb_frag_struct *frag; const struct skb_frag_struct *frag, *stale;
bool linearize = false; int gso_size, nr_frags, sum;
unsigned int size = 0;
u16 num_frags;
u16 gso_segs;
num_frags = skb_shinfo(skb)->nr_frags; /* check to see if TSO is enabled, if so we may get a repreive */
gso_segs = skb_shinfo(skb)->gso_segs; gso_size = skb_shinfo(skb)->gso_size;
if (unlikely(!gso_size))
return true;
if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO)) { /* no need to check if number of frags is less than 8 */
u16 j = 0; nr_frags = skb_shinfo(skb)->nr_frags;
if (nr_frags < I40E_MAX_BUFFER_TXD)
return false;
if (num_frags < (I40E_MAX_BUFFER_TXD)) /* We need to walk through the list and validate that each group
goto linearize_chk_done; * of 6 fragments totals at least gso_size. However we don't need
/* try the simple math, if we have too many frags per segment */ * to perform such validation on the first or last 6 since the first
if (DIV_ROUND_UP((num_frags + gso_segs), gso_segs) > * 6 cannot inherit any data from a descriptor before them, and the
I40E_MAX_BUFFER_TXD) { * last 6 cannot inherit any data from a descriptor after them.
linearize = true; */
goto linearize_chk_done; nr_frags -= I40E_MAX_BUFFER_TXD - 1;
} frag = &skb_shinfo(skb)->frags[0];
frag = &skb_shinfo(skb)->frags[0];
/* we might still have more fragments per segment */ /* Initialize size to the negative value of gso_size minus 1. We
do { * use this as the worst case scenerio in which the frag ahead
size += skb_frag_size(frag); * of us only provides one byte which is why we are limited to 6
frag++; j++; * descriptors for a single transmit as the header and previous
if ((size >= skb_shinfo(skb)->gso_size) && * fragment are already consuming 2 descriptors.
(j < I40E_MAX_BUFFER_TXD)) { */
size = (size % skb_shinfo(skb)->gso_size); sum = 1 - gso_size;
j = (size) ? 1 : 0;
} /* Add size of frags 1 through 5 to create our initial sum */
if (j == I40E_MAX_BUFFER_TXD) { sum += skb_frag_size(++frag);
linearize = true; sum += skb_frag_size(++frag);
break; sum += skb_frag_size(++frag);
} sum += skb_frag_size(++frag);
num_frags--; sum += skb_frag_size(++frag);
} while (num_frags);
} else { /* Walk through fragments adding latest fragment, testing it, and
if (num_frags >= I40E_MAX_BUFFER_TXD) * then removing stale fragments from the sum.
linearize = true; */
stale = &skb_shinfo(skb)->frags[0];
for (;;) {
sum += skb_frag_size(++frag);
/* if sum is negative we failed to make sufficient progress */
if (sum < 0)
return true;
/* use pre-decrement to avoid processing last fragment */
if (!--nr_frags)
break;
sum -= skb_frag_size(++stale);
} }
linearize_chk_done: return false;
return linearize;
} }
/** /**
...@@ -2095,6 +2107,12 @@ static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb, ...@@ -2095,6 +2107,12 @@ static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
prefetch(skb->data); prefetch(skb->data);
count = i40e_xmit_descriptor_count(skb); count = i40e_xmit_descriptor_count(skb);
if (i40e_chk_linearize(skb, count)) {
if (__skb_linearize(skb))
goto out_drop;
count = TXD_USE_COUNT(skb->len);
tx_ring->tx_stats.tx_linearize++;
}
/* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD, /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
* + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD, * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
...@@ -2130,11 +2148,6 @@ static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb, ...@@ -2130,11 +2148,6 @@ static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
else if (tso) else if (tso)
tx_flags |= I40E_TX_FLAGS_TSO; tx_flags |= I40E_TX_FLAGS_TSO;
if (i40e_chk_linearize(skb, tx_flags)) {
if (skb_linearize(skb))
goto out_drop;
tx_ring->tx_stats.tx_linearize++;
}
skb_tx_timestamp(skb); skb_tx_timestamp(skb);
/* always enable CRC insertion offload */ /* always enable CRC insertion offload */
......
...@@ -327,6 +327,7 @@ int i40evf_napi_poll(struct napi_struct *napi, int budget); ...@@ -327,6 +327,7 @@ int i40evf_napi_poll(struct napi_struct *napi, int budget);
void i40evf_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector); void i40evf_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector);
u32 i40evf_get_tx_pending(struct i40e_ring *ring, bool in_sw); u32 i40evf_get_tx_pending(struct i40e_ring *ring, bool in_sw);
int __i40evf_maybe_stop_tx(struct i40e_ring *tx_ring, int size); int __i40evf_maybe_stop_tx(struct i40e_ring *tx_ring, int size);
bool __i40evf_chk_linearize(struct sk_buff *skb);
/** /**
* i40e_get_head - Retrieve head from head writeback * i40e_get_head - Retrieve head from head writeback
...@@ -382,4 +383,22 @@ static inline int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size) ...@@ -382,4 +383,22 @@ static inline int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
return 0; return 0;
return __i40evf_maybe_stop_tx(tx_ring, size); return __i40evf_maybe_stop_tx(tx_ring, size);
} }
/**
* i40e_chk_linearize - Check if there are more than 8 fragments per packet
* @skb: send buffer
* @count: number of buffers used
*
* Note: Our HW can't scatter-gather more than 8 fragments to build
* a packet on the wire and so we need to figure out the cases where we
* need to linearize the skb.
**/
static inline bool i40e_chk_linearize(struct sk_buff *skb, int count)
{
/* we can only support up to 8 data buffers for a single send */
if (likely(count <= I40E_MAX_BUFFER_TXD))
return false;
return __i40evf_chk_linearize(skb);
}
#endif /* _I40E_TXRX_H_ */ #endif /* _I40E_TXRX_H_ */
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment