Commit f8212f97 authored by Alexander Duyck's avatar Alexander Duyck Committed by David S. Miller

ixgbe: enable HW RSC for 82599

This patch enables hardware receive side coalescing for 82599 hardware.
82599 can merge multiple frames from the same TCP/IP flow into a single
structure that can span one ore more descriptors.  The accumulated data is
arranged similar to how jumbo frames are arranged with the exception that
other packets can be interlaced inbetween.  To overcome this issue a next
pointer is included in the written back descriptor which indicates the next
descriptor in the writeback sequence.

This feature sets the NETIF_F_LRO flag and clearing it via the ethtool set
flags operation will also disable hardware RSC.
Signed-off-by: default avatarAlexander Duyck <alexander.h.duyck@intel.com>
Signed-off-by: default avatarJeff Kirsher <jeffrey.t.kirsher@intel.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 45a5ead0
...@@ -147,6 +147,7 @@ struct ixgbe_ring { ...@@ -147,6 +147,7 @@ struct ixgbe_ring {
u16 work_limit; /* max work per interrupt */ u16 work_limit; /* max work per interrupt */
u16 rx_buf_len; u16 rx_buf_len;
u64 rsc_count; /* stat for coalesced packets */
}; };
enum ixgbe_ring_f_enum { enum ixgbe_ring_f_enum {
...@@ -294,6 +295,8 @@ struct ixgbe_adapter { ...@@ -294,6 +295,8 @@ struct ixgbe_adapter {
#define IXGBE_FLAG_IN_WATCHDOG_TASK (u32)(1 << 23) #define IXGBE_FLAG_IN_WATCHDOG_TASK (u32)(1 << 23)
#define IXGBE_FLAG_IN_SFP_LINK_TASK (u32)(1 << 24) #define IXGBE_FLAG_IN_SFP_LINK_TASK (u32)(1 << 24)
#define IXGBE_FLAG_IN_SFP_MOD_TASK (u32)(1 << 25) #define IXGBE_FLAG_IN_SFP_MOD_TASK (u32)(1 << 25)
#define IXGBE_FLAG_RSC_CAPABLE (u32)(1 << 26)
#define IXGBE_FLAG_RSC_ENABLED (u32)(1 << 27)
/* default to trying for four seconds */ /* default to trying for four seconds */
#define IXGBE_TRY_LINK_TIMEOUT (4 * HZ) #define IXGBE_TRY_LINK_TIMEOUT (4 * HZ)
...@@ -325,6 +328,7 @@ struct ixgbe_adapter { ...@@ -325,6 +328,7 @@ struct ixgbe_adapter {
struct timer_list sfp_timer; struct timer_list sfp_timer;
struct work_struct multispeed_fiber_task; struct work_struct multispeed_fiber_task;
struct work_struct sfp_config_module_task; struct work_struct sfp_config_module_task;
u64 rsc_count;
u32 wol; u32 wol;
u16 eeprom_version; u16 eeprom_version;
}; };
......
...@@ -67,6 +67,7 @@ static struct ixgbe_stats ixgbe_gstrings_stats[] = { ...@@ -67,6 +67,7 @@ static struct ixgbe_stats ixgbe_gstrings_stats[] = {
{"rx_over_errors", IXGBE_STAT(net_stats.rx_over_errors)}, {"rx_over_errors", IXGBE_STAT(net_stats.rx_over_errors)},
{"rx_crc_errors", IXGBE_STAT(net_stats.rx_crc_errors)}, {"rx_crc_errors", IXGBE_STAT(net_stats.rx_crc_errors)},
{"rx_frame_errors", IXGBE_STAT(net_stats.rx_frame_errors)}, {"rx_frame_errors", IXGBE_STAT(net_stats.rx_frame_errors)},
{"hw_rsc_count", IXGBE_STAT(rsc_count)},
{"rx_fifo_errors", IXGBE_STAT(net_stats.rx_fifo_errors)}, {"rx_fifo_errors", IXGBE_STAT(net_stats.rx_fifo_errors)},
{"rx_missed_errors", IXGBE_STAT(net_stats.rx_missed_errors)}, {"rx_missed_errors", IXGBE_STAT(net_stats.rx_missed_errors)},
{"tx_aborted_errors", IXGBE_STAT(net_stats.tx_aborted_errors)}, {"tx_aborted_errors", IXGBE_STAT(net_stats.tx_aborted_errors)},
...@@ -1127,6 +1128,27 @@ static int ixgbe_set_coalesce(struct net_device *netdev, ...@@ -1127,6 +1128,27 @@ static int ixgbe_set_coalesce(struct net_device *netdev,
return 0; return 0;
} }
static int ixgbe_set_flags(struct net_device *netdev, u32 data)
{
struct ixgbe_adapter *adapter = netdev_priv(netdev);
ethtool_op_set_flags(netdev, data);
if (!(adapter->flags & IXGBE_FLAG_RSC_CAPABLE))
return 0;
/* if state changes we need to update adapter->flags and reset */
if ((!!(data & ETH_FLAG_LRO)) !=
(!!(adapter->flags & IXGBE_FLAG_RSC_ENABLED))) {
adapter->flags ^= IXGBE_FLAG_RSC_ENABLED;
if (netif_running(netdev))
ixgbe_reinit_locked(adapter);
else
ixgbe_reset(adapter);
}
return 0;
}
static const struct ethtool_ops ixgbe_ethtool_ops = { static const struct ethtool_ops ixgbe_ethtool_ops = {
.get_settings = ixgbe_get_settings, .get_settings = ixgbe_get_settings,
...@@ -1161,7 +1183,7 @@ static const struct ethtool_ops ixgbe_ethtool_ops = { ...@@ -1161,7 +1183,7 @@ static const struct ethtool_ops ixgbe_ethtool_ops = {
.get_coalesce = ixgbe_get_coalesce, .get_coalesce = ixgbe_get_coalesce,
.set_coalesce = ixgbe_set_coalesce, .set_coalesce = ixgbe_set_coalesce,
.get_flags = ethtool_op_get_flags, .get_flags = ethtool_op_get_flags,
.set_flags = ethtool_op_set_flags, .set_flags = ixgbe_set_flags,
}; };
void ixgbe_set_ethtool_ops(struct net_device *netdev) void ixgbe_set_ethtool_ops(struct net_device *netdev)
......
...@@ -615,6 +615,40 @@ static inline u16 ixgbe_get_pkt_info(union ixgbe_adv_rx_desc *rx_desc) ...@@ -615,6 +615,40 @@ static inline u16 ixgbe_get_pkt_info(union ixgbe_adv_rx_desc *rx_desc)
return rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; return rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
} }
static inline u32 ixgbe_get_rsc_count(union ixgbe_adv_rx_desc *rx_desc)
{
return (le32_to_cpu(rx_desc->wb.lower.lo_dword.data) &
IXGBE_RXDADV_RSCCNT_MASK) >>
IXGBE_RXDADV_RSCCNT_SHIFT;
}
/**
* ixgbe_transform_rsc_queue - change rsc queue into a full packet
* @skb: pointer to the last skb in the rsc queue
*
* This function changes a queue full of hw rsc buffers into a completed
* packet. It uses the ->prev pointers to find the first packet and then
* turns it into the frag list owner.
**/
static inline struct sk_buff *ixgbe_transform_rsc_queue(struct sk_buff *skb)
{
unsigned int frag_list_size = 0;
while (skb->prev) {
struct sk_buff *prev = skb->prev;
frag_list_size += skb->len;
skb->prev = NULL;
skb = prev;
}
skb_shinfo(skb)->frag_list = skb->next;
skb->next = NULL;
skb->len += frag_list_size;
skb->data_len += frag_list_size;
skb->truesize += frag_list_size;
return skb;
}
static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
struct ixgbe_ring *rx_ring, struct ixgbe_ring *rx_ring,
int *work_done, int work_to_do) int *work_done, int work_to_do)
...@@ -624,7 +658,7 @@ static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, ...@@ -624,7 +658,7 @@ static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
union ixgbe_adv_rx_desc *rx_desc, *next_rxd; union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
struct ixgbe_rx_buffer *rx_buffer_info, *next_buffer; struct ixgbe_rx_buffer *rx_buffer_info, *next_buffer;
struct sk_buff *skb; struct sk_buff *skb;
unsigned int i; unsigned int i, rsc_count = 0;
u32 len, staterr; u32 len, staterr;
u16 hdr_info; u16 hdr_info;
bool cleaned = false; bool cleaned = false;
...@@ -690,20 +724,38 @@ static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, ...@@ -690,20 +724,38 @@ static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
i++; i++;
if (i == rx_ring->count) if (i == rx_ring->count)
i = 0; i = 0;
next_buffer = &rx_ring->rx_buffer_info[i];
next_rxd = IXGBE_RX_DESC_ADV(*rx_ring, i); next_rxd = IXGBE_RX_DESC_ADV(*rx_ring, i);
prefetch(next_rxd); prefetch(next_rxd);
cleaned_count++; cleaned_count++;
if (adapter->flags & IXGBE_FLAG_RSC_CAPABLE)
rsc_count = ixgbe_get_rsc_count(rx_desc);
if (rsc_count) {
u32 nextp = (staterr & IXGBE_RXDADV_NEXTP_MASK) >>
IXGBE_RXDADV_NEXTP_SHIFT;
next_buffer = &rx_ring->rx_buffer_info[nextp];
rx_ring->rsc_count += (rsc_count - 1);
} else {
next_buffer = &rx_ring->rx_buffer_info[i];
}
if (staterr & IXGBE_RXD_STAT_EOP) { if (staterr & IXGBE_RXD_STAT_EOP) {
if (skb->prev)
skb = ixgbe_transform_rsc_queue(skb);
rx_ring->stats.packets++; rx_ring->stats.packets++;
rx_ring->stats.bytes += skb->len; rx_ring->stats.bytes += skb->len;
} else { } else {
if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
rx_buffer_info->skb = next_buffer->skb; rx_buffer_info->skb = next_buffer->skb;
rx_buffer_info->dma = next_buffer->dma; rx_buffer_info->dma = next_buffer->dma;
next_buffer->skb = skb; next_buffer->skb = skb;
next_buffer->dma = 0; next_buffer->dma = 0;
} else {
skb->next = next_buffer->skb;
skb->next->prev = skb;
}
adapter->non_eop_descs++; adapter->non_eop_descs++;
goto next_desc; goto next_desc;
} }
...@@ -733,7 +785,7 @@ static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, ...@@ -733,7 +785,7 @@ static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
/* use prefetched values */ /* use prefetched values */
rx_desc = next_rxd; rx_desc = next_rxd;
rx_buffer_info = next_buffer; rx_buffer_info = &rx_ring->rx_buffer_info[i];
staterr = le32_to_cpu(rx_desc->wb.upper.status_error); staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
} }
...@@ -1729,6 +1781,7 @@ static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) ...@@ -1729,6 +1781,7 @@ static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
u32 fctrl, hlreg0; u32 fctrl, hlreg0;
u32 reta = 0, mrqc = 0; u32 reta = 0, mrqc = 0;
u32 rdrxctl; u32 rdrxctl;
u32 rscctrl;
int rx_buf_len; int rx_buf_len;
/* Decide whether to use packet split mode or not */ /* Decide whether to use packet split mode or not */
...@@ -1746,7 +1799,8 @@ static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) ...@@ -1746,7 +1799,8 @@ static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
} }
} else { } else {
if (netdev->mtu <= ETH_DATA_LEN) if (!(adapter->flags & IXGBE_FLAG_RSC_ENABLED) &&
(netdev->mtu <= ETH_DATA_LEN))
rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE; rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
else else
rx_buf_len = ALIGN(max_frame, 1024); rx_buf_len = ALIGN(max_frame, 1024);
...@@ -1868,8 +1922,38 @@ static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) ...@@ -1868,8 +1922,38 @@ static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
if (hw->mac.type == ixgbe_mac_82599EB) { if (hw->mac.type == ixgbe_mac_82599EB) {
rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
} }
if (adapter->flags & IXGBE_FLAG_RSC_ENABLED) {
/* Enable 82599 HW-RSC */
for (i = 0; i < adapter->num_rx_queues; i++) {
j = adapter->rx_ring[i].reg_idx;
rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(j));
rscctrl |= IXGBE_RSCCTL_RSCEN;
/*
* if packet split is enabled we can only support up
* to max frags + 1 descriptors.
*/
if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED)
#if (MAX_SKB_FRAGS < 3)
rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
#elif (MAX_SKB_FRAGS < 7)
rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
#elif (MAX_SKB_FRAGS < 15)
rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
#else
rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
#endif
else
rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(j), rscctrl);
}
/* Disable RSC for ACK packets */
IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
(IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
}
} }
static void ixgbe_vlan_rx_add_vid(struct net_device *netdev, u16 vid) static void ixgbe_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
...@@ -2438,8 +2522,13 @@ static void ixgbe_clean_rx_ring(struct ixgbe_adapter *adapter, ...@@ -2438,8 +2522,13 @@ static void ixgbe_clean_rx_ring(struct ixgbe_adapter *adapter,
rx_buffer_info->dma = 0; rx_buffer_info->dma = 0;
} }
if (rx_buffer_info->skb) { if (rx_buffer_info->skb) {
dev_kfree_skb(rx_buffer_info->skb); struct sk_buff *skb = rx_buffer_info->skb;
rx_buffer_info->skb = NULL; rx_buffer_info->skb = NULL;
do {
struct sk_buff *this = skb;
skb = skb->prev;
dev_kfree_skb(this);
} while (skb);
} }
if (!rx_buffer_info->page) if (!rx_buffer_info->page)
continue; continue;
...@@ -3180,8 +3269,11 @@ static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter) ...@@ -3180,8 +3269,11 @@ static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter)
adapter->ring_feature[RING_F_DCB].indices = IXGBE_MAX_DCB_INDICES; adapter->ring_feature[RING_F_DCB].indices = IXGBE_MAX_DCB_INDICES;
if (hw->mac.type == ixgbe_mac_82598EB) if (hw->mac.type == ixgbe_mac_82598EB)
adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82598; adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82598;
else if (hw->mac.type == ixgbe_mac_82599EB) else if (hw->mac.type == ixgbe_mac_82599EB) {
adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82599; adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82599;
adapter->flags |= IXGBE_FLAG_RSC_CAPABLE;
adapter->flags |= IXGBE_FLAG_RSC_ENABLED;
}
#ifdef CONFIG_IXGBE_DCB #ifdef CONFIG_IXGBE_DCB
/* Configure DCB traffic classes */ /* Configure DCB traffic classes */
...@@ -3765,9 +3857,13 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter) ...@@ -3765,9 +3857,13 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter)
u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot; u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot;
if (hw->mac.type == ixgbe_mac_82599EB) { if (hw->mac.type == ixgbe_mac_82599EB) {
u64 rsc_count = 0;
for (i = 0; i < 16; i++) for (i = 0; i < 16; i++)
adapter->hw_rx_no_dma_resources += adapter->hw_rx_no_dma_resources +=
IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
for (i = 0; i < adapter->num_rx_queues; i++)
rsc_count += adapter->rx_ring[i].rsc_count;
adapter->rsc_count = rsc_count;
} }
adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
...@@ -4742,6 +4838,9 @@ static int __devinit ixgbe_probe(struct pci_dev *pdev, ...@@ -4742,6 +4838,9 @@ static int __devinit ixgbe_probe(struct pci_dev *pdev,
if (pci_using_dac) if (pci_using_dac)
netdev->features |= NETIF_F_HIGHDMA; netdev->features |= NETIF_F_HIGHDMA;
if (adapter->flags & IXGBE_FLAG_RSC_ENABLED)
netdev->features |= NETIF_F_LRO;
/* make sure the EEPROM is good */ /* make sure the EEPROM is good */
if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) { if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) {
dev_err(&pdev->dev, "The EEPROM Checksum Is Not Valid\n"); dev_err(&pdev->dev, "The EEPROM Checksum Is Not Valid\n");
......
...@@ -443,6 +443,21 @@ ...@@ -443,6 +443,21 @@
#define IXGBE_SECTXCTRL_STORE_FORWARD_ENABLE 0x4 #define IXGBE_SECTXCTRL_STORE_FORWARD_ENABLE 0x4
/* HW RSC registers */
#define IXGBE_RSCCTL(_i) (((_i) < 64) ? (0x0102C + ((_i) * 0x40)) : \
(0x0D02C + ((_i - 64) * 0x40)))
#define IXGBE_RSCDBU 0x03028
#define IXGBE_RSCCTL_RSCEN 0x01
#define IXGBE_RSCCTL_MAXDESC_1 0x00
#define IXGBE_RSCCTL_MAXDESC_4 0x04
#define IXGBE_RSCCTL_MAXDESC_8 0x08
#define IXGBE_RSCCTL_MAXDESC_16 0x0C
#define IXGBE_RXDADV_RSCCNT_SHIFT 17
#define IXGBE_GPIE_RSC_DELAY_SHIFT 11
#define IXGBE_RXDADV_RSCCNT_MASK 0x001E0000
#define IXGBE_RSCDBU_RSCACKDIS 0x00000080
#define IXGBE_RDRXCTL_RSCFRSTSIZE 0x003E0000
/* DCB registers */ /* DCB registers */
#define IXGBE_RTRPCS 0x02430 #define IXGBE_RTRPCS 0x02430
#define IXGBE_RTTDCS 0x04900 #define IXGBE_RTTDCS 0x04900
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment