Skip to content

L
linux
Commit 61aee772 authored by David S. Miller's avatar David S. Miller
Browse files
Options

Merge branch 'mtk_eth_soc-xdp' 

Lorenzo Bianconi says:

====================
mtk_eth_soc: add xdp support

Introduce XDP support for mtk_eth_soc driver if rx hwlro is not
enabled in the chipset (e.g. mt7986).
Supported XDP verdicts:
- XDP_PASS
- XDP_DROP
- XDP_REDIRECT
- XDP_TX
- ndo_xdp_xmit
Rely on page_pool allocator for single page buffers in order to keep
them dma mapped and add skb recycling support.

Changes since v3:
- add missing rcu_read_lock()/rcu_read_unlock()
- introduce mtk_page_pool_enabled() utility routine

Changes since v2:
- fix leftover sparse warning
- add page_pool ethtool stats

Changes since v1:
- do not allocate mtk_xdp_stats array on the stack in mtk_rx_poll
- add rcu annotation to bpf program
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 502c6f8c 84b9cd38
Hide whitespace changes
Inline Side-by-side
Showing with 537 additions and 71 deletions
drivers/net/ethernet/mediatek/Kconfig
View file @ 61aee772
......@@ -17,6 +17,8 @@ config NET_MEDIATEK_SOC
select PINCTRL
select PHYLINK
select DIMLIB
select PAGE_POOL
select PAGE_POOL_STATS
help
This driver supports the gigabit ethernet MACs in the
MediaTek SoC family.
......
drivers/net/ethernet/mediatek/mtk_eth_soc.c
View file @ 61aee772
......@@ -34,6 +34,10 @@ MODULE_PARM_DESC(msg_level, "Message level (-1=defaults,0=none,...,16=all)");
#define MTK_ETHTOOL_STAT(x) { #x, \
offsetof(struct mtk_hw_stats, x) / sizeof(u64) }
#define MTK_ETHTOOL_XDP_STAT(x) { #x, \
offsetof(struct mtk_hw_stats, xdp_stats.x) / \
sizeof(u64) }
static const struct mtk_reg_map mtk_reg_map = {
.tx_irq_mask = 0x1a1c,
.tx_irq_status = 0x1a18,
......@@ -141,6 +145,13 @@ static const struct mtk_ethtool_stats {
MTK_ETHTOOL_STAT(rx_long_errors),
MTK_ETHTOOL_STAT(rx_checksum_errors),
MTK_ETHTOOL_STAT(rx_flow_control_packets),
MTK_ETHTOOL_XDP_STAT(rx_xdp_redirect),
MTK_ETHTOOL_XDP_STAT(rx_xdp_pass),
MTK_ETHTOOL_XDP_STAT(rx_xdp_drop),
MTK_ETHTOOL_XDP_STAT(rx_xdp_tx),
MTK_ETHTOOL_XDP_STAT(rx_xdp_tx_errors),
MTK_ETHTOOL_XDP_STAT(tx_xdp_xmit),
MTK_ETHTOOL_XDP_STAT(tx_xdp_xmit_errors),
};
static const char * const mtk_clks_source_name[] = {
......@@ -1020,15 +1031,26 @@ static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf,
}
}
tx_buf->flags = 0;
if (tx_buf->skb &&
(tx_buf->skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC)) {
if (napi)
napi_consume_skb(tx_buf->skb, napi);
if (tx_buf->type == MTK_TYPE_SKB) {
if (tx_buf->data &&
tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) {
struct sk_buff *skb = tx_buf->data;
if (napi)
napi_consume_skb(skb, napi);
else
dev_kfree_skb_any(skb);
}
} else if (tx_buf->data) {
struct xdp_frame *xdpf = tx_buf->data;
if (napi && tx_buf->type == MTK_TYPE_XDP_TX)
xdp_return_frame_rx_napi(xdpf);
else
dev_kfree_skb_any(tx_buf->skb);
xdp_return_frame(xdpf);
}
tx_buf->skb = NULL;
tx_buf->flags = 0;
tx_buf->data = NULL;
}
static void setup_tx_buf(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf,
......@@ -1045,7 +1067,7 @@ static void setup_tx_buf(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf,
dma_unmap_addr_set(tx_buf, dma_addr1, mapped_addr);
dma_unmap_len_set(tx_buf, dma_len1, size);
} else {
tx_buf->skb = (struct sk_buff *)MTK_DMA_DUMMY_DESC;
tx_buf->data = (void *)MTK_DMA_DUMMY_DESC;
txd->txd1 = mapped_addr;
txd->txd2 = TX_DMA_PLEN0(size);
dma_unmap_addr_set(tx_buf, dma_addr0, mapped_addr);
......@@ -1221,7 +1243,7 @@ static int mtk_tx_map(struct sk_buff *skb, struct net_device *dev,
soc->txrx.txd_size);
if (new_desc)
memset(tx_buf, 0, sizeof(*tx_buf));
tx_buf->skb = (struct sk_buff *)MTK_DMA_DUMMY_DESC;
tx_buf->data = (void *)MTK_DMA_DUMMY_DESC;
tx_buf->flags |= MTK_TX_FLAGS_PAGE0;
tx_buf->flags |= (!mac->id) ? MTK_TX_FLAGS_FPORT0 :
MTK_TX_FLAGS_FPORT1;
......@@ -1235,7 +1257,8 @@ static int mtk_tx_map(struct sk_buff *skb, struct net_device *dev,
}
/* store skb to cleanup */
itx_buf->skb = skb;
itx_buf->type = MTK_TYPE_SKB;
itx_buf->data = skb;
if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
if (k & 0x1)
......@@ -1432,11 +1455,259 @@ static void mtk_update_rx_cpu_idx(struct mtk_eth *eth)
}
}
static bool mtk_page_pool_enabled(struct mtk_eth *eth)
{
return !eth->hwlro;
}
static struct page_pool *mtk_create_page_pool(struct mtk_eth *eth,
struct xdp_rxq_info *xdp_q,
int id, int size)
{
struct page_pool_params pp_params = {
.order = 0,
.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
.pool_size = size,
.nid = NUMA_NO_NODE,
.dev = eth->dma_dev,
.offset = MTK_PP_HEADROOM,
.max_len = MTK_PP_MAX_BUF_SIZE,
};
struct page_pool *pp;
int err;
pp_params.dma_dir = rcu_access_pointer(eth->prog) ? DMA_BIDIRECTIONAL
: DMA_FROM_DEVICE;
pp = page_pool_create(&pp_params);
if (IS_ERR(pp))
return pp;
err = __xdp_rxq_info_reg(xdp_q, &eth->dummy_dev, eth->rx_napi.napi_id,
id, PAGE_SIZE);
if (err < 0)
goto err_free_pp;
err = xdp_rxq_info_reg_mem_model(xdp_q, MEM_TYPE_PAGE_POOL, pp);
if (err)
goto err_unregister_rxq;
return pp;
err_unregister_rxq:
xdp_rxq_info_unreg(xdp_q);
err_free_pp:
page_pool_destroy(pp);
return ERR_PTR(err);
}
static void *mtk_page_pool_get_buff(struct page_pool *pp, dma_addr_t *dma_addr,
gfp_t gfp_mask)
{
struct page *page;
page = page_pool_alloc_pages(pp, gfp_mask | __GFP_NOWARN);
if (!page)
return NULL;
*dma_addr = page_pool_get_dma_addr(page) + MTK_PP_HEADROOM;
return page_address(page);
}
static void mtk_rx_put_buff(struct mtk_rx_ring *ring, void *data, bool napi)
{
if (ring->page_pool)
page_pool_put_full_page(ring->page_pool,
virt_to_head_page(data), napi);
else
skb_free_frag(data);
}
static int mtk_xdp_submit_frame(struct mtk_eth *eth, struct xdp_frame *xdpf,
struct net_device *dev, bool dma_map)
{
const struct mtk_soc_data *soc = eth->soc;
struct mtk_tx_ring *ring = &eth->tx_ring;
struct mtk_tx_dma_desc_info txd_info = {
.size = xdpf->len,
.first = true,
.last = true,
};
struct mtk_mac *mac = netdev_priv(dev);
struct mtk_tx_dma *txd, *txd_pdma;
int err = 0, index = 0, n_desc = 1;
struct mtk_tx_buf *tx_buf;
if (unlikely(test_bit(MTK_RESETTING, &eth->state)))
return -EBUSY;
if (unlikely(atomic_read(&ring->free_count) <= 1))
return -EBUSY;
spin_lock(&eth->page_lock);
txd = ring->next_free;
if (txd == ring->last_free) {
err = -ENOMEM;
goto out;
}
tx_buf = mtk_desc_to_tx_buf(ring, txd, soc->txrx.txd_size);
memset(tx_buf, 0, sizeof(*tx_buf));
if (dma_map) { /* ndo_xdp_xmit */
txd_info.addr = dma_map_single(eth->dma_dev, xdpf->data,
txd_info.size, DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(eth->dma_dev, txd_info.addr))) {
err = -ENOMEM;
goto out;
}
tx_buf->flags |= MTK_TX_FLAGS_SINGLE0;
} else {
struct page *page = virt_to_head_page(xdpf->data);
txd_info.addr = page_pool_get_dma_addr(page) +
sizeof(*xdpf) + xdpf->headroom;
dma_sync_single_for_device(eth->dma_dev, txd_info.addr,
txd_info.size,
DMA_BIDIRECTIONAL);
}
mtk_tx_set_dma_desc(dev, txd, &txd_info);
tx_buf->flags |= !mac->id ? MTK_TX_FLAGS_FPORT0 : MTK_TX_FLAGS_FPORT1;
txd_pdma = qdma_to_pdma(ring, txd);
setup_tx_buf(eth, tx_buf, txd_pdma, txd_info.addr, txd_info.size,
index++);
/* store xdpf for cleanup */
tx_buf->type = dma_map ? MTK_TYPE_XDP_NDO : MTK_TYPE_XDP_TX;
tx_buf->data = xdpf;
if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
if (index & 1)
txd_pdma->txd2 |= TX_DMA_LS0;
else
txd_pdma->txd2 |= TX_DMA_LS1;
}
ring->next_free = mtk_qdma_phys_to_virt(ring, txd->txd2);
atomic_sub(n_desc, &ring->free_count);
/* make sure that all changes to the dma ring are flushed before we
* continue
*/
wmb();
if (MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
mtk_w32(eth, txd->txd2, soc->reg_map->qdma.ctx_ptr);
} else {
int idx;
idx = txd_to_idx(ring, txd, soc->txrx.txd_size);
mtk_w32(eth, NEXT_DESP_IDX(idx, ring->dma_size),
MT7628_TX_CTX_IDX0);
}
out:
spin_unlock(&eth->page_lock);
return err;
}
static int mtk_xdp_xmit(struct net_device *dev, int num_frame,
struct xdp_frame **frames, u32 flags)
{
struct mtk_mac *mac = netdev_priv(dev);
struct mtk_hw_stats *hw_stats = mac->hw_stats;
struct mtk_eth *eth = mac->hw;
int i, nxmit = 0;
if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
return -EINVAL;
for (i = 0; i < num_frame; i++) {
if (mtk_xdp_submit_frame(eth, frames[i], dev, true))
break;
nxmit++;
}
u64_stats_update_begin(&hw_stats->syncp);
hw_stats->xdp_stats.tx_xdp_xmit += nxmit;
hw_stats->xdp_stats.tx_xdp_xmit_errors += num_frame - nxmit;
u64_stats_update_end(&hw_stats->syncp);
return nxmit;
}
static u32 mtk_xdp_run(struct mtk_eth *eth, struct mtk_rx_ring *ring,
struct xdp_buff *xdp, struct net_device *dev)
{
struct mtk_mac *mac = netdev_priv(dev);
struct mtk_hw_stats *hw_stats = mac->hw_stats;
u64 *count = &hw_stats->xdp_stats.rx_xdp_drop;
struct bpf_prog *prog;
u32 act = XDP_PASS;
rcu_read_lock();
prog = rcu_dereference(eth->prog);
if (!prog)
goto out;
act = bpf_prog_run_xdp(prog, xdp);
switch (act) {
case XDP_PASS:
count = &hw_stats->xdp_stats.rx_xdp_pass;
goto update_stats;
case XDP_REDIRECT:
if (unlikely(xdp_do_redirect(dev, xdp, prog))) {
act = XDP_DROP;
break;
}
count = &hw_stats->xdp_stats.rx_xdp_redirect;
goto update_stats;
case XDP_TX: {
struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
if (mtk_xdp_submit_frame(eth, xdpf, dev, false)) {
count = &hw_stats->xdp_stats.rx_xdp_tx_errors;
act = XDP_DROP;
break;
}
count = &hw_stats->xdp_stats.rx_xdp_tx;
goto update_stats;
}
default:
bpf_warn_invalid_xdp_action(dev, prog, act);
fallthrough;
case XDP_ABORTED:
trace_xdp_exception(dev, prog, act);
fallthrough;
case XDP_DROP:
break;
}
page_pool_put_full_page(ring->page_pool,
virt_to_head_page(xdp->data), true);
update_stats:
u64_stats_update_begin(&hw_stats->syncp);
*count = *count + 1;
u64_stats_update_end(&hw_stats->syncp);
out:
rcu_read_unlock();
return act;
}
static int mtk_poll_rx(struct napi_struct *napi, int budget,
struct mtk_eth *eth)
{
struct dim_sample dim_sample = {};
struct mtk_rx_ring *ring;
bool xdp_flush = false;
int idx;
struct sk_buff *skb;
u8 *data, *new_data;
......@@ -1477,41 +1748,86 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
if (unlikely(test_bit(MTK_RESETTING, &eth->state)))
goto release_desc;
pktlen = RX_DMA_GET_PLEN0(trxd.rxd2);
/* alloc new buffer */
if (ring->frag_size <= PAGE_SIZE)
new_data = napi_alloc_frag(ring->frag_size);
else
new_data = mtk_max_lro_buf_alloc(GFP_ATOMIC);
if (unlikely(!new_data)) {
netdev->stats.rx_dropped++;
goto release_desc;
}
dma_addr = dma_map_single(eth->dma_dev,
new_data + NET_SKB_PAD +
eth->ip_align,
ring->buf_size,
DMA_FROM_DEVICE);
if (unlikely(dma_mapping_error(eth->dma_dev, dma_addr))) {
skb_free_frag(new_data);
netdev->stats.rx_dropped++;
goto release_desc;
}
if (ring->page_pool) {
struct page *page = virt_to_head_page(data);
struct xdp_buff xdp;
u32 ret;
new_data = mtk_page_pool_get_buff(ring->page_pool,
&dma_addr,
GFP_ATOMIC);
if (unlikely(!new_data)) {
netdev->stats.rx_dropped++;
goto release_desc;
}
dma_unmap_single(eth->dma_dev, trxd.rxd1,
ring->buf_size, DMA_FROM_DEVICE);
dma_sync_single_for_cpu(eth->dma_dev,
page_pool_get_dma_addr(page) + MTK_PP_HEADROOM,
pktlen, page_pool_get_dma_dir(ring->page_pool));
/* receive data */
skb = build_skb(data, ring->frag_size);
if (unlikely(!skb)) {
skb_free_frag(data);
netdev->stats.rx_dropped++;
goto skip_rx;
xdp_init_buff(&xdp, PAGE_SIZE, &ring->xdp_q);
xdp_prepare_buff(&xdp, data, MTK_PP_HEADROOM, pktlen,
false);
xdp_buff_clear_frags_flag(&xdp);
ret = mtk_xdp_run(eth, ring, &xdp, netdev);
if (ret == XDP_REDIRECT)
xdp_flush = true;
if (ret != XDP_PASS)
goto skip_rx;
skb = build_skb(data, PAGE_SIZE);
if (unlikely(!skb)) {
page_pool_put_full_page(ring->page_pool,
page, true);
netdev->stats.rx_dropped++;
goto skip_rx;
}
skb_reserve(skb, xdp.data - xdp.data_hard_start);
skb_put(skb, xdp.data_end - xdp.data);
skb_mark_for_recycle(skb);
} else {
if (ring->frag_size <= PAGE_SIZE)
new_data = napi_alloc_frag(ring->frag_size);
else
new_data = mtk_max_lro_buf_alloc(GFP_ATOMIC);
if (unlikely(!new_data)) {
netdev->stats.rx_dropped++;
goto release_desc;
}
dma_addr = dma_map_single(eth->dma_dev,
new_data + NET_SKB_PAD + eth->ip_align,
ring->buf_size, DMA_FROM_DEVICE);
if (unlikely(dma_mapping_error(eth->dma_dev,
dma_addr))) {
skb_free_frag(new_data);
netdev->stats.rx_dropped++;
goto release_desc;
}
dma_unmap_single(eth->dma_dev, trxd.rxd1,
ring->buf_size, DMA_FROM_DEVICE);
skb = build_skb(data, ring->frag_size);
if (unlikely(!skb)) {
netdev->stats.rx_dropped++;
skb_free_frag(data);
goto skip_rx;
}
skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
skb_put(skb, pktlen);
}
skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
pktlen = RX_DMA_GET_PLEN0(trxd.rxd2);
skb->dev = netdev;
skb_put(skb, pktlen);
bytes += skb->len;
if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
rxdcsum = &trxd.rxd3;
......@@ -1523,7 +1839,6 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
else
skb_checksum_none_assert(skb);
skb->protocol = eth_type_trans(skb, netdev);
bytes += pktlen;
hash = trxd.rxd4 & MTK_RXD4_FOE_ENTRY;
if (hash != MTK_RXD4_FOE_ENTRY) {
......@@ -1561,7 +1876,6 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
skip_rx:
ring->data[idx] = new_data;
rxd->rxd1 = (unsigned int)dma_addr;
release_desc:
if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
rxd->rxd2 = RX_DMA_LSO;
......@@ -1569,7 +1883,6 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
rxd->rxd2 = RX_DMA_PREP_PLEN0(ring->buf_size);
ring->calc_idx = idx;
done++;
}
......@@ -1588,6 +1901,9 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
&dim_sample);
net_dim(&eth->rx_dim, dim_sample);
if (xdp_flush)
xdp_do_flush_map();
return done;
}
......@@ -1596,9 +1912,8 @@ static int mtk_poll_tx_qdma(struct mtk_eth *eth, int budget,
{
const struct mtk_reg_map *reg_map = eth->soc->reg_map;
struct mtk_tx_ring *ring = &eth->tx_ring;
struct mtk_tx_dma *desc;
struct sk_buff *skb;
struct mtk_tx_buf *tx_buf;
struct mtk_tx_dma *desc;
u32 cpu, dma;
cpu = ring->last_free_ptr;
......@@ -1619,15 +1934,21 @@ static int mtk_poll_tx_qdma(struct mtk_eth *eth, int budget,
if (tx_buf->flags & MTK_TX_FLAGS_FPORT1)
mac = 1;
skb = tx_buf->skb;
if (!skb)
if (!tx_buf->data)
break;
if (skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC) {
if (tx_buf->type == MTK_TYPE_SKB &&
tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) {
struct sk_buff *skb = tx_buf->data;
bytes[mac] += skb->len;
done[mac]++;
budget--;
} else if (tx_buf->type == MTK_TYPE_XDP_TX ||
tx_buf->type == MTK_TYPE_XDP_NDO) {
budget--;
}
mtk_tx_unmap(eth, tx_buf, true);
ring->last_free = desc;
......@@ -1646,9 +1967,8 @@ static int mtk_poll_tx_pdma(struct mtk_eth *eth, int budget,
unsigned int *done, unsigned int *bytes)
{
struct mtk_tx_ring *ring = &eth->tx_ring;
struct mtk_tx_dma *desc;
struct sk_buff *skb;
struct mtk_tx_buf *tx_buf;
struct mtk_tx_dma *desc;
u32 cpu, dma;
cpu = ring->cpu_idx;
......@@ -1656,14 +1976,18 @@ static int mtk_poll_tx_pdma(struct mtk_eth *eth, int budget,
while ((cpu != dma) && budget) {
tx_buf = &ring->buf[cpu];
skb = tx_buf->skb;
if (!skb)
if (!tx_buf->data)
break;
if (skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC) {
if (tx_buf->type == MTK_TYPE_SKB &&
tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) {
struct sk_buff *skb = tx_buf->data;
bytes[0] += skb->len;
done[0]++;
budget--;
} else if (tx_buf->type == MTK_TYPE_XDP_TX ||
tx_buf->type == MTK_TYPE_XDP_NDO) {
budget--;
}
mtk_tx_unmap(eth, tx_buf, true);
......@@ -1933,13 +2257,15 @@ static int mtk_rx_alloc(struct mtk_eth *eth, int ring_no, int rx_flag)
if (!ring->data)
return -ENOMEM;
for (i = 0; i < rx_dma_size; i++) {
if (ring->frag_size <= PAGE_SIZE)
ring->data[i] = netdev_alloc_frag(ring->frag_size);
else
ring->data[i] = mtk_max_lro_buf_alloc(GFP_KERNEL);
if (!ring->data[i])
return -ENOMEM;
if (mtk_page_pool_enabled(eth)) {
struct page_pool *pp;
pp = mtk_create_page_pool(eth, &ring->xdp_q, ring_no,
rx_dma_size);
if (IS_ERR(pp))
return PTR_ERR(pp);
ring->page_pool = pp;
}
ring->dma = dma_alloc_coherent(eth->dma_dev,
......@@ -1950,16 +2276,33 @@ static int mtk_rx_alloc(struct mtk_eth *eth, int ring_no, int rx_flag)
for (i = 0; i < rx_dma_size; i++) {
struct mtk_rx_dma_v2 *rxd;
dma_addr_t dma_addr = dma_map_single(eth->dma_dev,
ring->data[i] + NET_SKB_PAD + eth->ip_align,
ring->buf_size,
DMA_FROM_DEVICE);
if (unlikely(dma_mapping_error(eth->dma_dev, dma_addr)))
return -ENOMEM;
dma_addr_t dma_addr;
void *data;
rxd = ring->dma + i * eth->soc->txrx.rxd_size;
if (ring->page_pool) {
data = mtk_page_pool_get_buff(ring->page_pool,
&dma_addr, GFP_KERNEL);
if (!data)
return -ENOMEM;
} else {
if (ring->frag_size <= PAGE_SIZE)
data = netdev_alloc_frag(ring->frag_size);
else
data = mtk_max_lro_buf_alloc(GFP_KERNEL);
if (!data)
return -ENOMEM;
dma_addr = dma_map_single(eth->dma_dev,
data + NET_SKB_PAD + eth->ip_align,
ring->buf_size, DMA_FROM_DEVICE);
if (unlikely(dma_mapping_error(eth->dma_dev,
dma_addr)))
return -ENOMEM;
}
rxd->rxd1 = (unsigned int)dma_addr;
ring->data[i] = data;
if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
rxd->rxd2 = RX_DMA_LSO;
......@@ -1975,6 +2318,7 @@ static int mtk_rx_alloc(struct mtk_eth *eth, int ring_no, int rx_flag)
rxd->rxd8 = 0;
}
}
ring->dma_size = rx_dma_size;
ring->calc_idx_update = false;
ring->calc_idx = rx_dma_size - 1;
......@@ -2026,7 +2370,7 @@ static void mtk_rx_clean(struct mtk_eth *eth, struct mtk_rx_ring *ring)
dma_unmap_single(eth->dma_dev, rxd->rxd1,
ring->buf_size, DMA_FROM_DEVICE);
skb_free_frag(ring->data[i]);
mtk_rx_put_buff(ring, ring->data[i], false);
}
kfree(ring->data);
ring->data = NULL;
......@@ -2038,6 +2382,13 @@ static void mtk_rx_clean(struct mtk_eth *eth, struct mtk_rx_ring *ring)
ring->dma, ring->phys);
ring->dma = NULL;
}
if (ring->page_pool) {
if (xdp_rxq_info_is_reg(&ring->xdp_q))
xdp_rxq_info_unreg(&ring->xdp_q);
page_pool_destroy(ring->page_pool);
ring->page_pool = NULL;
}
}
static int mtk_hwlro_rx_init(struct mtk_eth *eth)
......@@ -2645,6 +2996,48 @@ static int mtk_stop(struct net_device *dev)
return 0;
}
static int mtk_xdp_setup(struct net_device *dev, struct bpf_prog *prog,
struct netlink_ext_ack *extack)
{
struct mtk_mac *mac = netdev_priv(dev);
struct mtk_eth *eth = mac->hw;
struct bpf_prog *old_prog;
bool need_update;
if (eth->hwlro) {
NL_SET_ERR_MSG_MOD(extack, "XDP not supported with HWLRO");
return -EOPNOTSUPP;
}
if (dev->mtu > MTK_PP_MAX_BUF_SIZE) {
NL_SET_ERR_MSG_MOD(extack, "MTU too large for XDP");
return -EOPNOTSUPP;
}
need_update = !!eth->prog != !!prog;
if (netif_running(dev) && need_update)
mtk_stop(dev);
old_prog = rcu_replace_pointer(eth->prog, prog, lockdep_rtnl_is_held());
if (old_prog)
bpf_prog_put(old_prog);
if (netif_running(dev) && need_update)
return mtk_open(dev);
return 0;
}
static int mtk_xdp(struct net_device *dev, struct netdev_bpf *xdp)
{
switch (xdp->command) {
case XDP_SETUP_PROG:
return mtk_xdp_setup(dev, xdp->prog, xdp->extack);
default:
return -EINVAL;
}
}
static void ethsys_reset(struct mtk_eth *eth, u32 reset_bits)
{
regmap_update_bits(eth->ethsys, ETHSYS_RSTCTRL,
......@@ -2940,6 +3333,12 @@ static int mtk_change_mtu(struct net_device *dev, int new_mtu)
struct mtk_eth *eth = mac->hw;
u32 mcr_cur, mcr_new;
if (rcu_access_pointer(eth->prog) &&
length > MTK_PP_MAX_BUF_SIZE) {
netdev_err(dev, "Invalid MTU for XDP mode\n");
return -EINVAL;
}
if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
mcr_cur = mtk_r32(mac->hw, MTK_MAC_MCR(mac->id));
mcr_new = mcr_cur & ~MAC_MCR_MAX_RX_MASK;
......@@ -3129,11 +3528,18 @@ static void mtk_get_strings(struct net_device *dev, u32 stringset, u8 *data)
int i;
switch (stringset) {
case ETH_SS_STATS:
case ETH_SS_STATS: {
struct mtk_mac *mac = netdev_priv(dev);
for (i = 0; i < ARRAY_SIZE(mtk_ethtool_stats); i++) {
memcpy(data, mtk_ethtool_stats[i].str, ETH_GSTRING_LEN);
data += ETH_GSTRING_LEN;
}
if (mtk_page_pool_enabled(mac->hw))
page_pool_ethtool_stats_get_strings(data);
break;
}
default:
break;
}
}
......@@ -3141,13 +3547,35 @@ static void mtk_get_strings(struct net_device *dev, u32 stringset, u8 *data)
static int mtk_get_sset_count(struct net_device *dev, int sset)
{
switch (sset) {
case ETH_SS_STATS:
return ARRAY_SIZE(mtk_ethtool_stats);
case ETH_SS_STATS: {
int count = ARRAY_SIZE(mtk_ethtool_stats);
struct mtk_mac *mac = netdev_priv(dev);
if (mtk_page_pool_enabled(mac->hw))
count += page_pool_ethtool_stats_get_count();
return count;
}
default:
return -EOPNOTSUPP;
}
}
static void mtk_ethtool_pp_stats(struct mtk_eth *eth, u64 *data)
{
struct page_pool_stats stats = {};
int i;
for (i = 0; i < ARRAY_SIZE(eth->rx_ring); i++) {
struct mtk_rx_ring *ring = &eth->rx_ring[i];
if (!ring->page_pool)
continue;
page_pool_get_stats(ring->page_pool, &stats);
}
page_pool_ethtool_stats_get(data, &stats);
}
static void mtk_get_ethtool_stats(struct net_device *dev,
struct ethtool_stats *stats, u64 *data)
{
......@@ -3175,6 +3603,8 @@ static void mtk_get_ethtool_stats(struct net_device *dev,
for (i = 0; i < ARRAY_SIZE(mtk_ethtool_stats); i++)
*data_dst++ = *(data_src + mtk_ethtool_stats[i].offset);
if (mtk_page_pool_enabled(mac->hw))
mtk_ethtool_pp_stats(mac->hw, data_dst);
} while (u64_stats_fetch_retry_irq(&hwstats->syncp, start));
}
......@@ -3267,6 +3697,8 @@ static const struct net_device_ops mtk_netdev_ops = {
.ndo_poll_controller = mtk_poll_controller,
#endif
.ndo_setup_tc = mtk_eth_setup_tc,
.ndo_bpf = mtk_xdp,
.ndo_xdp_xmit = mtk_xdp_xmit,
};
static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np)
......
drivers/net/ethernet/mediatek/mtk_eth_soc.h
View file @ 61aee772
......@@ -18,6 +18,8 @@
#include <linux/rhashtable.h>
#include <linux/dim.h>
#include <linux/bitfield.h>
#include <net/page_pool.h>
#include <linux/bpf_trace.h>
#include "mtk_ppe.h"
#define MTK_QDMA_PAGE_SIZE 2048
......@@ -49,6 +51,11 @@
#define MTK_HW_FEATURES_MT7628 (NETIF_F_SG | NETIF_F_RXCSUM)
#define NEXT_DESP_IDX(X, Y) (((X) + 1) & ((Y) - 1))
#define MTK_PP_HEADROOM XDP_PACKET_HEADROOM
#define MTK_PP_PAD (MTK_PP_HEADROOM + \
SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
#define MTK_PP_MAX_BUF_SIZE (PAGE_SIZE - MTK_PP_PAD)
#define MTK_QRX_OFFSET 0x10
#define MTK_MAX_RX_RING_NUM 4
......@@ -563,6 +570,16 @@ struct mtk_tx_dma_v2 {
struct mtk_eth;
struct mtk_mac;
struct mtk_xdp_stats {
u64 rx_xdp_redirect;
u64 rx_xdp_pass;
u64 rx_xdp_drop;
u64 rx_xdp_tx;
u64 rx_xdp_tx_errors;
u64 tx_xdp_xmit;
u64 tx_xdp_xmit_errors;
};
/* struct mtk_hw_stats - the structure that holds the traffic statistics.
* @stats_lock: make sure that stats operations are atomic
* @reg_offset: the status register offset of the SoC
......@@ -586,6 +603,8 @@ struct mtk_hw_stats {
u64 rx_checksum_errors;
u64 rx_flow_control_packets;
struct mtk_xdp_stats xdp_stats;
spinlock_t stats_lock;
u32 reg_offset;
struct u64_stats_sync syncp;
......@@ -677,6 +696,12 @@ enum mtk_dev_state {
MTK_RESETTING
};
enum mtk_tx_buf_type {
MTK_TYPE_SKB,
MTK_TYPE_XDP_TX,
MTK_TYPE_XDP_NDO,
};
/* struct mtk_tx_buf - This struct holds the pointers to the memory pointed at
* by the TX descriptor s
* @skb: The SKB pointer of the packet being sent
......@@ -686,7 +711,9 @@ enum mtk_dev_state {
* @dma_len1: The length of the second segment
*/
struct mtk_tx_buf {
struct sk_buff *skb;
enum mtk_tx_buf_type type;
void *data;
u32 flags;
DEFINE_DMA_UNMAP_ADDR(dma_addr0);
DEFINE_DMA_UNMAP_LEN(dma_len0);
......@@ -745,6 +772,9 @@ struct mtk_rx_ring {
bool calc_idx_update;
u16 calc_idx;
u32 crx_idx_reg;
/* page_pool */
struct page_pool *page_pool;
struct xdp_rxq_info xdp_q;
};
enum mkt_eth_capabilities {
......@@ -1078,6 +1108,8 @@ struct mtk_eth {
struct mtk_ppe *ppe;
struct rhashtable flow_table;
struct bpf_prog __rcu *prog;
};
/* struct mtk_mac - the structure that holds the info about the MACs of the
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7