Skip to content

L
linux
Commit 02f7f61e authored by David S. Miller's avatar David S. Miller
Browse files
Options

Merge branch 'net-mvpp2-improve-the-interrupt-usage' 

Antoine Tenart says:

====================
net: mvpp2: improve the interrupt usage

This series aims to improve the interrupts descriptions and usage in the
Marvell PPv2 driver.

- Before the series interrupts were named after their s/w usage,
  which in fact can be configured. The series rename all those
  interrupts and add a description of the ones left over.

- In PPv2 the interrupts are mapped to vectors. Those vectors were
  directly mapped to a given CPU, and per-cpu accesses were done. While
  this worked on our cases, the registers accesses mapped to the vectors
  are not actually linked to a given CPU. They instead are linked to
  what is called a "s/w thread". The series modify this so that the s/w
  threads are used instead of the CPU numbers, by adding an indirection.
  This means we now can have systems with more CPUs than s/w threads.

This is based on today's net-next, and was tested on various boards
using both versions of the PPv2 engine.

Two more patches will be coming, to update the device trees describing a
PPv2 engine. The patches are ready, but will go through a different
tree. I'll send them once this series will be accepted. This is not an
issue as the PPv2 driver keeps the dt bindings backward compatibility.
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents f543305d 1068549c
Hide whitespace changes
Inline Side-by-side
Showing with 315 additions and 188 deletions
Documentation/devicetree/bindings/net/marvell-pp2.txt
View file @ 02f7f61e
......@@ -31,7 +31,7 @@ required.
Required properties (port):
- interrupts: interrupt for the port
- interrupts: interrupt(s) for the port
- port-id: ID of the port from the MAC point of view
- gop-port-id: only for marvell,armada-7k-pp2, ID of the port from the
GOP (Group Of Ports) point of view. This ID is used to index the
......@@ -43,10 +43,12 @@ Optional properties (port):
- marvell,loopback: port is loopback mode
- phy: a phandle to a phy node defining the PHY address (as the reg
property, a single integer).
- interrupt-names: if more than a single interrupt for rx is given, must
be the name associated to the interrupts listed. Valid
names are: "tx-cpu0", "tx-cpu1", "tx-cpu2", "tx-cpu3",
"rx-shared", "link".
- interrupt-names: if more than a single interrupt for is given, must be the
name associated to the interrupts listed. Valid names are:
"hifX", with X in [0..8], and "link". The names "tx-cpu0",
"tx-cpu1", "tx-cpu2", "tx-cpu3" and "rx-shared" are supported
for backward compatibility but shouldn't be used for new
additions.
- marvell,system-controller: a phandle to the system controller.
Example for marvell,armada-375-pp2:
......@@ -89,9 +91,14 @@ cpm_ethernet: ethernet@0 {
<ICU_GRP_NSR 43 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 47 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 51 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 55 IRQ_TYPE_LEVEL_HIGH>;
interrupt-names = "tx-cpu0", "tx-cpu1", "tx-cpu2",
"tx-cpu3", "rx-shared";
<ICU_GRP_NSR 55 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 59 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 63 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 67 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 71 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 129 IRQ_TYPE_LEVEL_HIGH>;
interrupt-names = "hif0", "hif1", "hif2", "hif3", "hif4",
"hif5", "hif6", "hif7", "hif8", "link";
port-id = <0>;
gop-port-id = <0>;
};
......@@ -101,9 +108,14 @@ cpm_ethernet: ethernet@0 {
<ICU_GRP_NSR 44 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 48 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 52 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 56 IRQ_TYPE_LEVEL_HIGH>;
interrupt-names = "tx-cpu0", "tx-cpu1", "tx-cpu2",
"tx-cpu3", "rx-shared";
<ICU_GRP_NSR 56 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 60 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 64 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 68 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 72 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 128 IRQ_TYPE_LEVEL_HIGH>;
interrupt-names = "hif0", "hif1", "hif2", "hif3", "hif4",
"hif5", "hif6", "hif7", "hif8", "link";
port-id = <1>;
gop-port-id = <2>;
};
......@@ -113,9 +125,14 @@ cpm_ethernet: ethernet@0 {
<ICU_GRP_NSR 45 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 49 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 53 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 57 IRQ_TYPE_LEVEL_HIGH>;
interrupt-names = "tx-cpu0", "tx-cpu1", "tx-cpu2",
"tx-cpu3", "rx-shared";
<ICU_GRP_NSR 57 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 61 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 65 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 69 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 73 IRQ_TYPE_LEVEL_HIGH>,
<ICU_GRP_NSR 127 IRQ_TYPE_LEVEL_HIGH>;
interrupt-names = "hif0", "hif1", "hif2", "hif3", "hif4",
"hif5", "hif6", "hif7", "hif8", "link";
port-id = <2>;
gop-port-id = <3>;
};
......
drivers/net/ethernet/marvell/mvpp2/mvpp2.h
View file @ 02f7f61e
......@@ -253,7 +253,8 @@
#define MVPP2_ISR_ENABLE_INTERRUPT(mask) ((mask) & 0xffff)
#define MVPP2_ISR_DISABLE_INTERRUPT(mask) (((mask) << 16) & 0xffff0000)
#define MVPP2_ISR_RX_TX_CAUSE_REG(port) (0x5480 + 4 * (port))
#define MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK 0xffff
#define MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK(version) \
((version) == MVPP21 ? 0xffff : 0xff)
#define MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK 0xff0000
#define MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_OFFSET 16
#define MVPP2_CAUSE_RX_FIFO_OVERRUN_MASK BIT(24)
......@@ -613,6 +614,7 @@
/* Port flags */
#define MVPP2_F_LOOPBACK BIT(0)
#define MVPP2_F_DT_COMPAT BIT(1)
/* Marvell tag types */
enum mvpp2_tag_type {
......@@ -662,7 +664,7 @@ enum mvpp2_prs_l3_cast {
#define MVPP21_ADDR_SPACE_SZ 0
#define MVPP22_ADDR_SPACE_SZ SZ_64K
#define MVPP2_MAX_THREADS 8
#define MVPP2_MAX_THREADS 9
#define MVPP2_MAX_QVECS MVPP2_MAX_THREADS
/* GMAC MIB Counters register definitions */
......@@ -734,6 +736,11 @@ struct mvpp2 {
int port_count;
struct mvpp2_port *port_list[MVPP2_MAX_PORTS];
/* Number of Tx threads used */
unsigned int nthreads;
/* Map of threads needing locking */
unsigned long lock_map;
/* Aggregated TXQs */
struct mvpp2_tx_queue *aggr_txqs;
......@@ -823,6 +830,12 @@ struct mvpp2_port {
/* Per-CPU port control */
struct mvpp2_port_pcpu __percpu *pcpu;
/* Protect the BM refills and the Tx paths when a thread is used on more
* than a single CPU.
*/
spinlock_t bm_lock[MVPP2_MAX_THREADS];
spinlock_t tx_lock[MVPP2_MAX_THREADS];
/* Flags */
unsigned long flags;
......@@ -969,7 +982,7 @@ struct mvpp2_txq_pcpu_buf {
/* Per-CPU Tx queue control */
struct mvpp2_txq_pcpu {
int cpu;
unsigned int thread;
/* Number of Tx DMA descriptors in the descriptor ring */
int size;
......@@ -1095,14 +1108,6 @@ struct mvpp2_bm_pool {
void mvpp2_write(struct mvpp2 *priv, u32 offset, u32 data);
u32 mvpp2_read(struct mvpp2 *priv, u32 offset);
u32 mvpp2_read_relaxed(struct mvpp2 *priv, u32 offset);
void mvpp2_percpu_write(struct mvpp2 *priv, int cpu, u32 offset, u32 data);
u32 mvpp2_percpu_read(struct mvpp2 *priv, int cpu, u32 offset);
void mvpp2_percpu_write_relaxed(struct mvpp2 *priv, int cpu, u32 offset,
u32 data);
void mvpp2_dbgfs_init(struct mvpp2 *priv, const char *name);
void mvpp2_dbgfs_cleanup(struct mvpp2 *priv);
......
drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
View file @ 02f7f61e
......@@ -82,13 +82,19 @@ u32 mvpp2_read(struct mvpp2 *priv, u32 offset)
return readl(priv->swth_base[0] + offset);
}
u32 mvpp2_read_relaxed(struct mvpp2 *priv, u32 offset)
static u32 mvpp2_read_relaxed(struct mvpp2 *priv, u32 offset)
{
return readl_relaxed(priv->swth_base[0] + offset);
}
static inline u32 mvpp2_cpu_to_thread(struct mvpp2 *priv, int cpu)
{
return cpu % priv->nthreads;
}
/* These accessors should be used to access:
*
* - per-CPU registers, where each CPU has its own copy of the
* - per-thread registers, where each thread has its own copy of the
* register.
*
* MVPP2_BM_VIRT_ALLOC_REG
......@@ -104,8 +110,8 @@ u32 mvpp2_read_relaxed(struct mvpp2 *priv, u32 offset)
* MVPP2_TXQ_SENT_REG
* MVPP2_RXQ_NUM_REG
*
* - global registers that must be accessed through a specific CPU
* window, because they are related to an access to a per-CPU
* - global registers that must be accessed through a specific thread
* window, because they are related to an access to a per-thread
* register
*
* MVPP2_BM_PHY_ALLOC_REG (related to MVPP2_BM_VIRT_ALLOC_REG)
......@@ -122,28 +128,28 @@ u32 mvpp2_read_relaxed(struct mvpp2 *priv, u32 offset)
* MVPP2_TXQ_PREF_BUF_REG (related to MVPP2_TXQ_NUM_REG)
* MVPP2_TXQ_PREF_BUF_REG (related to MVPP2_TXQ_NUM_REG)
*/
void mvpp2_percpu_write(struct mvpp2 *priv, int cpu,
static void mvpp2_thread_write(struct mvpp2 *priv, unsigned int thread,
u32 offset, u32 data)
{
writel(data, priv->swth_base[cpu] + offset);
writel(data, priv->swth_base[thread] + offset);
}
u32 mvpp2_percpu_read(struct mvpp2 *priv, int cpu,
static u32 mvpp2_thread_read(struct mvpp2 *priv, unsigned int thread,
u32 offset)
{
return readl(priv->swth_base[cpu] + offset);
return readl(priv->swth_base[thread] + offset);
}
void mvpp2_percpu_write_relaxed(struct mvpp2 *priv, int cpu,
static void mvpp2_thread_write_relaxed(struct mvpp2 *priv, unsigned int thread,
u32 offset, u32 data)
{
writel_relaxed(data, priv->swth_base[cpu] + offset);
writel_relaxed(data, priv->swth_base[thread] + offset);
}
static u32 mvpp2_percpu_read_relaxed(struct mvpp2 *priv, int cpu,
static u32 mvpp2_thread_read_relaxed(struct mvpp2 *priv, unsigned int thread,
u32 offset)
{
return readl_relaxed(priv->swth_base[cpu] + offset);
return readl_relaxed(priv->swth_base[thread] + offset);
}
static dma_addr_t mvpp2_txdesc_dma_addr_get(struct mvpp2_port *port,
......@@ -385,17 +391,17 @@ static void mvpp2_bm_bufs_get_addrs(struct device *dev, struct mvpp2 *priv,
dma_addr_t *dma_addr,
phys_addr_t *phys_addr)
{
int cpu = get_cpu();
unsigned int thread = mvpp2_cpu_to_thread(priv, get_cpu());
*dma_addr = mvpp2_percpu_read(priv, cpu,
*dma_addr = mvpp2_thread_read(priv, thread,
MVPP2_BM_PHY_ALLOC_REG(bm_pool->id));
*phys_addr = mvpp2_percpu_read(priv, cpu, MVPP2_BM_VIRT_ALLOC_REG);
*phys_addr = mvpp2_thread_read(priv, thread, MVPP2_BM_VIRT_ALLOC_REG);
if (priv->hw_version == MVPP22) {
u32 val;
u32 dma_addr_highbits, phys_addr_highbits;
val = mvpp2_percpu_read(priv, cpu, MVPP22_BM_ADDR_HIGH_ALLOC);
val = mvpp2_thread_read(priv, thread, MVPP22_BM_ADDR_HIGH_ALLOC);
dma_addr_highbits = (val & MVPP22_BM_ADDR_HIGH_PHYS_MASK);
phys_addr_highbits = (val & MVPP22_BM_ADDR_HIGH_VIRT_MASK) >>
MVPP22_BM_ADDR_HIGH_VIRT_SHIFT;
......@@ -626,7 +632,11 @@ static inline void mvpp2_bm_pool_put(struct mvpp2_port *port, int pool,
dma_addr_t buf_dma_addr,
phys_addr_t buf_phys_addr)
{
int cpu = get_cpu();
unsigned int thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
unsigned long flags = 0;
if (test_bit(thread, &port->priv->lock_map))
spin_lock_irqsave(&port->bm_lock[thread], flags);
if (port->priv->hw_version == MVPP22) {
u32 val = 0;
......@@ -640,7 +650,7 @@ static inline void mvpp2_bm_pool_put(struct mvpp2_port *port, int pool,
<< MVPP22_BM_ADDR_HIGH_VIRT_RLS_SHIFT) &
MVPP22_BM_ADDR_HIGH_VIRT_RLS_MASK;
mvpp2_percpu_write_relaxed(port->priv, cpu,
mvpp2_thread_write_relaxed(port->priv, thread,
MVPP22_BM_ADDR_HIGH_RLS_REG, val);
}
......@@ -649,11 +659,14 @@ static inline void mvpp2_bm_pool_put(struct mvpp2_port *port, int pool,
* descriptor. Instead of storing the virtual address, we
* store the physical address
*/
mvpp2_percpu_write_relaxed(port->priv, cpu,
mvpp2_thread_write_relaxed(port->priv, thread,
MVPP2_BM_VIRT_RLS_REG, buf_phys_addr);
mvpp2_percpu_write_relaxed(port->priv, cpu,
mvpp2_thread_write_relaxed(port->priv, thread,
MVPP2_BM_PHY_RLS_REG(pool), buf_dma_addr);
if (test_bit(thread, &port->priv->lock_map))
spin_unlock_irqrestore(&port->bm_lock[thread], flags);
put_cpu();
}
......@@ -886,7 +899,7 @@ static inline void mvpp2_qvec_interrupt_disable(struct mvpp2_queue_vector *qvec)
MVPP2_ISR_DISABLE_INTERRUPT(qvec->sw_thread_mask));
}
/* Mask the current CPU's Rx/Tx interrupts
/* Mask the current thread's Rx/Tx interrupts
* Called by on_each_cpu(), guaranteed to run with migration disabled,
* using smp_processor_id() is OK.
*/
......@@ -894,11 +907,16 @@ static void mvpp2_interrupts_mask(void *arg)
{
struct mvpp2_port *port = arg;
mvpp2_percpu_write(port->priv, smp_processor_id(),
/* If the thread isn't used, don't do anything */
if (smp_processor_id() > port->priv->nthreads)
return;
mvpp2_thread_write(port->priv,
mvpp2_cpu_to_thread(port->priv, smp_processor_id()),
MVPP2_ISR_RX_TX_MASK_REG(port->id), 0);
}
/* Unmask the current CPU's Rx/Tx interrupts.
/* Unmask the current thread's Rx/Tx interrupts.
* Called by on_each_cpu(), guaranteed to run with migration disabled,
* using smp_processor_id() is OK.
*/
......@@ -907,12 +925,17 @@ static void mvpp2_interrupts_unmask(void *arg)
struct mvpp2_port *port = arg;
u32 val;
/* If the thread isn't used, don't do anything */
if (smp_processor_id() > port->priv->nthreads)
return;
val = MVPP2_CAUSE_MISC_SUM_MASK |
MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK;
MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK(port->priv->hw_version);
if (port->has_tx_irqs)
val |= MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK;
mvpp2_percpu_write(port->priv, smp_processor_id(),
mvpp2_thread_write(port->priv,
mvpp2_cpu_to_thread(port->priv, smp_processor_id()),
MVPP2_ISR_RX_TX_MASK_REG(port->id), val);
}
......@@ -928,7 +951,7 @@ mvpp2_shared_interrupt_mask_unmask(struct mvpp2_port *port, bool mask)
if (mask)
val = 0;
else
val = MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK;
val = MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK(MVPP22);
for (i = 0; i < port->nqvecs; i++) {
struct mvpp2_queue_vector *v = port->qvecs + i;
......@@ -936,7 +959,7 @@ mvpp2_shared_interrupt_mask_unmask(struct mvpp2_port *port, bool mask)
if (v->type != MVPP2_QUEUE_VECTOR_SHARED)
continue;
mvpp2_percpu_write(port->priv, v->sw_thread_id,
mvpp2_thread_write(port->priv, v->sw_thread_id,
MVPP2_ISR_RX_TX_MASK_REG(port->id), val);
}
}
......@@ -1624,7 +1647,8 @@ mvpp2_txq_next_desc_get(struct mvpp2_tx_queue *txq)
static void mvpp2_aggr_txq_pend_desc_add(struct mvpp2_port *port, int pending)
{
/* aggregated access - relevant TXQ number is written in TX desc */
mvpp2_percpu_write(port->priv, smp_processor_id(),
mvpp2_thread_write(port->priv,
mvpp2_cpu_to_thread(port->priv, smp_processor_id()),
MVPP2_AGGR_TXQ_UPDATE_REG, pending);
}
......@@ -1634,14 +1658,15 @@ static void mvpp2_aggr_txq_pend_desc_add(struct mvpp2_port *port, int pending)
* Called only from mvpp2_tx(), so migration is disabled, using
* smp_processor_id() is OK.
*/
static int mvpp2_aggr_desc_num_check(struct mvpp2 *priv,
static int mvpp2_aggr_desc_num_check(struct mvpp2_port *port,
struct mvpp2_tx_queue *aggr_txq, int num)
{
if ((aggr_txq->count + num) > MVPP2_AGGR_TXQ_SIZE) {
/* Update number of occupied aggregated Tx descriptors */
int cpu = smp_processor_id();
u32 val = mvpp2_read_relaxed(priv,
MVPP2_AGGR_TXQ_STATUS_REG(cpu));
unsigned int thread =
mvpp2_cpu_to_thread(port->priv, smp_processor_id());
u32 val = mvpp2_read_relaxed(port->priv,
MVPP2_AGGR_TXQ_STATUS_REG(thread));
aggr_txq->count = val & MVPP2_AGGR_TXQ_PENDING_MASK;
......@@ -1657,16 +1682,17 @@ static int mvpp2_aggr_desc_num_check(struct mvpp2 *priv,
* only by mvpp2_tx(), so migration is disabled, using
* smp_processor_id() is OK.
*/
static int mvpp2_txq_alloc_reserved_desc(struct mvpp2 *priv,
static int mvpp2_txq_alloc_reserved_desc(struct mvpp2_port *port,
struct mvpp2_tx_queue *txq, int num)
{
unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id());
struct mvpp2 *priv = port->priv;
u32 val;
int cpu = smp_processor_id();
val = (txq->id << MVPP2_TXQ_RSVD_REQ_Q_OFFSET) | num;
mvpp2_percpu_write_relaxed(priv, cpu, MVPP2_TXQ_RSVD_REQ_REG, val);
mvpp2_thread_write_relaxed(priv, thread, MVPP2_TXQ_RSVD_REQ_REG, val);
val = mvpp2_percpu_read_relaxed(priv, cpu, MVPP2_TXQ_RSVD_RSLT_REG);
val = mvpp2_thread_read_relaxed(priv, thread, MVPP2_TXQ_RSVD_RSLT_REG);
return val & MVPP2_TXQ_RSVD_RSLT_MASK;
}
......@@ -1674,12 +1700,13 @@ static int mvpp2_txq_alloc_reserved_desc(struct mvpp2 *priv,
/* Check if there are enough reserved descriptors for transmission.
* If not, request chunk of reserved descriptors and check again.
*/
static int mvpp2_txq_reserved_desc_num_proc(struct mvpp2 *priv,
static int mvpp2_txq_reserved_desc_num_proc(struct mvpp2_port *port,
struct mvpp2_tx_queue *txq,
struct mvpp2_txq_pcpu *txq_pcpu,
int num)
{
int req, cpu, desc_count;
int req, desc_count;
unsigned int thread;
if (txq_pcpu->reserved_num >= num)
return 0;
......@@ -1690,10 +1717,10 @@ static int mvpp2_txq_reserved_desc_num_proc(struct mvpp2 *priv,
desc_count = 0;
/* Compute total of used descriptors */
for_each_present_cpu(cpu) {
for (thread = 0; thread < port->priv->nthreads; thread++) {
struct mvpp2_txq_pcpu *txq_pcpu_aux;
txq_pcpu_aux = per_cpu_ptr(txq->pcpu, cpu);
txq_pcpu_aux = per_cpu_ptr(txq->pcpu, thread);
desc_count += txq_pcpu_aux->count;
desc_count += txq_pcpu_aux->reserved_num;
}
......@@ -1702,10 +1729,10 @@ static int mvpp2_txq_reserved_desc_num_proc(struct mvpp2 *priv,
desc_count += req;
if (desc_count >
(txq->size - (num_present_cpus() * MVPP2_CPU_DESC_CHUNK)))
(txq->size - (MVPP2_MAX_THREADS * MVPP2_CPU_DESC_CHUNK)))
return -ENOMEM;
txq_pcpu->reserved_num += mvpp2_txq_alloc_reserved_desc(priv, txq, req);
txq_pcpu->reserved_num += mvpp2_txq_alloc_reserved_desc(port, txq, req);
/* OK, the descriptor could have been updated: check again. */
if (txq_pcpu->reserved_num < num)
......@@ -1759,7 +1786,7 @@ static u32 mvpp2_txq_desc_csum(int l3_offs, int l3_proto,
/* Get number of sent descriptors and decrement counter.
* The number of sent descriptors is returned.
* Per-CPU access
* Per-thread access
*
* Called only from mvpp2_txq_done(), called from mvpp2_tx()
* (migration disabled) and from the TX completion tasklet (migration
......@@ -1771,7 +1798,8 @@ static inline int mvpp2_txq_sent_desc_proc(struct mvpp2_port *port,
u32 val;
/* Reading status reg resets transmitted descriptor counter */
val = mvpp2_percpu_read_relaxed(port->priv, smp_processor_id(),
val = mvpp2_thread_read_relaxed(port->priv,
mvpp2_cpu_to_thread(port->priv, smp_processor_id()),
MVPP2_TXQ_SENT_REG(txq->id));
return (val & MVPP2_TRANSMITTED_COUNT_MASK) >>
......@@ -1786,10 +1814,15 @@ static void mvpp2_txq_sent_counter_clear(void *arg)
struct mvpp2_port *port = arg;
int queue;
/* If the thread isn't used, don't do anything */
if (smp_processor_id() > port->priv->nthreads)
return;
for (queue = 0; queue < port->ntxqs; queue++) {
int id = port->txqs[queue]->id;
mvpp2_percpu_read(port->priv, smp_processor_id(),
mvpp2_thread_read(port->priv,
mvpp2_cpu_to_thread(port->priv, smp_processor_id()),
MVPP2_TXQ_SENT_REG(id));
}
}
......@@ -1849,13 +1882,13 @@ static void mvpp2_txp_max_tx_size_set(struct mvpp2_port *port)
static void mvpp2_rx_pkts_coal_set(struct mvpp2_port *port,
struct mvpp2_rx_queue *rxq)
{
int cpu = get_cpu();
unsigned int thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
if (rxq->pkts_coal > MVPP2_OCCUPIED_THRESH_MASK)
rxq->pkts_coal = MVPP2_OCCUPIED_THRESH_MASK;
mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_NUM_REG, rxq->id);
mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_THRESH_REG,
mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_NUM_REG, rxq->id);
mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_THRESH_REG,
rxq->pkts_coal);
put_cpu();
......@@ -1865,15 +1898,15 @@ static void mvpp2_rx_pkts_coal_set(struct mvpp2_port *port,
static void mvpp2_tx_pkts_coal_set(struct mvpp2_port *port,
struct mvpp2_tx_queue *txq)
{
int cpu = get_cpu();
unsigned int thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
u32 val;
if (txq->done_pkts_coal > MVPP2_TXQ_THRESH_MASK)
txq->done_pkts_coal = MVPP2_TXQ_THRESH_MASK;
val = (txq->done_pkts_coal << MVPP2_TXQ_THRESH_OFFSET);
mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_NUM_REG, txq->id);
mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_THRESH_REG, val);
mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id);
mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_THRESH_REG, val);
put_cpu();
}
......@@ -1974,7 +2007,7 @@ static void mvpp2_txq_done(struct mvpp2_port *port, struct mvpp2_tx_queue *txq,
struct netdev_queue *nq = netdev_get_tx_queue(port->dev, txq->log_id);
int tx_done;
if (txq_pcpu->cpu != smp_processor_id())
if (txq_pcpu->thread != mvpp2_cpu_to_thread(port->priv, smp_processor_id()))
netdev_err(port->dev, "wrong cpu on the end of Tx processing\n");
tx_done = mvpp2_txq_sent_desc_proc(port, txq);
......@@ -1990,7 +2023,7 @@ static void mvpp2_txq_done(struct mvpp2_port *port, struct mvpp2_tx_queue *txq,
}
static unsigned int mvpp2_tx_done(struct mvpp2_port *port, u32 cause,
int cpu)
unsigned int thread)
{
struct mvpp2_tx_queue *txq;
struct mvpp2_txq_pcpu *txq_pcpu;
......@@ -2001,7 +2034,7 @@ static unsigned int mvpp2_tx_done(struct mvpp2_port *port, u32 cause,
if (!txq)
break;
txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
if (txq_pcpu->count) {
mvpp2_txq_done(port, txq, txq_pcpu);
......@@ -2017,8 +2050,8 @@ static unsigned int mvpp2_tx_done(struct mvpp2_port *port, u32 cause,
/* Allocate and initialize descriptors for aggr TXQ */
static int mvpp2_aggr_txq_init(struct platform_device *pdev,
struct mvpp2_tx_queue *aggr_txq, int cpu,
struct mvpp2 *priv)
struct mvpp2_tx_queue *aggr_txq,
unsigned int thread, struct mvpp2 *priv)
{
u32 txq_dma;
......@@ -2033,7 +2066,7 @@ static int mvpp2_aggr_txq_init(struct platform_device *pdev,
/* Aggr TXQ no reset WA */
aggr_txq->next_desc_to_proc = mvpp2_read(priv,
MVPP2_AGGR_TXQ_INDEX_REG(cpu));
MVPP2_AGGR_TXQ_INDEX_REG(thread));
/* Set Tx descriptors queue starting address indirect
* access
......@@ -2044,8 +2077,8 @@ static int mvpp2_aggr_txq_init(struct platform_device *pdev,
txq_dma = aggr_txq->descs_dma >>
MVPP22_AGGR_TXQ_DESC_ADDR_OFFS;
mvpp2_write(priv, MVPP2_AGGR_TXQ_DESC_ADDR_REG(cpu), txq_dma);
mvpp2_write(priv, MVPP2_AGGR_TXQ_DESC_SIZE_REG(cpu),
mvpp2_write(priv, MVPP2_AGGR_TXQ_DESC_ADDR_REG(thread), txq_dma);
mvpp2_write(priv, MVPP2_AGGR_TXQ_DESC_SIZE_REG(thread),
MVPP2_AGGR_TXQ_SIZE);
return 0;
......@@ -2056,8 +2089,8 @@ static int mvpp2_rxq_init(struct mvpp2_port *port,
struct mvpp2_rx_queue *rxq)
{
unsigned int thread;
u32 rxq_dma;
int cpu;
rxq->size = port->rx_ring_size;
......@@ -2074,15 +2107,15 @@ static int mvpp2_rxq_init(struct mvpp2_port *port,
mvpp2_write(port->priv, MVPP2_RXQ_STATUS_REG(rxq->id), 0);
/* Set Rx descriptors queue starting address - indirect access */
cpu = get_cpu();
mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_NUM_REG, rxq->id);
thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_NUM_REG, rxq->id);
if (port->priv->hw_version == MVPP21)
rxq_dma = rxq->descs_dma;
else
rxq_dma = rxq->descs_dma >> MVPP22_DESC_ADDR_OFFS;
mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_DESC_ADDR_REG, rxq_dma);
mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_DESC_SIZE_REG, rxq->size);
mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_INDEX_REG, 0);
mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_ADDR_REG, rxq_dma);
mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_SIZE_REG, rxq->size);
mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_INDEX_REG, 0);
put_cpu();
/* Set Offset */
......@@ -2127,7 +2160,7 @@ static void mvpp2_rxq_drop_pkts(struct mvpp2_port *port,
static void mvpp2_rxq_deinit(struct mvpp2_port *port,
struct mvpp2_rx_queue *rxq)
{
int cpu;
unsigned int thread;
mvpp2_rxq_drop_pkts(port, rxq);
......@@ -2146,10 +2179,10 @@ static void mvpp2_rxq_deinit(struct mvpp2_port *port,
* free descriptor number
*/
mvpp2_write(port->priv, MVPP2_RXQ_STATUS_REG(rxq->id), 0);
cpu = get_cpu();
mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_NUM_REG, rxq->id);
mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_DESC_ADDR_REG, 0);
mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_DESC_SIZE_REG, 0);
thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_NUM_REG, rxq->id);
mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_ADDR_REG, 0);
mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_SIZE_REG, 0);
put_cpu();
}
......@@ -2158,7 +2191,8 @@ static int mvpp2_txq_init(struct mvpp2_port *port,
struct mvpp2_tx_queue *txq)
{
u32 val;
int cpu, desc, desc_per_txq, tx_port_num;
unsigned int thread;
int desc, desc_per_txq, tx_port_num;
struct mvpp2_txq_pcpu *txq_pcpu;
txq->size = port->tx_ring_size;
......@@ -2173,18 +2207,18 @@ static int mvpp2_txq_init(struct mvpp2_port *port,
txq->last_desc = txq->size - 1;
/* Set Tx descriptors queue starting address - indirect access */
cpu = get_cpu();
mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_NUM_REG, txq->id);
mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_DESC_ADDR_REG,
thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id);
mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_ADDR_REG,
txq->descs_dma);
mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_DESC_SIZE_REG,
mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_SIZE_REG,
txq->size & MVPP2_TXQ_DESC_SIZE_MASK);
mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_INDEX_REG, 0);
mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_RSVD_CLR_REG,
mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_INDEX_REG, 0);
mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_RSVD_CLR_REG,
txq->id << MVPP2_TXQ_RSVD_CLR_OFFSET);
val = mvpp2_percpu_read(port->priv, cpu, MVPP2_TXQ_PENDING_REG);
val = mvpp2_thread_read(port->priv, thread, MVPP2_TXQ_PENDING_REG);
val &= ~MVPP2_TXQ_PENDING_MASK;
mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_PENDING_REG, val);
mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PENDING_REG, val);
/* Calculate base address in prefetch buffer. We reserve 16 descriptors
* for each existing TXQ.
......@@ -2195,7 +2229,7 @@ static int mvpp2_txq_init(struct mvpp2_port *port,
desc = (port->id * MVPP2_MAX_TXQ * desc_per_txq) +
(txq->log_id * desc_per_txq);
mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_PREF_BUF_REG,
mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG,
MVPP2_PREF_BUF_PTR(desc) | MVPP2_PREF_BUF_SIZE_16 |
MVPP2_PREF_BUF_THRESH(desc_per_txq / 2));
put_cpu();
......@@ -2214,8 +2248,8 @@ static int mvpp2_txq_init(struct mvpp2_port *port,
mvpp2_write(port->priv, MVPP2_TXQ_SCHED_TOKEN_SIZE_REG(txq->log_id),
val);
for_each_present_cpu(cpu) {
txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
for (thread = 0; thread < port->priv->nthreads; thread++) {
txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
txq_pcpu->size = txq->size;
txq_pcpu->buffs = kmalloc_array(txq_pcpu->size,
sizeof(*txq_pcpu->buffs),
......@@ -2249,10 +2283,10 @@ static void mvpp2_txq_deinit(struct mvpp2_port *port,
struct mvpp2_tx_queue *txq)
{
struct mvpp2_txq_pcpu *txq_pcpu;
int cpu;
unsigned int thread;
for_each_present_cpu(cpu) {
txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
for (thread = 0; thread < port->priv->nthreads; thread++) {
txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
kfree(txq_pcpu->buffs);
if (txq_pcpu->tso_headers)
......@@ -2278,10 +2312,10 @@ static void mvpp2_txq_deinit(struct mvpp2_port *port,
mvpp2_write(port->priv, MVPP2_TXQ_SCHED_TOKEN_CNTR_REG(txq->id), 0);
/* Set Tx descriptors queue starting address and size */
cpu = get_cpu();
mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_NUM_REG, txq->id);
mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_DESC_ADDR_REG, 0);
mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_DESC_SIZE_REG, 0);
thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id);
mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_ADDR_REG, 0);
mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_SIZE_REG, 0);
put_cpu();
}
......@@ -2289,14 +2323,14 @@ static void mvpp2_txq_deinit(struct mvpp2_port *port,
static void mvpp2_txq_clean(struct mvpp2_port *port, struct mvpp2_tx_queue *txq)
{
struct mvpp2_txq_pcpu *txq_pcpu;
int delay, pending, cpu;
int delay, pending;
unsigned int thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
u32 val;
cpu = get_cpu();
mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_NUM_REG, txq->id);
val = mvpp2_percpu_read(port->priv, cpu, MVPP2_TXQ_PREF_BUF_REG);
mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id);
val = mvpp2_thread_read(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG);
val |= MVPP2_TXQ_DRAIN_EN_MASK;
mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_PREF_BUF_REG, val);
mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG, val);
/* The napi queue has been stopped so wait for all packets
* to be transmitted.
......@@ -2312,17 +2346,17 @@ static void mvpp2_txq_clean(struct mvpp2_port *port, struct mvpp2_tx_queue *txq)
mdelay(1);
delay++;
pending = mvpp2_percpu_read(port->priv, cpu,
pending = mvpp2_thread_read(port->priv, thread,
MVPP2_TXQ_PENDING_REG);
pending &= MVPP2_TXQ_PENDING_MASK;
} while (pending);
val &= ~MVPP2_TXQ_DRAIN_EN_MASK;
mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_PREF_BUF_REG, val);
mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG, val);
put_cpu();
for_each_present_cpu(cpu) {
txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
for (thread = 0; thread < port->priv->nthreads; thread++) {
txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
/* Release all packets */
mvpp2_txq_bufs_free(port, txq, txq_pcpu, txq_pcpu->count);
......@@ -2503,16 +2537,20 @@ static void mvpp2_tx_proc_cb(unsigned long data)
{
struct net_device *dev = (struct net_device *)data;
struct mvpp2_port *port = netdev_priv(dev);
struct mvpp2_port_pcpu *port_pcpu = this_cpu_ptr(port->pcpu);
struct mvpp2_port_pcpu *port_pcpu;
unsigned int tx_todo, cause;
port_pcpu = per_cpu_ptr(port->pcpu,
mvpp2_cpu_to_thread(port->priv, smp_processor_id()));
if (!netif_running(dev))
return;
port_pcpu->timer_scheduled = false;
/* Process all the Tx queues */
cause = (1 << port->ntxqs) - 1;
tx_todo = mvpp2_tx_done(port, cause, smp_processor_id());
tx_todo = mvpp2_tx_done(port, cause,
mvpp2_cpu_to_thread(port->priv, smp_processor_id()));
/* Set the timer in case not all the packets were processed */
if (tx_todo)
......@@ -2728,7 +2766,8 @@ static inline void
tx_desc_unmap_put(struct mvpp2_port *port, struct mvpp2_tx_queue *txq,
struct mvpp2_tx_desc *desc)
{
struct mvpp2_txq_pcpu *txq_pcpu = this_cpu_ptr(txq->pcpu);
unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id());
struct mvpp2_txq_pcpu *txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
dma_addr_t buf_dma_addr =
mvpp2_txdesc_dma_addr_get(port, desc);
......@@ -2745,7 +2784,8 @@ static int mvpp2_tx_frag_process(struct mvpp2_port *port, struct sk_buff *skb,
struct mvpp2_tx_queue *aggr_txq,
struct mvpp2_tx_queue *txq)
{
struct mvpp2_txq_pcpu *txq_pcpu = this_cpu_ptr(txq->pcpu);
unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id());
struct mvpp2_txq_pcpu *txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
struct mvpp2_tx_desc *tx_desc;
int i;
dma_addr_t buf_dma_addr;
......@@ -2864,9 +2904,8 @@ static int mvpp2_tx_tso(struct sk_buff *skb, struct net_device *dev,
int i, len, descs = 0;
/* Check number of available descriptors */
if (mvpp2_aggr_desc_num_check(port->priv, aggr_txq,
tso_count_descs(skb)) ||
mvpp2_txq_reserved_desc_num_proc(port->priv, txq, txq_pcpu,
if (mvpp2_aggr_desc_num_check(port, aggr_txq, tso_count_descs(skb)) ||
mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu,
tso_count_descs(skb)))
return 0;
......@@ -2913,14 +2952,21 @@ static int mvpp2_tx(struct sk_buff *skb, struct net_device *dev)
struct mvpp2_txq_pcpu *txq_pcpu;
struct mvpp2_tx_desc *tx_desc;
dma_addr_t buf_dma_addr;
unsigned long flags = 0;
unsigned int thread;
int frags = 0;
u16 txq_id;
u32 tx_cmd;
thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id());
txq_id = skb_get_queue_mapping(skb);
txq = port->txqs[txq_id];
txq_pcpu = this_cpu_ptr(txq->pcpu);
aggr_txq = &port->priv->aggr_txqs[smp_processor_id()];
txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
aggr_txq = &port->priv->aggr_txqs[thread];
if (test_bit(thread, &port->priv->lock_map))
spin_lock_irqsave(&port->tx_lock[thread], flags);
if (skb_is_gso(skb)) {
frags = mvpp2_tx_tso(skb, dev, txq, aggr_txq, txq_pcpu);
......@@ -2929,9 +2975,8 @@ static int mvpp2_tx(struct sk_buff *skb, struct net_device *dev)
frags = skb_shinfo(skb)->nr_frags + 1;
/* Check number of available descriptors */
if (mvpp2_aggr_desc_num_check(port->priv, aggr_txq, frags) ||
mvpp2_txq_reserved_desc_num_proc(port->priv, txq,
txq_pcpu, frags)) {
if (mvpp2_aggr_desc_num_check(port, aggr_txq, frags) ||
mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu, frags)) {
frags = 0;
goto out;
}
......@@ -2973,7 +3018,7 @@ static int mvpp2_tx(struct sk_buff *skb, struct net_device *dev)
out:
if (frags > 0) {
struct mvpp2_pcpu_stats *stats = this_cpu_ptr(port->stats);
struct mvpp2_pcpu_stats *stats = per_cpu_ptr(port->stats, thread);
struct netdev_queue *nq = netdev_get_tx_queue(dev, txq_id);
txq_pcpu->reserved_num -= frags;
......@@ -3003,11 +3048,14 @@ static int mvpp2_tx(struct sk_buff *skb, struct net_device *dev)
/* Set the timer in case not all frags were processed */
if (!port->has_tx_irqs && txq_pcpu->count <= frags &&
txq_pcpu->count > 0) {
struct mvpp2_port_pcpu *port_pcpu = this_cpu_ptr(port->pcpu);
struct mvpp2_port_pcpu *port_pcpu = per_cpu_ptr(port->pcpu, thread);
mvpp2_timer_set(port_pcpu);
}
if (test_bit(thread, &port->priv->lock_map))
spin_unlock_irqrestore(&port->tx_lock[thread], flags);
return NETDEV_TX_OK;
}
......@@ -3027,7 +3075,7 @@ static int mvpp2_poll(struct napi_struct *napi, int budget)
int rx_done = 0;
struct mvpp2_port *port = netdev_priv(napi->dev);
struct mvpp2_queue_vector *qv;
int cpu = smp_processor_id();
unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id());
qv = container_of(napi, struct mvpp2_queue_vector, napi);
......@@ -3041,7 +3089,7 @@ static int mvpp2_poll(struct napi_struct *napi, int budget)
*
* Each CPU has its own Rx/Tx cause register
*/
cause_rx_tx = mvpp2_percpu_read_relaxed(port->priv, qv->sw_thread_id,
cause_rx_tx = mvpp2_thread_read_relaxed(port->priv, qv->sw_thread_id,
MVPP2_ISR_RX_TX_CAUSE_REG(port->id));
cause_misc = cause_rx_tx & MVPP2_CAUSE_MISC_SUM_MASK;
......@@ -3050,7 +3098,7 @@ static int mvpp2_poll(struct napi_struct *napi, int budget)
/* Clear the cause register */
mvpp2_write(port->priv, MVPP2_ISR_MISC_CAUSE_REG, 0);
mvpp2_percpu_write(port->priv, cpu,
mvpp2_thread_write(port->priv, thread,
MVPP2_ISR_RX_TX_CAUSE_REG(port->id),
cause_rx_tx & ~MVPP2_CAUSE_MISC_SUM_MASK);
}
......@@ -3062,7 +3110,8 @@ static int mvpp2_poll(struct napi_struct *napi, int budget)
}
/* Process RX packets */
cause_rx = cause_rx_tx & MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK;
cause_rx = cause_rx_tx &
MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK(port->priv->hw_version);
cause_rx <<= qv->first_rxq;
cause_rx |= qv->pending_cause_rx;
while (cause_rx && budget > 0) {
......@@ -3137,7 +3186,7 @@ static void mvpp2_start_dev(struct mvpp2_port *port)
for (i = 0; i < port->nqvecs; i++)
napi_enable(&port->qvecs[i].napi);
/* Enable interrupts on all CPUs */
/* Enable interrupts on all threads */
mvpp2_interrupts_enable(port);
if (port->priv->hw_version == MVPP22)
......@@ -3167,7 +3216,7 @@ static void mvpp2_stop_dev(struct mvpp2_port *port)
{
int i;
/* Disable interrupts on all CPUs */
/* Disable interrupts on all threads */
mvpp2_interrupts_disable(port);
for (i = 0; i < port->nqvecs; i++)
......@@ -3247,9 +3296,18 @@ static int mvpp2_irqs_init(struct mvpp2_port *port)
if (err)
goto err;
if (qv->type == MVPP2_QUEUE_VECTOR_PRIVATE)
irq_set_affinity_hint(qv->irq,
cpumask_of(qv->sw_thread_id));
if (qv->type == MVPP2_QUEUE_VECTOR_PRIVATE) {
unsigned long mask = 0;
unsigned int cpu;
for_each_present_cpu(cpu) {
if (mvpp2_cpu_to_thread(port->priv, cpu) ==
qv->sw_thread_id)
mask |= BIT(cpu);
}
irq_set_affinity_hint(qv->irq, to_cpumask(&mask));
}
}
return 0;
......@@ -3393,11 +3451,11 @@ static int mvpp2_stop(struct net_device *dev)
{
struct mvpp2_port *port = netdev_priv(dev);
struct mvpp2_port_pcpu *port_pcpu;
int cpu;
unsigned int thread;
mvpp2_stop_dev(port);
/* Mask interrupts on all CPUs */
/* Mask interrupts on all threads */
on_each_cpu(mvpp2_interrupts_mask, port, 1);
mvpp2_shared_interrupt_mask_unmask(port, true);
......@@ -3408,8 +3466,8 @@ static int mvpp2_stop(struct net_device *dev)
mvpp2_irqs_deinit(port);
if (!port->has_tx_irqs) {
for_each_present_cpu(cpu) {
port_pcpu = per_cpu_ptr(port->pcpu, cpu);
for (thread = 0; thread < port->priv->nthreads; thread++) {
port_pcpu = per_cpu_ptr(port->pcpu, thread);
hrtimer_cancel(&port_pcpu->tx_done_timer);
port_pcpu->timer_scheduled = false;
......@@ -3554,7 +3612,7 @@ mvpp2_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
{
struct mvpp2_port *port = netdev_priv(dev);
unsigned int start;
int cpu;
unsigned int cpu;
for_each_possible_cpu(cpu) {
struct mvpp2_pcpu_stats *cpu_stats;
......@@ -3981,12 +4039,18 @@ static int mvpp2_simple_queue_vectors_init(struct mvpp2_port *port,
static int mvpp2_multi_queue_vectors_init(struct mvpp2_port *port,
struct device_node *port_node)
{
struct mvpp2 *priv = port->priv;
struct mvpp2_queue_vector *v;
int i, ret;
port->nqvecs = num_possible_cpus();
if (queue_mode == MVPP2_QDIST_SINGLE_MODE)
port->nqvecs += 1;
switch (queue_mode) {
case MVPP2_QDIST_SINGLE_MODE:
port->nqvecs = priv->nthreads + 1;
break;
case MVPP2_QDIST_MULTI_MODE:
port->nqvecs = priv->nthreads;
break;
}
for (i = 0; i < port->nqvecs; i++) {
char irqname[16];
......@@ -3998,7 +4062,10 @@ static int mvpp2_multi_queue_vectors_init(struct mvpp2_port *port,
v->sw_thread_id = i;
v->sw_thread_mask = BIT(i);
snprintf(irqname, sizeof(irqname), "tx-cpu%d", i);
if (port->flags & MVPP2_F_DT_COMPAT)
snprintf(irqname, sizeof(irqname), "tx-cpu%d", i);
else
snprintf(irqname, sizeof(irqname), "hif%d", i);
if (queue_mode == MVPP2_QDIST_MULTI_MODE) {
v->first_rxq = i * MVPP2_DEFAULT_RXQ;
......@@ -4008,7 +4075,9 @@ static int mvpp2_multi_queue_vectors_init(struct mvpp2_port *port,
v->first_rxq = 0;
v->nrxqs = port->nrxqs;
v->type = MVPP2_QUEUE_VECTOR_SHARED;
strncpy(irqname, "rx-shared", sizeof(irqname));
if (port->flags & MVPP2_F_DT_COMPAT)
strncpy(irqname, "rx-shared", sizeof(irqname));
}
if (port_node)
......@@ -4085,7 +4154,8 @@ static int mvpp2_port_init(struct mvpp2_port *port)
struct device *dev = port->dev->dev.parent;
struct mvpp2 *priv = port->priv;
struct mvpp2_txq_pcpu *txq_pcpu;
int queue, cpu, err;
unsigned int thread;
int queue, err;
/* Checks for hardware constraints */
if (port->first_rxq + port->nrxqs >
......@@ -4129,9 +4199,9 @@ static int mvpp2_port_init(struct mvpp2_port *port)
txq->id = queue_phy_id;
txq->log_id = queue;
txq->done_pkts_coal = MVPP2_TXDONE_COAL_PKTS_THRESH;
for_each_present_cpu(cpu) {
txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
txq_pcpu->cpu = cpu;
for (thread = 0; thread < priv->nthreads; thread++) {
txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
txq_pcpu->thread = thread;
}
port->txqs[queue] = txq;
......@@ -4204,24 +4274,51 @@ static int mvpp2_port_init(struct mvpp2_port *port)
return err;
}
/* Checks if the port DT description has the TX interrupts
* described. On PPv2.1, there are no such interrupts. On PPv2.2,
* there are available, but we need to keep support for old DTs.
static bool mvpp22_port_has_legacy_tx_irqs(struct device_node *port_node,
unsigned long *flags)
{
char *irqs[5] = { "rx-shared", "tx-cpu0", "tx-cpu1", "tx-cpu2",
"tx-cpu3" };
int i;
for (i = 0; i < 5; i++)
if (of_property_match_string(port_node, "interrupt-names",
irqs[i]) < 0)
return false;
*flags |= MVPP2_F_DT_COMPAT;
return true;
}
/* Checks if the port dt description has the required Tx interrupts:
* - PPv2.1: there are no such interrupts.
* - PPv2.2:
* - The old DTs have: "rx-shared", "tx-cpuX" with X in [0...3]
* - The new ones have: "hifX" with X in [0..8]
*
* All those variants are supported to keep the backward compatibility.
*/
static bool mvpp2_port_has_tx_irqs(struct mvpp2 *priv,
struct device_node *port_node)
static bool mvpp2_port_has_irqs(struct mvpp2 *priv,
struct device_node *port_node,
unsigned long *flags)
{
char *irqs[5] = { "rx-shared", "tx-cpu0", "tx-cpu1",
"tx-cpu2", "tx-cpu3" };
int ret, i;
char name[5];
int i;
/* ACPI */
if (!port_node)
return true;
if (priv->hw_version == MVPP21)
return false;
for (i = 0; i < 5; i++) {
ret = of_property_match_string(port_node, "interrupt-names",
irqs[i]);
if (ret < 0)
if (mvpp22_port_has_legacy_tx_irqs(port_node, flags))
return true;
for (i = 0; i < MVPP2_MAX_THREADS; i++) {
snprintf(name, 5, "hif%d", i);
if (of_property_match_string(port_node, "interrupt-names",
name) < 0)
return false;
}
......@@ -4598,23 +4695,21 @@ static int mvpp2_port_probe(struct platform_device *pdev,
struct resource *res;
struct phylink *phylink;
char *mac_from = "";
unsigned int ntxqs, nrxqs;
unsigned int ntxqs, nrxqs, thread;
unsigned long flags = 0;
bool has_tx_irqs;
u32 id;
int features;
int phy_mode;
int err, i, cpu;
int err, i;
if (port_node) {
has_tx_irqs = mvpp2_port_has_tx_irqs(priv, port_node);
} else {
has_tx_irqs = true;
queue_mode = MVPP2_QDIST_MULTI_MODE;
has_tx_irqs = mvpp2_port_has_irqs(priv, port_node, &flags);
if (!has_tx_irqs && queue_mode == MVPP2_QDIST_MULTI_MODE) {
dev_err(&pdev->dev,
"not enough IRQs to support multi queue mode\n");
return -EINVAL;
}
if (!has_tx_irqs)
queue_mode = MVPP2_QDIST_SINGLE_MODE;
ntxqs = MVPP2_MAX_TXQ;
if (priv->hw_version == MVPP22 && queue_mode == MVPP2_QDIST_MULTI_MODE)
nrxqs = MVPP2_DEFAULT_RXQ * num_possible_cpus();
......@@ -4662,6 +4757,7 @@ static int mvpp2_port_probe(struct platform_device *pdev,
port->nrxqs = nrxqs;
port->priv = priv;
port->has_tx_irqs = has_tx_irqs;
port->flags = flags;
err = mvpp2_queue_vectors_init(port, port_node);
if (err)
......@@ -4758,8 +4854,8 @@ static int mvpp2_port_probe(struct platform_device *pdev,
}
if (!port->has_tx_irqs) {
for_each_present_cpu(cpu) {
port_pcpu = per_cpu_ptr(port->pcpu, cpu);
for (thread = 0; thread < priv->nthreads; thread++) {
port_pcpu = per_cpu_ptr(port->pcpu, thread);
hrtimer_init(&port_pcpu->tx_done_timer, CLOCK_MONOTONIC,
HRTIMER_MODE_REL_PINNED);
......@@ -5043,13 +5139,13 @@ static int mvpp2_init(struct platform_device *pdev, struct mvpp2 *priv)
}
/* Allocate and initialize aggregated TXQs */
priv->aggr_txqs = devm_kcalloc(&pdev->dev, num_present_cpus(),
priv->aggr_txqs = devm_kcalloc(&pdev->dev, MVPP2_MAX_THREADS,
sizeof(*priv->aggr_txqs),
GFP_KERNEL);
if (!priv->aggr_txqs)
return -ENOMEM;
for_each_present_cpu(i) {
for (i = 0; i < MVPP2_MAX_THREADS; i++) {
priv->aggr_txqs[i].id = i;
priv->aggr_txqs[i].size = MVPP2_AGGR_TXQ_SIZE;
err = mvpp2_aggr_txq_init(pdev, &priv->aggr_txqs[i], i, priv);
......@@ -5096,7 +5192,7 @@ static int mvpp2_probe(struct platform_device *pdev)
struct mvpp2 *priv;
struct resource *res;
void __iomem *base;
int i;
int i, shared;
int err;
priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
......@@ -5161,6 +5257,15 @@ static int mvpp2_probe(struct platform_device *pdev)
mvpp2_setup_bm_pool();
priv->nthreads = min_t(unsigned int, num_present_cpus(),
MVPP2_MAX_THREADS);
shared = num_present_cpus() - priv->nthreads;
if (shared > 0)
bitmap_fill(&priv->lock_map,
min_t(int, shared, MVPP2_MAX_THREADS));
for (i = 0; i < MVPP2_MAX_THREADS; i++) {
u32 addr_space_sz;
......@@ -5335,7 +5440,7 @@ static int mvpp2_remove(struct platform_device *pdev)
mvpp2_bm_pool_destroy(pdev, priv, bm_pool);
}
for_each_present_cpu(i) {
for (i = 0; i < MVPP2_MAX_THREADS; i++) {
struct mvpp2_tx_queue *aggr_txq = &priv->aggr_txqs[i];
dma_free_coherent(&pdev->dev,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7