Commit 63f545ed authored by Brett Creeley's avatar Brett Creeley Committed by Jeff Kirsher

ice: Add support for adaptive interrupt moderation

Currently the driver does not support adaptive/dynamic interrupt
moderation. This patch adds support for this. Also, adaptive/dynamic
interrupt moderation is turned on by default upon driver load.

In order to support adaptive interrupt moderation, two functions were
added, ice_update_itr() and ice_itr_divisor(). These are used to
determine the current packet load and to determine a divisor based
on link speed respectively.

This patch also adds the ICE_ITR_GRAN_S define that is used in the
hot-path when setting a new ITR value. The shift is used to pet two
birds with one hand, set the ITR value while re-enabling the
interrupt. Also, the ICE_ITR_GRAN_S is defined as 1 because the device
has a ITR granularity of 2usecs.
Signed-off-by: default avatarBrett Creeley <brett.creeley@intel.com>
Signed-off-by: default avatarAnirudh Venkataramanan <anirudh.venkataramanan@intel.com>
Tested-by: default avatarAndrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: default avatarJeff Kirsher <jeffrey.t.kirsher@intel.com>
parent 9be1d6f8
...@@ -110,6 +110,7 @@ ...@@ -110,6 +110,7 @@
#define GLINT_DYN_CTL_CLEARPBA_M BIT(1) #define GLINT_DYN_CTL_CLEARPBA_M BIT(1)
#define GLINT_DYN_CTL_SWINT_TRIG_M BIT(2) #define GLINT_DYN_CTL_SWINT_TRIG_M BIT(2)
#define GLINT_DYN_CTL_ITR_INDX_S 3 #define GLINT_DYN_CTL_ITR_INDX_S 3
#define GLINT_DYN_CTL_INTERVAL_S 5
#define GLINT_DYN_CTL_SW_ITR_INDX_M ICE_M(0x3, 25) #define GLINT_DYN_CTL_SW_ITR_INDX_M ICE_M(0x3, 25)
#define GLINT_DYN_CTL_INTENA_MSK_M BIT(31) #define GLINT_DYN_CTL_INTENA_MSK_M BIT(31)
#define GLINT_ITR(_i, _INT) (0x00154000 + ((_i) * 8192 + (_INT) * 4)) #define GLINT_ITR(_i, _INT) (0x00154000 + ((_i) * 8192 + (_INT) * 4))
......
...@@ -1717,22 +1717,34 @@ static u32 ice_intrl_usec_to_reg(u8 intrl, u8 gran) ...@@ -1717,22 +1717,34 @@ static u32 ice_intrl_usec_to_reg(u8 intrl, u8 gran)
static void static void
ice_cfg_itr(struct ice_hw *hw, struct ice_q_vector *q_vector, u16 vector) ice_cfg_itr(struct ice_hw *hw, struct ice_q_vector *q_vector, u16 vector)
{ {
u8 itr_gran = hw->itr_gran;
if (q_vector->num_ring_rx) { if (q_vector->num_ring_rx) {
struct ice_ring_container *rc = &q_vector->rx; struct ice_ring_container *rc = &q_vector->rx;
rc->itr = ITR_TO_REG(ICE_DFLT_RX_ITR, itr_gran); /* if this value is set then don't overwrite with default */
if (!rc->itr_setting)
rc->itr_setting = ICE_DFLT_RX_ITR;
rc->target_itr = ITR_TO_REG(rc->itr_setting);
rc->next_update = jiffies + 1;
rc->current_itr = rc->target_itr;
rc->latency_range = ICE_LOW_LATENCY; rc->latency_range = ICE_LOW_LATENCY;
wr32(hw, GLINT_ITR(rc->itr_idx, vector), rc->itr); wr32(hw, GLINT_ITR(rc->itr_idx, vector),
ITR_REG_ALIGN(rc->current_itr) >> ICE_ITR_GRAN_S);
} }
if (q_vector->num_ring_tx) { if (q_vector->num_ring_tx) {
struct ice_ring_container *rc = &q_vector->tx; struct ice_ring_container *rc = &q_vector->tx;
rc->itr = ITR_TO_REG(ICE_DFLT_TX_ITR, itr_gran); /* if this value is set then don't overwrite with default */
if (!rc->itr_setting)
rc->itr_setting = ICE_DFLT_TX_ITR;
rc->target_itr = ITR_TO_REG(rc->itr_setting);
rc->next_update = jiffies + 1;
rc->current_itr = rc->target_itr;
rc->latency_range = ICE_LOW_LATENCY; rc->latency_range = ICE_LOW_LATENCY;
wr32(hw, GLINT_ITR(rc->itr_idx, vector), rc->itr); wr32(hw, GLINT_ITR(rc->itr_idx, vector),
ITR_REG_ALIGN(rc->current_itr) >> ICE_ITR_GRAN_S);
} }
} }
......
...@@ -1389,7 +1389,6 @@ static int ice_req_irq_msix_misc(struct ice_pf *pf) ...@@ -1389,7 +1389,6 @@ static int ice_req_irq_msix_misc(struct ice_pf *pf)
{ {
struct ice_hw *hw = &pf->hw; struct ice_hw *hw = &pf->hw;
int oicr_idx, err = 0; int oicr_idx, err = 0;
u8 itr_gran;
u32 val; u32 val;
if (!pf->int_name[0]) if (!pf->int_name[0])
...@@ -1453,10 +1452,8 @@ static int ice_req_irq_msix_misc(struct ice_pf *pf) ...@@ -1453,10 +1452,8 @@ static int ice_req_irq_msix_misc(struct ice_pf *pf)
PFINT_MBX_CTL_CAUSE_ENA_M); PFINT_MBX_CTL_CAUSE_ENA_M);
wr32(hw, PFINT_MBX_CTL, val); wr32(hw, PFINT_MBX_CTL, val);
itr_gran = hw->itr_gran;
wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->hw_oicr_idx), wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->hw_oicr_idx),
ITR_TO_REG(ICE_ITR_8K, itr_gran)); ITR_REG_ALIGN(ICE_ITR_8K) >> ICE_ITR_GRAN_S);
ice_flush(hw); ice_flush(hw);
ice_irq_dynamic_ena(hw, NULL, NULL); ice_irq_dynamic_ena(hw, NULL, NULL);
...@@ -1997,6 +1994,23 @@ static int ice_init_interrupt_scheme(struct ice_pf *pf) ...@@ -1997,6 +1994,23 @@ static int ice_init_interrupt_scheme(struct ice_pf *pf)
return 0; return 0;
} }
/**
* ice_verify_itr_gran - verify driver's assumption of ITR granularity
* @pf: pointer to the PF structure
*
* There is no error returned here because the driver will be able to handle a
* different ITR granularity, but interrupt moderation will not be accurate if
* the driver's assumptions are not verified. This assumption is made so we can
* use constants in the hot path instead of accessing structure members.
*/
static void ice_verify_itr_gran(struct ice_pf *pf)
{
if (pf->hw.itr_gran != (ICE_ITR_GRAN_S << 1))
dev_warn(&pf->pdev->dev,
"%d ITR granularity assumption is invalid, actual ITR granularity is %d. Interrupt moderation will be inaccurate!\n",
(ICE_ITR_GRAN_S << 1), pf->hw.itr_gran);
}
/** /**
* ice_verify_cacheline_size - verify driver's assumption of 64 Byte cache lines * ice_verify_cacheline_size - verify driver's assumption of 64 Byte cache lines
* @pf: pointer to the PF structure * @pf: pointer to the PF structure
...@@ -2163,6 +2177,7 @@ static int ice_probe(struct pci_dev *pdev, ...@@ -2163,6 +2177,7 @@ static int ice_probe(struct pci_dev *pdev,
mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period));
ice_verify_cacheline_size(pf); ice_verify_cacheline_size(pf);
ice_verify_itr_gran(pf);
return 0; return 0;
......
...@@ -1052,6 +1052,69 @@ static int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) ...@@ -1052,6 +1052,69 @@ static int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget)
return failure ? budget : (int)total_rx_pkts; return failure ? budget : (int)total_rx_pkts;
} }
/**
* ice_buildreg_itr - build value for writing to the GLINT_DYN_CTL register
* @itr_idx: interrupt throttling index
* @reg_itr: interrupt throttling value adjusted based on ITR granularity
*/
static u32 ice_buildreg_itr(int itr_idx, u16 reg_itr)
{
return GLINT_DYN_CTL_INTENA_M | GLINT_DYN_CTL_CLEARPBA_M |
(itr_idx << GLINT_DYN_CTL_ITR_INDX_S) |
(reg_itr << GLINT_DYN_CTL_INTERVAL_S);
}
/**
* ice_update_ena_itr - Update ITR and re-enable MSIX interrupt
* @vsi: the VSI associated with the q_vector
* @q_vector: q_vector for which ITR is being updated and interrupt enabled
*/
static void
ice_update_ena_itr(struct ice_vsi *vsi, struct ice_q_vector *q_vector)
{
struct ice_hw *hw = &vsi->back->hw;
struct ice_ring_container *rc;
u32 itr_val;
/* This block of logic allows us to get away with only updating
* one ITR value with each interrupt. The idea is to perform a
* pseudo-lazy update with the following criteria.
*
* 1. Rx is given higher priority than Tx if both are in same state
* 2. If we must reduce an ITR that is given highest priority.
* 3. We then give priority to increasing ITR based on amount.
*/
if (q_vector->rx.target_itr < q_vector->rx.current_itr) {
rc = &q_vector->rx;
/* Rx ITR needs to be reduced, this is highest priority */
itr_val = ice_buildreg_itr(rc->itr_idx, rc->target_itr);
rc->current_itr = rc->target_itr;
} else if ((q_vector->tx.target_itr < q_vector->tx.current_itr) ||
((q_vector->rx.target_itr - q_vector->rx.current_itr) <
(q_vector->tx.target_itr - q_vector->tx.current_itr))) {
rc = &q_vector->tx;
/* Tx ITR needs to be reduced, this is second priority
* Tx ITR needs to be increased more than Rx, fourth priority
*/
itr_val = ice_buildreg_itr(rc->itr_idx, rc->target_itr);
rc->current_itr = rc->target_itr;
} else if (q_vector->rx.current_itr != q_vector->rx.target_itr) {
rc = &q_vector->rx;
/* Rx ITR needs to be increased, third priority */
itr_val = ice_buildreg_itr(rc->itr_idx, rc->target_itr);
rc->current_itr = rc->target_itr;
} else {
/* Still have to re-enable the interrupts */
itr_val = ice_buildreg_itr(ICE_ITR_NONE, 0);
}
if (!test_bit(__ICE_DOWN, vsi->state)) {
int vector = vsi->hw_base_vector + q_vector->v_idx;
wr32(hw, GLINT_DYN_CTL(vector), itr_val);
}
}
/** /**
* ice_napi_poll - NAPI polling Rx/Tx cleanup routine * ice_napi_poll - NAPI polling Rx/Tx cleanup routine
* @napi: napi struct with our devices info in it * @napi: napi struct with our devices info in it
...@@ -1108,7 +1171,7 @@ int ice_napi_poll(struct napi_struct *napi, int budget) ...@@ -1108,7 +1171,7 @@ int ice_napi_poll(struct napi_struct *napi, int budget)
*/ */
if (likely(napi_complete_done(napi, work_done))) if (likely(napi_complete_done(napi, work_done)))
if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
ice_irq_dynamic_ena(&vsi->back->hw, vsi, q_vector); ice_update_ena_itr(vsi, q_vector);
return min(work_done, budget - 1); return min(work_done, budget - 1);
} }
......
...@@ -116,16 +116,15 @@ enum ice_rx_dtype { ...@@ -116,16 +116,15 @@ enum ice_rx_dtype {
/* indices into GLINT_ITR registers */ /* indices into GLINT_ITR registers */
#define ICE_RX_ITR ICE_IDX_ITR0 #define ICE_RX_ITR ICE_IDX_ITR0
#define ICE_TX_ITR ICE_IDX_ITR1 #define ICE_TX_ITR ICE_IDX_ITR1
#define ICE_ITR_DYNAMIC 0x8000 /* use top bit as a flag */ #define ICE_ITR_8K 124
#define ICE_ITR_8K 125
#define ICE_ITR_20K 50 #define ICE_ITR_20K 50
#define ICE_DFLT_TX_ITR ICE_ITR_20K #define ICE_DFLT_TX_ITR (ICE_ITR_20K | ICE_ITR_DYNAMIC)
#define ICE_DFLT_RX_ITR ICE_ITR_20K #define ICE_DFLT_RX_ITR (ICE_ITR_20K | ICE_ITR_DYNAMIC)
/* apply ITR granularity translation to program the register. itr_gran is either #define ICE_ITR_DYNAMIC 0x8000 /* used as flag for itr_setting */
* 2 or 4 usecs so we need to divide by 2 first then shift by that value #define ITR_TO_REG(setting) ((setting) & ~ICE_ITR_DYNAMIC)
*/ #define ICE_ITR_GRAN_S 1 /* Assume ITR granularity is 2us */
#define ITR_TO_REG(val, itr_gran) (((val) & ~ICE_ITR_DYNAMIC) >> \ #define ICE_ITR_MASK 0x1FFE /* ITR register value alignment mask */
((itr_gran) / 2)) #define ITR_REG_ALIGN(setting) __ALIGN_MASK(setting, ~ICE_ITR_MASK)
#define ICE_DFLT_INTRL 0 #define ICE_DFLT_INTRL 0
...@@ -180,13 +179,20 @@ enum ice_latency_range { ...@@ -180,13 +179,20 @@ enum ice_latency_range {
}; };
struct ice_ring_container { struct ice_ring_container {
/* array of pointers to rings */ /* head of linked-list of rings */
struct ice_ring *ring; struct ice_ring *ring;
unsigned long next_update; /* jiffies value of next queue update */
unsigned int total_bytes; /* total bytes processed this int */ unsigned int total_bytes; /* total bytes processed this int */
unsigned int total_pkts; /* total packets processed this int */ unsigned int total_pkts; /* total packets processed this int */
enum ice_latency_range latency_range; enum ice_latency_range latency_range;
int itr_idx; /* index in the interrupt vector */ int itr_idx; /* index in the interrupt vector */
u16 itr; u16 target_itr; /* value in usecs divided by the hw->itr_gran */
u16 current_itr; /* value in usecs divided by the hw->itr_gran */
/* high bit set means dynamic ITR, rest is used to store user
* readable ITR value in usecs and must be converted before programming
* to a register.
*/
u16 itr_setting;
}; };
/* iterator for handling rings in ring container */ /* iterator for handling rings in ring container */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment