Commit ed4cb414 authored by Eric Anholt's avatar Eric Anholt Committed by Dave Airlie

i915: Add support for MSI and interrupt mitigation.

Previous attempts at interrupt mitigation had been foiled by i915_wait_irq's
failure to update the sarea seqno value when the status page indicated that
the seqno had already been passed.  MSI support has been seen to cut CPU
costs by up to 40% in some workloads by avoiding other expensive interrupt
handlers for frequent graphics interrupts.
Signed-off-by: default avatarEric Anholt <eric@anholt.net>
Signed-off-by: default avatarDave Airlie <airlied@redhat.com>
parent 585fb111
...@@ -63,7 +63,7 @@ int drm_irq_by_busid(struct drm_device *dev, void *data, ...@@ -63,7 +63,7 @@ int drm_irq_by_busid(struct drm_device *dev, void *data,
p->devnum != PCI_SLOT(dev->pdev->devfn) || p->funcnum != PCI_FUNC(dev->pdev->devfn)) p->devnum != PCI_SLOT(dev->pdev->devfn) || p->funcnum != PCI_FUNC(dev->pdev->devfn))
return -EINVAL; return -EINVAL;
p->irq = dev->irq; p->irq = dev->pdev->irq;
DRM_DEBUG("%d:%d:%d => IRQ %d\n", p->busnum, p->devnum, p->funcnum, DRM_DEBUG("%d:%d:%d => IRQ %d\n", p->busnum, p->devnum, p->funcnum,
p->irq); p->irq);
...@@ -89,7 +89,7 @@ static int drm_irq_install(struct drm_device * dev) ...@@ -89,7 +89,7 @@ static int drm_irq_install(struct drm_device * dev)
if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ)) if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
return -EINVAL; return -EINVAL;
if (dev->irq == 0) if (dev->pdev->irq == 0)
return -EINVAL; return -EINVAL;
mutex_lock(&dev->struct_mutex); mutex_lock(&dev->struct_mutex);
...@@ -107,7 +107,7 @@ static int drm_irq_install(struct drm_device * dev) ...@@ -107,7 +107,7 @@ static int drm_irq_install(struct drm_device * dev)
dev->irq_enabled = 1; dev->irq_enabled = 1;
mutex_unlock(&dev->struct_mutex); mutex_unlock(&dev->struct_mutex);
DRM_DEBUG("irq=%d\n", dev->irq); DRM_DEBUG("irq=%d\n", dev->pdev->irq);
if (drm_core_check_feature(dev, DRIVER_IRQ_VBL)) { if (drm_core_check_feature(dev, DRIVER_IRQ_VBL)) {
init_waitqueue_head(&dev->vbl_queue); init_waitqueue_head(&dev->vbl_queue);
...@@ -127,8 +127,12 @@ static int drm_irq_install(struct drm_device * dev) ...@@ -127,8 +127,12 @@ static int drm_irq_install(struct drm_device * dev)
if (drm_core_check_feature(dev, DRIVER_IRQ_SHARED)) if (drm_core_check_feature(dev, DRIVER_IRQ_SHARED))
sh_flags = IRQF_SHARED; sh_flags = IRQF_SHARED;
ret = request_irq(dev->irq, dev->driver->irq_handler, ret = request_irq(dev->pdev->irq, dev->driver->irq_handler,
sh_flags, dev->devname, dev); sh_flags, dev->devname, dev);
/* Expose the device irq number to drivers that want to export it for
* whatever reason.
*/
dev->irq = dev->pdev->irq;
if (ret < 0) { if (ret < 0) {
mutex_lock(&dev->struct_mutex); mutex_lock(&dev->struct_mutex);
dev->irq_enabled = 0; dev->irq_enabled = 0;
...@@ -164,11 +168,11 @@ int drm_irq_uninstall(struct drm_device * dev) ...@@ -164,11 +168,11 @@ int drm_irq_uninstall(struct drm_device * dev)
if (!irq_enabled) if (!irq_enabled)
return -EINVAL; return -EINVAL;
DRM_DEBUG("irq=%d\n", dev->irq); DRM_DEBUG("irq=%d\n", dev->pdev->irq);
dev->driver->irq_uninstall(dev); dev->driver->irq_uninstall(dev);
free_irq(dev->irq, dev); free_irq(dev->pdev->irq, dev);
dev->locked_tasklet_func = NULL; dev->locked_tasklet_func = NULL;
...@@ -201,7 +205,7 @@ int drm_control(struct drm_device *dev, void *data, ...@@ -201,7 +205,7 @@ int drm_control(struct drm_device *dev, void *data,
if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ)) if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
return 0; return 0;
if (dev->if_version < DRM_IF_VERSION(1, 2) && if (dev->if_version < DRM_IF_VERSION(1, 2) &&
ctl->irq != dev->irq) ctl->irq != dev->pdev->irq)
return -EINVAL; return -EINVAL;
return drm_irq_install(dev); return drm_irq_install(dev);
case DRM_UNINST_HANDLER: case DRM_UNINST_HANDLER:
...@@ -239,7 +243,7 @@ int drm_wait_vblank(struct drm_device *dev, void *data, struct drm_file *file_pr ...@@ -239,7 +243,7 @@ int drm_wait_vblank(struct drm_device *dev, void *data, struct drm_file *file_pr
int ret = 0; int ret = 0;
unsigned int flags, seq; unsigned int flags, seq;
if ((!dev->irq) || (!dev->irq_enabled)) if ((!dev->pdev->irq) || (!dev->irq_enabled))
return -EINVAL; return -EINVAL;
if (vblwait->request.type & if (vblwait->request.type &
......
...@@ -84,7 +84,7 @@ static int i915_dma_cleanup(struct drm_device * dev) ...@@ -84,7 +84,7 @@ static int i915_dma_cleanup(struct drm_device * dev)
* may not have been called from userspace and after dev_private * may not have been called from userspace and after dev_private
* is freed, it's too late. * is freed, it's too late.
*/ */
if (dev->irq) if (dev->irq_enabled)
drm_irq_uninstall(dev); drm_irq_uninstall(dev);
if (dev_priv->ring.virtual_start) { if (dev_priv->ring.virtual_start) {
...@@ -644,7 +644,7 @@ static int i915_getparam(struct drm_device *dev, void *data, ...@@ -644,7 +644,7 @@ static int i915_getparam(struct drm_device *dev, void *data,
switch (param->param) { switch (param->param) {
case I915_PARAM_IRQ_ACTIVE: case I915_PARAM_IRQ_ACTIVE:
value = dev->irq ? 1 : 0; value = dev->irq_enabled;
break; break;
case I915_PARAM_ALLOW_BATCHBUFFER: case I915_PARAM_ALLOW_BATCHBUFFER:
value = dev_priv->allow_batchbuffer ? 1 : 0; value = dev_priv->allow_batchbuffer ? 1 : 0;
...@@ -763,6 +763,20 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) ...@@ -763,6 +763,20 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
ret = drm_addmap(dev, base, size, _DRM_REGISTERS, ret = drm_addmap(dev, base, size, _DRM_REGISTERS,
_DRM_KERNEL | _DRM_DRIVER, _DRM_KERNEL | _DRM_DRIVER,
&dev_priv->mmio_map); &dev_priv->mmio_map);
/* On the 945G/GM, the chipset reports the MSI capability on the
* integrated graphics even though the support isn't actually there
* according to the published specs. It doesn't appear to function
* correctly in testing on 945G.
* This may be a side effect of MSI having been made available for PEG
* and the registers being closely associated.
*/
if (!IS_I945G(dev) && !IS_I945GM(dev))
pci_enable_msi(dev->pdev);
spin_lock_init(&dev_priv->user_irq_lock);
return ret; return ret;
} }
...@@ -770,6 +784,9 @@ int i915_driver_unload(struct drm_device *dev) ...@@ -770,6 +784,9 @@ int i915_driver_unload(struct drm_device *dev)
{ {
struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_private *dev_priv = dev->dev_private;
if (dev->pdev->msi_enabled)
pci_disable_msi(dev->pdev);
if (dev_priv->mmio_map) if (dev_priv->mmio_map)
drm_rmmap(dev, dev_priv->mmio_map); drm_rmmap(dev, dev_priv->mmio_map);
......
...@@ -105,6 +105,12 @@ typedef struct drm_i915_private { ...@@ -105,6 +105,12 @@ typedef struct drm_i915_private {
wait_queue_head_t irq_queue; wait_queue_head_t irq_queue;
atomic_t irq_received; atomic_t irq_received;
atomic_t irq_emitted; atomic_t irq_emitted;
/** Protects user_irq_refcount and irq_mask_reg */
spinlock_t user_irq_lock;
/** Refcount for i915_user_irq_get() versus i915_user_irq_put(). */
int user_irq_refcount;
/** Cached value of IMR to avoid reads in updating the bitfield */
u32 irq_mask_reg;
int tex_lru_log_granularity; int tex_lru_log_granularity;
int allow_batchbuffer; int allow_batchbuffer;
......
...@@ -33,6 +33,31 @@ ...@@ -33,6 +33,31 @@
#define MAX_NOPID ((u32)~0) #define MAX_NOPID ((u32)~0)
/** These are the interrupts used by the driver */
#define I915_INTERRUPT_ENABLE_MASK (I915_USER_INTERRUPT | \
I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT | \
I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT)
static inline void
i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
{
if ((dev_priv->irq_mask_reg & mask) != 0) {
dev_priv->irq_mask_reg &= ~mask;
I915_WRITE(IMR, dev_priv->irq_mask_reg);
(void) I915_READ(IMR);
}
}
static inline void
i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
{
if ((dev_priv->irq_mask_reg & mask) != mask) {
dev_priv->irq_mask_reg |= mask;
I915_WRITE(IMR, dev_priv->irq_mask_reg);
(void) I915_READ(IMR);
}
}
/** /**
* Emit blits for scheduled buffer swaps. * Emit blits for scheduled buffer swaps.
* *
...@@ -229,46 +254,50 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS) ...@@ -229,46 +254,50 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
{ {
struct drm_device *dev = (struct drm_device *) arg; struct drm_device *dev = (struct drm_device *) arg;
drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
u16 temp;
u32 pipea_stats, pipeb_stats; u32 pipea_stats, pipeb_stats;
u32 iir;
pipea_stats = I915_READ(PIPEASTAT); pipea_stats = I915_READ(PIPEASTAT);
pipeb_stats = I915_READ(PIPEBSTAT); pipeb_stats = I915_READ(PIPEBSTAT);
temp = I915_READ16(IIR); if (dev->pdev->msi_enabled)
I915_WRITE(IMR, ~0);
temp &= (I915_USER_INTERRUPT | iir = I915_READ(IIR);
I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT);
DRM_DEBUG("%s flag=%08x\n", __FUNCTION__, temp); DRM_DEBUG("iir=%08x\n", iir);
if (temp == 0) if (iir == 0) {
if (dev->pdev->msi_enabled) {
I915_WRITE(IMR, dev_priv->irq_mask_reg);
(void) I915_READ(IMR);
}
return IRQ_NONE; return IRQ_NONE;
}
I915_WRITE16(IIR, temp); I915_WRITE(IIR, iir);
(void) I915_READ16(IIR); if (dev->pdev->msi_enabled)
DRM_READMEMORYBARRIER(); I915_WRITE(IMR, dev_priv->irq_mask_reg);
(void) I915_READ(IIR); /* Flush posted writes */
dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
if (temp & I915_USER_INTERRUPT) if (iir & I915_USER_INTERRUPT)
DRM_WAKEUP(&dev_priv->irq_queue); DRM_WAKEUP(&dev_priv->irq_queue);
if (temp & (I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT | if (iir & (I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT)) { I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT)) {
int vblank_pipe = dev_priv->vblank_pipe; int vblank_pipe = dev_priv->vblank_pipe;
if ((vblank_pipe & if ((vblank_pipe &
(DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B)) (DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B))
== (DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B)) { == (DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B)) {
if (temp & I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT) if (iir & I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT)
atomic_inc(&dev->vbl_received); atomic_inc(&dev->vbl_received);
if (temp & I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT) if (iir & I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT)
atomic_inc(&dev->vbl_received2); atomic_inc(&dev->vbl_received2);
} else if (((temp & I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT) && } else if (((iir & I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT) &&
(vblank_pipe & DRM_I915_VBLANK_PIPE_A)) || (vblank_pipe & DRM_I915_VBLANK_PIPE_A)) ||
((temp & I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT) && ((iir & I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT) &&
(vblank_pipe & DRM_I915_VBLANK_PIPE_B))) (vblank_pipe & DRM_I915_VBLANK_PIPE_B)))
atomic_inc(&dev->vbl_received); atomic_inc(&dev->vbl_received);
...@@ -314,6 +343,27 @@ static int i915_emit_irq(struct drm_device * dev) ...@@ -314,6 +343,27 @@ static int i915_emit_irq(struct drm_device * dev)
return dev_priv->counter; return dev_priv->counter;
} }
static void i915_user_irq_get(struct drm_device *dev)
{
drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
spin_lock(&dev_priv->user_irq_lock);
if (dev->irq_enabled && (++dev_priv->user_irq_refcount == 1))
i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
spin_unlock(&dev_priv->user_irq_lock);
}
static void i915_user_irq_put(struct drm_device *dev)
{
drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
spin_lock(&dev_priv->user_irq_lock);
BUG_ON(dev->irq_enabled && dev_priv->user_irq_refcount <= 0);
if (dev->irq_enabled && (--dev_priv->user_irq_refcount == 0))
i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
spin_unlock(&dev_priv->user_irq_lock);
}
static int i915_wait_irq(struct drm_device * dev, int irq_nr) static int i915_wait_irq(struct drm_device * dev, int irq_nr)
{ {
drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
...@@ -322,13 +372,17 @@ static int i915_wait_irq(struct drm_device * dev, int irq_nr) ...@@ -322,13 +372,17 @@ static int i915_wait_irq(struct drm_device * dev, int irq_nr)
DRM_DEBUG("irq_nr=%d breadcrumb=%d\n", irq_nr, DRM_DEBUG("irq_nr=%d breadcrumb=%d\n", irq_nr,
READ_BREADCRUMB(dev_priv)); READ_BREADCRUMB(dev_priv));
if (READ_BREADCRUMB(dev_priv) >= irq_nr) if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
return 0; return 0;
}
dev_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; dev_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
i915_user_irq_get(dev);
DRM_WAIT_ON(ret, dev_priv->irq_queue, 3 * DRM_HZ, DRM_WAIT_ON(ret, dev_priv->irq_queue, 3 * DRM_HZ,
READ_BREADCRUMB(dev_priv) >= irq_nr); READ_BREADCRUMB(dev_priv) >= irq_nr);
i915_user_irq_put(dev);
if (ret == -EBUSY) { if (ret == -EBUSY) {
DRM_ERROR("EBUSY -- rec: %d emitted: %d\n", DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
...@@ -413,20 +467,6 @@ int i915_irq_wait(struct drm_device *dev, void *data, ...@@ -413,20 +467,6 @@ int i915_irq_wait(struct drm_device *dev, void *data,
return i915_wait_irq(dev, irqwait->irq_seq); return i915_wait_irq(dev, irqwait->irq_seq);
} }
static void i915_enable_interrupt (struct drm_device *dev)
{
drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
u16 flag;
flag = 0;
if (dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_A)
flag |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
if (dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_B)
flag |= I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
I915_WRITE16(IER, I915_USER_INTERRUPT | flag);
}
/* Set the vblank monitor pipe /* Set the vblank monitor pipe
*/ */
int i915_vblank_pipe_set(struct drm_device *dev, void *data, int i915_vblank_pipe_set(struct drm_device *dev, void *data,
...@@ -434,6 +474,7 @@ int i915_vblank_pipe_set(struct drm_device *dev, void *data, ...@@ -434,6 +474,7 @@ int i915_vblank_pipe_set(struct drm_device *dev, void *data,
{ {
drm_i915_private_t *dev_priv = dev->dev_private; drm_i915_private_t *dev_priv = dev->dev_private;
drm_i915_vblank_pipe_t *pipe = data; drm_i915_vblank_pipe_t *pipe = data;
u32 enable_mask = 0, disable_mask = 0;
if (!dev_priv) { if (!dev_priv) {
DRM_ERROR("called with no initialization\n"); DRM_ERROR("called with no initialization\n");
...@@ -445,9 +486,20 @@ int i915_vblank_pipe_set(struct drm_device *dev, void *data, ...@@ -445,9 +486,20 @@ int i915_vblank_pipe_set(struct drm_device *dev, void *data,
return -EINVAL; return -EINVAL;
} }
dev_priv->vblank_pipe = pipe->pipe; if (pipe->pipe & DRM_I915_VBLANK_PIPE_A)
enable_mask |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
else
disable_mask |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
if (pipe->pipe & DRM_I915_VBLANK_PIPE_B)
enable_mask |= I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
else
disable_mask |= I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
i915_enable_interrupt (dev); i915_enable_irq(dev_priv, enable_mask);
i915_disable_irq(dev_priv, disable_mask);
dev_priv->vblank_pipe = pipe->pipe;
return 0; return 0;
} }
...@@ -464,7 +516,7 @@ int i915_vblank_pipe_get(struct drm_device *dev, void *data, ...@@ -464,7 +516,7 @@ int i915_vblank_pipe_get(struct drm_device *dev, void *data,
return -EINVAL; return -EINVAL;
} }
flag = I915_READ(IER); flag = I915_READ(IMR);
pipe->pipe = 0; pipe->pipe = 0;
if (flag & I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT) if (flag & I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT)
pipe->pipe |= DRM_I915_VBLANK_PIPE_A; pipe->pipe |= DRM_I915_VBLANK_PIPE_A;
...@@ -586,9 +638,9 @@ void i915_driver_irq_preinstall(struct drm_device * dev) ...@@ -586,9 +638,9 @@ void i915_driver_irq_preinstall(struct drm_device * dev)
{ {
drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
I915_WRITE16(HWSTAM, 0xfffe); I915_WRITE(HWSTAM, 0xfffe);
I915_WRITE16(IMR, 0x0); I915_WRITE(IMR, 0x0);
I915_WRITE16(IER, 0x0); I915_WRITE(IER, 0x0);
} }
void i915_driver_irq_postinstall(struct drm_device * dev) void i915_driver_irq_postinstall(struct drm_device * dev)
...@@ -601,7 +653,18 @@ void i915_driver_irq_postinstall(struct drm_device * dev) ...@@ -601,7 +653,18 @@ void i915_driver_irq_postinstall(struct drm_device * dev)
if (!dev_priv->vblank_pipe) if (!dev_priv->vblank_pipe)
dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A; dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A;
i915_enable_interrupt(dev);
/* Set initial unmasked IRQs to just the selected vblank pipes. */
dev_priv->irq_mask_reg = ~0;
if (dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_A)
dev_priv->irq_mask_reg &= ~I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
if (dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_B)
dev_priv->irq_mask_reg &= ~I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
I915_WRITE(IMR, dev_priv->irq_mask_reg);
I915_WRITE(IER, I915_INTERRUPT_ENABLE_MASK);
(void) I915_READ(IER);
DRM_INIT_WAITQUEUE(&dev_priv->irq_queue); DRM_INIT_WAITQUEUE(&dev_priv->irq_queue);
} }
...@@ -613,10 +676,10 @@ void i915_driver_irq_uninstall(struct drm_device * dev) ...@@ -613,10 +676,10 @@ void i915_driver_irq_uninstall(struct drm_device * dev)
if (!dev_priv) if (!dev_priv)
return; return;
I915_WRITE16(HWSTAM, 0xffff); I915_WRITE(HWSTAM, 0xffff);
I915_WRITE16(IMR, 0xffff); I915_WRITE(IMR, 0xffff);
I915_WRITE16(IER, 0x0); I915_WRITE(IER, 0x0);
temp = I915_READ16(IIR); temp = I915_READ(IIR);
I915_WRITE16(IIR, temp); I915_WRITE(IIR, temp);
} }
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment