Commit 09246732 authored by Chris Wilson's avatar Chris Wilson Committed by Daniel Vetter

drm/i915: Write RING_TAIL once per-request

Ignoring the legacy DRI1 code, and a couple of special cases (to be
discussed later), all access to the ring is mediated through requests.
The first write to a ring will grab a seqno and mark the ring as having
an outstanding_lazy_request. Either through explicitly adding a request
after an execbuffer or through an implicit wait (either by the CPU or by
a semaphore), that sequence of writes will be terminated with a request.
So we can ellide all the intervening writes to the tail register and
send the entire command stream to the GPU at once. This will reduce the
number of *serialising* writes to the tail register by a factor or 3-5
times (depending upon architecture and number of workarounds, context
switches, etc involved). This becomes even more noticeable when the
register write is overloaded with a number of debugging tools. The
astute reader will wonder if it is then possible to overflow the ring
with a single command. It is not. When we start a command sequence to
the ring, we check for available space and issue a wait in case we have
not. The ring wait will in this case be forced to flush the outstanding
register write and then poll the ACTHD for sufficient space to continue.

The exception to the rule where everything is inside a request are a few
initialisation cases where we may want to write GPU commands via the CS
before userspace wakes up and page flips.
Signed-off-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: default avatarDaniel Vetter <daniel.vetter@ffwll.ch>
parent ad0d6dc4
...@@ -52,7 +52,7 @@ ...@@ -52,7 +52,7 @@
intel_ring_emit(LP_RING(dev_priv), x) intel_ring_emit(LP_RING(dev_priv), x)
#define ADVANCE_LP_RING() \ #define ADVANCE_LP_RING() \
intel_ring_advance(LP_RING(dev_priv)) __intel_ring_advance(LP_RING(dev_priv))
/** /**
* Lock test for when it's just for synchronization of ring access. * Lock test for when it's just for synchronization of ring access.
......
...@@ -7698,7 +7698,7 @@ static int intel_gen2_queue_flip(struct drm_device *dev, ...@@ -7698,7 +7698,7 @@ static int intel_gen2_queue_flip(struct drm_device *dev,
intel_ring_emit(ring, 0); /* aux display base address, unused */ intel_ring_emit(ring, 0); /* aux display base address, unused */
intel_mark_page_flip_active(intel_crtc); intel_mark_page_flip_active(intel_crtc);
intel_ring_advance(ring); __intel_ring_advance(ring);
return 0; return 0;
err_unpin: err_unpin:
...@@ -7740,7 +7740,7 @@ static int intel_gen3_queue_flip(struct drm_device *dev, ...@@ -7740,7 +7740,7 @@ static int intel_gen3_queue_flip(struct drm_device *dev,
intel_ring_emit(ring, MI_NOOP); intel_ring_emit(ring, MI_NOOP);
intel_mark_page_flip_active(intel_crtc); intel_mark_page_flip_active(intel_crtc);
intel_ring_advance(ring); __intel_ring_advance(ring);
return 0; return 0;
err_unpin: err_unpin:
...@@ -7789,7 +7789,7 @@ static int intel_gen4_queue_flip(struct drm_device *dev, ...@@ -7789,7 +7789,7 @@ static int intel_gen4_queue_flip(struct drm_device *dev,
intel_ring_emit(ring, pf | pipesrc); intel_ring_emit(ring, pf | pipesrc);
intel_mark_page_flip_active(intel_crtc); intel_mark_page_flip_active(intel_crtc);
intel_ring_advance(ring); __intel_ring_advance(ring);
return 0; return 0;
err_unpin: err_unpin:
...@@ -7834,7 +7834,7 @@ static int intel_gen6_queue_flip(struct drm_device *dev, ...@@ -7834,7 +7834,7 @@ static int intel_gen6_queue_flip(struct drm_device *dev,
intel_ring_emit(ring, pf | pipesrc); intel_ring_emit(ring, pf | pipesrc);
intel_mark_page_flip_active(intel_crtc); intel_mark_page_flip_active(intel_crtc);
intel_ring_advance(ring); __intel_ring_advance(ring);
return 0; return 0;
err_unpin: err_unpin:
...@@ -7913,7 +7913,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev, ...@@ -7913,7 +7913,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
intel_ring_emit(ring, (MI_NOOP)); intel_ring_emit(ring, (MI_NOOP));
intel_mark_page_flip_active(intel_crtc); intel_mark_page_flip_active(intel_crtc);
intel_ring_advance(ring); __intel_ring_advance(ring);
return 0; return 0;
err_unpin: err_unpin:
......
...@@ -41,6 +41,16 @@ static inline int ring_space(struct intel_ring_buffer *ring) ...@@ -41,6 +41,16 @@ static inline int ring_space(struct intel_ring_buffer *ring)
return space; return space;
} }
void __intel_ring_advance(struct intel_ring_buffer *ring)
{
struct drm_i915_private *dev_priv = ring->dev->dev_private;
ring->tail &= ring->size - 1;
if (dev_priv->gpu_error.stop_rings & intel_ring_flag(ring))
return;
ring->write_tail(ring, ring->tail);
}
static int static int
gen2_render_ring_flush(struct intel_ring_buffer *ring, gen2_render_ring_flush(struct intel_ring_buffer *ring,
u32 invalidate_domains, u32 invalidate_domains,
...@@ -631,7 +641,7 @@ gen6_add_request(struct intel_ring_buffer *ring) ...@@ -631,7 +641,7 @@ gen6_add_request(struct intel_ring_buffer *ring)
intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
intel_ring_emit(ring, ring->outstanding_lazy_seqno); intel_ring_emit(ring, ring->outstanding_lazy_seqno);
intel_ring_emit(ring, MI_USER_INTERRUPT); intel_ring_emit(ring, MI_USER_INTERRUPT);
intel_ring_advance(ring); __intel_ring_advance(ring);
return 0; return 0;
} }
...@@ -744,7 +754,7 @@ pc_render_add_request(struct intel_ring_buffer *ring) ...@@ -744,7 +754,7 @@ pc_render_add_request(struct intel_ring_buffer *ring)
intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
intel_ring_emit(ring, ring->outstanding_lazy_seqno); intel_ring_emit(ring, ring->outstanding_lazy_seqno);
intel_ring_emit(ring, 0); intel_ring_emit(ring, 0);
intel_ring_advance(ring); __intel_ring_advance(ring);
return 0; return 0;
} }
...@@ -965,7 +975,7 @@ i9xx_add_request(struct intel_ring_buffer *ring) ...@@ -965,7 +975,7 @@ i9xx_add_request(struct intel_ring_buffer *ring)
intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
intel_ring_emit(ring, ring->outstanding_lazy_seqno); intel_ring_emit(ring, ring->outstanding_lazy_seqno);
intel_ring_emit(ring, MI_USER_INTERRUPT); intel_ring_emit(ring, MI_USER_INTERRUPT);
intel_ring_advance(ring); __intel_ring_advance(ring);
return 0; return 0;
} }
...@@ -1414,6 +1424,9 @@ static int ring_wait_for_space(struct intel_ring_buffer *ring, int n) ...@@ -1414,6 +1424,9 @@ static int ring_wait_for_space(struct intel_ring_buffer *ring, int n)
if (ret != -ENOSPC) if (ret != -ENOSPC)
return ret; return ret;
/* force the tail write in case we have been skipping them */
__intel_ring_advance(ring);
trace_i915_ring_wait_begin(ring); trace_i915_ring_wait_begin(ring);
/* With GEM the hangcheck timer should kick us out of the loop, /* With GEM the hangcheck timer should kick us out of the loop,
* leaving it early runs the risk of corrupting GEM state (due * leaving it early runs the risk of corrupting GEM state (due
...@@ -1568,17 +1581,6 @@ void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno) ...@@ -1568,17 +1581,6 @@ void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno)
ring->hangcheck.seqno = seqno; ring->hangcheck.seqno = seqno;
} }
void intel_ring_advance(struct intel_ring_buffer *ring)
{
struct drm_i915_private *dev_priv = ring->dev->dev_private;
ring->tail &= ring->size - 1;
if (dev_priv->gpu_error.stop_rings & intel_ring_flag(ring))
return;
ring->write_tail(ring, ring->tail);
}
static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring, static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
u32 value) u32 value)
{ {
......
...@@ -239,7 +239,12 @@ static inline void intel_ring_emit(struct intel_ring_buffer *ring, ...@@ -239,7 +239,12 @@ static inline void intel_ring_emit(struct intel_ring_buffer *ring,
iowrite32(data, ring->virtual_start + ring->tail); iowrite32(data, ring->virtual_start + ring->tail);
ring->tail += 4; ring->tail += 4;
} }
void intel_ring_advance(struct intel_ring_buffer *ring); static inline void intel_ring_advance(struct intel_ring_buffer *ring)
{
ring->tail &= ring->size - 1;
}
void __intel_ring_advance(struct intel_ring_buffer *ring);
int __must_check intel_ring_idle(struct intel_ring_buffer *ring); int __must_check intel_ring_idle(struct intel_ring_buffer *ring);
void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno); void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno);
int intel_ring_flush_all_caches(struct intel_ring_buffer *ring); int intel_ring_flush_all_caches(struct intel_ring_buffer *ring);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment