Commit b1e3177b authored by Chris Wilson's avatar Chris Wilson

drm/i915: Coordinate i915_active with its own mutex

Forgo the struct_mutex serialisation for i915_active, and interpose its
own mutex handling for active/retire.

This is a multi-layered sleight-of-hand. First, we had to ensure that no
active/retire callbacks accidentally inverted the mutex ordering rules,
nor assumed that they were themselves serialised by struct_mutex. More
challenging though, is the rule over updating elements of the active
rbtree. Instead of the whole i915_active now being serialised by
struct_mutex, allocations/rotations of the tree are serialised by the
i915_active.mutex and individual nodes are serialised by the caller
using the i915_timeline.mutex (we need to use nested spinlocks to
interact with the dma_fence callback lists).

The pain point here is that instead of a single mutex around execbuf, we
now have to take a mutex for active tracker (one for each vma, context,
etc) and a couple of spinlocks for each fence update. The improvement in
fine grained locking allowing for multiple concurrent clients
(eventually!) should be worth it in typical loads.

v2: Add some comments that barely elucidate anything :(
Signed-off-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: default avatarTvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20191004134015.13204-6-chris@chris-wilson.co.uk
parent 274cbf20
...@@ -257,7 +257,7 @@ intel_frontbuffer_get(struct drm_i915_gem_object *obj) ...@@ -257,7 +257,7 @@ intel_frontbuffer_get(struct drm_i915_gem_object *obj)
front->obj = obj; front->obj = obj;
kref_init(&front->ref); kref_init(&front->ref);
atomic_set(&front->bits, 0); atomic_set(&front->bits, 0);
i915_active_init(i915, &front->write, i915_active_init(&front->write,
frontbuffer_active, frontbuffer_active,
i915_active_may_sleep(frontbuffer_retire)); i915_active_may_sleep(frontbuffer_retire));
......
...@@ -1360,8 +1360,7 @@ void intel_overlay_setup(struct drm_i915_private *dev_priv) ...@@ -1360,8 +1360,7 @@ void intel_overlay_setup(struct drm_i915_private *dev_priv)
overlay->contrast = 75; overlay->contrast = 75;
overlay->saturation = 146; overlay->saturation = 146;
i915_active_init(dev_priv, i915_active_init(&overlay->last_flip,
&overlay->last_flip,
NULL, intel_overlay_last_flip_retire); NULL, intel_overlay_last_flip_retire);
ret = get_registers(overlay, OVERLAY_NEEDS_PHYSICAL(dev_priv)); ret = get_registers(overlay, OVERLAY_NEEDS_PHYSICAL(dev_priv));
......
...@@ -868,20 +868,18 @@ static int context_barrier_task(struct i915_gem_context *ctx, ...@@ -868,20 +868,18 @@ static int context_barrier_task(struct i915_gem_context *ctx,
void (*task)(void *data), void (*task)(void *data),
void *data) void *data)
{ {
struct drm_i915_private *i915 = ctx->i915;
struct context_barrier_task *cb; struct context_barrier_task *cb;
struct i915_gem_engines_iter it; struct i915_gem_engines_iter it;
struct intel_context *ce; struct intel_context *ce;
int err = 0; int err = 0;
lockdep_assert_held(&i915->drm.struct_mutex);
GEM_BUG_ON(!task); GEM_BUG_ON(!task);
cb = kmalloc(sizeof(*cb), GFP_KERNEL); cb = kmalloc(sizeof(*cb), GFP_KERNEL);
if (!cb) if (!cb)
return -ENOMEM; return -ENOMEM;
i915_active_init(i915, &cb->base, NULL, cb_retire); i915_active_init(&cb->base, NULL, cb_retire);
err = i915_active_acquire(&cb->base); err = i915_active_acquire(&cb->base);
if (err) { if (err) {
kfree(cb); kfree(cb);
......
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
#define __I915_GEM_OBJECT_TYPES_H__ #define __I915_GEM_OBJECT_TYPES_H__
#include <drm/drm_gem.h> #include <drm/drm_gem.h>
#include <uapi/drm/i915_drm.h>
#include "i915_active.h" #include "i915_active.h"
#include "i915_selftest.h" #include "i915_selftest.h"
......
...@@ -16,14 +16,11 @@ static void call_idle_barriers(struct intel_engine_cs *engine) ...@@ -16,14 +16,11 @@ static void call_idle_barriers(struct intel_engine_cs *engine)
struct llist_node *node, *next; struct llist_node *node, *next;
llist_for_each_safe(node, next, llist_del_all(&engine->barrier_tasks)) { llist_for_each_safe(node, next, llist_del_all(&engine->barrier_tasks)) {
struct i915_active_request *active = struct dma_fence_cb *cb =
container_of((struct list_head *)node, container_of((struct list_head *)node,
typeof(*active), link); typeof(*cb), node);
INIT_LIST_HEAD(&active->link); cb->func(NULL, cb);
RCU_INIT_POINTER(active->request, NULL);
active->retire(active, NULL);
} }
} }
......
...@@ -240,7 +240,7 @@ intel_context_init(struct intel_context *ce, ...@@ -240,7 +240,7 @@ intel_context_init(struct intel_context *ce,
mutex_init(&ce->pin_mutex); mutex_init(&ce->pin_mutex);
i915_active_init(ctx->i915, &ce->active, i915_active_init(&ce->active,
__intel_context_active, __intel_context_retire); __intel_context_active, __intel_context_retire);
} }
...@@ -307,7 +307,7 @@ int intel_context_prepare_remote_request(struct intel_context *ce, ...@@ -307,7 +307,7 @@ int intel_context_prepare_remote_request(struct intel_context *ce,
return err; return err;
/* Queue this switch after current activity by this context. */ /* Queue this switch after current activity by this context. */
err = i915_active_request_set(&tl->last_request, rq); err = i915_active_fence_set(&tl->last_request, rq);
mutex_unlock(&tl->mutex); mutex_unlock(&tl->mutex);
if (err) if (err)
return err; return err;
......
...@@ -95,7 +95,7 @@ node_create(struct intel_engine_pool *pool, size_t sz) ...@@ -95,7 +95,7 @@ node_create(struct intel_engine_pool *pool, size_t sz)
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
node->pool = pool; node->pool = pool;
i915_active_init(engine->i915, &node->active, pool_active, pool_retire); i915_active_init(&node->active, pool_active, pool_retire);
obj = i915_gem_object_create_internal(engine->i915, sz); obj = i915_gem_object_create_internal(engine->i915, sz);
if (IS_ERR(obj)) { if (IS_ERR(obj)) {
......
...@@ -844,10 +844,10 @@ static bool __intel_gt_unset_wedged(struct intel_gt *gt) ...@@ -844,10 +844,10 @@ static bool __intel_gt_unset_wedged(struct intel_gt *gt)
*/ */
spin_lock_irqsave(&timelines->lock, flags); spin_lock_irqsave(&timelines->lock, flags);
list_for_each_entry(tl, &timelines->active_list, link) { list_for_each_entry(tl, &timelines->active_list, link) {
struct i915_request *rq; struct dma_fence *fence;
rq = i915_active_request_get_unlocked(&tl->last_request); fence = i915_active_fence_get(&tl->last_request);
if (!rq) if (!fence)
continue; continue;
spin_unlock_irqrestore(&timelines->lock, flags); spin_unlock_irqrestore(&timelines->lock, flags);
...@@ -859,8 +859,8 @@ static bool __intel_gt_unset_wedged(struct intel_gt *gt) ...@@ -859,8 +859,8 @@ static bool __intel_gt_unset_wedged(struct intel_gt *gt)
* (I915_FENCE_TIMEOUT) so this wait should not be unbounded * (I915_FENCE_TIMEOUT) so this wait should not be unbounded
* in the worst case. * in the worst case.
*/ */
dma_fence_default_wait(&rq->fence, false, MAX_SCHEDULE_TIMEOUT); dma_fence_default_wait(fence, false, MAX_SCHEDULE_TIMEOUT);
i915_request_put(rq); dma_fence_put(fence);
/* Restart iteration after droping lock */ /* Restart iteration after droping lock */
spin_lock_irqsave(&timelines->lock, flags); spin_lock_irqsave(&timelines->lock, flags);
......
...@@ -178,8 +178,7 @@ cacheline_alloc(struct intel_timeline_hwsp *hwsp, unsigned int cacheline) ...@@ -178,8 +178,7 @@ cacheline_alloc(struct intel_timeline_hwsp *hwsp, unsigned int cacheline)
cl->hwsp = hwsp; cl->hwsp = hwsp;
cl->vaddr = page_pack_bits(vaddr, cacheline); cl->vaddr = page_pack_bits(vaddr, cacheline);
i915_active_init(hwsp->gt->i915, &cl->active, i915_active_init(&cl->active, __cacheline_active, __cacheline_retire);
__cacheline_active, __cacheline_retire);
return cl; return cl;
} }
...@@ -255,7 +254,7 @@ int intel_timeline_init(struct intel_timeline *timeline, ...@@ -255,7 +254,7 @@ int intel_timeline_init(struct intel_timeline *timeline,
mutex_init(&timeline->mutex); mutex_init(&timeline->mutex);
INIT_ACTIVE_REQUEST(&timeline->last_request, &timeline->mutex); INIT_ACTIVE_FENCE(&timeline->last_request, &timeline->mutex);
INIT_LIST_HEAD(&timeline->requests); INIT_LIST_HEAD(&timeline->requests);
i915_syncmap_init(&timeline->sync); i915_syncmap_init(&timeline->sync);
...@@ -443,7 +442,7 @@ __intel_timeline_get_seqno(struct intel_timeline *tl, ...@@ -443,7 +442,7 @@ __intel_timeline_get_seqno(struct intel_timeline *tl,
* free it after the current request is retired, which ensures that * free it after the current request is retired, which ensures that
* all writes into the cacheline from previous requests are complete. * all writes into the cacheline from previous requests are complete.
*/ */
err = i915_active_ref(&tl->hwsp_cacheline->active, tl, rq); err = i915_active_ref(&tl->hwsp_cacheline->active, tl, &rq->fence);
if (err) if (err)
goto err_cacheline; goto err_cacheline;
......
...@@ -58,12 +58,13 @@ struct intel_timeline { ...@@ -58,12 +58,13 @@ struct intel_timeline {
*/ */
struct list_head requests; struct list_head requests;
/* Contains an RCU guarded pointer to the last request. No reference is /*
* Contains an RCU guarded pointer to the last request. No reference is
* held to the request, users must carefully acquire a reference to * held to the request, users must carefully acquire a reference to
* the request using i915_active_request_get_request_rcu(), or hold the * the request using i915_active_fence_get(), or manage the RCU
* struct_mutex. * protection themselves (cf the i915_active_fence API).
*/ */
struct i915_active_request last_request; struct i915_active_fence last_request;
/** /**
* We track the most recent seqno that we wait on in every context so * We track the most recent seqno that we wait on in every context so
......
...@@ -47,24 +47,20 @@ static int context_sync(struct intel_context *ce) ...@@ -47,24 +47,20 @@ static int context_sync(struct intel_context *ce)
mutex_lock(&tl->mutex); mutex_lock(&tl->mutex);
do { do {
struct i915_request *rq; struct dma_fence *fence;
long timeout; long timeout;
rcu_read_lock(); fence = i915_active_fence_get(&tl->last_request);
rq = rcu_dereference(tl->last_request.request); if (!fence)
if (rq)
rq = i915_request_get_rcu(rq);
rcu_read_unlock();
if (!rq)
break; break;
timeout = i915_request_wait(rq, 0, HZ / 10); timeout = dma_fence_wait_timeout(fence, false, HZ / 10);
if (timeout < 0) if (timeout < 0)
err = timeout; err = timeout;
else else
i915_request_retire_upto(rq); i915_request_retire_upto(to_request(fence));
i915_request_put(rq); dma_fence_put(fence);
} while (!err); } while (!err);
mutex_unlock(&tl->mutex); mutex_unlock(&tl->mutex);
......
...@@ -1172,9 +1172,13 @@ static struct i915_request *dummy_request(struct intel_engine_cs *engine) ...@@ -1172,9 +1172,13 @@ static struct i915_request *dummy_request(struct intel_engine_cs *engine)
if (!rq) if (!rq)
return NULL; return NULL;
INIT_LIST_HEAD(&rq->active_list);
rq->engine = engine; rq->engine = engine;
spin_lock_init(&rq->lock);
INIT_LIST_HEAD(&rq->fence.cb_list);
rq->fence.lock = &rq->lock;
rq->fence.ops = &i915_fence_ops;
i915_sched_node_init(&rq->sched); i915_sched_node_init(&rq->sched);
/* mark this request as permanently incomplete */ /* mark this request as permanently incomplete */
...@@ -1267,8 +1271,8 @@ static int live_suppress_wait_preempt(void *arg) ...@@ -1267,8 +1271,8 @@ static int live_suppress_wait_preempt(void *arg)
} }
/* Disable NEWCLIENT promotion */ /* Disable NEWCLIENT promotion */
__i915_active_request_set(&i915_request_timeline(rq[i])->last_request, __i915_active_fence_set(&i915_request_timeline(rq[i])->last_request,
dummy); &dummy->fence);
i915_request_add(rq[i]); i915_request_add(rq[i]);
} }
......
...@@ -15,7 +15,7 @@ void mock_timeline_init(struct intel_timeline *timeline, u64 context) ...@@ -15,7 +15,7 @@ void mock_timeline_init(struct intel_timeline *timeline, u64 context)
mutex_init(&timeline->mutex); mutex_init(&timeline->mutex);
INIT_ACTIVE_REQUEST(&timeline->last_request, &timeline->mutex); INIT_ACTIVE_FENCE(&timeline->last_request, &timeline->mutex);
INIT_LIST_HEAD(&timeline->requests); INIT_LIST_HEAD(&timeline->requests);
i915_syncmap_init(&timeline->sync); i915_syncmap_init(&timeline->sync);
......
...@@ -385,11 +385,8 @@ intel_gvt_workload_req_alloc(struct intel_vgpu_workload *workload) ...@@ -385,11 +385,8 @@ intel_gvt_workload_req_alloc(struct intel_vgpu_workload *workload)
{ {
struct intel_vgpu *vgpu = workload->vgpu; struct intel_vgpu *vgpu = workload->vgpu;
struct intel_vgpu_submission *s = &vgpu->submission; struct intel_vgpu_submission *s = &vgpu->submission;
struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
struct i915_request *rq; struct i915_request *rq;
lockdep_assert_held(&dev_priv->drm.struct_mutex);
if (workload->req) if (workload->req)
return 0; return 0;
......
...@@ -12,8 +12,6 @@ ...@@ -12,8 +12,6 @@
#include "i915_active.h" #include "i915_active.h"
#include "i915_globals.h" #include "i915_globals.h"
#define BKL(ref) (&(ref)->i915->drm.struct_mutex)
/* /*
* Active refs memory management * Active refs memory management
* *
...@@ -27,35 +25,35 @@ static struct i915_global_active { ...@@ -27,35 +25,35 @@ static struct i915_global_active {
} global; } global;
struct active_node { struct active_node {
struct i915_active_request base; struct i915_active_fence base;
struct i915_active *ref; struct i915_active *ref;
struct rb_node node; struct rb_node node;
u64 timeline; u64 timeline;
}; };
static inline struct active_node * static inline struct active_node *
node_from_active(struct i915_active_request *active) node_from_active(struct i915_active_fence *active)
{ {
return container_of(active, struct active_node, base); return container_of(active, struct active_node, base);
} }
#define take_preallocated_barriers(x) llist_del_all(&(x)->preallocated_barriers) #define take_preallocated_barriers(x) llist_del_all(&(x)->preallocated_barriers)
static inline bool is_barrier(const struct i915_active_request *active) static inline bool is_barrier(const struct i915_active_fence *active)
{ {
return IS_ERR(rcu_access_pointer(active->request)); return IS_ERR(rcu_access_pointer(active->fence));
} }
static inline struct llist_node *barrier_to_ll(struct active_node *node) static inline struct llist_node *barrier_to_ll(struct active_node *node)
{ {
GEM_BUG_ON(!is_barrier(&node->base)); GEM_BUG_ON(!is_barrier(&node->base));
return (struct llist_node *)&node->base.link; return (struct llist_node *)&node->base.cb.node;
} }
static inline struct intel_engine_cs * static inline struct intel_engine_cs *
__barrier_to_engine(struct active_node *node) __barrier_to_engine(struct active_node *node)
{ {
return (struct intel_engine_cs *)READ_ONCE(node->base.link.prev); return (struct intel_engine_cs *)READ_ONCE(node->base.cb.node.prev);
} }
static inline struct intel_engine_cs * static inline struct intel_engine_cs *
...@@ -68,7 +66,7 @@ barrier_to_engine(struct active_node *node) ...@@ -68,7 +66,7 @@ barrier_to_engine(struct active_node *node)
static inline struct active_node *barrier_from_ll(struct llist_node *x) static inline struct active_node *barrier_from_ll(struct llist_node *x)
{ {
return container_of((struct list_head *)x, return container_of((struct list_head *)x,
struct active_node, base.link); struct active_node, base.cb.node);
} }
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) && IS_ENABLED(CONFIG_DEBUG_OBJECTS) #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) && IS_ENABLED(CONFIG_DEBUG_OBJECTS)
...@@ -147,15 +145,18 @@ __active_retire(struct i915_active *ref) ...@@ -147,15 +145,18 @@ __active_retire(struct i915_active *ref)
if (!retire) if (!retire)
return; return;
GEM_BUG_ON(rcu_access_pointer(ref->excl)); GEM_BUG_ON(rcu_access_pointer(ref->excl.fence));
rbtree_postorder_for_each_entry_safe(it, n, &root, node) { rbtree_postorder_for_each_entry_safe(it, n, &root, node) {
GEM_BUG_ON(i915_active_request_isset(&it->base)); GEM_BUG_ON(i915_active_fence_isset(&it->base));
kmem_cache_free(global.slab_cache, it); kmem_cache_free(global.slab_cache, it);
} }
/* After the final retire, the entire struct may be freed */ /* After the final retire, the entire struct may be freed */
if (ref->retire) if (ref->retire)
ref->retire(ref); ref->retire(ref);
/* ... except if you wait on it, you must manage your own references! */
wake_up_var(ref);
} }
static void static void
...@@ -189,12 +190,20 @@ active_retire(struct i915_active *ref) ...@@ -189,12 +190,20 @@ active_retire(struct i915_active *ref)
} }
static void static void
node_retire(struct i915_active_request *base, struct i915_request *rq) node_retire(struct dma_fence *fence, struct dma_fence_cb *cb)
{ {
active_retire(node_from_active(base)->ref); i915_active_fence_cb(fence, cb);
active_retire(container_of(cb, struct active_node, base.cb)->ref);
} }
static struct i915_active_request * static void
excl_retire(struct dma_fence *fence, struct dma_fence_cb *cb)
{
i915_active_fence_cb(fence, cb);
active_retire(container_of(cb, struct i915_active, excl.cb));
}
static struct i915_active_fence *
active_instance(struct i915_active *ref, struct intel_timeline *tl) active_instance(struct i915_active *ref, struct intel_timeline *tl)
{ {
struct active_node *node, *prealloc; struct active_node *node, *prealloc;
...@@ -238,7 +247,7 @@ active_instance(struct i915_active *ref, struct intel_timeline *tl) ...@@ -238,7 +247,7 @@ active_instance(struct i915_active *ref, struct intel_timeline *tl)
} }
node = prealloc; node = prealloc;
i915_active_request_init(&node->base, &tl->mutex, NULL, node_retire); __i915_active_fence_init(&node->base, &tl->mutex, NULL, node_retire);
node->ref = ref; node->ref = ref;
node->timeline = idx; node->timeline = idx;
...@@ -253,8 +262,7 @@ active_instance(struct i915_active *ref, struct intel_timeline *tl) ...@@ -253,8 +262,7 @@ active_instance(struct i915_active *ref, struct intel_timeline *tl)
return &node->base; return &node->base;
} }
void __i915_active_init(struct drm_i915_private *i915, void __i915_active_init(struct i915_active *ref,
struct i915_active *ref,
int (*active)(struct i915_active *ref), int (*active)(struct i915_active *ref),
void (*retire)(struct i915_active *ref), void (*retire)(struct i915_active *ref),
struct lock_class_key *key) struct lock_class_key *key)
...@@ -263,19 +271,18 @@ void __i915_active_init(struct drm_i915_private *i915, ...@@ -263,19 +271,18 @@ void __i915_active_init(struct drm_i915_private *i915,
debug_active_init(ref); debug_active_init(ref);
ref->i915 = i915;
ref->flags = 0; ref->flags = 0;
ref->active = active; ref->active = active;
ref->retire = ptr_unpack_bits(retire, &bits, 2); ref->retire = ptr_unpack_bits(retire, &bits, 2);
if (bits & I915_ACTIVE_MAY_SLEEP) if (bits & I915_ACTIVE_MAY_SLEEP)
ref->flags |= I915_ACTIVE_RETIRE_SLEEPS; ref->flags |= I915_ACTIVE_RETIRE_SLEEPS;
ref->excl = NULL;
ref->tree = RB_ROOT; ref->tree = RB_ROOT;
ref->cache = NULL; ref->cache = NULL;
init_llist_head(&ref->preallocated_barriers); init_llist_head(&ref->preallocated_barriers);
atomic_set(&ref->count, 0); atomic_set(&ref->count, 0);
__mutex_init(&ref->mutex, "i915_active", key); __mutex_init(&ref->mutex, "i915_active", key);
__i915_active_fence_init(&ref->excl, &ref->mutex, NULL, excl_retire);
INIT_WORK(&ref->work, active_work); INIT_WORK(&ref->work, active_work);
} }
...@@ -329,9 +336,9 @@ __active_del_barrier(struct i915_active *ref, struct active_node *node) ...@@ -329,9 +336,9 @@ __active_del_barrier(struct i915_active *ref, struct active_node *node)
int i915_active_ref(struct i915_active *ref, int i915_active_ref(struct i915_active *ref,
struct intel_timeline *tl, struct intel_timeline *tl,
struct i915_request *rq) struct dma_fence *fence)
{ {
struct i915_active_request *active; struct i915_active_fence *active;
int err; int err;
lockdep_assert_held(&tl->mutex); lockdep_assert_held(&tl->mutex);
...@@ -354,66 +361,44 @@ int i915_active_ref(struct i915_active *ref, ...@@ -354,66 +361,44 @@ int i915_active_ref(struct i915_active *ref,
* request that we want to emit on the kernel_context. * request that we want to emit on the kernel_context.
*/ */
__active_del_barrier(ref, node_from_active(active)); __active_del_barrier(ref, node_from_active(active));
RCU_INIT_POINTER(active->request, NULL); RCU_INIT_POINTER(active->fence, NULL);
INIT_LIST_HEAD(&active->link); atomic_dec(&ref->count);
} else {
if (!i915_active_request_isset(active))
atomic_inc(&ref->count);
} }
GEM_BUG_ON(!atomic_read(&ref->count)); if (!__i915_active_fence_set(active, fence))
__i915_active_request_set(active, rq); atomic_inc(&ref->count);
out: out:
i915_active_release(ref); i915_active_release(ref);
return err; return err;
} }
static void excl_cb(struct dma_fence *f, struct dma_fence_cb *cb)
{
struct i915_active *ref = container_of(cb, typeof(*ref), excl_cb);
RCU_INIT_POINTER(ref->excl, NULL);
dma_fence_put(f);
active_retire(ref);
}
void i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f) void i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f)
{ {
/* We expect the caller to manage the exclusive timeline ordering */ /* We expect the caller to manage the exclusive timeline ordering */
GEM_BUG_ON(i915_active_is_idle(ref)); GEM_BUG_ON(i915_active_is_idle(ref));
dma_fence_get(f); /*
* As we don't know which mutex the caller is using, we told a small
rcu_read_lock(); * lie to the debug code that it is using the i915_active.mutex;
if (rcu_access_pointer(ref->excl)) { * and now we must stick to that lie.
struct dma_fence *old; */
mutex_acquire(&ref->mutex.dep_map, 0, 0, _THIS_IP_);
old = dma_fence_get_rcu_safe(&ref->excl); if (!__i915_active_fence_set(&ref->excl, f))
if (old) { atomic_inc(&ref->count);
if (dma_fence_remove_callback(old, &ref->excl_cb)) mutex_release(&ref->mutex.dep_map, 0, _THIS_IP_);
atomic_dec(&ref->count); }
dma_fence_put(old);
}
}
rcu_read_unlock();
atomic_inc(&ref->count);
rcu_assign_pointer(ref->excl, f);
if (dma_fence_add_callback(f, &ref->excl_cb, excl_cb)) { bool i915_active_acquire_if_busy(struct i915_active *ref)
RCU_INIT_POINTER(ref->excl, NULL); {
atomic_dec(&ref->count); debug_active_assert(ref);
dma_fence_put(f); return atomic_add_unless(&ref->count, 1, 0);
}
} }
int i915_active_acquire(struct i915_active *ref) int i915_active_acquire(struct i915_active *ref)
{ {
int err; int err;
debug_active_assert(ref); if (i915_active_acquire_if_busy(ref))
if (atomic_add_unless(&ref->count, 1, 0))
return 0; return 0;
err = mutex_lock_interruptible(&ref->mutex); err = mutex_lock_interruptible(&ref->mutex);
...@@ -438,121 +423,57 @@ void i915_active_release(struct i915_active *ref) ...@@ -438,121 +423,57 @@ void i915_active_release(struct i915_active *ref)
active_retire(ref); active_retire(ref);
} }
static void __active_ungrab(struct i915_active *ref) static void enable_signaling(struct i915_active_fence *active)
{
clear_and_wake_up_bit(I915_ACTIVE_GRAB_BIT, &ref->flags);
}
bool i915_active_trygrab(struct i915_active *ref)
{ {
debug_active_assert(ref); struct dma_fence *fence;
if (test_and_set_bit(I915_ACTIVE_GRAB_BIT, &ref->flags))
return false;
if (!atomic_add_unless(&ref->count, 1, 0)) {
__active_ungrab(ref);
return false;
}
return true; fence = i915_active_fence_get(active);
} if (!fence)
return;
void i915_active_ungrab(struct i915_active *ref)
{
GEM_BUG_ON(!test_bit(I915_ACTIVE_GRAB_BIT, &ref->flags));
active_retire(ref);
__active_ungrab(ref);
}
static int excl_wait(struct i915_active *ref)
{
struct dma_fence *old;
int err = 0;
if (!rcu_access_pointer(ref->excl))
return 0;
rcu_read_lock();
old = dma_fence_get_rcu_safe(&ref->excl);
rcu_read_unlock();
if (old) {
err = dma_fence_wait(old, true);
dma_fence_put(old);
}
return err; dma_fence_enable_sw_signaling(fence);
dma_fence_put(fence);
} }
int i915_active_wait(struct i915_active *ref) int i915_active_wait(struct i915_active *ref)
{ {
struct active_node *it, *n; struct active_node *it, *n;
int err; int err = 0;
might_sleep(); might_sleep();
might_lock(&ref->mutex);
if (i915_active_is_idle(ref))
return 0;
err = mutex_lock_interruptible(&ref->mutex);
if (err)
return err;
if (!atomic_add_unless(&ref->count, 1, 0)) { if (!i915_active_acquire_if_busy(ref))
mutex_unlock(&ref->mutex);
return 0; return 0;
}
err = excl_wait(ref);
if (err)
goto out;
/* Flush lazy signals */
enable_signaling(&ref->excl);
rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
if (is_barrier(&it->base)) { /* unconnected idle-barrier */ if (is_barrier(&it->base)) /* unconnected idle barrier */
err = -EBUSY; continue;
break;
}
err = i915_active_request_retire(&it->base, BKL(ref)); enable_signaling(&it->base);
if (err)
break;
} }
/* Any fence added after the wait begins will not be auto-signaled */
out: i915_active_release(ref);
__active_retire(ref);
if (err) if (err)
return err; return err;
if (wait_on_bit(&ref->flags, I915_ACTIVE_GRAB_BIT, TASK_KILLABLE)) if (wait_var_event_interruptible(ref, i915_active_is_idle(ref)))
return -EINTR; return -EINTR;
flush_work(&ref->work);
if (!i915_active_is_idle(ref))
return -EBUSY;
return 0; return 0;
} }
int i915_request_await_active_request(struct i915_request *rq,
struct i915_active_request *active)
{
struct i915_request *barrier =
i915_active_request_raw(active, &rq->i915->drm.struct_mutex);
return barrier ? i915_request_await_dma_fence(rq, &barrier->fence) : 0;
}
int i915_request_await_active(struct i915_request *rq, struct i915_active *ref) int i915_request_await_active(struct i915_request *rq, struct i915_active *ref)
{ {
int err = 0; int err = 0;
if (rcu_access_pointer(ref->excl)) { if (rcu_access_pointer(ref->excl.fence)) {
struct dma_fence *fence; struct dma_fence *fence;
rcu_read_lock(); rcu_read_lock();
fence = dma_fence_get_rcu_safe(&ref->excl); fence = dma_fence_get_rcu_safe(&ref->excl.fence);
rcu_read_unlock(); rcu_read_unlock();
if (fence) { if (fence) {
err = i915_request_await_dma_fence(rq, fence); err = i915_request_await_dma_fence(rq, fence);
...@@ -578,7 +499,7 @@ void i915_active_fini(struct i915_active *ref) ...@@ -578,7 +499,7 @@ void i915_active_fini(struct i915_active *ref)
static inline bool is_idle_barrier(struct active_node *node, u64 idx) static inline bool is_idle_barrier(struct active_node *node, u64 idx)
{ {
return node->timeline == idx && !i915_active_request_isset(&node->base); return node->timeline == idx && !i915_active_fence_isset(&node->base);
} }
static struct active_node *reuse_idle_barrier(struct i915_active *ref, u64 idx) static struct active_node *reuse_idle_barrier(struct i915_active *ref, u64 idx)
...@@ -698,13 +619,13 @@ int i915_active_acquire_preallocate_barrier(struct i915_active *ref, ...@@ -698,13 +619,13 @@ int i915_active_acquire_preallocate_barrier(struct i915_active *ref,
node->base.lock = node->base.lock =
&engine->kernel_context->timeline->mutex; &engine->kernel_context->timeline->mutex;
#endif #endif
RCU_INIT_POINTER(node->base.request, NULL); RCU_INIT_POINTER(node->base.fence, NULL);
node->base.retire = node_retire; node->base.cb.func = node_retire;
node->timeline = idx; node->timeline = idx;
node->ref = ref; node->ref = ref;
} }
if (!i915_active_request_isset(&node->base)) { if (!i915_active_fence_isset(&node->base)) {
/* /*
* Mark this as being *our* unconnected proto-node. * Mark this as being *our* unconnected proto-node.
* *
...@@ -714,8 +635,8 @@ int i915_active_acquire_preallocate_barrier(struct i915_active *ref, ...@@ -714,8 +635,8 @@ int i915_active_acquire_preallocate_barrier(struct i915_active *ref,
* and then we can use the rb_node and list pointers * and then we can use the rb_node and list pointers
* for our tracking of the pending barrier. * for our tracking of the pending barrier.
*/ */
RCU_INIT_POINTER(node->base.request, ERR_PTR(-EAGAIN)); RCU_INIT_POINTER(node->base.fence, ERR_PTR(-EAGAIN));
node->base.link.prev = (void *)engine; node->base.cb.node.prev = (void *)engine;
atomic_inc(&ref->count); atomic_inc(&ref->count);
} }
...@@ -782,44 +703,113 @@ void i915_request_add_active_barriers(struct i915_request *rq) ...@@ -782,44 +703,113 @@ void i915_request_add_active_barriers(struct i915_request *rq)
{ {
struct intel_engine_cs *engine = rq->engine; struct intel_engine_cs *engine = rq->engine;
struct llist_node *node, *next; struct llist_node *node, *next;
unsigned long flags;
GEM_BUG_ON(intel_engine_is_virtual(engine)); GEM_BUG_ON(intel_engine_is_virtual(engine));
GEM_BUG_ON(i915_request_timeline(rq) != engine->kernel_context->timeline); GEM_BUG_ON(i915_request_timeline(rq) != engine->kernel_context->timeline);
node = llist_del_all(&engine->barrier_tasks);
if (!node)
return;
/* /*
* Attach the list of proto-fences to the in-flight request such * Attach the list of proto-fences to the in-flight request such
* that the parent i915_active will be released when this request * that the parent i915_active will be released when this request
* is retired. * is retired.
*/ */
llist_for_each_safe(node, next, llist_del_all(&engine->barrier_tasks)) { spin_lock_irqsave(&rq->lock, flags);
RCU_INIT_POINTER(barrier_from_ll(node)->base.request, rq); llist_for_each_safe(node, next, node) {
RCU_INIT_POINTER(barrier_from_ll(node)->base.fence, &rq->fence);
smp_wmb(); /* serialise with reuse_idle_barrier */ smp_wmb(); /* serialise with reuse_idle_barrier */
list_add_tail((struct list_head *)node, &rq->active_list); list_add_tail((struct list_head *)node, &rq->fence.cb_list);
}
spin_unlock_irqrestore(&rq->lock, flags);
}
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
#define active_is_held(active) lockdep_is_held((active)->lock)
#else
#define active_is_held(active) true
#endif
/*
* __i915_active_fence_set: Update the last active fence along its timeline
* @active: the active tracker
* @fence: the new fence (under construction)
*
* Records the new @fence as the last active fence along its timeline in
* this active tracker, moving the tracking callbacks from the previous
* fence onto this one. Returns the previous fence (if not already completed),
* which the caller must ensure is executed before the new fence. To ensure
* that the order of fences within the timeline of the i915_active_fence is
* maintained, it must be locked by the caller.
*/
struct dma_fence *
__i915_active_fence_set(struct i915_active_fence *active,
struct dma_fence *fence)
{
struct dma_fence *prev;
unsigned long flags;
/* NB: must be serialised by an outer timeline mutex (active->lock) */
spin_lock_irqsave(fence->lock, flags);
GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags));
prev = rcu_dereference_protected(active->fence, active_is_held(active));
if (prev) {
GEM_BUG_ON(prev == fence);
spin_lock_nested(prev->lock, SINGLE_DEPTH_NESTING);
__list_del_entry(&active->cb.node);
spin_unlock(prev->lock); /* serialise with prev->cb_list */
/*
* active->fence is reset by the callback from inside
* interrupt context. We need to serialise our list
* manipulation with the fence->lock to prevent the prev
* being lost inside an interrupt (it can't be replaced as
* no other caller is allowed to enter __i915_active_fence_set
* as we hold the timeline lock). After serialising with
* the callback, we need to double check which ran first,
* our list_del() [decoupling prev from the callback] or
* the callback...
*/
prev = rcu_access_pointer(active->fence);
} }
rcu_assign_pointer(active->fence, fence);
list_add_tail(&active->cb.node, &fence->cb_list);
spin_unlock_irqrestore(fence->lock, flags);
return prev;
} }
int i915_active_request_set(struct i915_active_request *active, int i915_active_fence_set(struct i915_active_fence *active,
struct i915_request *rq) struct i915_request *rq)
{ {
int err; struct dma_fence *fence;
int err = 0;
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
lockdep_assert_held(active->lock); lockdep_assert_held(active->lock);
#endif #endif
/* Must maintain ordering wrt previous active requests */ /* Must maintain timeline ordering wrt previous active requests */
err = i915_request_await_active_request(rq, active); rcu_read_lock();
if (err) fence = __i915_active_fence_set(active, &rq->fence);
return err; if (fence) /* but the previous fence may not belong to that timeline! */
fence = dma_fence_get_rcu(fence);
rcu_read_unlock();
if (fence) {
err = i915_request_await_dma_fence(rq, fence);
dma_fence_put(fence);
}
__i915_active_request_set(active, rq); return err;
return 0;
} }
void i915_active_retire_noop(struct i915_active_request *active, void i915_active_noop(struct dma_fence *fence, struct dma_fence_cb *cb)
struct i915_request *request)
{ {
/* Space left intentionally blank */ i915_active_fence_cb(fence, cb);
} }
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
......
...@@ -12,6 +12,10 @@ ...@@ -12,6 +12,10 @@
#include "i915_active_types.h" #include "i915_active_types.h"
#include "i915_request.h" #include "i915_request.h"
struct i915_request;
struct intel_engine_cs;
struct intel_timeline;
/* /*
* We treat requests as fences. This is not be to confused with our * We treat requests as fences. This is not be to confused with our
* "fence registers" but pipeline synchronisation objects ala GL_ARB_sync. * "fence registers" but pipeline synchronisation objects ala GL_ARB_sync.
...@@ -28,308 +32,108 @@ ...@@ -28,308 +32,108 @@
* write access so that we can perform concurrent read operations between * write access so that we can perform concurrent read operations between
* the CPU and GPU engines, as well as waiting for all rendering to * the CPU and GPU engines, as well as waiting for all rendering to
* complete, or waiting for the last GPU user of a "fence register". The * complete, or waiting for the last GPU user of a "fence register". The
* object then embeds a #i915_active_request to track the most recent (in * object then embeds a #i915_active_fence to track the most recent (in
* retirement order) request relevant for the desired mode of access. * retirement order) request relevant for the desired mode of access.
* The #i915_active_request is updated with i915_active_request_set() to * The #i915_active_fence is updated with i915_active_fence_set() to
* track the most recent fence request, typically this is done as part of * track the most recent fence request, typically this is done as part of
* i915_vma_move_to_active(). * i915_vma_move_to_active().
* *
* When the #i915_active_request completes (is retired), it will * When the #i915_active_fence completes (is retired), it will
* signal its completion to the owner through a callback as well as mark * signal its completion to the owner through a callback as well as mark
* itself as idle (i915_active_request.request == NULL). The owner * itself as idle (i915_active_fence.request == NULL). The owner
* can then perform any action, such as delayed freeing of an active * can then perform any action, such as delayed freeing of an active
* resource including itself. * resource including itself.
*/ */
void i915_active_retire_noop(struct i915_active_request *active, void i915_active_noop(struct dma_fence *fence, struct dma_fence_cb *cb);
struct i915_request *request);
/** /**
* i915_active_request_init - prepares the activity tracker for use * __i915_active_fence_init - prepares the activity tracker for use
* @active - the active tracker * @active - the active tracker
* @rq - initial request to track, can be NULL * @fence - initial fence to track, can be NULL
* @func - a callback when then the tracker is retired (becomes idle), * @func - a callback when then the tracker is retired (becomes idle),
* can be NULL * can be NULL
* *
* i915_active_request_init() prepares the embedded @active struct for use as * i915_active_fence_init() prepares the embedded @active struct for use as
* an activity tracker, that is for tracking the last known active request * an activity tracker, that is for tracking the last known active fence
* associated with it. When the last request becomes idle, when it is retired * associated with it. When the last fence becomes idle, when it is retired
* after completion, the optional callback @func is invoked. * after completion, the optional callback @func is invoked.
*/ */
static inline void static inline void
i915_active_request_init(struct i915_active_request *active, __i915_active_fence_init(struct i915_active_fence *active,
struct mutex *lock, struct mutex *lock,
struct i915_request *rq, void *fence,
i915_active_retire_fn retire) dma_fence_func_t fn)
{ {
RCU_INIT_POINTER(active->request, rq); RCU_INIT_POINTER(active->fence, fence);
INIT_LIST_HEAD(&active->link); active->cb.func = fn ?: i915_active_noop;
active->retire = retire ?: i915_active_retire_noop;
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
active->lock = lock; active->lock = lock;
#endif #endif
} }
#define INIT_ACTIVE_REQUEST(name, lock) \ #define INIT_ACTIVE_FENCE(A, LOCK) \
i915_active_request_init((name), (lock), NULL, NULL) __i915_active_fence_init((A), (LOCK), NULL, NULL)
/**
* i915_active_request_set - updates the tracker to watch the current request
* @active - the active tracker
* @request - the request to watch
*
* __i915_active_request_set() watches the given @request for completion. Whilst
* that @request is busy, the @active reports busy. When that @request is
* retired, the @active tracker is updated to report idle.
*/
static inline void
__i915_active_request_set(struct i915_active_request *active,
struct i915_request *request)
{
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
lockdep_assert_held(active->lock);
#endif
list_move(&active->link, &request->active_list);
rcu_assign_pointer(active->request, request);
}
int __must_check struct dma_fence *
i915_active_request_set(struct i915_active_request *active, __i915_active_fence_set(struct i915_active_fence *active,
struct i915_request *rq); struct dma_fence *fence);
/** /**
* i915_active_request_raw - return the active request * i915_active_fence_set - updates the tracker to watch the current fence
* @active - the active tracker * @active - the active tracker
* @rq - the request to watch
* *
* i915_active_request_raw() returns the current request being tracked, or NULL. * i915_active_fence_set() watches the given @rq for completion. While
* It does not obtain a reference on the request for the caller, so the caller * that @rq is busy, the @active reports busy. When that @rq is signaled
* must hold struct_mutex. * (or else retired) the @active tracker is updated to report idle.
*/ */
static inline struct i915_request * int __must_check
i915_active_request_raw(const struct i915_active_request *active, i915_active_fence_set(struct i915_active_fence *active,
struct mutex *mutex) struct i915_request *rq);
{
return rcu_dereference_protected(active->request,
lockdep_is_held(mutex));
}
/**
* i915_active_request_peek - report the active request being monitored
* @active - the active tracker
*
* i915_active_request_peek() returns the current request being tracked if
* still active, or NULL. It does not obtain a reference on the request
* for the caller, so the caller must hold struct_mutex.
*/
static inline struct i915_request *
i915_active_request_peek(const struct i915_active_request *active,
struct mutex *mutex)
{
struct i915_request *request;
request = i915_active_request_raw(active, mutex);
if (!request || i915_request_completed(request))
return NULL;
return request;
}
/**
* i915_active_request_get - return a reference to the active request
* @active - the active tracker
*
* i915_active_request_get() returns a reference to the active request, or NULL
* if the active tracker is idle. The caller must hold struct_mutex.
*/
static inline struct i915_request *
i915_active_request_get(const struct i915_active_request *active,
struct mutex *mutex)
{
return i915_request_get(i915_active_request_peek(active, mutex));
}
/**
* __i915_active_request_get_rcu - return a reference to the active request
* @active - the active tracker
*
* __i915_active_request_get() returns a reference to the active request,
* or NULL if the active tracker is idle. The caller must hold the RCU read
* lock, but the returned pointer is safe to use outside of RCU.
*/
static inline struct i915_request *
__i915_active_request_get_rcu(const struct i915_active_request *active)
{
/*
* Performing a lockless retrieval of the active request is super
* tricky. SLAB_TYPESAFE_BY_RCU merely guarantees that the backing
* slab of request objects will not be freed whilst we hold the
* RCU read lock. It does not guarantee that the request itself
* will not be freed and then *reused*. Viz,
*
* Thread A Thread B
*
* rq = active.request
* retire(rq) -> free(rq);
* (rq is now first on the slab freelist)
* active.request = NULL
*
* rq = new submission on a new object
* ref(rq)
*
* To prevent the request from being reused whilst the caller
* uses it, we take a reference like normal. Whilst acquiring
* the reference we check that it is not in a destroyed state
* (refcnt == 0). That prevents the request being reallocated
* whilst the caller holds on to it. To check that the request
* was not reallocated as we acquired the reference we have to
* check that our request remains the active request across
* the lookup, in the same manner as a seqlock. The visibility
* of the pointer versus the reference counting is controlled
* by using RCU barriers (rcu_dereference and rcu_assign_pointer).
*
* In the middle of all that, we inspect whether the request is
* complete. Retiring is lazy so the request may be completed long
* before the active tracker is updated. Querying whether the
* request is complete is far cheaper (as it involves no locked
* instructions setting cachelines to exclusive) than acquiring
* the reference, so we do it first. The RCU read lock ensures the
* pointer dereference is valid, but does not ensure that the
* seqno nor HWS is the right one! However, if the request was
* reallocated, that means the active tracker's request was complete.
* If the new request is also complete, then both are and we can
* just report the active tracker is idle. If the new request is
* incomplete, then we acquire a reference on it and check that
* it remained the active request.
*
* It is then imperative that we do not zero the request on
* reallocation, so that we can chase the dangling pointers!
* See i915_request_alloc().
*/
do {
struct i915_request *request;
request = rcu_dereference(active->request);
if (!request || i915_request_completed(request))
return NULL;
/*
* An especially silly compiler could decide to recompute the
* result of i915_request_completed, more specifically
* re-emit the load for request->fence.seqno. A race would catch
* a later seqno value, which could flip the result from true to
* false. Which means part of the instructions below might not
* be executed, while later on instructions are executed. Due to
* barriers within the refcounting the inconsistency can't reach
* past the call to i915_request_get_rcu, but not executing
* that while still executing i915_request_put() creates
* havoc enough. Prevent this with a compiler barrier.
*/
barrier();
request = i915_request_get_rcu(request);
/*
* What stops the following rcu_access_pointer() from occurring
* before the above i915_request_get_rcu()? If we were
* to read the value before pausing to get the reference to
* the request, we may not notice a change in the active
* tracker.
*
* The rcu_access_pointer() is a mere compiler barrier, which
* means both the CPU and compiler are free to perform the
* memory read without constraint. The compiler only has to
* ensure that any operations after the rcu_access_pointer()
* occur afterwards in program order. This means the read may
* be performed earlier by an out-of-order CPU, or adventurous
* compiler.
*
* The atomic operation at the heart of
* i915_request_get_rcu(), see dma_fence_get_rcu(), is
* atomic_inc_not_zero() which is only a full memory barrier
* when successful. That is, if i915_request_get_rcu()
* returns the request (and so with the reference counted
* incremented) then the following read for rcu_access_pointer()
* must occur after the atomic operation and so confirm
* that this request is the one currently being tracked.
*
* The corresponding write barrier is part of
* rcu_assign_pointer().
*/
if (!request || request == rcu_access_pointer(active->request))
return rcu_pointer_handoff(request);
i915_request_put(request);
} while (1);
}
/** /**
* i915_active_request_get_unlocked - return a reference to the active request * i915_active_fence_get - return a reference to the active fence
* @active - the active tracker * @active - the active tracker
* *
* i915_active_request_get_unlocked() returns a reference to the active request, * i915_active_fence_get() returns a reference to the active fence,
* or NULL if the active tracker is idle. The reference is obtained under RCU, * or NULL if the active tracker is idle. The reference is obtained under RCU,
* so no locking is required by the caller. * so no locking is required by the caller.
* *
* The reference should be freed with i915_request_put(). * The reference should be freed with dma_fence_put().
*/ */
static inline struct i915_request * static inline struct dma_fence *
i915_active_request_get_unlocked(const struct i915_active_request *active) i915_active_fence_get(struct i915_active_fence *active)
{ {
struct i915_request *request; struct dma_fence *fence;
rcu_read_lock(); rcu_read_lock();
request = __i915_active_request_get_rcu(active); fence = dma_fence_get_rcu_safe(&active->fence);
rcu_read_unlock(); rcu_read_unlock();
return request; return fence;
} }
/** /**
* i915_active_request_isset - report whether the active tracker is assigned * i915_active_fence_isset - report whether the active tracker is assigned
* @active - the active tracker * @active - the active tracker
* *
* i915_active_request_isset() returns true if the active tracker is currently * i915_active_fence_isset() returns true if the active tracker is currently
* assigned to a request. Due to the lazy retiring, that request may be idle * assigned to a fence. Due to the lazy retiring, that fence may be idle
* and this may report stale information. * and this may report stale information.
*/ */
static inline bool static inline bool
i915_active_request_isset(const struct i915_active_request *active) i915_active_fence_isset(const struct i915_active_fence *active)
{ {
return rcu_access_pointer(active->request); return rcu_access_pointer(active->fence);
} }
/** static inline void
* i915_active_request_retire - waits until the request is retired i915_active_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
* @active - the active request on which to wait
*
* i915_active_request_retire() waits until the request is completed,
* and then ensures that at least the retirement handler for this
* @active tracker is called before returning. If the @active
* tracker is idle, the function returns immediately.
*/
static inline int __must_check
i915_active_request_retire(struct i915_active_request *active,
struct mutex *mutex)
{ {
struct i915_request *request; struct i915_active_fence *active =
long ret; container_of(cb, typeof(*active), cb);
request = i915_active_request_raw(active, mutex);
if (!request)
return 0;
ret = i915_request_wait(request,
I915_WAIT_INTERRUPTIBLE,
MAX_SCHEDULE_TIMEOUT);
if (ret < 0)
return ret;
list_del_init(&active->link); RCU_INIT_POINTER(active->fence, NULL);
RCU_INIT_POINTER(active->request, NULL);
active->retire(active, request);
return 0;
} }
/* /*
...@@ -358,47 +162,40 @@ i915_active_request_retire(struct i915_active_request *active, ...@@ -358,47 +162,40 @@ i915_active_request_retire(struct i915_active_request *active,
* synchronisation. * synchronisation.
*/ */
void __i915_active_init(struct drm_i915_private *i915, void __i915_active_init(struct i915_active *ref,
struct i915_active *ref,
int (*active)(struct i915_active *ref), int (*active)(struct i915_active *ref),
void (*retire)(struct i915_active *ref), void (*retire)(struct i915_active *ref),
struct lock_class_key *key); struct lock_class_key *key);
#define i915_active_init(i915, ref, active, retire) do { \ #define i915_active_init(ref, active, retire) do { \
static struct lock_class_key __key; \ static struct lock_class_key __key; \
\ \
__i915_active_init(i915, ref, active, retire, &__key); \ __i915_active_init(ref, active, retire, &__key); \
} while (0) } while (0)
int i915_active_ref(struct i915_active *ref, int i915_active_ref(struct i915_active *ref,
struct intel_timeline *tl, struct intel_timeline *tl,
struct i915_request *rq); struct dma_fence *fence);
static inline int static inline int
i915_active_add_request(struct i915_active *ref, struct i915_request *rq) i915_active_add_request(struct i915_active *ref, struct i915_request *rq)
{ {
return i915_active_ref(ref, i915_request_timeline(rq), rq); return i915_active_ref(ref, i915_request_timeline(rq), &rq->fence);
} }
void i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f); void i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f);
static inline bool i915_active_has_exclusive(struct i915_active *ref) static inline bool i915_active_has_exclusive(struct i915_active *ref)
{ {
return rcu_access_pointer(ref->excl); return rcu_access_pointer(ref->excl.fence);
} }
int i915_active_wait(struct i915_active *ref); int i915_active_wait(struct i915_active *ref);
int i915_request_await_active(struct i915_request *rq, int i915_request_await_active(struct i915_request *rq, struct i915_active *ref);
struct i915_active *ref);
int i915_request_await_active_request(struct i915_request *rq,
struct i915_active_request *active);
int i915_active_acquire(struct i915_active *ref); int i915_active_acquire(struct i915_active *ref);
bool i915_active_acquire_if_busy(struct i915_active *ref);
void i915_active_release(struct i915_active *ref); void i915_active_release(struct i915_active *ref);
void __i915_active_release_nested(struct i915_active *ref, int subclass);
bool i915_active_trygrab(struct i915_active *ref);
void i915_active_ungrab(struct i915_active *ref);
static inline bool static inline bool
i915_active_is_idle(const struct i915_active *ref) i915_active_is_idle(const struct i915_active *ref)
......
...@@ -17,17 +17,9 @@ ...@@ -17,17 +17,9 @@
#include "i915_utils.h" #include "i915_utils.h"
struct drm_i915_private; struct i915_active_fence {
struct i915_active_request; struct dma_fence __rcu *fence;
struct i915_request; struct dma_fence_cb cb;
typedef void (*i915_active_retire_fn)(struct i915_active_request *,
struct i915_request *);
struct i915_active_request {
struct i915_request __rcu *request;
struct list_head link;
i915_active_retire_fn retire;
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
/* /*
* Incorporeal! * Incorporeal!
...@@ -53,20 +45,17 @@ struct active_node; ...@@ -53,20 +45,17 @@ struct active_node;
#define i915_active_may_sleep(fn) ptr_pack_bits(&(fn), I915_ACTIVE_MAY_SLEEP, 2) #define i915_active_may_sleep(fn) ptr_pack_bits(&(fn), I915_ACTIVE_MAY_SLEEP, 2)
struct i915_active { struct i915_active {
struct drm_i915_private *i915; atomic_t count;
struct mutex mutex;
struct active_node *cache; struct active_node *cache;
struct rb_root tree; struct rb_root tree;
struct mutex mutex;
atomic_t count;
/* Preallocated "exclusive" node */ /* Preallocated "exclusive" node */
struct dma_fence __rcu *excl; struct i915_active_fence excl;
struct dma_fence_cb excl_cb;
unsigned long flags; unsigned long flags;
#define I915_ACTIVE_RETIRE_SLEEPS BIT(0) #define I915_ACTIVE_RETIRE_SLEEPS BIT(0)
#define I915_ACTIVE_GRAB_BIT 1
int (*active)(struct i915_active *ref); int (*active)(struct i915_active *ref);
void (*retire)(struct i915_active *ref); void (*retire)(struct i915_active *ref);
......
...@@ -892,28 +892,38 @@ wait_for_timelines(struct intel_gt *gt, unsigned int wait, long timeout) ...@@ -892,28 +892,38 @@ wait_for_timelines(struct intel_gt *gt, unsigned int wait, long timeout)
spin_lock_irqsave(&timelines->lock, flags); spin_lock_irqsave(&timelines->lock, flags);
list_for_each_entry(tl, &timelines->active_list, link) { list_for_each_entry(tl, &timelines->active_list, link) {
struct i915_request *rq; struct dma_fence *fence;
rq = i915_active_request_get_unlocked(&tl->last_request); fence = i915_active_fence_get(&tl->last_request);
if (!rq) if (!fence)
continue; continue;
spin_unlock_irqrestore(&timelines->lock, flags); spin_unlock_irqrestore(&timelines->lock, flags);
/* if (!dma_fence_is_i915(fence)) {
* "Race-to-idle". timeout = dma_fence_wait_timeout(fence,
* flags & I915_WAIT_INTERRUPTIBLE,
* Switching to the kernel context is often used a synchronous timeout);
* step prior to idling, e.g. in suspend for flushing all } else {
* current operations to memory before sleeping. These we struct i915_request *rq = to_request(fence);
* want to complete as quickly as possible to avoid prolonged
* stalls, so allow the gpu to boost to maximum clocks. /*
*/ * "Race-to-idle".
if (wait & I915_WAIT_FOR_IDLE_BOOST) *
gen6_rps_boost(rq); * Switching to the kernel context is often used as
* a synchronous step prior to idling, e.g. in suspend
* for flushing all current operations to memory before
* sleeping. These we want to complete as quickly as
* possible to avoid prolonged stalls, so allow the gpu
* to boost to maximum clocks.
*/
if (flags & I915_WAIT_FOR_IDLE_BOOST)
gen6_rps_boost(rq);
timeout = i915_request_wait(rq, flags, timeout);
}
timeout = i915_request_wait(rq, wait, timeout); dma_fence_put(fence);
i915_request_put(rq);
if (timeout < 0) if (timeout < 0)
return timeout; return timeout;
......
...@@ -1861,7 +1861,6 @@ static const struct i915_vma_ops pd_vma_ops = { ...@@ -1861,7 +1861,6 @@ static const struct i915_vma_ops pd_vma_ops = {
static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size) static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size)
{ {
struct drm_i915_private *i915 = ppgtt->base.vm.i915;
struct i915_ggtt *ggtt = ppgtt->base.vm.gt->ggtt; struct i915_ggtt *ggtt = ppgtt->base.vm.gt->ggtt;
struct i915_vma *vma; struct i915_vma *vma;
...@@ -1872,7 +1871,7 @@ static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size) ...@@ -1872,7 +1871,7 @@ static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size)
if (!vma) if (!vma)
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
i915_active_init(i915, &vma->active, NULL, NULL); i915_active_init(&vma->active, NULL, NULL);
mutex_init(&vma->pages_mutex); mutex_init(&vma->pages_mutex);
vma->vm = i915_vm_get(&ggtt->vm); vma->vm = i915_vm_get(&ggtt->vm);
......
...@@ -1299,7 +1299,7 @@ capture_vma(struct capture_vma *next, ...@@ -1299,7 +1299,7 @@ capture_vma(struct capture_vma *next,
if (!c) if (!c)
return next; return next;
if (!i915_active_trygrab(&vma->active)) { if (!i915_active_acquire_if_busy(&vma->active)) {
kfree(c); kfree(c);
return next; return next;
} }
...@@ -1439,7 +1439,7 @@ gem_record_rings(struct i915_gpu_state *error, struct compress *compress) ...@@ -1439,7 +1439,7 @@ gem_record_rings(struct i915_gpu_state *error, struct compress *compress)
*this->slot = *this->slot =
i915_error_object_create(i915, vma, compress); i915_error_object_create(i915, vma, compress);
i915_active_ungrab(&vma->active); i915_active_release(&vma->active);
i915_vma_put(vma); i915_vma_put(vma);
capture = this->next; capture = this->next;
......
...@@ -218,8 +218,6 @@ static void remove_from_engine(struct i915_request *rq) ...@@ -218,8 +218,6 @@ static void remove_from_engine(struct i915_request *rq)
static bool i915_request_retire(struct i915_request *rq) static bool i915_request_retire(struct i915_request *rq)
{ {
struct i915_active_request *active, *next;
if (!i915_request_completed(rq)) if (!i915_request_completed(rq))
return false; return false;
...@@ -244,35 +242,6 @@ static bool i915_request_retire(struct i915_request *rq) ...@@ -244,35 +242,6 @@ static bool i915_request_retire(struct i915_request *rq)
&i915_request_timeline(rq)->requests)); &i915_request_timeline(rq)->requests));
rq->ring->head = rq->postfix; rq->ring->head = rq->postfix;
/*
* Walk through the active list, calling retire on each. This allows
* objects to track their GPU activity and mark themselves as idle
* when their *last* active request is completed (updating state
* tracking lists for eviction, active references for GEM, etc).
*
* As the ->retire() may free the node, we decouple it first and
* pass along the auxiliary information (to avoid dereferencing
* the node after the callback).
*/
list_for_each_entry_safe(active, next, &rq->active_list, link) {
/*
* In microbenchmarks or focusing upon time inside the kernel,
* we may spend an inordinate amount of time simply handling
* the retirement of requests and processing their callbacks.
* Of which, this loop itself is particularly hot due to the
* cache misses when jumping around the list of
* i915_active_request. So we try to keep this loop as
* streamlined as possible and also prefetch the next
* i915_active_request to try and hide the likely cache miss.
*/
prefetchw(next);
INIT_LIST_HEAD(&active->link);
RCU_INIT_POINTER(active->request, NULL);
active->retire(active, rq);
}
local_irq_disable(); local_irq_disable();
/* /*
...@@ -704,7 +673,6 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) ...@@ -704,7 +673,6 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp)
rq->flags = 0; rq->flags = 0;
rq->execution_mask = ALL_ENGINES; rq->execution_mask = ALL_ENGINES;
INIT_LIST_HEAD(&rq->active_list);
INIT_LIST_HEAD(&rq->execute_cb); INIT_LIST_HEAD(&rq->execute_cb);
/* /*
...@@ -743,7 +711,6 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) ...@@ -743,7 +711,6 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp)
ce->ring->emit = rq->head; ce->ring->emit = rq->head;
/* Make sure we didn't add ourselves to external state before freeing */ /* Make sure we didn't add ourselves to external state before freeing */
GEM_BUG_ON(!list_empty(&rq->active_list));
GEM_BUG_ON(!list_empty(&rq->sched.signalers_list)); GEM_BUG_ON(!list_empty(&rq->sched.signalers_list));
GEM_BUG_ON(!list_empty(&rq->sched.waiters_list)); GEM_BUG_ON(!list_empty(&rq->sched.waiters_list));
...@@ -1174,8 +1141,8 @@ __i915_request_add_to_timeline(struct i915_request *rq) ...@@ -1174,8 +1141,8 @@ __i915_request_add_to_timeline(struct i915_request *rq)
* precludes optimising to use semaphores serialisation of a single * precludes optimising to use semaphores serialisation of a single
* timeline across engines. * timeline across engines.
*/ */
prev = rcu_dereference_protected(timeline->last_request.request, prev = to_request(__i915_active_fence_set(&timeline->last_request,
lockdep_is_held(&timeline->mutex)); &rq->fence));
if (prev && !i915_request_completed(prev)) { if (prev && !i915_request_completed(prev)) {
if (is_power_of_2(prev->engine->mask | rq->engine->mask)) if (is_power_of_2(prev->engine->mask | rq->engine->mask))
i915_sw_fence_await_sw_fence(&rq->submit, i915_sw_fence_await_sw_fence(&rq->submit,
...@@ -1200,7 +1167,6 @@ __i915_request_add_to_timeline(struct i915_request *rq) ...@@ -1200,7 +1167,6 @@ __i915_request_add_to_timeline(struct i915_request *rq)
* us, the timeline will hold its seqno which is later than ours. * us, the timeline will hold its seqno which is later than ours.
*/ */
GEM_BUG_ON(timeline->seqno != rq->fence.seqno); GEM_BUG_ON(timeline->seqno != rq->fence.seqno);
__i915_active_request_set(&timeline->last_request, rq);
return prev; return prev;
} }
......
...@@ -211,7 +211,6 @@ struct i915_request { ...@@ -211,7 +211,6 @@ struct i915_request {
* on the active_list (of their final request). * on the active_list (of their final request).
*/ */
struct i915_capture_list *capture_list; struct i915_capture_list *capture_list;
struct list_head active_list;
/** Time at which this request was emitted, in jiffies. */ /** Time at which this request was emitted, in jiffies. */
unsigned long emitted_jiffies; unsigned long emitted_jiffies;
......
...@@ -120,8 +120,7 @@ vma_create(struct drm_i915_gem_object *obj, ...@@ -120,8 +120,7 @@ vma_create(struct drm_i915_gem_object *obj,
vma->size = obj->base.size; vma->size = obj->base.size;
vma->display_alignment = I915_GTT_MIN_ALIGNMENT; vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
i915_active_init(vm->i915, &vma->active, i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire);
__i915_vma_active, __i915_vma_retire);
/* Declare ourselves safe for use inside shrinkers */ /* Declare ourselves safe for use inside shrinkers */
if (IS_ENABLED(CONFIG_LOCKDEP)) { if (IS_ENABLED(CONFIG_LOCKDEP)) {
...@@ -1148,6 +1147,7 @@ int __i915_vma_unbind(struct i915_vma *vma) ...@@ -1148,6 +1147,7 @@ int __i915_vma_unbind(struct i915_vma *vma)
if (ret) if (ret)
return ret; return ret;
GEM_BUG_ON(i915_vma_is_active(vma));
if (i915_vma_is_pinned(vma)) { if (i915_vma_is_pinned(vma)) {
vma_print_allocator(vma, "is pinned"); vma_print_allocator(vma, "is pinned");
return -EBUSY; return -EBUSY;
......
...@@ -68,7 +68,7 @@ static struct live_active *__live_alloc(struct drm_i915_private *i915) ...@@ -68,7 +68,7 @@ static struct live_active *__live_alloc(struct drm_i915_private *i915)
return NULL; return NULL;
kref_init(&active->ref); kref_init(&active->ref);
i915_active_init(i915, &active->base, __live_active, __live_retire); i915_active_init(&active->base, __live_active, __live_retire);
return active; return active;
} }
...@@ -146,19 +146,13 @@ static int live_active_wait(void *arg) ...@@ -146,19 +146,13 @@ static int live_active_wait(void *arg)
{ {
struct drm_i915_private *i915 = arg; struct drm_i915_private *i915 = arg;
struct live_active *active; struct live_active *active;
intel_wakeref_t wakeref;
int err = 0; int err = 0;
/* Check that we get a callback when requests retire upon waiting */ /* Check that we get a callback when requests retire upon waiting */
mutex_lock(&i915->drm.struct_mutex);
wakeref = intel_runtime_pm_get(&i915->runtime_pm);
active = __live_active_setup(i915); active = __live_active_setup(i915);
if (IS_ERR(active)) { if (IS_ERR(active))
err = PTR_ERR(active); return PTR_ERR(active);
goto err;
}
i915_active_wait(&active->base); i915_active_wait(&active->base);
if (!READ_ONCE(active->retired)) { if (!READ_ONCE(active->retired)) {
...@@ -168,11 +162,9 @@ static int live_active_wait(void *arg) ...@@ -168,11 +162,9 @@ static int live_active_wait(void *arg)
__live_put(active); __live_put(active);
mutex_lock(&i915->drm.struct_mutex);
if (igt_flush_test(i915, I915_WAIT_LOCKED)) if (igt_flush_test(i915, I915_WAIT_LOCKED))
err = -EIO; err = -EIO;
err:
intel_runtime_pm_put(&i915->runtime_pm, wakeref);
mutex_unlock(&i915->drm.struct_mutex); mutex_unlock(&i915->drm.struct_mutex);
return err; return err;
...@@ -182,23 +174,19 @@ static int live_active_retire(void *arg) ...@@ -182,23 +174,19 @@ static int live_active_retire(void *arg)
{ {
struct drm_i915_private *i915 = arg; struct drm_i915_private *i915 = arg;
struct live_active *active; struct live_active *active;
intel_wakeref_t wakeref;
int err = 0; int err = 0;
/* Check that we get a callback when requests are indirectly retired */ /* Check that we get a callback when requests are indirectly retired */
mutex_lock(&i915->drm.struct_mutex);
wakeref = intel_runtime_pm_get(&i915->runtime_pm);
active = __live_active_setup(i915); active = __live_active_setup(i915);
if (IS_ERR(active)) { if (IS_ERR(active))
err = PTR_ERR(active); return PTR_ERR(active);
goto err;
}
/* waits for & retires all requests */ /* waits for & retires all requests */
mutex_lock(&i915->drm.struct_mutex);
if (igt_flush_test(i915, I915_WAIT_LOCKED)) if (igt_flush_test(i915, I915_WAIT_LOCKED))
err = -EIO; err = -EIO;
mutex_unlock(&i915->drm.struct_mutex);
if (!READ_ONCE(active->retired)) { if (!READ_ONCE(active->retired)) {
pr_err("i915_active not retired after flushing!\n"); pr_err("i915_active not retired after flushing!\n");
...@@ -207,10 +195,6 @@ static int live_active_retire(void *arg) ...@@ -207,10 +195,6 @@ static int live_active_retire(void *arg)
__live_put(active); __live_put(active);
err:
intel_runtime_pm_put(&i915->runtime_pm, wakeref);
mutex_unlock(&i915->drm.struct_mutex);
return err; return err;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment