Commit d6ea8886 authored by Dave Airlie's avatar Dave Airlie

drm/ttm: Add a bo list reserve fastpath (v2)

Makes it possible to reserve a list of buffer objects with a single
spin lock / unlock if there is no contention.
Should improve cpu usage on SMP kernels.

v2: Initialize private list members on reserve and don't call
ttm_bo_list_ref_sub() with zero put_count.
Signed-off-by: default avatarThomas Hellstrom <thellstrom@vmware.com>
Signed-off-by: default avatarDave Airlie <airlied@redhat.com>
parent 27641c3f
...@@ -169,7 +169,7 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible) ...@@ -169,7 +169,7 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible)
} }
EXPORT_SYMBOL(ttm_bo_wait_unreserved); EXPORT_SYMBOL(ttm_bo_wait_unreserved);
static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
{ {
struct ttm_bo_device *bdev = bo->bdev; struct ttm_bo_device *bdev = bo->bdev;
struct ttm_mem_type_manager *man; struct ttm_mem_type_manager *man;
...@@ -191,11 +191,7 @@ static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) ...@@ -191,11 +191,7 @@ static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
} }
} }
/** int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
* Call with the lru_lock held.
*/
static int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
{ {
int put_count = 0; int put_count = 0;
...@@ -267,6 +263,15 @@ static void ttm_bo_ref_bug(struct kref *list_kref) ...@@ -267,6 +263,15 @@ static void ttm_bo_ref_bug(struct kref *list_kref)
BUG(); BUG();
} }
void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count,
bool never_free)
{
while (count--)
kref_put(&bo->list_kref,
(never_free || (count >= 0)) ? ttm_bo_ref_bug :
ttm_bo_release_list);
}
int ttm_bo_reserve(struct ttm_buffer_object *bo, int ttm_bo_reserve(struct ttm_buffer_object *bo,
bool interruptible, bool interruptible,
bool no_wait, bool use_sequence, uint32_t sequence) bool no_wait, bool use_sequence, uint32_t sequence)
...@@ -282,8 +287,7 @@ int ttm_bo_reserve(struct ttm_buffer_object *bo, ...@@ -282,8 +287,7 @@ int ttm_bo_reserve(struct ttm_buffer_object *bo,
put_count = ttm_bo_del_from_lru(bo); put_count = ttm_bo_del_from_lru(bo);
spin_unlock(&glob->lru_lock); spin_unlock(&glob->lru_lock);
while (put_count--) ttm_bo_list_ref_sub(bo, put_count, true);
kref_put(&bo->list_kref, ttm_bo_ref_bug);
return ret; return ret;
} }
...@@ -496,8 +500,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) ...@@ -496,8 +500,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
spin_unlock(&glob->lru_lock); spin_unlock(&glob->lru_lock);
ttm_bo_cleanup_memtype_use(bo); ttm_bo_cleanup_memtype_use(bo);
while (put_count--) ttm_bo_list_ref_sub(bo, put_count, true);
kref_put(&bo->list_kref, ttm_bo_ref_bug);
return; return;
} else { } else {
...@@ -580,8 +583,7 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, ...@@ -580,8 +583,7 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
spin_unlock(&glob->lru_lock); spin_unlock(&glob->lru_lock);
ttm_bo_cleanup_memtype_use(bo); ttm_bo_cleanup_memtype_use(bo);
while (put_count--) ttm_bo_list_ref_sub(bo, put_count, true);
kref_put(&bo->list_kref, ttm_bo_ref_bug);
return 0; return 0;
} }
...@@ -802,8 +804,7 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, ...@@ -802,8 +804,7 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
BUG_ON(ret != 0); BUG_ON(ret != 0);
while (put_count--) ttm_bo_list_ref_sub(bo, put_count, true);
kref_put(&bo->list_kref, ttm_bo_ref_bug);
ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu); ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu);
ttm_bo_unreserve(bo); ttm_bo_unreserve(bo);
...@@ -1783,8 +1784,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) ...@@ -1783,8 +1784,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
put_count = ttm_bo_del_from_lru(bo); put_count = ttm_bo_del_from_lru(bo);
spin_unlock(&glob->lru_lock); spin_unlock(&glob->lru_lock);
while (put_count--) ttm_bo_list_ref_sub(bo, put_count, true);
kref_put(&bo->list_kref, ttm_bo_ref_bug);
/** /**
* Wait for GPU, then move to system cached. * Wait for GPU, then move to system cached.
......
...@@ -32,6 +32,72 @@ ...@@ -32,6 +32,72 @@
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/module.h> #include <linux/module.h>
static void ttm_eu_backoff_reservation_locked(struct list_head *list)
{
struct ttm_validate_buffer *entry;
list_for_each_entry(entry, list, head) {
struct ttm_buffer_object *bo = entry->bo;
if (!entry->reserved)
continue;
if (entry->removed) {
ttm_bo_add_to_lru(bo);
entry->removed = false;
}
entry->reserved = false;
atomic_set(&bo->reserved, 0);
wake_up_all(&bo->event_queue);
}
}
static void ttm_eu_del_from_lru_locked(struct list_head *list)
{
struct ttm_validate_buffer *entry;
list_for_each_entry(entry, list, head) {
struct ttm_buffer_object *bo = entry->bo;
if (!entry->reserved)
continue;
if (!entry->removed) {
entry->put_count = ttm_bo_del_from_lru(bo);
entry->removed = true;
}
}
}
static void ttm_eu_list_ref_sub(struct list_head *list)
{
struct ttm_validate_buffer *entry;
list_for_each_entry(entry, list, head) {
struct ttm_buffer_object *bo = entry->bo;
if (entry->put_count) {
ttm_bo_list_ref_sub(bo, entry->put_count, true);
entry->put_count = 0;
}
}
}
static int ttm_eu_wait_unreserved_locked(struct list_head *list,
struct ttm_buffer_object *bo)
{
struct ttm_bo_global *glob = bo->glob;
int ret;
ttm_eu_del_from_lru_locked(list);
spin_unlock(&glob->lru_lock);
ret = ttm_bo_wait_unreserved(bo, true);
spin_lock(&glob->lru_lock);
if (unlikely(ret != 0))
ttm_eu_backoff_reservation_locked(list);
return ret;
}
void ttm_eu_backoff_reservation(struct list_head *list) void ttm_eu_backoff_reservation(struct list_head *list)
{ {
struct ttm_validate_buffer *entry; struct ttm_validate_buffer *entry;
...@@ -61,35 +127,71 @@ EXPORT_SYMBOL(ttm_eu_backoff_reservation); ...@@ -61,35 +127,71 @@ EXPORT_SYMBOL(ttm_eu_backoff_reservation);
int ttm_eu_reserve_buffers(struct list_head *list, uint32_t val_seq) int ttm_eu_reserve_buffers(struct list_head *list, uint32_t val_seq)
{ {
struct ttm_bo_global *glob;
struct ttm_validate_buffer *entry; struct ttm_validate_buffer *entry;
int ret; int ret;
if (list_empty(list))
return 0;
list_for_each_entry(entry, list, head) {
entry->reserved = false;
entry->put_count = 0;
entry->removed = false;
}
entry = list_first_entry(list, struct ttm_validate_buffer, head);
glob = entry->bo->glob;
retry: retry:
spin_lock(&glob->lru_lock);
list_for_each_entry(entry, list, head) { list_for_each_entry(entry, list, head) {
struct ttm_buffer_object *bo = entry->bo; struct ttm_buffer_object *bo = entry->bo;
entry->reserved = false; retry_this_bo:
ret = ttm_bo_reserve(bo, true, false, true, val_seq); ret = ttm_bo_reserve_locked(bo, true, true, true, val_seq);
if (ret != 0) { switch (ret) {
ttm_eu_backoff_reservation(list); case 0:
if (ret == -EAGAIN) { break;
case -EBUSY:
ret = ttm_eu_wait_unreserved_locked(list, bo);
if (unlikely(ret != 0)) {
spin_unlock(&glob->lru_lock);
ttm_eu_list_ref_sub(list);
return ret;
}
goto retry_this_bo;
case -EAGAIN:
ttm_eu_backoff_reservation_locked(list);
spin_unlock(&glob->lru_lock);
ttm_eu_list_ref_sub(list);
ret = ttm_bo_wait_unreserved(bo, true); ret = ttm_bo_wait_unreserved(bo, true);
if (unlikely(ret != 0)) if (unlikely(ret != 0))
return ret; return ret;
goto retry; goto retry;
} else default:
ttm_eu_backoff_reservation_locked(list);
spin_unlock(&glob->lru_lock);
ttm_eu_list_ref_sub(list);
return ret; return ret;
} }
entry->reserved = true; entry->reserved = true;
if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { if (unlikely(atomic_read(&bo->cpu_writers) > 0)) {
ttm_eu_backoff_reservation(list); ttm_eu_backoff_reservation_locked(list);
spin_unlock(&glob->lru_lock);
ttm_eu_list_ref_sub(list);
ret = ttm_bo_wait_cpu(bo, false); ret = ttm_bo_wait_cpu(bo, false);
if (ret) if (ret)
return ret; return ret;
goto retry; goto retry;
} }
} }
ttm_eu_del_from_lru_locked(list);
spin_unlock(&glob->lru_lock);
ttm_eu_list_ref_sub(list);
return 0; return 0;
} }
EXPORT_SYMBOL(ttm_eu_reserve_buffers); EXPORT_SYMBOL(ttm_eu_reserve_buffers);
......
...@@ -364,6 +364,44 @@ extern int ttm_bo_validate(struct ttm_buffer_object *bo, ...@@ -364,6 +364,44 @@ extern int ttm_bo_validate(struct ttm_buffer_object *bo,
*/ */
extern void ttm_bo_unref(struct ttm_buffer_object **bo); extern void ttm_bo_unref(struct ttm_buffer_object **bo);
/**
* ttm_bo_list_ref_sub
*
* @bo: The buffer object.
* @count: The number of references with which to decrease @bo::list_kref;
* @never_free: The refcount should not reach zero with this operation.
*
* Release @count lru list references to this buffer object.
*/
extern void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count,
bool never_free);
/**
* ttm_bo_add_to_lru
*
* @bo: The buffer object.
*
* Add this bo to the relevant mem type lru and, if it's backed by
* system pages (ttms) to the swap list.
* This function must be called with struct ttm_bo_global::lru_lock held, and
* is typically called immediately prior to unreserving a bo.
*/
extern void ttm_bo_add_to_lru(struct ttm_buffer_object *bo);
/**
* ttm_bo_del_from_lru
*
* @bo: The buffer object.
*
* Remove this bo from all lru lists used to lookup and reserve an object.
* This function must be called with struct ttm_bo_global::lru_lock held,
* and is usually called just immediately after the bo has been reserved to
* avoid recursive reservation from lru lists.
*/
extern int ttm_bo_del_from_lru(struct ttm_buffer_object *bo);
/** /**
* ttm_bo_lock_delayed_workqueue * ttm_bo_lock_delayed_workqueue
* *
......
...@@ -864,6 +864,20 @@ extern int ttm_bo_reserve(struct ttm_buffer_object *bo, ...@@ -864,6 +864,20 @@ extern int ttm_bo_reserve(struct ttm_buffer_object *bo,
bool interruptible, bool interruptible,
bool no_wait, bool use_sequence, uint32_t sequence); bool no_wait, bool use_sequence, uint32_t sequence);
/**
* ttm_bo_reserve_locked:
*
* Similar to ttm_bo_reserve, but must be called with the glob::lru_lock
* spinlock held, and will not remove reserved buffers from the lru lists.
* The function may release the LRU spinlock if it needs to sleep.
*/
extern int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
bool interruptible,
bool no_wait, bool use_sequence,
uint32_t sequence);
/** /**
* ttm_bo_unreserve * ttm_bo_unreserve
* *
......
...@@ -41,7 +41,9 @@ ...@@ -41,7 +41,9 @@
* @bo: refcounted buffer object pointer. * @bo: refcounted buffer object pointer.
* @new_sync_obj_arg: New sync_obj_arg for @bo, to be used once * @new_sync_obj_arg: New sync_obj_arg for @bo, to be used once
* adding a new sync object. * adding a new sync object.
* @reservied: Indicates whether @bo has been reserved for validation. * @reserved: Indicates whether @bo has been reserved for validation.
* @removed: Indicates whether @bo has been removed from lru lists.
* @put_count: Number of outstanding references on bo::list_kref.
*/ */
struct ttm_validate_buffer { struct ttm_validate_buffer {
...@@ -49,6 +51,8 @@ struct ttm_validate_buffer { ...@@ -49,6 +51,8 @@ struct ttm_validate_buffer {
struct ttm_buffer_object *bo; struct ttm_buffer_object *bo;
void *new_sync_obj_arg; void *new_sync_obj_arg;
bool reserved; bool reserved;
bool removed;
int put_count;
}; };
/** /**
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment