Skip to content

L
linux
Commit 75192758 authored by Matthew Brost's avatar Matthew Brost
Browse files
Options

drm/xe: Add ops_execute function which returns a fence 

Add ops_execute function which returns a fence. This will be helpful to
initiate all binds (VM bind IOCTL, rebinds in exec IOCTL, rebinds in
preempt rebind worker, and rebinds in pagefaults) via a gpuva ops list.
Returning a fence is needed in various paths.

v2:
 - Rebase

Cc: Oak Zeng <oak.zeng@intel.com>
Signed-off-by: default avatarMatthew Brost <matthew.brost@intel.com>
Reviewed-by: default avatarOak Zeng <oak.zeng@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20240425045513.1913039-3-matthew.brost@intel.com
parent 77f2ef3f
Hide whitespace changes
Inline Side-by-side
Showing with 111 additions and 100 deletions
drivers/gpu/drm/xe/xe_vm.c
View file @ 75192758
...@@ -1732,16 +1732,17 @@ find_ufence_get(struct xe_sync_entry *syncs, u32 num_syncs) ...@@ -1732,16 +1732,17 @@ find_ufence_get(struct xe_sync_entry *syncs, u32 num_syncs)
return NULL; return NULL;
} }
static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, static struct dma_fence *
struct xe_exec_queue *q, struct xe_sync_entry *syncs, xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q,
u32 num_syncs, bool immediate, bool first_op, struct xe_bo *bo, struct xe_sync_entry *syncs, u32 num_syncs,
bool last_op) bool immediate, bool first_op, bool last_op)
{ {
struct dma_fence *fence; struct dma_fence *fence;
struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q); struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
struct xe_user_fence *ufence; struct xe_user_fence *ufence;
xe_vm_assert_held(vm); xe_vm_assert_held(vm);
xe_bo_assert_held(bo);
ufence = find_ufence_get(syncs, num_syncs); ufence = find_ufence_get(syncs, num_syncs);
if (vma->ufence && ufence) if (vma->ufence && ufence)
...@@ -1753,7 +1754,7 @@ static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, ...@@ -1753,7 +1754,7 @@ static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma,
fence = xe_vm_bind_vma(vma, q, syncs, num_syncs, first_op, fence = xe_vm_bind_vma(vma, q, syncs, num_syncs, first_op,
last_op); last_op);
if (IS_ERR(fence)) if (IS_ERR(fence))
return PTR_ERR(fence); return fence;
} else { } else {
int i; int i;
...@@ -1768,26 +1769,14 @@ static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, ...@@ -1768,26 +1769,14 @@ static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma,
if (last_op) if (last_op)
xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence); xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
dma_fence_put(fence);
return 0;
}
static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q,
struct xe_bo *bo, struct xe_sync_entry *syncs,
u32 num_syncs, bool immediate, bool first_op,
bool last_op)
{
xe_vm_assert_held(vm);
xe_bo_assert_held(bo);
return __xe_vm_bind(vm, vma, q, syncs, num_syncs, immediate, first_op, return fence;
last_op);
} }
static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, static struct dma_fence *
struct xe_exec_queue *q, struct xe_sync_entry *syncs, xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
u32 num_syncs, bool first_op, bool last_op) struct xe_exec_queue *q, struct xe_sync_entry *syncs,
u32 num_syncs, bool first_op, bool last_op)
{ {
struct dma_fence *fence; struct dma_fence *fence;
struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q); struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
...@@ -1797,14 +1786,13 @@ static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, ...@@ -1797,14 +1786,13 @@ static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
fence = xe_vm_unbind_vma(vma, q, syncs, num_syncs, first_op, last_op); fence = xe_vm_unbind_vma(vma, q, syncs, num_syncs, first_op, last_op);
if (IS_ERR(fence)) if (IS_ERR(fence))
return PTR_ERR(fence); return fence;
xe_vma_destroy(vma, fence); xe_vma_destroy(vma, fence);
if (last_op) if (last_op)
xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence); xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
dma_fence_put(fence);
return 0; return fence;
} }
#define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE | \ #define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE | \
...@@ -1947,10 +1935,11 @@ static const u32 region_to_mem_type[] = { ...@@ -1947,10 +1935,11 @@ static const u32 region_to_mem_type[] = {
XE_PL_VRAM1, XE_PL_VRAM1,
}; };
static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma, static struct dma_fence *
struct xe_exec_queue *q, u32 region, xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
struct xe_sync_entry *syncs, u32 num_syncs, struct xe_exec_queue *q, u32 region,
bool first_op, bool last_op) struct xe_sync_entry *syncs, u32 num_syncs,
bool first_op, bool last_op)
{ {
struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q); struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
int err; int err;
...@@ -1960,27 +1949,24 @@ static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma, ...@@ -1960,27 +1949,24 @@ static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
if (!xe_vma_has_no_bo(vma)) { if (!xe_vma_has_no_bo(vma)) {
err = xe_bo_migrate(xe_vma_bo(vma), region_to_mem_type[region]); err = xe_bo_migrate(xe_vma_bo(vma), region_to_mem_type[region]);
if (err) if (err)
return err; return ERR_PTR(err);
} }
if (vma->tile_mask != (vma->tile_present & ~vma->tile_invalidated)) { if (vma->tile_mask != (vma->tile_present & ~vma->tile_invalidated)) {
return xe_vm_bind(vm, vma, q, xe_vma_bo(vma), syncs, num_syncs, return xe_vm_bind(vm, vma, q, xe_vma_bo(vma), syncs, num_syncs,
true, first_op, last_op); true, first_op, last_op);
} else { } else {
struct dma_fence *fence =
xe_exec_queue_last_fence_get(wait_exec_queue, vm);
int i; int i;
/* Nothing to do, signal fences now */ /* Nothing to do, signal fences now */
if (last_op) { if (last_op) {
for (i = 0; i < num_syncs; i++) { for (i = 0; i < num_syncs; i++)
struct dma_fence *fence =
xe_exec_queue_last_fence_get(wait_exec_queue, vm);
xe_sync_entry_signal(&syncs[i], fence); xe_sync_entry_signal(&syncs[i], fence);
dma_fence_put(fence);
}
} }
return 0; return fence;
} }
} }
...@@ -2433,10 +2419,10 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, ...@@ -2433,10 +2419,10 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
return 0; return 0;
} }
static int op_execute(struct xe_vm *vm, struct xe_vma *vma, static struct dma_fence *op_execute(struct xe_vm *vm, struct xe_vma *vma,
struct xe_vma_op *op) struct xe_vma_op *op)
{ {
int err; struct dma_fence *fence = NULL;
lockdep_assert_held_write(&vm->lock); lockdep_assert_held_write(&vm->lock);
...@@ -2445,11 +2431,11 @@ static int op_execute(struct xe_vm *vm, struct xe_vma *vma, ...@@ -2445,11 +2431,11 @@ static int op_execute(struct xe_vm *vm, struct xe_vma *vma,
switch (op->base.op) { switch (op->base.op) {
case DRM_GPUVA_OP_MAP: case DRM_GPUVA_OP_MAP:
err = xe_vm_bind(vm, vma, op->q, xe_vma_bo(vma), fence = xe_vm_bind(vm, vma, op->q, xe_vma_bo(vma),
op->syncs, op->num_syncs, op->syncs, op->num_syncs,
op->map.immediate || !xe_vm_in_fault_mode(vm), op->map.immediate || !xe_vm_in_fault_mode(vm),
op->flags & XE_VMA_OP_FIRST, op->flags & XE_VMA_OP_FIRST,
op->flags & XE_VMA_OP_LAST); op->flags & XE_VMA_OP_LAST);
break; break;
case DRM_GPUVA_OP_REMAP: case DRM_GPUVA_OP_REMAP:
{ {
...@@ -2459,37 +2445,39 @@ static int op_execute(struct xe_vm *vm, struct xe_vma *vma, ...@@ -2459,37 +2445,39 @@ static int op_execute(struct xe_vm *vm, struct xe_vma *vma,
if (!op->remap.unmap_done) { if (!op->remap.unmap_done) {
if (prev || next) if (prev || next)
vma->gpuva.flags |= XE_VMA_FIRST_REBIND; vma->gpuva.flags |= XE_VMA_FIRST_REBIND;
err = xe_vm_unbind(vm, vma, op->q, op->syncs, fence = xe_vm_unbind(vm, vma, op->q, op->syncs,
op->num_syncs, op->num_syncs,
op->flags & XE_VMA_OP_FIRST, op->flags & XE_VMA_OP_FIRST,
op->flags & XE_VMA_OP_LAST && op->flags & XE_VMA_OP_LAST &&
!prev && !next); !prev && !next);
if (err) if (IS_ERR(fence))
break; break;
op->remap.unmap_done = true; op->remap.unmap_done = true;
} }
if (prev) { if (prev) {
op->remap.prev->gpuva.flags |= XE_VMA_LAST_REBIND; op->remap.prev->gpuva.flags |= XE_VMA_LAST_REBIND;
err = xe_vm_bind(vm, op->remap.prev, op->q, dma_fence_put(fence);
xe_vma_bo(op->remap.prev), op->syncs, fence = xe_vm_bind(vm, op->remap.prev, op->q,
op->num_syncs, true, false, xe_vma_bo(op->remap.prev), op->syncs,
op->flags & XE_VMA_OP_LAST && !next); op->num_syncs, true, false,
op->flags & XE_VMA_OP_LAST && !next);
op->remap.prev->gpuva.flags &= ~XE_VMA_LAST_REBIND; op->remap.prev->gpuva.flags &= ~XE_VMA_LAST_REBIND;
if (err) if (IS_ERR(fence))
break; break;
op->remap.prev = NULL; op->remap.prev = NULL;
} }
if (next) { if (next) {
op->remap.next->gpuva.flags |= XE_VMA_LAST_REBIND; op->remap.next->gpuva.flags |= XE_VMA_LAST_REBIND;
err = xe_vm_bind(vm, op->remap.next, op->q, dma_fence_put(fence);
xe_vma_bo(op->remap.next), fence = xe_vm_bind(vm, op->remap.next, op->q,
op->syncs, op->num_syncs, xe_vma_bo(op->remap.next),
true, false, op->syncs, op->num_syncs,
op->flags & XE_VMA_OP_LAST); true, false,
op->flags & XE_VMA_OP_LAST);
op->remap.next->gpuva.flags &= ~XE_VMA_LAST_REBIND; op->remap.next->gpuva.flags &= ~XE_VMA_LAST_REBIND;
if (err) if (IS_ERR(fence))
break; break;
op->remap.next = NULL; op->remap.next = NULL;
} }
...@@ -2497,34 +2485,36 @@ static int op_execute(struct xe_vm *vm, struct xe_vma *vma, ...@@ -2497,34 +2485,36 @@ static int op_execute(struct xe_vm *vm, struct xe_vma *vma,
break; break;
} }
case DRM_GPUVA_OP_UNMAP: case DRM_GPUVA_OP_UNMAP:
err = xe_vm_unbind(vm, vma, op->q, op->syncs, fence = xe_vm_unbind(vm, vma, op->q, op->syncs,
op->num_syncs, op->flags & XE_VMA_OP_FIRST, op->num_syncs, op->flags & XE_VMA_OP_FIRST,
op->flags & XE_VMA_OP_LAST); op->flags & XE_VMA_OP_LAST);
break; break;
case DRM_GPUVA_OP_PREFETCH: case DRM_GPUVA_OP_PREFETCH:
err = xe_vm_prefetch(vm, vma, op->q, op->prefetch.region, fence = xe_vm_prefetch(vm, vma, op->q, op->prefetch.region,
op->syncs, op->num_syncs, op->syncs, op->num_syncs,
op->flags & XE_VMA_OP_FIRST, op->flags & XE_VMA_OP_FIRST,
op->flags & XE_VMA_OP_LAST); op->flags & XE_VMA_OP_LAST);
break; break;
default: default:
drm_warn(&vm->xe->drm, "NOT POSSIBLE"); drm_warn(&vm->xe->drm, "NOT POSSIBLE");
} }
if (err) if (IS_ERR(fence))
trace_xe_vma_fail(vma); trace_xe_vma_fail(vma);
return err; return fence;
} }
static int __xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma, static struct dma_fence *
struct xe_vma_op *op) __xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma,
struct xe_vma_op *op)
{ {
struct dma_fence *fence;
int err; int err;
retry_userptr: retry_userptr:
err = op_execute(vm, vma, op); fence = op_execute(vm, vma, op);
if (err == -EAGAIN) { if (IS_ERR(fence) && PTR_ERR(fence) == -EAGAIN) {
lockdep_assert_held_write(&vm->lock); lockdep_assert_held_write(&vm->lock);
if (op->base.op == DRM_GPUVA_OP_REMAP) { if (op->base.op == DRM_GPUVA_OP_REMAP) {
...@@ -2541,22 +2531,24 @@ static int __xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma, ...@@ -2541,22 +2531,24 @@ static int __xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma,
if (!err) if (!err)
goto retry_userptr; goto retry_userptr;
fence = ERR_PTR(err);
trace_xe_vma_fail(vma); trace_xe_vma_fail(vma);
} }
} }
return err; return fence;
} }
static int xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op) static struct dma_fence *
xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op)
{ {
int ret = 0; struct dma_fence *fence = ERR_PTR(-ENOMEM);
lockdep_assert_held_write(&vm->lock); lockdep_assert_held_write(&vm->lock);
switch (op->base.op) { switch (op->base.op) {
case DRM_GPUVA_OP_MAP: case DRM_GPUVA_OP_MAP:
ret = __xe_vma_op_execute(vm, op->map.vma, op); fence = __xe_vma_op_execute(vm, op->map.vma, op);
break; break;
case DRM_GPUVA_OP_REMAP: case DRM_GPUVA_OP_REMAP:
{ {
...@@ -2569,23 +2561,23 @@ static int xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op) ...@@ -2569,23 +2561,23 @@ static int xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op)
else else
vma = op->remap.next; vma = op->remap.next;
ret = __xe_vma_op_execute(vm, vma, op); fence = __xe_vma_op_execute(vm, vma, op);
break; break;
} }
case DRM_GPUVA_OP_UNMAP: case DRM_GPUVA_OP_UNMAP:
ret = __xe_vma_op_execute(vm, gpuva_to_vma(op->base.unmap.va), fence = __xe_vma_op_execute(vm, gpuva_to_vma(op->base.unmap.va),
op); op);
break; break;
case DRM_GPUVA_OP_PREFETCH: case DRM_GPUVA_OP_PREFETCH:
ret = __xe_vma_op_execute(vm, fence = __xe_vma_op_execute(vm,
gpuva_to_vma(op->base.prefetch.va), gpuva_to_vma(op->base.prefetch.va),
op); op);
break; break;
default: default:
drm_warn(&vm->xe->drm, "NOT POSSIBLE"); drm_warn(&vm->xe->drm, "NOT POSSIBLE");
} }
return ret; return fence;
} }
static void xe_vma_op_cleanup(struct xe_vm *vm, struct xe_vma_op *op) static void xe_vma_op_cleanup(struct xe_vm *vm, struct xe_vma_op *op)
...@@ -2760,11 +2752,35 @@ static int vm_bind_ioctl_ops_lock_and_prep(struct drm_exec *exec, ...@@ -2760,11 +2752,35 @@ static int vm_bind_ioctl_ops_lock_and_prep(struct drm_exec *exec,
return 0; return 0;
} }
static struct dma_fence *ops_execute(struct xe_vm *vm,
struct list_head *ops_list,
bool cleanup)
{
struct xe_vma_op *op, *next;
struct dma_fence *fence = NULL;
list_for_each_entry_safe(op, next, ops_list, link) {
if (!IS_ERR(fence)) {
dma_fence_put(fence);
fence = xe_vma_op_execute(vm, op);
}
if (IS_ERR(fence)) {
drm_warn(&vm->xe->drm, "VM op(%d) failed with %ld",
op->base.op, PTR_ERR(fence));
fence = ERR_PTR(-ENOSPC);
}
if (cleanup)
xe_vma_op_cleanup(vm, op);
}
return fence;
}
static int vm_bind_ioctl_ops_execute(struct xe_vm *vm, static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
struct list_head *ops_list) struct list_head *ops_list)
{ {
struct drm_exec exec; struct drm_exec exec;
struct xe_vma_op *op, *next; struct dma_fence *fence;
int err; int err;
lockdep_assert_held_write(&vm->lock); lockdep_assert_held_write(&vm->lock);
...@@ -2777,19 +2793,14 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm, ...@@ -2777,19 +2793,14 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
if (err) if (err)
goto unlock; goto unlock;
list_for_each_entry_safe(op, next, ops_list, link) { fence = ops_execute(vm, ops_list, true);
err = xe_vma_op_execute(vm, op); if (IS_ERR(fence)) {
if (err) { err = PTR_ERR(fence);
drm_warn(&vm->xe->drm, "VM op(%d) failed with %d", /* FIXME: Killing VM rather than proper error handling */
op->base.op, err); xe_vm_kill(vm, false);
/* goto unlock;
* FIXME: Killing VM rather than proper error handling } else {
*/ dma_fence_put(fence);
xe_vm_kill(vm, false);
err = -ENOSPC;
goto unlock;
}
xe_vma_op_cleanup(vm, op);
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7