Commit c98171cc authored by Felix Kuehling's avatar Felix Kuehling Committed by Alex Deucher

drm/amdgpu: Handle GPUVM fault storms

When many wavefronts cause VM faults at the same time, it can
overwhelm the interrupt handler and cause IH ring overflows before
the driver can notify or kill the faulting application.

As a workaround I'm introducing limited per-VM fault credit. After
that number of VM faults have occurred, further VM faults are
filtered out at the prescreen stage of processing.

This depends on the PASID in the interrupt packet, so it currently
only works for KFD contexts.
Signed-off-by: default avatarFelix Kuehling <Felix.Kuehling@amd.com>
Reviewed-by: default avatarAlex Deucher <alexander.deucher@amd.com>
Signed-off-by: default avatarAlex Deucher <alexander.deucher@amd.com>
parent 1bab0fc0
...@@ -2682,6 +2682,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, ...@@ -2682,6 +2682,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
} }
INIT_KFIFO(vm->faults); INIT_KFIFO(vm->faults);
vm->fault_credit = 16;
return 0; return 0;
...@@ -2775,6 +2776,36 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) ...@@ -2775,6 +2776,36 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
amdgpu_vm_free_reserved_vmid(adev, vm, i); amdgpu_vm_free_reserved_vmid(adev, vm, i);
} }
/**
* amdgpu_vm_pasid_fault_credit - Check fault credit for given PASID
*
* @adev: amdgpu_device pointer
* @pasid: PASID do identify the VM
*
* This function is expected to be called in interrupt context. Returns
* true if there was fault credit, false otherwise
*/
bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
unsigned int pasid)
{
struct amdgpu_vm *vm;
spin_lock(&adev->vm_manager.pasid_lock);
vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
spin_unlock(&adev->vm_manager.pasid_lock);
if (!vm)
/* VM not found, can't track fault credit */
return true;
/* No lock needed. only accessed by IRQ handler */
if (!vm->fault_credit)
/* Too many faults in this VM */
return false;
vm->fault_credit--;
return true;
}
/** /**
* amdgpu_vm_manager_init - init the VM manager * amdgpu_vm_manager_init - init the VM manager
* *
......
...@@ -165,8 +165,11 @@ struct amdgpu_vm { ...@@ -165,8 +165,11 @@ struct amdgpu_vm {
/* Flag to indicate ATS support from PTE for GFX9 */ /* Flag to indicate ATS support from PTE for GFX9 */
bool pte_support_ats; bool pte_support_ats;
/* Up to 128 pending page faults */ /* Up to 128 pending retry page faults */
DECLARE_KFIFO(faults, u64, 128); DECLARE_KFIFO(faults, u64, 128);
/* Limit non-retry fault storms */
unsigned int fault_credit;
}; };
struct amdgpu_vm_id { struct amdgpu_vm_id {
...@@ -244,6 +247,8 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev); ...@@ -244,6 +247,8 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
int vm_context, unsigned int pasid); int vm_context, unsigned int pasid);
void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm); void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
unsigned int pasid);
void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
struct list_head *validated, struct list_head *validated,
struct amdgpu_bo_list_entry *entry); struct amdgpu_bo_list_entry *entry);
......
...@@ -237,8 +237,23 @@ static u32 cik_ih_get_wptr(struct amdgpu_device *adev) ...@@ -237,8 +237,23 @@ static u32 cik_ih_get_wptr(struct amdgpu_device *adev)
*/ */
static bool cik_ih_prescreen_iv(struct amdgpu_device *adev) static bool cik_ih_prescreen_iv(struct amdgpu_device *adev)
{ {
/* Process all interrupts */ u32 ring_index = adev->irq.ih.rptr >> 2;
return true; u16 pasid;
switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
case 146:
case 147:
pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
return true;
break;
default:
/* Not a VM fault */
return true;
}
adev->irq.ih.rptr += 16;
return false;
} }
/** /**
......
...@@ -216,8 +216,23 @@ static u32 cz_ih_get_wptr(struct amdgpu_device *adev) ...@@ -216,8 +216,23 @@ static u32 cz_ih_get_wptr(struct amdgpu_device *adev)
*/ */
static bool cz_ih_prescreen_iv(struct amdgpu_device *adev) static bool cz_ih_prescreen_iv(struct amdgpu_device *adev)
{ {
/* Process all interrupts */ u32 ring_index = adev->irq.ih.rptr >> 2;
return true; u16 pasid;
switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
case 146:
case 147:
pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
return true;
break;
default:
/* Not a VM fault */
return true;
}
adev->irq.ih.rptr += 16;
return false;
} }
/** /**
......
...@@ -216,8 +216,23 @@ static u32 iceland_ih_get_wptr(struct amdgpu_device *adev) ...@@ -216,8 +216,23 @@ static u32 iceland_ih_get_wptr(struct amdgpu_device *adev)
*/ */
static bool iceland_ih_prescreen_iv(struct amdgpu_device *adev) static bool iceland_ih_prescreen_iv(struct amdgpu_device *adev)
{ {
/* Process all interrupts */ u32 ring_index = adev->irq.ih.rptr >> 2;
return true; u16 pasid;
switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
case 146:
case 147:
pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
return true;
break;
default:
/* Not a VM fault */
return true;
}
adev->irq.ih.rptr += 16;
return false;
} }
/** /**
......
...@@ -227,8 +227,23 @@ static u32 tonga_ih_get_wptr(struct amdgpu_device *adev) ...@@ -227,8 +227,23 @@ static u32 tonga_ih_get_wptr(struct amdgpu_device *adev)
*/ */
static bool tonga_ih_prescreen_iv(struct amdgpu_device *adev) static bool tonga_ih_prescreen_iv(struct amdgpu_device *adev)
{ {
/* Process all interrupts */ u32 ring_index = adev->irq.ih.rptr >> 2;
return true; u16 pasid;
switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
case 146:
case 147:
pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
return true;
break;
default:
/* Not a VM fault */
return true;
}
adev->irq.ih.rptr += 16;
return false;
} }
/** /**
......
...@@ -260,15 +260,18 @@ static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev) ...@@ -260,15 +260,18 @@ static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
return true; return true;
} }
/* Not a retry fault */
if (!(dw5 & 0x80))
return true;
pasid = dw3 & 0xffff; pasid = dw3 & 0xffff;
/* No PASID, can't identify faulting process */ /* No PASID, can't identify faulting process */
if (!pasid) if (!pasid)
return true; return true;
/* Not a retry fault, check fault credit */
if (!(dw5 & 0x80)) {
if (!amdgpu_vm_pasid_fault_credit(adev, pasid))
goto ignore_iv;
return true;
}
addr = ((u64)(dw5 & 0xf) << 44) | ((u64)dw4 << 12); addr = ((u64)(dw5 & 0xf) << 44) | ((u64)dw4 << 12);
key = AMDGPU_VM_FAULT(pasid, addr); key = AMDGPU_VM_FAULT(pasid, addr);
r = amdgpu_ih_add_fault(adev, key); r = amdgpu_ih_add_fault(adev, key);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment