drm/nouveau/fence/nv84-: put processes to sleep while waiting on fences

Signed-off-by: Ben Skeggs <bskeggs@redhat.com>

drivers/gpu/drm/nouveau/nouveau_dma.h

@@ -191,7 +191,7 @@ WIND_RING(struct nouveau_channel *chan)
 #define NV84_SUBCHAN_SEMAPHORE_TRIGGER_WRITE_LONG                    0x00000002
 #define NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_GEQUAL                0x00000004
 #define NVC0_SUBCHAN_SEMAPHORE_TRIGGER_YIELD                         0x00001000
-#define NV84_SUBCHAN_NOTIFY_INTR                                     0x00000020
+#define NV84_SUBCHAN_UEVENT                                          0x00000020
 #define NV84_SUBCHAN_WRCACHE_FLUSH                                   0x00000024
 #define NV10_SUBCHAN_REF_CNT                                         0x00000050
 #define NVSW_SUBCHAN_PAGE_FLIP                                       0x00000054

drivers/gpu/drm/nouveau/nouveau_fence.c

@@ -33,6 +33,8 @@
 #include "nouveau_dma.h"
 #include "nouveau_fence.h"
 
+#include <engine/fifo.h>
+
 void
 nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
 {
@@ -107,13 +109,87 @@ nouveau_fence_done(struct nouveau_fence *fence)
 	return !fence->channel;
 }
 
+struct nouveau_fence_uevent {
+	struct nouveau_eventh handler;
+	struct nouveau_fence_priv *priv;
+};
+
+static int
+nouveau_fence_wait_uevent_handler(struct nouveau_eventh *event, int index)
+{
+	struct nouveau_fence_uevent *uevent =
+		container_of(event, struct nouveau_fence_uevent, handler);
+	wake_up_all(&uevent->priv->waiting);
+	return NVKM_EVENT_KEEP;
+}
+
+static int
+nouveau_fence_wait_uevent(struct nouveau_fence *fence, bool intr)
+
+{
+	struct nouveau_channel *chan = fence->channel;
+	struct nouveau_fifo *pfifo = nouveau_fifo(chan->drm->device);
+	struct nouveau_fence_priv *priv = chan->drm->fence;
+	struct nouveau_fence_uevent uevent = {
+		.handler.func = nouveau_fence_wait_uevent_handler,
+		.priv = priv,
+	};
+	int ret = 0;
+
+	nouveau_event_get(pfifo->uevent, 0, &uevent.handler);
+
+	if (fence->timeout) {
+		unsigned long timeout = fence->timeout - jiffies;
+
+		if (time_before(jiffies, fence->timeout)) {
+			if (intr) {
+				ret = wait_event_interruptible_timeout(
+						priv->waiting,
+						nouveau_fence_done(fence),
+						timeout);
+			} else {
+				ret = wait_event_timeout(priv->waiting,
+						nouveau_fence_done(fence),
+						timeout);
+			}
+		}
+
+		if (ret >= 0) {
+			fence->timeout = jiffies + ret;
+			if (time_after_eq(jiffies, fence->timeout))
+				ret = -EBUSY;
+		}
+	} else {
+		if (intr) {
+			ret = wait_event_interruptible(priv->waiting,
+					nouveau_fence_done(fence));
+		} else {
+			wait_event(priv->waiting, nouveau_fence_done(fence));
+		}
+	}
+
+	nouveau_event_put(pfifo->uevent, 0, &uevent.handler);
+	if (unlikely(ret < 0))
+		return ret;
+
+	return 0;
+}
+
 int
 nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
 {
+	struct nouveau_channel *chan = fence->channel;
+	struct nouveau_fence_priv *priv = chan ? chan->drm->fence : NULL;
 	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
 	ktime_t t;
 	int ret = 0;
 
+	while (priv && priv->uevent && lazy && !nouveau_fence_done(fence)) {
+		ret = nouveau_fence_wait_uevent(fence, intr);
+		if (ret < 0)
+			return ret;
+	}
+
 	while (!nouveau_fence_done(fence)) {
 		if (fence->timeout && time_after_eq(jiffies, fence->timeout)) {
 			ret = -EBUSY;

drivers/gpu/drm/nouveau/nouveau_fence.h

@@ -43,6 +43,9 @@ struct nouveau_fence_priv {
 	int  (*sync)(struct nouveau_fence *, struct nouveau_channel *,
 		     struct nouveau_channel *);
 	u32  (*read)(struct nouveau_channel *);
+
+	wait_queue_head_t waiting;
+	bool uevent;
 };
 
 #define nouveau_fence(drm) ((struct nouveau_fence_priv *)(drm)->fence)

drivers/gpu/drm/nouveau/nv84_fence.c

@@ -47,15 +47,16 @@ nv84_fence_emit(struct nouveau_fence *fence)
 {
 	struct nouveau_channel *chan = fence->channel;
 	struct nouveau_fifo_chan *fifo = (void *)chan->object;
-	int ret = RING_SPACE(chan, 7);
+	int ret = RING_SPACE(chan, 8);
 	if (ret == 0) {
 		BEGIN_NV04(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 1);
 		OUT_RING  (chan, NvSema);
-		BEGIN_NV04(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
+		BEGIN_NV04(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 5);
 		OUT_RING  (chan, upper_32_bits(fifo->chid * 16));
 		OUT_RING  (chan, lower_32_bits(fifo->chid * 16));
 		OUT_RING  (chan, fence->sequence);
 		OUT_RING  (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_WRITE_LONG);
+		OUT_RING  (chan, 0x00000000);
 		FIRE_RING (chan);
 	}
 	return ret;
@@ -174,6 +175,9 @@ nv84_fence_create(struct nouveau_drm *drm)
 	priv->base.sync = nv84_fence_sync;
 	priv->base.read = nv84_fence_read;
 
+	init_waitqueue_head(&priv->base.waiting);
+	priv->base.uevent = true;
+
 	ret = nouveau_gpuobj_new(drm->device, NULL, chan * 16, 0x1000, 0,
 				&priv->mem);
 	if (ret)

drivers/gpu/drm/nouveau/nvc0_fence.c

@@ -62,13 +62,14 @@ nvc0_fence_emit(struct nouveau_fence *fence)
 	u64 addr = fctx->vma.offset + fifo->chid * 16;
 	int ret;
 
-	ret = RING_SPACE(chan, 5);
+	ret = RING_SPACE(chan, 6);
 	if (ret == 0) {
-		BEGIN_NVC0(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
+		BEGIN_NVC0(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 5);
 		OUT_RING  (chan, upper_32_bits(addr));
 		OUT_RING  (chan, lower_32_bits(addr));
 		OUT_RING  (chan, fence->sequence);
 		OUT_RING  (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_WRITE_LONG);
+		OUT_RING  (chan, 0x00000000);
 		FIRE_RING (chan);
 	}
 
@@ -217,6 +218,9 @@ nvc0_fence_create(struct nouveau_drm *drm)
 	priv->base.sync = nvc0_fence_sync;
 	priv->base.read = nvc0_fence_read;
 
+	init_waitqueue_head(&priv->base.waiting);
+	priv->base.uevent = true;
+
 	ret = nouveau_bo_new(drm->dev, 16 * (pfifo->max + 1), 0,
 			     TTM_PL_FLAG_VRAM, 0, 0, NULL, &priv->bo);
 	if (ret == 0) {