Commit 3f0e64d0 authored by Jens Axboe's avatar Jens Axboe

io_uring: move SQPOLL post-wakeup ring need wakeup flag into wake handler

We need to decouple the clearing on wakeup from the the inline schedule,
as that is going to be required for handling multiple rings in one
thread.

Wrap our wakeup handler so we can clear it when we get the wakeup, by
definition that is when we no longer need the flag set.
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 6a779382
...@@ -6625,6 +6625,23 @@ static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx) ...@@ -6625,6 +6625,23 @@ static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
spin_unlock_irq(&ctx->completion_lock); spin_unlock_irq(&ctx->completion_lock);
} }
static int io_sq_wake_function(struct wait_queue_entry *wqe, unsigned mode,
int sync, void *key)
{
struct io_ring_ctx *ctx = container_of(wqe, struct io_ring_ctx, sqo_wait_entry);
int ret;
ret = autoremove_wake_function(wqe, mode, sync, key);
if (ret) {
unsigned long flags;
spin_lock_irqsave(&ctx->completion_lock, flags);
ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
spin_unlock_irqrestore(&ctx->completion_lock, flags);
}
return ret;
}
static int io_sq_thread(void *data) static int io_sq_thread(void *data)
{ {
struct io_ring_ctx *ctx = data; struct io_ring_ctx *ctx = data;
...@@ -6633,6 +6650,7 @@ static int io_sq_thread(void *data) ...@@ -6633,6 +6650,7 @@ static int io_sq_thread(void *data)
int ret = 0; int ret = 0;
init_wait(&ctx->sqo_wait_entry); init_wait(&ctx->sqo_wait_entry);
ctx->sqo_wait_entry.func = io_sq_wake_function;
complete(&ctx->sq_thread_comp); complete(&ctx->sq_thread_comp);
...@@ -6715,7 +6733,6 @@ static int io_sq_thread(void *data) ...@@ -6715,7 +6733,6 @@ static int io_sq_thread(void *data)
schedule(); schedule();
finish_wait(ctx->sqo_wait, &ctx->sqo_wait_entry); finish_wait(ctx->sqo_wait, &ctx->sqo_wait_entry);
io_ring_clear_wakeup_flag(ctx);
ret = 0; ret = 0;
continue; continue;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment