Commit 2b2ed975 authored by Jens Axboe's avatar Jens Axboe

io_uring: fix bad inflight accounting for SETUP_IOPOLL|SETUP_SQTHREAD

We currently assume that submissions from the sqthread are successful,
and if IO polling is enabled, we use that value for knowing how many
completions to look for. But if we overflowed the CQ ring or some
requests simply got errored and already completed, they won't be
available for polling.

For the case of IO polling and SQTHREAD usage, look at the pending
poll list. If it ever hits empty then we know that we don't have
anymore pollable requests inflight. For that case, simply reset
the inflight count to zero.
Reported-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Reviewed-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 498ccd9e
...@@ -874,19 +874,11 @@ static void io_iopoll_reap_events(struct io_ring_ctx *ctx) ...@@ -874,19 +874,11 @@ static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
mutex_unlock(&ctx->uring_lock); mutex_unlock(&ctx->uring_lock);
} }
static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events, static int __io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
long min) long min)
{ {
int iters, ret = 0; int iters = 0, ret = 0;
/*
* We disallow the app entering submit/complete with polling, but we
* still need to lock the ring to prevent racing with polled issue
* that got punted to a workqueue.
*/
mutex_lock(&ctx->uring_lock);
iters = 0;
do { do {
int tmin = 0; int tmin = 0;
...@@ -922,6 +914,21 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events, ...@@ -922,6 +914,21 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
ret = 0; ret = 0;
} while (min && !*nr_events && !need_resched()); } while (min && !*nr_events && !need_resched());
return ret;
}
static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
long min)
{
int ret;
/*
* We disallow the app entering submit/complete with polling, but we
* still need to lock the ring to prevent racing with polled issue
* that got punted to a workqueue.
*/
mutex_lock(&ctx->uring_lock);
ret = __io_iopoll_check(ctx, nr_events, min);
mutex_unlock(&ctx->uring_lock); mutex_unlock(&ctx->uring_lock);
return ret; return ret;
} }
...@@ -2657,7 +2664,20 @@ static int io_sq_thread(void *data) ...@@ -2657,7 +2664,20 @@ static int io_sq_thread(void *data)
unsigned nr_events = 0; unsigned nr_events = 0;
if (ctx->flags & IORING_SETUP_IOPOLL) { if (ctx->flags & IORING_SETUP_IOPOLL) {
io_iopoll_check(ctx, &nr_events, 0); /*
* inflight is the count of the maximum possible
* entries we submitted, but it can be smaller
* if we dropped some of them. If we don't have
* poll entries available, then we know that we
* have nothing left to poll for. Reset the
* inflight count to zero in that case.
*/
mutex_lock(&ctx->uring_lock);
if (!list_empty(&ctx->poll_list))
__io_iopoll_check(ctx, &nr_events, 0);
else
inflight = 0;
mutex_unlock(&ctx->uring_lock);
} else { } else {
/* /*
* Normal IO, just pretend everything completed. * Normal IO, just pretend everything completed.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment