Commit d625c6ee authored by Jens Axboe's avatar Jens Axboe

io_uring: read opcode and user_data from SQE exactly once

If we defer a request, we can't be reading the opcode again. Ensure that
the user_data and opcode fields are stable. For the user_data we already
have a place for it, for the opcode we can fill a one byte hold and store
that as well. For both of them, assign them when we originally read the
SQE in io_get_sqring(). Any code that uses sqe->opcode or sqe->user_data
is switched to req->opcode and req->user_data.
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent b29472ee
...@@ -384,6 +384,7 @@ struct io_kiocb { ...@@ -384,6 +384,7 @@ struct io_kiocb {
bool has_user; bool has_user;
bool in_async; bool in_async;
bool needs_fixed_file; bool needs_fixed_file;
u8 opcode;
struct io_ring_ctx *ctx; struct io_ring_ctx *ctx;
union { union {
...@@ -597,12 +598,10 @@ static void __io_commit_cqring(struct io_ring_ctx *ctx) ...@@ -597,12 +598,10 @@ static void __io_commit_cqring(struct io_ring_ctx *ctx)
} }
} }
static inline bool io_sqe_needs_user(const struct io_uring_sqe *sqe) static inline bool io_req_needs_user(struct io_kiocb *req)
{ {
u8 opcode = READ_ONCE(sqe->opcode); return !(req->opcode == IORING_OP_READ_FIXED ||
req->opcode == IORING_OP_WRITE_FIXED);
return !(opcode == IORING_OP_READ_FIXED ||
opcode == IORING_OP_WRITE_FIXED);
} }
static inline bool io_prep_async_work(struct io_kiocb *req, static inline bool io_prep_async_work(struct io_kiocb *req,
...@@ -611,7 +610,7 @@ static inline bool io_prep_async_work(struct io_kiocb *req, ...@@ -611,7 +610,7 @@ static inline bool io_prep_async_work(struct io_kiocb *req,
bool do_hashed = false; bool do_hashed = false;
if (req->sqe) { if (req->sqe) {
switch (req->sqe->opcode) { switch (req->opcode) {
case IORING_OP_WRITEV: case IORING_OP_WRITEV:
case IORING_OP_WRITE_FIXED: case IORING_OP_WRITE_FIXED:
/* only regular files should be hashed for writes */ /* only regular files should be hashed for writes */
...@@ -634,7 +633,7 @@ static inline bool io_prep_async_work(struct io_kiocb *req, ...@@ -634,7 +633,7 @@ static inline bool io_prep_async_work(struct io_kiocb *req,
req->work.flags |= IO_WQ_WORK_UNBOUND; req->work.flags |= IO_WQ_WORK_UNBOUND;
break; break;
} }
if (io_sqe_needs_user(req->sqe)) if (io_req_needs_user(req))
req->work.flags |= IO_WQ_WORK_NEEDS_USER; req->work.flags |= IO_WQ_WORK_NEEDS_USER;
} }
...@@ -1005,7 +1004,7 @@ static void io_fail_links(struct io_kiocb *req) ...@@ -1005,7 +1004,7 @@ static void io_fail_links(struct io_kiocb *req)
trace_io_uring_fail_link(req, link); trace_io_uring_fail_link(req, link);
if ((req->flags & REQ_F_LINK_TIMEOUT) && if ((req->flags & REQ_F_LINK_TIMEOUT) &&
link->sqe->opcode == IORING_OP_LINK_TIMEOUT) { link->opcode == IORING_OP_LINK_TIMEOUT) {
io_link_cancel_timeout(link); io_link_cancel_timeout(link);
} else { } else {
io_cqring_fill_event(link, -ECANCELED); io_cqring_fill_event(link, -ECANCELED);
...@@ -1648,7 +1647,7 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req, ...@@ -1648,7 +1647,7 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
* for that purpose and instead let the caller pass in the read/write * for that purpose and instead let the caller pass in the read/write
* flag. * flag.
*/ */
opcode = READ_ONCE(sqe->opcode); opcode = req->opcode;
if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) { if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
*iovec = NULL; *iovec = NULL;
return io_import_fixed(req->ctx, rw, sqe, iter); return io_import_fixed(req->ctx, rw, sqe, iter);
...@@ -3082,7 +3081,7 @@ static int io_req_defer_prep(struct io_kiocb *req) ...@@ -3082,7 +3081,7 @@ static int io_req_defer_prep(struct io_kiocb *req)
struct iov_iter iter; struct iov_iter iter;
ssize_t ret; ssize_t ret;
switch (io->sqe.opcode) { switch (req->opcode) {
case IORING_OP_READV: case IORING_OP_READV:
case IORING_OP_READ_FIXED: case IORING_OP_READ_FIXED:
/* ensure prep does right import */ /* ensure prep does right import */
...@@ -3181,11 +3180,10 @@ __attribute__((nonnull)) ...@@ -3181,11 +3180,10 @@ __attribute__((nonnull))
static int io_issue_sqe(struct io_kiocb *req, struct io_kiocb **nxt, static int io_issue_sqe(struct io_kiocb *req, struct io_kiocb **nxt,
bool force_nonblock) bool force_nonblock)
{ {
int ret, opcode;
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
int ret;
opcode = READ_ONCE(req->sqe->opcode); switch (req->opcode) {
switch (opcode) {
case IORING_OP_NOP: case IORING_OP_NOP:
ret = io_nop(req); ret = io_nop(req);
break; break;
...@@ -3322,11 +3320,9 @@ static bool io_req_op_valid(int op) ...@@ -3322,11 +3320,9 @@ static bool io_req_op_valid(int op)
return op >= IORING_OP_NOP && op < IORING_OP_LAST; return op >= IORING_OP_NOP && op < IORING_OP_LAST;
} }
static int io_op_needs_file(const struct io_uring_sqe *sqe) static int io_req_needs_file(struct io_kiocb *req)
{ {
int op = READ_ONCE(sqe->opcode); switch (req->opcode) {
switch (op) {
case IORING_OP_NOP: case IORING_OP_NOP:
case IORING_OP_POLL_REMOVE: case IORING_OP_POLL_REMOVE:
case IORING_OP_TIMEOUT: case IORING_OP_TIMEOUT:
...@@ -3335,7 +3331,7 @@ static int io_op_needs_file(const struct io_uring_sqe *sqe) ...@@ -3335,7 +3331,7 @@ static int io_op_needs_file(const struct io_uring_sqe *sqe)
case IORING_OP_LINK_TIMEOUT: case IORING_OP_LINK_TIMEOUT:
return 0; return 0;
default: default:
if (io_req_op_valid(op)) if (io_req_op_valid(req->opcode))
return 1; return 1;
return -EINVAL; return -EINVAL;
} }
...@@ -3362,7 +3358,7 @@ static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req) ...@@ -3362,7 +3358,7 @@ static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req)
if (flags & IOSQE_IO_DRAIN) if (flags & IOSQE_IO_DRAIN)
req->flags |= REQ_F_IO_DRAIN; req->flags |= REQ_F_IO_DRAIN;
ret = io_op_needs_file(req->sqe); ret = io_req_needs_file(req);
if (ret <= 0) if (ret <= 0)
return ret; return ret;
...@@ -3482,7 +3478,7 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) ...@@ -3482,7 +3478,7 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
link_list); link_list);
if (!nxt || nxt->sqe->opcode != IORING_OP_LINK_TIMEOUT) if (!nxt || nxt->opcode != IORING_OP_LINK_TIMEOUT)
return NULL; return NULL;
req->flags |= REQ_F_LINK_TIMEOUT; req->flags |= REQ_F_LINK_TIMEOUT;
...@@ -3584,8 +3580,6 @@ static bool io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state, ...@@ -3584,8 +3580,6 @@ static bool io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state,
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
int ret; int ret;
req->user_data = req->sqe->user_data;
/* enforce forwards compatibility on users */ /* enforce forwards compatibility on users */
if (unlikely(req->sqe->flags & ~SQE_VALID_FLAGS)) { if (unlikely(req->sqe->flags & ~SQE_VALID_FLAGS)) {
ret = -EINVAL; ret = -EINVAL;
...@@ -3717,6 +3711,8 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct io_kiocb *req) ...@@ -3717,6 +3711,8 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct io_kiocb *req)
*/ */
req->sequence = ctx->cached_sq_head; req->sequence = ctx->cached_sq_head;
req->sqe = &ctx->sq_sqes[head]; req->sqe = &ctx->sq_sqes[head];
req->opcode = READ_ONCE(req->sqe->opcode);
req->user_data = READ_ONCE(req->sqe->user_data);
ctx->cached_sq_head++; ctx->cached_sq_head++;
return true; return true;
} }
...@@ -3762,7 +3758,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, ...@@ -3762,7 +3758,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
break; break;
} }
if (io_sqe_needs_user(req->sqe) && !*mm) { if (io_req_needs_user(req) && !*mm) {
mm_fault = mm_fault || !mmget_not_zero(ctx->sqo_mm); mm_fault = mm_fault || !mmget_not_zero(ctx->sqo_mm);
if (!mm_fault) { if (!mm_fault) {
use_mm(ctx->sqo_mm); use_mm(ctx->sqo_mm);
...@@ -3778,8 +3774,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, ...@@ -3778,8 +3774,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
req->has_user = *mm != NULL; req->has_user = *mm != NULL;
req->in_async = async; req->in_async = async;
req->needs_fixed_file = async; req->needs_fixed_file = async;
trace_io_uring_submit_sqe(ctx, req->sqe->user_data, trace_io_uring_submit_sqe(ctx, req->user_data, true, async);
true, async);
if (!io_submit_sqe(req, statep, &link)) if (!io_submit_sqe(req, statep, &link))
break; break;
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment