Skip to content

L
linux
Commit 03b1230c authored by Jens Axboe's avatar Jens Axboe
Browse files
Options

io_uring: ensure async punted sendmsg/recvmsg requests copy data 

Just like commit f67676d1 for read/write requests, this one ensures
that the msghdr data is fully copied if we need to punt a recvmsg or
sendmsg system call to async context.
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent f67676d1
Hide whitespace changes
Inline Side-by-side
Showing with 156 additions and 64 deletions
fs/io_uring.c
View file @ 03b1230c
...@@ -308,6 +308,13 @@ struct io_timeout { ...@@ -308,6 +308,13 @@ struct io_timeout {
struct io_timeout_data *data; struct io_timeout_data *data;
}; };
struct io_async_msghdr {
struct iovec fast_iov[UIO_FASTIOV];
struct iovec *iov;
struct sockaddr __user *uaddr;
struct msghdr msg;
};
struct io_async_rw { struct io_async_rw {
struct iovec fast_iov[UIO_FASTIOV]; struct iovec fast_iov[UIO_FASTIOV];
struct iovec *iov; struct iovec *iov;
...@@ -319,6 +326,7 @@ struct io_async_ctx { ...@@ -319,6 +326,7 @@ struct io_async_ctx {
struct io_uring_sqe sqe; struct io_uring_sqe sqe;
union { union {
struct io_async_rw rw; struct io_async_rw rw;
struct io_async_msghdr msg;
}; };
}; };
...@@ -1991,12 +1999,25 @@ static int io_sync_file_range(struct io_kiocb *req, ...@@ -1991,12 +1999,25 @@ static int io_sync_file_range(struct io_kiocb *req,
return 0; return 0;
} }
static int io_sendmsg_prep(struct io_kiocb *req, struct io_async_ctx *io)
{
#if defined(CONFIG_NET) #if defined(CONFIG_NET)
static int io_send_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe, const struct io_uring_sqe *sqe = req->sqe;
struct io_kiocb **nxt, bool force_nonblock, struct user_msghdr __user *msg;
long (*fn)(struct socket *, struct user_msghdr __user *, unsigned flags;
unsigned int))
flags = READ_ONCE(sqe->msg_flags);
msg = (struct user_msghdr __user *)(unsigned long) READ_ONCE(sqe->addr);
return sendmsg_copy_msghdr(&io->msg.msg, msg, flags, &io->msg.iov);
#else
return 0;
#endif
}
static int io_sendmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
struct io_kiocb **nxt, bool force_nonblock)
{ {
#if defined(CONFIG_NET)
struct socket *sock; struct socket *sock;
int ret; int ret;
...@@ -2005,7 +2026,9 @@ static int io_send_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe, ...@@ -2005,7 +2026,9 @@ static int io_send_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
sock = sock_from_file(req->file, &ret); sock = sock_from_file(req->file, &ret);
if (sock) { if (sock) {
struct user_msghdr __user *msg; struct io_async_ctx io, *copy;
struct sockaddr_storage addr;
struct msghdr *kmsg;
unsigned flags; unsigned flags;
flags = READ_ONCE(sqe->msg_flags); flags = READ_ONCE(sqe->msg_flags);
...@@ -2014,32 +2037,59 @@ static int io_send_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe, ...@@ -2014,32 +2037,59 @@ static int io_send_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
else if (force_nonblock) else if (force_nonblock)
flags |= MSG_DONTWAIT; flags |= MSG_DONTWAIT;
msg = (struct user_msghdr __user *) (unsigned long) if (req->io) {
READ_ONCE(sqe->addr); kmsg = &req->io->msg.msg;
kmsg->msg_name = &addr;
} else {
kmsg = &io.msg.msg;
kmsg->msg_name = &addr;
io.msg.iov = io.msg.fast_iov;
ret = io_sendmsg_prep(req, &io);
if (ret)
goto out;
}
ret = fn(sock, msg, flags); ret = __sys_sendmsg_sock(sock, kmsg, flags);
if (force_nonblock && ret == -EAGAIN) if (force_nonblock && ret == -EAGAIN) {
copy = kmalloc(sizeof(*copy), GFP_KERNEL);
if (!copy) {
ret = -ENOMEM;
goto out;
}
memcpy(&copy->msg, &io.msg, sizeof(copy->msg));
req->io = copy;
memcpy(&req->io->sqe, req->sqe, sizeof(*req->sqe));
req->sqe = &req->io->sqe;
return ret; return ret;
}
if (ret == -ERESTARTSYS) if (ret == -ERESTARTSYS)
ret = -EINTR; ret = -EINTR;
} }
out:
io_cqring_add_event(req, ret); io_cqring_add_event(req, ret);
if (ret < 0 && (req->flags & REQ_F_LINK)) if (ret < 0 && (req->flags & REQ_F_LINK))
req->flags |= REQ_F_FAIL_LINK; req->flags |= REQ_F_FAIL_LINK;
io_put_req_find_next(req, nxt); io_put_req_find_next(req, nxt);
return 0; return 0;
} #else
return -EOPNOTSUPP;
#endif #endif
}
static int io_sendmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe, static int io_recvmsg_prep(struct io_kiocb *req, struct io_async_ctx *io)
struct io_kiocb **nxt, bool force_nonblock)
{ {
#if defined(CONFIG_NET) #if defined(CONFIG_NET)
return io_send_recvmsg(req, sqe, nxt, force_nonblock, const struct io_uring_sqe *sqe = req->sqe;
__sys_sendmsg_sock); struct user_msghdr __user *msg;
unsigned flags;
flags = READ_ONCE(sqe->msg_flags);
msg = (struct user_msghdr __user *)(unsigned long) READ_ONCE(sqe->addr);
return recvmsg_copy_msghdr(&io->msg.msg, msg, flags, &io->msg.uaddr,
&io->msg.iov);
#else #else
return -EOPNOTSUPP; return 0;
#endif #endif
} }
...@@ -2047,8 +2097,63 @@ static int io_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe, ...@@ -2047,8 +2097,63 @@ static int io_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
struct io_kiocb **nxt, bool force_nonblock) struct io_kiocb **nxt, bool force_nonblock)
{ {
#if defined(CONFIG_NET) #if defined(CONFIG_NET)
return io_send_recvmsg(req, sqe, nxt, force_nonblock, struct socket *sock;
__sys_recvmsg_sock); int ret;
if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
return -EINVAL;
sock = sock_from_file(req->file, &ret);
if (sock) {
struct user_msghdr __user *msg;
struct io_async_ctx io, *copy;
struct sockaddr_storage addr;
struct msghdr *kmsg;
unsigned flags;
flags = READ_ONCE(sqe->msg_flags);
if (flags & MSG_DONTWAIT)
req->flags |= REQ_F_NOWAIT;
else if (force_nonblock)
flags |= MSG_DONTWAIT;
msg = (struct user_msghdr __user *) (unsigned long)
READ_ONCE(sqe->addr);
if (req->io) {
kmsg = &req->io->msg.msg;
kmsg->msg_name = &addr;
} else {
kmsg = &io.msg.msg;
kmsg->msg_name = &addr;
io.msg.iov = io.msg.fast_iov;
ret = io_recvmsg_prep(req, &io);
if (ret)
goto out;
}
ret = __sys_recvmsg_sock(sock, kmsg, msg, io.msg.uaddr, flags);
if (force_nonblock && ret == -EAGAIN) {
copy = kmalloc(sizeof(*copy), GFP_KERNEL);
if (!copy) {
ret = -ENOMEM;
goto out;
}
memcpy(copy, &io, sizeof(*copy));
req->io = copy;
memcpy(&req->io->sqe, req->sqe, sizeof(*req->sqe));
req->sqe = &req->io->sqe;
return ret;
}
if (ret == -ERESTARTSYS)
ret = -EINTR;
}
out:
io_cqring_add_event(req, ret);
if (ret < 0 && (req->flags & REQ_F_LINK))
req->flags |= REQ_F_FAIL_LINK;
io_put_req_find_next(req, nxt);
return 0;
#else #else
return -EOPNOTSUPP; return -EOPNOTSUPP;
#endif #endif
...@@ -2721,6 +2826,12 @@ static int io_req_defer_prep(struct io_kiocb *req, struct io_async_ctx *io) ...@@ -2721,6 +2826,12 @@ static int io_req_defer_prep(struct io_kiocb *req, struct io_async_ctx *io)
case IORING_OP_WRITE_FIXED: case IORING_OP_WRITE_FIXED:
ret = io_write_prep(req, &iovec, &iter, true); ret = io_write_prep(req, &iovec, &iter, true);
break; break;
case IORING_OP_SENDMSG:
ret = io_sendmsg_prep(req, io);
break;
case IORING_OP_RECVMSG:
ret = io_recvmsg_prep(req, io);
break;
default: default:
req->io = io; req->io = io;
return 0; return 0;
......
include/linux/socket.h
View file @ 03b1230c
...@@ -378,12 +378,19 @@ extern int __sys_recvmmsg(int fd, struct mmsghdr __user *mmsg, ...@@ -378,12 +378,19 @@ extern int __sys_recvmmsg(int fd, struct mmsghdr __user *mmsg,
extern int __sys_sendmmsg(int fd, struct mmsghdr __user *mmsg, extern int __sys_sendmmsg(int fd, struct mmsghdr __user *mmsg,
unsigned int vlen, unsigned int flags, unsigned int vlen, unsigned int flags,
bool forbid_cmsg_compat); bool forbid_cmsg_compat);
extern long __sys_sendmsg_sock(struct socket *sock, extern long __sys_sendmsg_sock(struct socket *sock, struct msghdr *msg,
struct user_msghdr __user *msg,
unsigned int flags); unsigned int flags);
extern long __sys_recvmsg_sock(struct socket *sock, extern long __sys_recvmsg_sock(struct socket *sock, struct msghdr *msg,
struct user_msghdr __user *msg, struct user_msghdr __user *umsg,
struct sockaddr __user *uaddr,
unsigned int flags); unsigned int flags);
extern int sendmsg_copy_msghdr(struct msghdr *msg,
struct user_msghdr __user *umsg, unsigned flags,
struct iovec **iov);
extern int recvmsg_copy_msghdr(struct msghdr *msg,
struct user_msghdr __user *umsg, unsigned flags,
struct sockaddr __user **uaddr,
struct iovec **iov);
/* helpers which do the actual work for syscalls */ /* helpers which do the actual work for syscalls */
extern int __sys_recvfrom(int fd, void __user *ubuf, size_t size, extern int __sys_recvfrom(int fd, void __user *ubuf, size_t size,
......
net/socket.c
View file @ 03b1230c
...@@ -2346,9 +2346,9 @@ static int ____sys_sendmsg(struct socket *sock, struct msghdr *msg_sys, ...@@ -2346,9 +2346,9 @@ static int ____sys_sendmsg(struct socket *sock, struct msghdr *msg_sys,
return err; return err;
} }
static int sendmsg_copy_msghdr(struct msghdr *msg, int sendmsg_copy_msghdr(struct msghdr *msg,
struct user_msghdr __user *umsg, unsigned flags, struct user_msghdr __user *umsg, unsigned flags,
struct iovec **iov) struct iovec **iov)
{ {
int err; int err;
...@@ -2390,27 +2390,14 @@ static int ___sys_sendmsg(struct socket *sock, struct user_msghdr __user *msg, ...@@ -2390,27 +2390,14 @@ static int ___sys_sendmsg(struct socket *sock, struct user_msghdr __user *msg,
/* /*
* BSD sendmsg interface * BSD sendmsg interface
*/ */
long __sys_sendmsg_sock(struct socket *sock, struct user_msghdr __user *umsg, long __sys_sendmsg_sock(struct socket *sock, struct msghdr *msg,
unsigned int flags) unsigned int flags)
{ {
struct iovec iovstack[UIO_FASTIOV], *iov = iovstack;
struct sockaddr_storage address;
struct msghdr msg = { .msg_name = &address };
ssize_t err;
err = sendmsg_copy_msghdr(&msg, umsg, flags, &iov);
if (err)
return err;
/* disallow ancillary data requests from this path */ /* disallow ancillary data requests from this path */
if (msg.msg_control || msg.msg_controllen) { if (msg->msg_control || msg->msg_controllen)
err = -EINVAL; return -EINVAL;
goto out;
}
err = ____sys_sendmsg(sock, &msg, flags, NULL, 0); return ____sys_sendmsg(sock, msg, flags, NULL, 0);
out:
kfree(iov);
return err;
} }
long __sys_sendmsg(int fd, struct user_msghdr __user *msg, unsigned int flags, long __sys_sendmsg(int fd, struct user_msghdr __user *msg, unsigned int flags,
...@@ -2516,10 +2503,10 @@ SYSCALL_DEFINE4(sendmmsg, int, fd, struct mmsghdr __user *, mmsg, ...@@ -2516,10 +2503,10 @@ SYSCALL_DEFINE4(sendmmsg, int, fd, struct mmsghdr __user *, mmsg,
return __sys_sendmmsg(fd, mmsg, vlen, flags, true); return __sys_sendmmsg(fd, mmsg, vlen, flags, true);
} }
static int recvmsg_copy_msghdr(struct msghdr *msg, int recvmsg_copy_msghdr(struct msghdr *msg,
struct user_msghdr __user *umsg, unsigned flags, struct user_msghdr __user *umsg, unsigned flags,
struct sockaddr __user **uaddr, struct sockaddr __user **uaddr,
struct iovec **iov) struct iovec **iov)
{ {
ssize_t err; ssize_t err;
...@@ -2609,28 +2596,15 @@ static int ___sys_recvmsg(struct socket *sock, struct user_msghdr __user *msg, ...@@ -2609,28 +2596,15 @@ static int ___sys_recvmsg(struct socket *sock, struct user_msghdr __user *msg,
* BSD recvmsg interface * BSD recvmsg interface
*/ */
long __sys_recvmsg_sock(struct socket *sock, struct user_msghdr __user *umsg, long __sys_recvmsg_sock(struct socket *sock, struct msghdr *msg,
unsigned int flags) struct user_msghdr __user *umsg,
struct sockaddr __user *uaddr, unsigned int flags)
{ {
struct iovec iovstack[UIO_FASTIOV], *iov = iovstack;
struct sockaddr_storage address;
struct msghdr msg = { .msg_name = &address };
struct sockaddr __user *uaddr;
ssize_t err;
err = recvmsg_copy_msghdr(&msg, umsg, flags, &uaddr, &iov);
if (err)
return err;
/* disallow ancillary data requests from this path */ /* disallow ancillary data requests from this path */
if (msg.msg_control || msg.msg_controllen) { if (msg->msg_control || msg->msg_controllen)
err = -EINVAL; return -EINVAL;
goto out;
}
err = ____sys_recvmsg(sock, &msg, umsg, uaddr, flags, 0); return ____sys_recvmsg(sock, msg, umsg, uaddr, flags, 0);
out:
kfree(iov);
return err;
} }
long __sys_recvmsg(int fd, struct user_msghdr __user *msg, unsigned int flags, long __sys_recvmsg(int fd, struct user_msghdr __user *msg, unsigned int flags,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7