Merge tag 'io_uring-6.0-2022-08-26' of git://git.kernel.dk/linux-block

Pull io_uring fixes from Jens Axboe:

 - Add missing header file to the MAINTAINERS entry for io_uring (Ammar)

 - liburing and the kernel ship the same io_uring.h header, but one
   change we've had for a long time only in liburing is to ensure it's
   C++ safe. Add extern C around it, so we can more easily sync them in
   the future (Ammar)

 - Fix an off-by-one in the sync cancel added in this merge window (me)

 - Error handling fix for passthrough (Kanchan)

 - Fix for address saving for async execution for the zc tx support
   (Pavel)

 - Fix ordering for TCP zc notifications, so we always have them ordered
   correctly between "data was sent" and "data was acked". This isn't
   strictly needed with the notification slots, but we've been pondering
   disabling the slot support for 6.0 - and if we do, then we do require
   the ordering to be sane. Regardless of that, it's the sane thing to
   do in terms of API (Pavel)

 - Minor cleanup for indentation and lockdep annotation (Pavel)

* tag 'io_uring-6.0-2022-08-26' of git://git.kernel.dk/linux-block:
  io_uring/net: save address for sendzc async execution
  io_uring: conditional ->async_data allocation
  io_uring/notif: order notif vs send CQEs
  io_uring/net: fix indentation
  io_uring/net: fix zc send link failing
  io_uring/net: fix must_hold annotation
  io_uring: fix submission-failure handling for uring-cmd
  io_uring: fix off-by-one in sync cancelation file check
  io_uring: uapi: Add `extern "C"` in io_uring.h for liburing
  MAINTAINERS: Add `include/linux/io_uring_types.h`

MAINTAINERS

@@ -10659,6 +10659,7 @@ T:	git git://git.kernel.dk/linux-block
 T:	git git://git.kernel.dk/liburing
 F:	io_uring/
 F:	include/linux/io_uring.h
+F:	include/linux/io_uring_types.h
 F:	include/uapi/linux/io_uring.h
 F:	tools/io_uring/
 

include/uapi/linux/io_uring.h

@@ -12,6 +12,10 @@
 #include <linux/types.h>
 #include <linux/time_types.h>
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /*
  * IO submission data structure (Submission Queue Entry)
  */
@@ -661,4 +665,8 @@ struct io_uring_recvmsg_out {
 	__u32 flags;
 };
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif

io_uring/cancel.c

@@ -218,7 +218,7 @@ static int __io_sync_cancel(struct io_uring_task *tctx,
 	    (cd->flags & IORING_ASYNC_CANCEL_FD_FIXED)) {
 		unsigned long file_ptr;
 
-		if (unlikely(fd > ctx->nr_user_files))
+		if (unlikely(fd >= ctx->nr_user_files))
 			return -EBADF;
 		fd = array_index_nospec(fd, ctx->nr_user_files);
 		file_ptr = io_fixed_file_slot(&ctx->file_table, fd)->file_ptr;

io_uring/io_uring.c

@@ -1450,9 +1450,10 @@ int io_req_prep_async(struct io_kiocb *req)
 		return 0;
 	if (WARN_ON_ONCE(req_has_async_data(req)))
 		return -EFAULT;
+	if (!io_op_defs[req->opcode].manual_alloc) {
 		if (io_alloc_async_data(req))
 			return -EAGAIN;
-
+	}
 	return def->prep_async(req);
 }
 

io_uring/net.c

@@ -182,6 +182,37 @@ static int io_sendmsg_copy_hdr(struct io_kiocb *req,
 					&iomsg->free_iov);
 }
 
+int io_sendzc_prep_async(struct io_kiocb *req)
+{
+	struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc);
+	struct io_async_msghdr *io;
+	int ret;
+
+	if (!zc->addr || req_has_async_data(req))
+		return 0;
+	if (io_alloc_async_data(req))
+		return -ENOMEM;
+
+	io = req->async_data;
+	ret = move_addr_to_kernel(zc->addr, zc->addr_len, &io->addr);
+	return ret;
+}
+
+static int io_setup_async_addr(struct io_kiocb *req,
+			      struct sockaddr_storage *addr,
+			      unsigned int issue_flags)
+{
+	struct io_async_msghdr *io;
+
+	if (!addr || req_has_async_data(req))
+		return -EAGAIN;
+	if (io_alloc_async_data(req))
+		return -ENOMEM;
+	io = req->async_data;
+	memcpy(&io->addr, addr, sizeof(io->addr));
+	return -EAGAIN;
+}
+
 int io_sendmsg_prep_async(struct io_kiocb *req)
 {
 	int ret;
@@ -944,7 +975,7 @@ static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb,
 
 int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)
 {
-	struct sockaddr_storage address;
+	struct sockaddr_storage __address, *addr = NULL;
 	struct io_ring_ctx *ctx = req->ctx;
 	struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc);
 	struct io_notif_slot *notif_slot;
@@ -978,10 +1009,17 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)
 	msg.msg_namelen = 0;
 
 	if (zc->addr) {
-		ret = move_addr_to_kernel(zc->addr, zc->addr_len, &address);
+		if (req_has_async_data(req)) {
+			struct io_async_msghdr *io = req->async_data;
+
+			msg.msg_name = addr = &io->addr;
+		} else {
+			ret = move_addr_to_kernel(zc->addr, zc->addr_len, &__address);
 			if (unlikely(ret < 0))
 				return ret;
-		msg.msg_name = (struct sockaddr *)&address;
+			msg.msg_name = (struct sockaddr *)&__address;
+			addr = &__address;
+		}
 		msg.msg_namelen = zc->addr_len;
 	}
 
@@ -1013,16 +1051,18 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)
 
 	if (unlikely(ret < min_ret)) {
 		if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
-			return -EAGAIN;
+			return io_setup_async_addr(req, addr, issue_flags);
+
 		if (ret > 0 && io_net_retry(sock, msg.msg_flags)) {
 			zc->len -= ret;
 			zc->buf += ret;
 			zc->done_io += ret;
 			req->flags |= REQ_F_PARTIAL_IO;
-			return -EAGAIN;
+			return io_setup_async_addr(req, addr, issue_flags);
 		}
 		if (ret == -ERESTARTSYS)
 			ret = -EINTR;
+		req_set_fail(req);
 	} else if (zc->flags & IORING_RECVSEND_NOTIF_FLUSH) {
 		io_notif_slot_flush_submit(notif_slot, 0);
 	}

io_uring/net.h

@@ -31,6 +31,7 @@ struct io_async_connect {
 int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
 int io_shutdown(struct io_kiocb *req, unsigned int issue_flags);
 
+int io_sendzc_prep_async(struct io_kiocb *req);
 int io_sendmsg_prep_async(struct io_kiocb *req);
 void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req);
 int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);

io_uring/notif.c

@@ -73,7 +73,7 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx,
 }
 
 void io_notif_slot_flush(struct io_notif_slot *slot)
-	__must_hold(&ctx->uring_lock)
+	__must_hold(&slot->notif->ctx->uring_lock)
 {
 	struct io_kiocb *notif = slot->notif;
 	struct io_notif_data *nd = io_notif_to_data(notif);
@@ -81,8 +81,10 @@ void io_notif_slot_flush(struct io_notif_slot *slot)
 	slot->notif = NULL;
 
 	/* drop slot's master ref */
-	if (refcount_dec_and_test(&nd->uarg.refcnt))
-		io_notif_complete(notif);
+	if (refcount_dec_and_test(&nd->uarg.refcnt)) {
+		notif->io_task_work.func = __io_notif_complete_tw;
+		io_req_task_work_add(notif);
+	}
 }
 
 __cold int io_notif_unregister(struct io_ring_ctx *ctx)

io_uring/opdef.c

@@ -478,13 +478,15 @@ const struct io_op_def io_op_defs[] = {
 		.pollout		= 1,
 		.audit_skip		= 1,
 		.ioprio			= 1,
+		.manual_alloc		= 1,
 #if defined(CONFIG_NET)
+		.async_size		= sizeof(struct io_async_msghdr),
 		.prep			= io_sendzc_prep,
 		.issue			= io_sendzc,
+		.prep_async		= io_sendzc_prep_async,
 #else
 		.prep			= io_eopnotsupp_prep,
 #endif
-
 	},
 };
 

io_uring/opdef.h

@@ -25,6 +25,8 @@ struct io_op_def {
 	unsigned		ioprio : 1;
 	/* supports iopoll */
 	unsigned		iopoll : 1;
+	/* opcode specific path will handle ->async_data allocation if needed */
+	unsigned		manual_alloc : 1;
 	/* size of async data needed, if any */
 	unsigned short		async_size;
 

io_uring/uring_cmd.c

@@ -112,7 +112,7 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
 		if (ret < 0)
 			req_set_fail(req);
 		io_req_set_res(req, ret, 0);
-		return IOU_OK;
+		return ret;
 	}
 
 	return IOU_ISSUE_SKIP_COMPLETE;