From 165126dc5e23979721122dc5c7cfb28b1ca234cc Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 21 Sep 2024 01:59:47 -0600 Subject: [PATCH 01/79] io_uring/eventfd: abstract out ev_fd put helper We call this in two spot, have a helper for it. In preparation for extending this part. Link: https://lore.kernel.org/r/20240921080307.185186-2-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/eventfd.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/io_uring/eventfd.c b/io_uring/eventfd.c index e37fddd5d9ce..8b628ab6bbff 100644 --- a/io_uring/eventfd.c +++ b/io_uring/eventfd.c @@ -41,6 +41,12 @@ static void io_eventfd_do_signal(struct rcu_head *rcu) io_eventfd_free(rcu); } +static void io_eventfd_put(struct io_ev_fd *ev_fd) +{ + if (refcount_dec_and_test(&ev_fd->refs)) + call_rcu(&ev_fd->rcu, io_eventfd_free); +} + void io_eventfd_signal(struct io_ring_ctx *ctx) { struct io_ev_fd *ev_fd = NULL; @@ -77,8 +83,7 @@ void io_eventfd_signal(struct io_ring_ctx *ctx) } } out: - if (refcount_dec_and_test(&ev_fd->refs)) - call_rcu(&ev_fd->rcu, io_eventfd_free); + io_eventfd_put(ev_fd); } void io_eventfd_flush_signal(struct io_ring_ctx *ctx) @@ -152,8 +157,7 @@ int io_eventfd_unregister(struct io_ring_ctx *ctx) if (ev_fd) { ctx->has_evfd = false; rcu_assign_pointer(ctx->io_ev_fd, NULL); - if (refcount_dec_and_test(&ev_fd->refs)) - call_rcu(&ev_fd->rcu, io_eventfd_free); + io_eventfd_put(ev_fd); return 0; } From 3c90b80df5b574c2c61626fd40fa3b23be21fa26 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 21 Sep 2024 01:59:48 -0600 Subject: [PATCH 02/79] io_uring/eventfd: check for the need to async notifier earlier It's not necessary to do this post grabbing a reference. With that, we can drop the out goto path as well. Link: https://lore.kernel.org/r/20240921080307.185186-3-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/eventfd.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/io_uring/eventfd.c b/io_uring/eventfd.c index 8b628ab6bbff..829873806f9f 100644 --- a/io_uring/eventfd.c +++ b/io_uring/eventfd.c @@ -69,10 +69,10 @@ void io_eventfd_signal(struct io_ring_ctx *ctx) */ if (unlikely(!ev_fd)) return; + if (ev_fd->eventfd_async && !io_wq_current_is_worker()) + return; if (!refcount_inc_not_zero(&ev_fd->refs)) return; - if (ev_fd->eventfd_async && !io_wq_current_is_worker()) - goto out; if (likely(eventfd_signal_allowed())) { eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE); @@ -82,7 +82,6 @@ void io_eventfd_signal(struct io_ring_ctx *ctx) return; } } -out: io_eventfd_put(ev_fd); } From 60c5f15800f21883615689e2423217a9c8a1b502 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 21 Sep 2024 01:59:49 -0600 Subject: [PATCH 03/79] io_uring/eventfd: move actual signaling part into separate helper In preparation for using this from multiple spots, move the signaling into a helper. Link: https://lore.kernel.org/r/20240921080307.185186-4-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/eventfd.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/io_uring/eventfd.c b/io_uring/eventfd.c index 829873806f9f..58e76f4d1e00 100644 --- a/io_uring/eventfd.c +++ b/io_uring/eventfd.c @@ -47,6 +47,22 @@ static void io_eventfd_put(struct io_ev_fd *ev_fd) call_rcu(&ev_fd->rcu, io_eventfd_free); } +/* + * Returns true if the caller should put the ev_fd reference, false if not. + */ +static bool __io_eventfd_signal(struct io_ev_fd *ev_fd) +{ + if (eventfd_signal_allowed()) { + eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE); + return true; + } + if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops)) { + call_rcu_hurry(&ev_fd->rcu, io_eventfd_do_signal); + return false; + } + return true; +} + void io_eventfd_signal(struct io_ring_ctx *ctx) { struct io_ev_fd *ev_fd = NULL; @@ -73,16 +89,8 @@ void io_eventfd_signal(struct io_ring_ctx *ctx) return; if (!refcount_inc_not_zero(&ev_fd->refs)) return; - - if (likely(eventfd_signal_allowed())) { - eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE); - } else { - if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops)) { - call_rcu_hurry(&ev_fd->rcu, io_eventfd_do_signal); - return; - } - } - io_eventfd_put(ev_fd); + if (__io_eventfd_signal(ev_fd)) + io_eventfd_put(ev_fd); } void io_eventfd_flush_signal(struct io_ring_ctx *ctx) From 3ca5a356041438534ecbb74159df91736238c6b1 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 21 Sep 2024 01:59:50 -0600 Subject: [PATCH 04/79] io_uring/eventfd: move trigger check into a helper It's a bit hard to read what guards the triggering, move it into a helper and add a comment explaining it too. This additionally moves the ev_fd == NULL check in there as well. Link: https://lore.kernel.org/r/20240921080307.185186-5-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/eventfd.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/io_uring/eventfd.c b/io_uring/eventfd.c index 58e76f4d1e00..0946d3da88d3 100644 --- a/io_uring/eventfd.c +++ b/io_uring/eventfd.c @@ -63,6 +63,17 @@ static bool __io_eventfd_signal(struct io_ev_fd *ev_fd) return true; } +/* + * Trigger if eventfd_async isn't set, or if it's set and the caller is + * an async worker. If ev_fd isn't valid, obviously return false. + */ +static bool io_eventfd_trigger(struct io_ev_fd *ev_fd) +{ + if (ev_fd) + return !ev_fd->eventfd_async || io_wq_current_is_worker(); + return false; +} + void io_eventfd_signal(struct io_ring_ctx *ctx) { struct io_ev_fd *ev_fd = NULL; @@ -83,9 +94,7 @@ void io_eventfd_signal(struct io_ring_ctx *ctx) * completed between the NULL check of ctx->io_ev_fd at the start of * the function and rcu_read_lock. */ - if (unlikely(!ev_fd)) - return; - if (ev_fd->eventfd_async && !io_wq_current_is_worker()) + if (!io_eventfd_trigger(ev_fd)) return; if (!refcount_inc_not_zero(&ev_fd->refs)) return; From 83a4f865e273b83426eafdd3aa51334cc21ac0fd Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 21 Sep 2024 01:59:51 -0600 Subject: [PATCH 05/79] io_uring/eventfd: abstract out ev_fd grab + release helpers In preparation for needing the ev_fd grabbing (and releasing) from another path, abstract out two helpers for that. Link: https://lore.kernel.org/r/20240921080307.185186-6-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/eventfd.c | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/io_uring/eventfd.c b/io_uring/eventfd.c index 0946d3da88d3..d1fdecd0c458 100644 --- a/io_uring/eventfd.c +++ b/io_uring/eventfd.c @@ -47,6 +47,13 @@ static void io_eventfd_put(struct io_ev_fd *ev_fd) call_rcu(&ev_fd->rcu, io_eventfd_free); } +static void io_eventfd_release(struct io_ev_fd *ev_fd, bool put_ref) +{ + if (put_ref) + io_eventfd_put(ev_fd); + rcu_read_unlock(); +} + /* * Returns true if the caller should put the ev_fd reference, false if not. */ @@ -74,14 +81,18 @@ static bool io_eventfd_trigger(struct io_ev_fd *ev_fd) return false; } -void io_eventfd_signal(struct io_ring_ctx *ctx) +/* + * On success, returns with an ev_fd reference grabbed and the RCU read + * lock held. + */ +static struct io_ev_fd *io_eventfd_grab(struct io_ring_ctx *ctx) { - struct io_ev_fd *ev_fd = NULL; + struct io_ev_fd *ev_fd; if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) - return; + return NULL; - guard(rcu)(); + rcu_read_lock(); /* * rcu_dereference ctx->io_ev_fd once and use it for both for checking @@ -90,16 +101,24 @@ void io_eventfd_signal(struct io_ring_ctx *ctx) ev_fd = rcu_dereference(ctx->io_ev_fd); /* - * Check again if ev_fd exists incase an io_eventfd_unregister call + * Check again if ev_fd exists in case an io_eventfd_unregister call * completed between the NULL check of ctx->io_ev_fd at the start of * the function and rcu_read_lock. */ - if (!io_eventfd_trigger(ev_fd)) - return; - if (!refcount_inc_not_zero(&ev_fd->refs)) - return; - if (__io_eventfd_signal(ev_fd)) - io_eventfd_put(ev_fd); + if (io_eventfd_trigger(ev_fd) && refcount_inc_not_zero(&ev_fd->refs)) + return ev_fd; + + rcu_read_unlock(); + return NULL; +} + +void io_eventfd_signal(struct io_ring_ctx *ctx) +{ + struct io_ev_fd *ev_fd; + + ev_fd = io_eventfd_grab(ctx); + if (ev_fd) + io_eventfd_release(ev_fd, __io_eventfd_signal(ev_fd)); } void io_eventfd_flush_signal(struct io_ring_ctx *ctx) From f4bb2f65bb8154c1a2c2d7e01db0c98dffb5918f Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 21 Sep 2024 01:59:52 -0600 Subject: [PATCH 06/79] io_uring/eventfd: move ctx->evfd_last_cq_tail into io_ev_fd Everything else about the io_uring eventfd support is nicely kept private to that code, except the cached_cq_tail tracking. With everything else in place, move io_eventfd_flush_signal() to using the ev_fd grab+release helpers, which then enables the direct use of io_ev_fd for this tracking too. Link: https://lore.kernel.org/r/20240921080307.185186-7-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/eventfd.c | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/io_uring/eventfd.c b/io_uring/eventfd.c index d1fdecd0c458..fab936d31ba8 100644 --- a/io_uring/eventfd.c +++ b/io_uring/eventfd.c @@ -13,10 +13,12 @@ struct io_ev_fd { struct eventfd_ctx *cq_ev_fd; - unsigned int eventfd_async: 1; - struct rcu_head rcu; + unsigned int eventfd_async; + /* protected by ->completion_lock */ + unsigned last_cq_tail; refcount_t refs; atomic_t ops; + struct rcu_head rcu; }; enum { @@ -123,25 +125,31 @@ void io_eventfd_signal(struct io_ring_ctx *ctx) void io_eventfd_flush_signal(struct io_ring_ctx *ctx) { - bool skip; + struct io_ev_fd *ev_fd; - spin_lock(&ctx->completion_lock); + ev_fd = io_eventfd_grab(ctx); + if (ev_fd) { + bool skip, put_ref = true; - /* - * Eventfd should only get triggered when at least one event has been - * posted. Some applications rely on the eventfd notification count - * only changing IFF a new CQE has been added to the CQ ring. There's - * no depedency on 1:1 relationship between how many times this - * function is called (and hence the eventfd count) and number of CQEs - * posted to the CQ ring. - */ - skip = ctx->cached_cq_tail == ctx->evfd_last_cq_tail; - ctx->evfd_last_cq_tail = ctx->cached_cq_tail; - spin_unlock(&ctx->completion_lock); - if (skip) - return; + /* + * Eventfd should only get triggered when at least one event + * has been posted. Some applications rely on the eventfd + * notification count only changing IFF a new CQE has been + * added to the CQ ring. There's no dependency on 1:1 + * relationship between how many times this function is called + * (and hence the eventfd count) and number of CQEs posted to + * the CQ ring. + */ + spin_lock(&ctx->completion_lock); + skip = ctx->cached_cq_tail == ev_fd->last_cq_tail; + ev_fd->last_cq_tail = ctx->cached_cq_tail; + spin_unlock(&ctx->completion_lock); - io_eventfd_signal(ctx); + if (!skip) + put_ref = __io_eventfd_signal(ev_fd); + + io_eventfd_release(ev_fd, put_ref); + } } int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, @@ -172,7 +180,7 @@ int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, } spin_lock(&ctx->completion_lock); - ctx->evfd_last_cq_tail = ctx->cached_cq_tail; + ev_fd->last_cq_tail = ctx->cached_cq_tail; spin_unlock(&ctx->completion_lock); ev_fd->eventfd_async = eventfd_async; From 95d6c9229a04cc12d39034cd6be6446a55a85d6d Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 24 Sep 2024 05:57:30 -0600 Subject: [PATCH 07/79] io_uring/msg_ring: refactor a few helper functions Mostly just to skip them taking an io_kiocb, rather just pass in the ctx and io_msg directly. In preparation for being able to issue a MSG_RING request without having an io_kiocb. No functional changes in this patch. Link: https://lore.kernel.org/r/20240924115932.116167-2-axboe@kernel.dk Signed-off-by: Jens Axboe --- io_uring/msg_ring.c | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index 7fd9badcfaf8..b8c527f08cd5 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -116,14 +116,13 @@ static struct io_kiocb *io_msg_get_kiocb(struct io_ring_ctx *ctx) return kmem_cache_alloc(req_cachep, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO); } -static int io_msg_data_remote(struct io_kiocb *req) +static int io_msg_data_remote(struct io_ring_ctx *target_ctx, + struct io_msg *msg) { - struct io_ring_ctx *target_ctx = req->file->private_data; - struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); struct io_kiocb *target; u32 flags = 0; - target = io_msg_get_kiocb(req->ctx); + target = io_msg_get_kiocb(target_ctx); if (unlikely(!target)) return -ENOMEM; @@ -134,10 +133,9 @@ static int io_msg_data_remote(struct io_kiocb *req) msg->user_data); } -static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags) +static int __io_msg_ring_data(struct io_ring_ctx *target_ctx, + struct io_msg *msg, unsigned int issue_flags) { - struct io_ring_ctx *target_ctx = req->file->private_data; - struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); u32 flags = 0; int ret; @@ -149,7 +147,7 @@ static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags) return -EBADFD; if (io_msg_need_remote(target_ctx)) - return io_msg_data_remote(req); + return io_msg_data_remote(target_ctx, msg); if (msg->flags & IORING_MSG_RING_FLAGS_PASS) flags = msg->cqe_flags; @@ -166,6 +164,14 @@ static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags) return ret; } +static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags) +{ + struct io_ring_ctx *target_ctx = req->file->private_data; + struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); + + return __io_msg_ring_data(target_ctx, msg, issue_flags); +} + static struct file *io_msg_grab_file(struct io_kiocb *req, unsigned int issue_flags) { struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); @@ -271,10 +277,8 @@ static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags) return io_msg_install_complete(req, issue_flags); } -int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +static int __io_msg_ring_prep(struct io_msg *msg, const struct io_uring_sqe *sqe) { - struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); - if (unlikely(sqe->buf_index || sqe->personality)) return -EINVAL; @@ -291,6 +295,11 @@ int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } +int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +{ + return __io_msg_ring_prep(io_kiocb_to_cmd(req, struct io_msg), sqe); +} + int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags) { struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); From a377132154ab8404dafcc52e8bc0c73050a954c2 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 24 Sep 2024 05:57:31 -0600 Subject: [PATCH 08/79] io_uring/msg_ring: add support for sending a sync message Normally MSG_RING requires both a source and a destination ring. But some users don't always have a ring avilable to send a message from, yet they still need to notify a target ring. Add support for using io_uring_register(2) without having a source ring, using a file descriptor of -1 for that. Internally those are called blind registration opcodes. Implement IORING_REGISTER_SEND_MSG_RING as a blind opcode, which simply takes an sqe that the application can put on the stack and use the normal liburing helpers to initialize it. Then the app can call: io_uring_register(-1, IORING_REGISTER_SEND_MSG_RING, &sqe, 1); and get the same behavior in terms of the target, where a CQE is posted with the details given in the sqe. For now this takes a single sqe pointer argument, and hence arg must be set to that, and nr_args must be 1. Could easily be extended to take an array of sqes, but for now let's keep it simple. Link: https://lore.kernel.org/r/20240924115932.116167-3-axboe@kernel.dk Signed-off-by: Jens Axboe --- include/uapi/linux/io_uring.h | 3 +++ io_uring/msg_ring.c | 29 +++++++++++++++++++++++++++++ io_uring/msg_ring.h | 1 + io_uring/register.c | 30 ++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+) diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 1fe79e750470..86cb385fe0b5 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -612,6 +612,9 @@ enum io_uring_register_op { /* clone registered buffers from source ring to current ring */ IORING_REGISTER_CLONE_BUFFERS = 30, + /* send MSG_RING without having a ring */ + IORING_REGISTER_SEND_MSG_RING = 31, + /* this goes last */ IORING_REGISTER_LAST, diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index b8c527f08cd5..edea1ffd501c 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -331,6 +331,35 @@ int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags) return IOU_OK; } +int io_uring_sync_msg_ring(struct io_uring_sqe *sqe) +{ + struct io_msg io_msg = { }; + struct fd f; + int ret; + + ret = __io_msg_ring_prep(&io_msg, sqe); + if (unlikely(ret)) + return ret; + + /* + * Only data sending supported, not IORING_MSG_SEND_FD as that one + * doesn't make sense without a source ring to send files from. + */ + if (io_msg.cmd != IORING_MSG_DATA) + return -EINVAL; + + ret = -EBADF; + f = fdget(sqe->fd); + if (fd_file(f)) { + ret = -EBADFD; + if (io_is_uring_fops(fd_file(f))) + ret = __io_msg_ring_data(fd_file(f)->private_data, + &io_msg, IO_URING_F_UNLOCKED); + fdput(f); + } + return ret; +} + void io_msg_cache_free(const void *entry) { struct io_kiocb *req = (struct io_kiocb *) entry; diff --git a/io_uring/msg_ring.h b/io_uring/msg_ring.h index 3030f3942f0f..38e7f8f0c944 100644 --- a/io_uring/msg_ring.h +++ b/io_uring/msg_ring.h @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +int io_uring_sync_msg_ring(struct io_uring_sqe *sqe); int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags); void io_msg_ring_cleanup(struct io_kiocb *req); diff --git a/io_uring/register.c b/io_uring/register.c index eca26d4884d9..52b2f9b74af8 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -28,6 +28,7 @@ #include "kbuf.h" #include "napi.h" #include "eventfd.h" +#include "msg_ring.h" #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ IORING_REGISTER_LAST + IORING_OP_LAST) @@ -588,6 +589,32 @@ struct file *io_uring_register_get_file(unsigned int fd, bool registered) return ERR_PTR(-EOPNOTSUPP); } +/* + * "blind" registration opcodes are ones where there's no ring given, and + * hence the source fd must be -1. + */ +static int io_uring_register_blind(unsigned int opcode, void __user *arg, + unsigned int nr_args) +{ + switch (opcode) { + case IORING_REGISTER_SEND_MSG_RING: { + struct io_uring_sqe sqe; + + if (!arg || nr_args != 1) + return -EINVAL; + if (copy_from_user(&sqe, arg, sizeof(sqe))) + return -EFAULT; + /* no flags supported */ + if (sqe.flags) + return -EINVAL; + if (sqe.opcode == IORING_OP_MSG_RING) + return io_uring_sync_msg_ring(&sqe); + } + } + + return -EINVAL; +} + SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, void __user *, arg, unsigned int, nr_args) { @@ -602,6 +629,9 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, if (opcode >= IORING_REGISTER_LAST) return -EINVAL; + if (fd == -1) + return io_uring_register_blind(opcode, arg, nr_args); + file = io_uring_register_get_file(fd, use_registered_ring); if (IS_ERR(file)) return PTR_ERR(file); From 829ab73e7bca455e1a8718325177cfb98b63d0df Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 30 Sep 2024 13:52:54 -0600 Subject: [PATCH 09/79] io_uring/poll: remove 'ctx' argument from io_poll_req_delete() It's always req->ctx being used anyway, having this as a separate argument (that is then not even used) just makes it more confusing. Signed-off-by: Jens Axboe --- io_uring/poll.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/io_uring/poll.c b/io_uring/poll.c index 1f63b60e85e7..175c279e59ea 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -129,7 +129,7 @@ static void io_poll_req_insert(struct io_kiocb *req) spin_unlock(&hb->lock); } -static void io_poll_req_delete(struct io_kiocb *req, struct io_ring_ctx *ctx) +static void io_poll_req_delete(struct io_kiocb *req) { struct io_hash_table *table = &req->ctx->cancel_table; u32 index = hash_long(req->cqe.user_data, table->hash_bits); @@ -165,7 +165,7 @@ static void io_poll_tw_hash_eject(struct io_kiocb *req, struct io_tw_state *ts) hash_del(&req->hash_node); req->flags &= ~REQ_F_HASH_LOCKED; } else { - io_poll_req_delete(req, ctx); + io_poll_req_delete(req); } } From 085268829b07202cf7bf8ec1a8fb7fd9d8f6a41a Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 30 Sep 2024 14:22:36 -0600 Subject: [PATCH 10/79] io_uring/poll: get rid of unlocked cancel hash io_uring maintains two hash lists of inflight requests: 1) ctx->cancel_table_locked. This is used when the caller has the ctx->uring_lock held already. This is only an issue side parameter, as removal or task_work will always have it held. 2) ctx->cancel_table. This is used when the issuer does NOT have the ctx->uring_lock held, and relies on the table spinlocks for access. However, it's pretty trivial to simply grab the lock in the one spot where we care about it, for insertion. With that, we can kill the unlocked table (and get rid of the _locked postfix for the other one). Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 6 +- io_uring/fdinfo.c | 11 +-- io_uring/io_uring.c | 4 - io_uring/poll.c | 142 ++++++++------------------------- 4 files changed, 36 insertions(+), 127 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 4b9ba523978d..d8ca27da1341 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -291,7 +291,7 @@ struct io_ring_ctx { struct xarray io_bl_xa; - struct io_hash_table cancel_table_locked; + struct io_hash_table cancel_table; struct io_alloc_cache apoll_cache; struct io_alloc_cache netmsg_cache; struct io_alloc_cache rw_cache; @@ -342,7 +342,6 @@ struct io_ring_ctx { struct list_head io_buffers_comp; struct list_head cq_overflow_list; - struct io_hash_table cancel_table; struct hlist_head waitid_list; @@ -459,7 +458,6 @@ enum { REQ_F_DOUBLE_POLL_BIT, REQ_F_APOLL_MULTISHOT_BIT, REQ_F_CLEAR_POLLIN_BIT, - REQ_F_HASH_LOCKED_BIT, /* keep async read/write and isreg together and in order */ REQ_F_SUPPORT_NOWAIT_BIT, REQ_F_ISREG_BIT, @@ -534,8 +532,6 @@ enum { REQ_F_APOLL_MULTISHOT = IO_REQ_FLAG(REQ_F_APOLL_MULTISHOT_BIT), /* recvmsg special flag, clear EPOLLIN */ REQ_F_CLEAR_POLLIN = IO_REQ_FLAG(REQ_F_CLEAR_POLLIN_BIT), - /* hashed into ->cancel_hash_locked, protected by ->uring_lock */ - REQ_F_HASH_LOCKED = IO_REQ_FLAG(REQ_F_HASH_LOCKED_BIT), /* don't use lazy poll wake for this request */ REQ_F_POLL_NO_LAZY = IO_REQ_FLAG(REQ_F_POLL_NO_LAZY_BIT), /* file is pollable */ diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index 6b1247664b35..a6bac533edbe 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -190,22 +190,13 @@ __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *file) } seq_puts(m, "PollList:\n"); - for (i = 0; i < (1U << ctx->cancel_table.hash_bits); i++) { + for (i = 0; has_lock && i < (1U << ctx->cancel_table.hash_bits); i++) { struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i]; - struct io_hash_bucket *hbl = &ctx->cancel_table_locked.hbs[i]; struct io_kiocb *req; - spin_lock(&hb->lock); hlist_for_each_entry(req, &hb->list, hash_node) seq_printf(m, " op=%d, task_works=%d\n", req->opcode, task_work_pending(req->task)); - spin_unlock(&hb->lock); - - if (!has_lock) - continue; - hlist_for_each_entry(req, &hbl->list, hash_node) - seq_printf(m, " op=%d, task_works=%d\n", req->opcode, - task_work_pending(req->task)); } if (has_lock) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index b2736e3491b8..f4e069cd03a5 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -294,8 +294,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) hash_bits = clamp(hash_bits, 1, 8); if (io_alloc_hash_table(&ctx->cancel_table, hash_bits)) goto err; - if (io_alloc_hash_table(&ctx->cancel_table_locked, hash_bits)) - goto err; if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free, 0, GFP_KERNEL)) goto err; @@ -361,7 +359,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) io_alloc_cache_free(&ctx->msg_cache, io_msg_cache_free); io_futex_cache_free(ctx); kfree(ctx->cancel_table.hbs); - kfree(ctx->cancel_table_locked.hbs); xa_destroy(&ctx->io_bl_xa); kfree(ctx); return NULL; @@ -2774,7 +2771,6 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) io_wq_put_hash(ctx->hash_map); io_napi_free(ctx); kfree(ctx->cancel_table.hbs); - kfree(ctx->cancel_table_locked.hbs); xa_destroy(&ctx->io_bl_xa); kfree(ctx); } diff --git a/io_uring/poll.c b/io_uring/poll.c index 175c279e59ea..217d667e0622 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -122,28 +122,6 @@ static void io_poll_req_insert(struct io_kiocb *req) { struct io_hash_table *table = &req->ctx->cancel_table; u32 index = hash_long(req->cqe.user_data, table->hash_bits); - struct io_hash_bucket *hb = &table->hbs[index]; - - spin_lock(&hb->lock); - hlist_add_head(&req->hash_node, &hb->list); - spin_unlock(&hb->lock); -} - -static void io_poll_req_delete(struct io_kiocb *req) -{ - struct io_hash_table *table = &req->ctx->cancel_table; - u32 index = hash_long(req->cqe.user_data, table->hash_bits); - spinlock_t *lock = &table->hbs[index].lock; - - spin_lock(lock); - hash_del(&req->hash_node); - spin_unlock(lock); -} - -static void io_poll_req_insert_locked(struct io_kiocb *req) -{ - struct io_hash_table *table = &req->ctx->cancel_table_locked; - u32 index = hash_long(req->cqe.user_data, table->hash_bits); lockdep_assert_held(&req->ctx->uring_lock); @@ -154,19 +132,14 @@ static void io_poll_tw_hash_eject(struct io_kiocb *req, struct io_tw_state *ts) { struct io_ring_ctx *ctx = req->ctx; - if (req->flags & REQ_F_HASH_LOCKED) { - /* - * ->cancel_table_locked is protected by ->uring_lock in - * contrast to per bucket spinlocks. Likely, tctx_task_work() - * already grabbed the mutex for us, but there is a chance it - * failed. - */ - io_tw_lock(ctx, ts); - hash_del(&req->hash_node); - req->flags &= ~REQ_F_HASH_LOCKED; - } else { - io_poll_req_delete(req); - } + /* + * ->cancel_table_locked is protected by ->uring_lock in + * contrast to per bucket spinlocks. Likely, tctx_task_work() + * already grabbed the mutex for us, but there is a chance it + * failed. + */ + io_tw_lock(ctx, ts); + hash_del(&req->hash_node); } static void io_init_poll_iocb(struct io_poll *poll, __poll_t events) @@ -563,12 +536,13 @@ static bool io_poll_can_finish_inline(struct io_kiocb *req, return pt->owning || io_poll_get_ownership(req); } -static void io_poll_add_hash(struct io_kiocb *req) +static void io_poll_add_hash(struct io_kiocb *req, unsigned int issue_flags) { - if (req->flags & REQ_F_HASH_LOCKED) - io_poll_req_insert_locked(req); - else - io_poll_req_insert(req); + struct io_ring_ctx *ctx = req->ctx; + + io_ring_submit_lock(ctx, issue_flags); + io_poll_req_insert(req); + io_ring_submit_unlock(ctx, issue_flags); } /* @@ -605,11 +579,6 @@ static int __io_arm_poll_handler(struct io_kiocb *req, ipt->owning = issue_flags & IO_URING_F_UNLOCKED; atomic_set(&req->poll_refs, (int)ipt->owning); - /* io-wq doesn't hold uring_lock */ - if (issue_flags & IO_URING_F_UNLOCKED) - req->flags &= ~REQ_F_HASH_LOCKED; - - /* * Exclusive waits may only wake a limited amount of entries * rather than all of them, this may interfere with lazy @@ -638,7 +607,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, if (mask && ((poll->events & (EPOLLET|EPOLLONESHOT)) == (EPOLLET|EPOLLONESHOT))) { if (!io_poll_can_finish_inline(req, ipt)) { - io_poll_add_hash(req); + io_poll_add_hash(req, issue_flags); return 0; } io_poll_remove_entries(req); @@ -647,7 +616,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, return 1; } - io_poll_add_hash(req); + io_poll_add_hash(req, issue_flags); if (mask && (poll->events & EPOLLET) && io_poll_can_finish_inline(req, ipt)) { @@ -720,12 +689,6 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) __poll_t mask = POLLPRI | POLLERR | EPOLLET; int ret; - /* - * apoll requests already grab the mutex to complete in the tw handler, - * so removal from the mutex-backed hash is free, use it by default. - */ - req->flags |= REQ_F_HASH_LOCKED; - if (!def->pollin && !def->pollout) return IO_APOLL_ABORTED; if (!io_file_can_poll(req)) @@ -761,18 +724,22 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) return IO_APOLL_OK; } -static __cold bool io_poll_remove_all_table(struct task_struct *tsk, - struct io_hash_table *table, - bool cancel_all) +/* + * Returns true if we found and killed one or more poll requests + */ +__cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, + bool cancel_all) { - unsigned nr_buckets = 1U << table->hash_bits; + unsigned nr_buckets = 1U << ctx->cancel_table.hash_bits; struct hlist_node *tmp; struct io_kiocb *req; bool found = false; int i; + lockdep_assert_held(&ctx->uring_lock); + for (i = 0; i < nr_buckets; i++) { - struct io_hash_bucket *hb = &table->hbs[i]; + struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i]; spin_lock(&hb->lock); hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) { @@ -787,28 +754,13 @@ static __cold bool io_poll_remove_all_table(struct task_struct *tsk, return found; } -/* - * Returns true if we found and killed one or more poll requests - */ -__cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, - bool cancel_all) - __must_hold(&ctx->uring_lock) -{ - bool ret; - - ret = io_poll_remove_all_table(tsk, &ctx->cancel_table, cancel_all); - ret |= io_poll_remove_all_table(tsk, &ctx->cancel_table_locked, cancel_all); - return ret; -} - static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, struct io_cancel_data *cd, - struct io_hash_table *table, struct io_hash_bucket **out_bucket) { struct io_kiocb *req; - u32 index = hash_long(cd->data, table->hash_bits); - struct io_hash_bucket *hb = &table->hbs[index]; + u32 index = hash_long(cd->data, ctx->cancel_table.hash_bits); + struct io_hash_bucket *hb = &ctx->cancel_table.hbs[index]; *out_bucket = NULL; @@ -831,17 +783,16 @@ static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx, struct io_cancel_data *cd, - struct io_hash_table *table, struct io_hash_bucket **out_bucket) { - unsigned nr_buckets = 1U << table->hash_bits; + unsigned nr_buckets = 1U << ctx->cancel_table.hash_bits; struct io_kiocb *req; int i; *out_bucket = NULL; for (i = 0; i < nr_buckets; i++) { - struct io_hash_bucket *hb = &table->hbs[i]; + struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i]; spin_lock(&hb->lock); hlist_for_each_entry(req, &hb->list, hash_node) { @@ -866,17 +817,16 @@ static int io_poll_disarm(struct io_kiocb *req) return 0; } -static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, - struct io_hash_table *table) +static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) { struct io_hash_bucket *bucket; struct io_kiocb *req; if (cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP | IORING_ASYNC_CANCEL_ANY)) - req = io_poll_file_find(ctx, cd, table, &bucket); + req = io_poll_file_find(ctx, cd, &bucket); else - req = io_poll_find(ctx, false, cd, table, &bucket); + req = io_poll_find(ctx, false, cd, &bucket); if (req) io_poll_cancel_req(req); @@ -890,12 +840,8 @@ int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, { int ret; - ret = __io_poll_cancel(ctx, cd, &ctx->cancel_table); - if (ret != -ENOENT) - return ret; - io_ring_submit_lock(ctx, issue_flags); - ret = __io_poll_cancel(ctx, cd, &ctx->cancel_table_locked); + ret = __io_poll_cancel(ctx, cd); io_ring_submit_unlock(ctx, issue_flags); return ret; } @@ -972,13 +918,6 @@ int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) ipt.pt._qproc = io_poll_queue_proc; - /* - * If sqpoll or single issuer, there is no contention for ->uring_lock - * and we'll end up holding it in tw handlers anyway. - */ - if (req->ctx->flags & (IORING_SETUP_SQPOLL|IORING_SETUP_SINGLE_ISSUER)) - req->flags |= REQ_F_HASH_LOCKED; - ret = __io_arm_poll_handler(req, poll, &ipt, poll->events, issue_flags); if (ret > 0) { io_req_set_res(req, ipt.result_mask, 0); @@ -997,18 +936,7 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) int ret2, ret = 0; io_ring_submit_lock(ctx, issue_flags); - preq = io_poll_find(ctx, true, &cd, &ctx->cancel_table, &bucket); - ret2 = io_poll_disarm(preq); - if (bucket) - spin_unlock(&bucket->lock); - if (!ret2) - goto found; - if (ret2 != -ENOENT) { - ret = ret2; - goto out; - } - - preq = io_poll_find(ctx, true, &cd, &ctx->cancel_table_locked, &bucket); + preq = io_poll_find(ctx, true, &cd, &bucket); ret2 = io_poll_disarm(preq); if (bucket) spin_unlock(&bucket->lock); @@ -1016,8 +944,6 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) ret = ret2; goto out; } - -found: if (WARN_ON_ONCE(preq->opcode != IORING_OP_POLL_ADD)) { ret = -EFAULT; goto out; From 879ba46a38e67595b96c87428fbb718d63821da2 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 30 Sep 2024 14:35:52 -0600 Subject: [PATCH 11/79] io_uring/poll: get rid of io_poll_tw_hash_eject() It serves no purposes anymore, all it does is delete the hash list entry. task_work always has the ring locked. Signed-off-by: Jens Axboe --- io_uring/poll.c | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/io_uring/poll.c b/io_uring/poll.c index 217d667e0622..a0d1a09c5a20 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -128,20 +128,6 @@ static void io_poll_req_insert(struct io_kiocb *req) hlist_add_head(&req->hash_node, &table->hbs[index].list); } -static void io_poll_tw_hash_eject(struct io_kiocb *req, struct io_tw_state *ts) -{ - struct io_ring_ctx *ctx = req->ctx; - - /* - * ->cancel_table_locked is protected by ->uring_lock in - * contrast to per bucket spinlocks. Likely, tctx_task_work() - * already grabbed the mutex for us, but there is a chance it - * failed. - */ - io_tw_lock(ctx, ts); - hash_del(&req->hash_node); -} - static void io_init_poll_iocb(struct io_poll *poll, __poll_t events) { poll->head = NULL; @@ -336,7 +322,8 @@ void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts) return; } io_poll_remove_entries(req); - io_poll_tw_hash_eject(req, ts); + /* task_work always has ->uring_lock held */ + hash_del(&req->hash_node); if (req->opcode == IORING_OP_POLL_ADD) { if (ret == IOU_POLL_DONE) { From ba4366f57b117c2eab996642288e5c75646ccfc9 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 30 Sep 2024 14:29:06 -0600 Subject: [PATCH 12/79] io_uring/poll: get rid of per-hashtable bucket locks Any access to the table is protected by ctx->uring_lock now anyway, the per-bucket locking doesn't buy us anything. Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 1 - io_uring/cancel.c | 4 +--- io_uring/poll.c | 39 +++++++++------------------------- 3 files changed, 11 insertions(+), 33 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index d8ca27da1341..9c7e1d3f06e5 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -67,7 +67,6 @@ struct io_file_table { }; struct io_hash_bucket { - spinlock_t lock; struct hlist_head list; } ____cacheline_aligned_in_smp; diff --git a/io_uring/cancel.c b/io_uring/cancel.c index a6e58a20efdd..755dd5506a5f 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -236,10 +236,8 @@ void init_hash_table(struct io_hash_table *table, unsigned size) { unsigned int i; - for (i = 0; i < size; i++) { - spin_lock_init(&table->hbs[i].lock); + for (i = 0; i < size; i++) INIT_HLIST_HEAD(&table->hbs[i].list); - } } static int __io_sync_cancel(struct io_uring_task *tctx, diff --git a/io_uring/poll.c b/io_uring/poll.c index a0d1a09c5a20..2d6698fb7400 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -728,7 +728,6 @@ __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, for (i = 0; i < nr_buckets; i++) { struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i]; - spin_lock(&hb->lock); hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) { if (io_match_task_safe(req, tsk, cancel_all)) { hlist_del_init(&req->hash_node); @@ -736,22 +735,17 @@ __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, found = true; } } - spin_unlock(&hb->lock); } return found; } static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, - struct io_cancel_data *cd, - struct io_hash_bucket **out_bucket) + struct io_cancel_data *cd) { struct io_kiocb *req; u32 index = hash_long(cd->data, ctx->cancel_table.hash_bits); struct io_hash_bucket *hb = &ctx->cancel_table.hbs[index]; - *out_bucket = NULL; - - spin_lock(&hb->lock); hlist_for_each_entry(req, &hb->list, hash_node) { if (cd->data != req->cqe.user_data) continue; @@ -761,34 +755,25 @@ static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, if (io_cancel_match_sequence(req, cd->seq)) continue; } - *out_bucket = hb; return req; } - spin_unlock(&hb->lock); return NULL; } static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx, - struct io_cancel_data *cd, - struct io_hash_bucket **out_bucket) + struct io_cancel_data *cd) { unsigned nr_buckets = 1U << ctx->cancel_table.hash_bits; struct io_kiocb *req; int i; - *out_bucket = NULL; - for (i = 0; i < nr_buckets; i++) { struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i]; - spin_lock(&hb->lock); hlist_for_each_entry(req, &hb->list, hash_node) { - if (io_cancel_req_match(req, cd)) { - *out_bucket = hb; + if (io_cancel_req_match(req, cd)) return req; - } } - spin_unlock(&hb->lock); } return NULL; } @@ -806,20 +791,19 @@ static int io_poll_disarm(struct io_kiocb *req) static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) { - struct io_hash_bucket *bucket; struct io_kiocb *req; if (cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP | IORING_ASYNC_CANCEL_ANY)) - req = io_poll_file_find(ctx, cd, &bucket); + req = io_poll_file_find(ctx, cd); else - req = io_poll_find(ctx, false, cd, &bucket); + req = io_poll_find(ctx, false, cd); - if (req) + if (req) { io_poll_cancel_req(req); - if (bucket) - spin_unlock(&bucket->lock); - return req ? 0 : -ENOENT; + return 0; + } + return -ENOENT; } int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, @@ -918,15 +902,12 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) struct io_poll_update *poll_update = io_kiocb_to_cmd(req, struct io_poll_update); struct io_ring_ctx *ctx = req->ctx; struct io_cancel_data cd = { .ctx = ctx, .data = poll_update->old_user_data, }; - struct io_hash_bucket *bucket; struct io_kiocb *preq; int ret2, ret = 0; io_ring_submit_lock(ctx, issue_flags); - preq = io_poll_find(ctx, true, &cd, &bucket); + preq = io_poll_find(ctx, true, &cd); ret2 = io_poll_disarm(preq); - if (bucket) - spin_unlock(&bucket->lock); if (ret2) { ret = ret2; goto out; From 8abf47a8d61c9e8314ae4cfa27e18c8df67c37bc Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 30 Sep 2024 14:30:39 -0600 Subject: [PATCH 13/79] io_uring/cancel: get rid of init_hash_table() helper All it does is initialize the lists, just move the INIT_HLIST_HEAD() into the one caller. Signed-off-by: Jens Axboe --- io_uring/cancel.c | 8 -------- io_uring/cancel.h | 1 - io_uring/io_uring.c | 4 +++- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/io_uring/cancel.c b/io_uring/cancel.c index 755dd5506a5f..cc3475b22ae5 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -232,14 +232,6 @@ int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) return IOU_OK; } -void init_hash_table(struct io_hash_table *table, unsigned size) -{ - unsigned int i; - - for (i = 0; i < size; i++) - INIT_HLIST_HEAD(&table->hbs[i].list); -} - static int __io_sync_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd, int fd) { diff --git a/io_uring/cancel.h b/io_uring/cancel.h index b33995e00ba9..bbfea2cd00ea 100644 --- a/io_uring/cancel.h +++ b/io_uring/cancel.h @@ -20,7 +20,6 @@ int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags); int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd, unsigned int issue_flags); -void init_hash_table(struct io_hash_table *table, unsigned size); int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg); bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd); diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index f4e069cd03a5..6aac72b2958f 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -263,13 +263,15 @@ static int io_alloc_hash_table(struct io_hash_table *table, unsigned bits) { unsigned hash_buckets = 1U << bits; size_t hash_size = hash_buckets * sizeof(table->hbs[0]); + int i; table->hbs = kmalloc(hash_size, GFP_KERNEL); if (!table->hbs) return -ENOMEM; table->hash_bits = bits; - init_hash_table(table, hash_buckets); + for (i = 0; i < hash_buckets; i++) + INIT_HLIST_HEAD(&table->hbs[i].list); return 0; } From b6b3eb19dd86ecc3f188bd419f12cdfcfbeda5e7 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 30 Sep 2024 17:11:32 -0600 Subject: [PATCH 14/79] io_uring: move cancel hash tables to kvmalloc/kvfree Convert to using kvmalloc/kfree() for the hash tables, and while at it, make it handle low memory situations better. Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 6aac72b2958f..d7ad4ea5f40b 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -261,13 +261,19 @@ static __cold void io_fallback_req_func(struct work_struct *work) static int io_alloc_hash_table(struct io_hash_table *table, unsigned bits) { - unsigned hash_buckets = 1U << bits; - size_t hash_size = hash_buckets * sizeof(table->hbs[0]); + unsigned int hash_buckets; int i; - table->hbs = kmalloc(hash_size, GFP_KERNEL); - if (!table->hbs) - return -ENOMEM; + do { + hash_buckets = 1U << bits; + table->hbs = kvmalloc_array(hash_buckets, sizeof(table->hbs[0]), + GFP_KERNEL_ACCOUNT); + if (table->hbs) + break; + if (bits == 1) + return -ENOMEM; + bits--; + } while (1); table->hash_bits = bits; for (i = 0; i < hash_buckets; i++) @@ -360,7 +366,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) io_alloc_cache_free(&ctx->uring_cache, kfree); io_alloc_cache_free(&ctx->msg_cache, io_msg_cache_free); io_futex_cache_free(ctx); - kfree(ctx->cancel_table.hbs); + kvfree(ctx->cancel_table.hbs); xa_destroy(&ctx->io_bl_xa); kfree(ctx); return NULL; @@ -2772,7 +2778,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) if (ctx->hash_map) io_wq_put_hash(ctx->hash_map); io_napi_free(ctx); - kfree(ctx->cancel_table.hbs); + kvfree(ctx->cancel_table.hbs); xa_destroy(&ctx->io_bl_xa); kfree(ctx); } From 1e6e7602cc9fdeaf7e2593755409e8d50545ed69 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 18 Oct 2024 17:07:31 +0100 Subject: [PATCH 15/79] io_uring: kill io_llist_xchg io_llist_xchg is only used to set the list to NULL, which can also be done with llist_del_all(). Use the latter and kill io_llist_xchg. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/d6765112680d2e86a58b76166b7513391ff4e5d7.1729264960.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index d7ad4ea5f40b..c0358a8d85d2 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1081,20 +1081,6 @@ struct llist_node *io_handle_tw_list(struct llist_node *node, return node; } -/** - * io_llist_xchg - swap all entries in a lock-less list - * @head: the head of lock-less list to delete all entries - * @new: new entry as the head of the list - * - * If list is empty, return NULL, otherwise, return the pointer to the first entry. - * The order of entries returned is from the newest to the oldest added one. - */ -static inline struct llist_node *io_llist_xchg(struct llist_head *head, - struct llist_node *new) -{ - return xchg(&head->first, new); -} - static __cold void io_fallback_tw(struct io_uring_task *tctx, bool sync) { struct llist_node *node = llist_del_all(&tctx->task_list); @@ -1316,7 +1302,7 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts, * llists are in reverse order, flip it back the right way before * running the pending items. */ - node = llist_reverse_order(io_llist_xchg(&ctx->work_llist, NULL)); + node = llist_reverse_order(llist_del_all(&ctx->work_llist)); while (node) { struct llist_node *next = node->next; struct io_kiocb *req = container_of(node, struct io_kiocb, From 9b296c625ac1d2ca9b129743c3f886bf7a0f471d Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 18 Oct 2024 17:07:59 +0100 Subject: [PATCH 16/79] io_uring: static_key for !IORING_SETUP_NO_SQARRAY IORING_SETUP_NO_SQARRAY should be preferred and used by default by liburing, optimise flag checking in io_get_sqe() with a static key. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/c164a48542fbb080115e2377ecf160c758562742.1729264988.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index c0358a8d85d2..fa9d31034c62 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -70,6 +70,7 @@ #include #include #include +#include #include #define CREATE_TRACE_POINTS @@ -149,6 +150,8 @@ static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, static void io_queue_sqe(struct io_kiocb *req); +static __read_mostly DEFINE_STATIC_KEY_FALSE(io_key_has_sqarray); + struct kmem_cache *req_cachep; static struct workqueue_struct *iou_wq __ro_after_init; @@ -2254,7 +2257,8 @@ static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) unsigned mask = ctx->sq_entries - 1; unsigned head = ctx->cached_sq_head++ & mask; - if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) { + if (static_branch_unlikely(&io_key_has_sqarray) && + (!(ctx->flags & IORING_SETUP_NO_SQARRAY))) { head = READ_ONCE(ctx->sq_array[head]); if (unlikely(head >= ctx->sq_entries)) { /* drop invalid entries */ @@ -2758,6 +2762,9 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) } io_rings_free(ctx); + if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) + static_branch_dec(&io_key_has_sqarray); + percpu_ref_exit(&ctx->refs); free_uid(ctx->user); io_req_caches_free(ctx); @@ -3549,6 +3556,9 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, ctx->clockid = CLOCK_MONOTONIC; ctx->clock_offset = 0; + if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) + static_branch_inc(&io_key_has_sqarray); + if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) && !(ctx->flags & IORING_SETUP_IOPOLL) && !(ctx->flags & IORING_SETUP_SQPOLL)) From 2946f08ae9ed650b94e0ffebcdfdda8de76bd926 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 18 Oct 2024 17:14:00 +0100 Subject: [PATCH 17/79] io_uring: clean up cqe trace points We have too many helpers posting CQEs, instead of tracing completion events before filling in a CQE and thus having to pass all the data, set the CQE first, pass it to the tracing helper and let it extract everything it needs. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/b83c1ca9ee5aed2df0f3bb743bf5ed699cce4c86.1729267437.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 5 +++++ include/trace/events/io_uring.h | 24 +++++++++--------------- io_uring/io_uring.c | 4 ++-- io_uring/io_uring.h | 7 +++---- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 9c7e1d3f06e5..391087144666 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -662,4 +662,9 @@ struct io_overflow_cqe { struct io_uring_cqe cqe; }; +static inline bool io_ctx_cqe32(struct io_ring_ctx *ctx) +{ + return ctx->flags & IORING_SETUP_CQE32; +} + #endif diff --git a/include/trace/events/io_uring.h b/include/trace/events/io_uring.h index 412c9c210a32..fb81c533b310 100644 --- a/include/trace/events/io_uring.h +++ b/include/trace/events/io_uring.h @@ -315,20 +315,14 @@ TRACE_EVENT(io_uring_fail_link, * io_uring_complete - called when completing an SQE * * @ctx: pointer to a ring context structure - * @req: pointer to a submitted request - * @user_data: user data associated with the request - * @res: result of the request - * @cflags: completion flags - * @extra1: extra 64-bit data for CQE32 - * @extra2: extra 64-bit data for CQE32 - * + * @req: (optional) pointer to a submitted request + * @cqe: pointer to the filled in CQE being posted */ TRACE_EVENT(io_uring_complete, - TP_PROTO(void *ctx, void *req, u64 user_data, int res, unsigned cflags, - u64 extra1, u64 extra2), +TP_PROTO(struct io_ring_ctx *ctx, void *req, struct io_uring_cqe *cqe), - TP_ARGS(ctx, req, user_data, res, cflags, extra1, extra2), + TP_ARGS(ctx, req, cqe), TP_STRUCT__entry ( __field( void *, ctx ) @@ -343,11 +337,11 @@ TRACE_EVENT(io_uring_complete, TP_fast_assign( __entry->ctx = ctx; __entry->req = req; - __entry->user_data = user_data; - __entry->res = res; - __entry->cflags = cflags; - __entry->extra1 = extra1; - __entry->extra2 = extra2; + __entry->user_data = cqe->user_data; + __entry->res = cqe->res; + __entry->cflags = cqe->flags; + __entry->extra1 = io_ctx_cqe32(ctx) ? cqe->big_cqe[0] : 0; + __entry->extra2 = io_ctx_cqe32(ctx) ? cqe->big_cqe[1] : 0; ), TP_printk("ring %p, req %p, user_data 0x%llx, result %d, cflags 0x%x " diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index fa9d31034c62..58b401900b41 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -828,8 +828,6 @@ static bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res, * the ring. */ if (likely(io_get_cqe(ctx, &cqe))) { - trace_io_uring_complete(ctx, NULL, user_data, res, cflags, 0, 0); - WRITE_ONCE(cqe->user_data, user_data); WRITE_ONCE(cqe->res, res); WRITE_ONCE(cqe->flags, cflags); @@ -838,6 +836,8 @@ static bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res, WRITE_ONCE(cqe->big_cqe[0], 0); WRITE_ONCE(cqe->big_cqe[1], 0); } + + trace_io_uring_complete(ctx, NULL, cqe); return true; } return false; diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 70b6675941ff..9cd9a127e9ed 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -189,16 +189,15 @@ static __always_inline bool io_fill_cqe_req(struct io_ring_ctx *ctx, if (unlikely(!io_get_cqe(ctx, &cqe))) return false; - if (trace_io_uring_complete_enabled()) - trace_io_uring_complete(req->ctx, req, req->cqe.user_data, - req->cqe.res, req->cqe.flags, - req->big_cqe.extra1, req->big_cqe.extra2); memcpy(cqe, &req->cqe, sizeof(*cqe)); if (ctx->flags & IORING_SETUP_CQE32) { memcpy(cqe->big_cqe, &req->big_cqe, sizeof(*cqe)); memset(&req->big_cqe, 0, sizeof(req->big_cqe)); } + + if (trace_io_uring_complete_enabled()) + trace_io_uring_complete(req->ctx, req, cqe); return true; } From c919790060230ac2b1824bbf4d3b64eb51f471ff Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 16 Oct 2024 15:04:55 -0600 Subject: [PATCH 18/79] io_uring/rsrc: don't assign bvec twice in io_import_fixed() iter->bvec is already set to imu->bvec - remove the one dead assignment and turn the other one into an addition instead. Signed-off-by: Jens Axboe --- io_uring/rsrc.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 6f3b6de230bd..ca2ec8a018be 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1127,7 +1127,6 @@ int io_import_fixed(int ddir, struct iov_iter *iter, const struct bio_vec *bvec = imu->bvec; if (offset < bvec->bv_len) { - iter->bvec = bvec; iter->count -= offset; iter->iov_offset = offset; } else { @@ -1137,7 +1136,7 @@ int io_import_fixed(int ddir, struct iov_iter *iter, offset -= bvec->bv_len; seg_skip = 1 + (offset >> imu->folio_shift); - iter->bvec = bvec + seg_skip; + iter->bvec += seg_skip; iter->nr_segs -= seg_skip; iter->count -= bvec->bv_len + offset; iter->iov_offset = offset & ((1UL << imu->folio_shift) - 1); From 892d3e80e1b9fc09aefdfd4d31f10f3d018863a0 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 16 Oct 2024 15:48:38 -0600 Subject: [PATCH 19/79] io_uring/uring_cmd: get rid of using req->imu It's pretty pointless to use io_kiocb as intermediate storage for this, so split the validity check and the actual usage. The resource node is assigned upfront at prep time, to prevent it from going away. The actual import is never called with the ctx->uring_lock held, so grab it for the import. Signed-off-by: Jens Axboe --- io_uring/uring_cmd.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 39c3c816ec78..58d0b817d6ea 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -211,11 +211,15 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) struct io_ring_ctx *ctx = req->ctx; u16 index; - req->buf_index = READ_ONCE(sqe->buf_index); - if (unlikely(req->buf_index >= ctx->nr_user_bufs)) + index = READ_ONCE(sqe->buf_index); + if (unlikely(index >= ctx->nr_user_bufs)) return -EFAULT; - index = array_index_nospec(req->buf_index, ctx->nr_user_bufs); - req->imu = ctx->user_bufs[index]; + req->buf_index = array_index_nospec(index, ctx->nr_user_bufs); + /* + * Pi node upfront, prior to io_uring_cmd_import_fixed() + * being called. This prevents destruction of the mapped buffer + * we'll need at actual import time. + */ io_req_set_rsrc_node(req, ctx, 0); } ioucmd->cmd_op = READ_ONCE(sqe->cmd_op); @@ -272,8 +276,17 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, struct iov_iter *iter, void *ioucmd) { struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); + struct io_ring_ctx *ctx = req->ctx; - return io_import_fixed(rw, iter, req->imu, ubuf, len); + /* Must have had rsrc_node assigned at prep time */ + if (req->rsrc_node) { + struct io_mapped_ubuf *imu; + + imu = READ_ONCE(ctx->user_bufs[req->buf_index]); + return io_import_fixed(rw, iter, imu, ubuf, len); + } + + return -EFAULT; } EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed); From 003f82b58c99146dfb0c9ce1ee7ed59bc572959b Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 16 Oct 2024 15:49:49 -0600 Subject: [PATCH 20/79] io_uring/rw: get rid of using req->imu It's assigned in the same function that it's being used, get rid of it. A local variable will do just fine. Signed-off-by: Jens Axboe --- io_uring/rw.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/io_uring/rw.c b/io_uring/rw.c index 354c4e175654..d8b9e7a712f6 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -330,6 +330,7 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct io_ring_ctx *ctx = req->ctx; + struct io_mapped_ubuf *imu; struct io_async_rw *io; u16 index; int ret; @@ -341,11 +342,11 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe if (unlikely(req->buf_index >= ctx->nr_user_bufs)) return -EFAULT; index = array_index_nospec(req->buf_index, ctx->nr_user_bufs); - req->imu = ctx->user_bufs[index]; + imu = ctx->user_bufs[index]; io_req_set_rsrc_node(req, ctx, 0); io = req->async_data; - ret = io_import_fixed(ddir, &io->iter, req->imu, rw->addr, rw->len); + ret = io_import_fixed(ddir, &io->iter, imu, rw->addr, rw->len); iov_iter_save_state(&io->iter, &io->iter_state); return ret; } From 1caa00d6b61651e04c04c2b50b3e149f24c6764d Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 23 Oct 2024 07:14:22 -0600 Subject: [PATCH 21/79] io_uring: remove 'issue_flags' argument for io_req_set_rsrc_node() All callers already hold the ring lock and hence are passing '0', remove the argument and the conditional locking that it controlled. Suggested-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/net.c | 2 +- io_uring/rsrc.h | 8 ++------ io_uring/rw.c | 2 +- io_uring/uring_cmd.c | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/io_uring/net.c b/io_uring/net.c index 18507658a921..fb1f2c37f7d1 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -1261,7 +1261,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EFAULT; idx = array_index_nospec(idx, ctx->nr_user_bufs); req->imu = READ_ONCE(ctx->user_bufs[idx]); - io_req_set_rsrc_node(notif, ctx, 0); + io_req_set_rsrc_node(notif, ctx); } if (req->opcode == IORING_OP_SEND_ZC) { diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 8ed588036210..c50d4be4aa6d 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -107,14 +107,10 @@ static inline void __io_req_set_rsrc_node(struct io_kiocb *req, } static inline void io_req_set_rsrc_node(struct io_kiocb *req, - struct io_ring_ctx *ctx, - unsigned int issue_flags) + struct io_ring_ctx *ctx) { - if (!req->rsrc_node) { - io_ring_submit_lock(ctx, issue_flags); + if (!req->rsrc_node) __io_req_set_rsrc_node(req, ctx); - io_ring_submit_unlock(ctx, issue_flags); - } } static inline u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx) diff --git a/io_uring/rw.c b/io_uring/rw.c index d8b9e7a712f6..8080ffd6d571 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -343,7 +343,7 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe return -EFAULT; index = array_index_nospec(req->buf_index, ctx->nr_user_bufs); imu = ctx->user_bufs[index]; - io_req_set_rsrc_node(req, ctx, 0); + io_req_set_rsrc_node(req, ctx); io = req->async_data; ret = io_import_fixed(ddir, &io->iter, imu, rw->addr, rw->len); diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 58d0b817d6ea..6994f60d7ec7 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -220,7 +220,7 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) * being called. This prevents destruction of the mapped buffer * we'll need at actual import time. */ - io_req_set_rsrc_node(req, ctx, 0); + io_req_set_rsrc_node(req, ctx); } ioucmd->cmd_op = READ_ONCE(sqe->cmd_op); From 51c967c6c9ea6c4d480e4778ace5243db22aa27b Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 16 Oct 2024 07:39:31 -0600 Subject: [PATCH 22/79] io_uring/net: move send zc fixed buffer import to issue path Let's keep it close with the actual import, there's no reason to do this on the prep side. With that, we can drop one of the branches checking for whether or not IORING_RECVSEND_FIXED_BUF is set. As a side-effect, get rid of req->imu usage. Signed-off-by: Jens Axboe --- io_uring/net.c | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/io_uring/net.c b/io_uring/net.c index fb1f2c37f7d1..b9e7e496ae85 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -76,6 +76,7 @@ struct io_sr_msg { /* initialised and used only by !msg send variants */ u16 addr_len; u16 buf_group; + u16 buf_index; void __user *addr; void __user *msg_control; /* used only for send zerocopy */ @@ -1254,16 +1255,6 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) } } - if (zc->flags & IORING_RECVSEND_FIXED_BUF) { - unsigned idx = READ_ONCE(sqe->buf_index); - - if (unlikely(idx >= ctx->nr_user_bufs)) - return -EFAULT; - idx = array_index_nospec(idx, ctx->nr_user_bufs); - req->imu = READ_ONCE(ctx->user_bufs[idx]); - io_req_set_rsrc_node(notif, ctx); - } - if (req->opcode == IORING_OP_SEND_ZC) { if (READ_ONCE(sqe->__pad3[0])) return -EINVAL; @@ -1279,6 +1270,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) zc->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); zc->len = READ_ONCE(sqe->len); zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL | MSG_ZEROCOPY; + zc->buf_index = READ_ONCE(sqe->buf_index); if (zc->msg_flags & MSG_DONTWAIT) req->flags |= REQ_F_NOWAIT; @@ -1339,13 +1331,31 @@ static int io_sg_from_iter(struct sk_buff *skb, return ret; } -static int io_send_zc_import(struct io_kiocb *req, struct io_async_msghdr *kmsg) +static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); + struct io_async_msghdr *kmsg = req->async_data; int ret; if (sr->flags & IORING_RECVSEND_FIXED_BUF) { - ret = io_import_fixed(ITER_SOURCE, &kmsg->msg.msg_iter, req->imu, + struct io_ring_ctx *ctx = req->ctx; + struct io_mapped_ubuf *imu; + int idx; + + ret = -EFAULT; + io_ring_submit_lock(ctx, issue_flags); + if (sr->buf_index < ctx->nr_user_bufs) { + idx = array_index_nospec(sr->buf_index, ctx->nr_user_bufs); + imu = READ_ONCE(ctx->user_bufs[idx]); + io_req_set_rsrc_node(sr->notif, ctx); + ret = 0; + } + io_ring_submit_unlock(ctx, issue_flags); + + if (unlikely(ret)) + return ret; + + ret = io_import_fixed(ITER_SOURCE, &kmsg->msg.msg_iter, imu, (u64)(uintptr_t)sr->buf, sr->len); if (unlikely(ret)) return ret; @@ -1382,7 +1392,7 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) return -EAGAIN; if (!zc->done_io) { - ret = io_send_zc_import(req, kmsg); + ret = io_send_zc_import(req, issue_flags); if (unlikely(ret)) return ret; } From e6d43739d0ee49a39505d696ba6a656f47c2bd39 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 16 Oct 2024 15:54:06 -0600 Subject: [PATCH 23/79] io_uring: kill 'imu' from struct io_kiocb It's no longer being used, remove it. Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 391087144666..6d3ee71bd832 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -613,9 +613,6 @@ struct io_kiocb { struct task_struct *task; union { - /* store used ubuf, so we can prevent reloading */ - struct io_mapped_ubuf *imu; - /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */ struct io_buffer *kbuf; From 93db98f6f1d62c9e58787f6beb62245ddb91f354 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 22 Oct 2024 15:43:12 +0100 Subject: [PATCH 24/79] io_uring/net: split send and sendmsg prep helpers A preparation patch splitting io_sendmsg_prep_setup into two separate helpers for send and sendmsg variants. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/1a2319471ba040e053b7f1d22f4af510d1118eca.1729607201.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/net.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/io_uring/net.c b/io_uring/net.c index b9e7e496ae85..aa256aa46409 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -385,16 +385,11 @@ static int io_send_setup(struct io_kiocb *req) return 0; } -static int io_sendmsg_prep_setup(struct io_kiocb *req, int is_msg) +static int io_sendmsg_setup(struct io_kiocb *req) { - struct io_async_msghdr *kmsg; + struct io_async_msghdr *kmsg = req->async_data; int ret; - kmsg = io_msg_alloc_async(req); - if (unlikely(!kmsg)) - return -ENOMEM; - if (!is_msg) - return io_send_setup(req); ret = io_sendmsg_copy_hdr(req, kmsg); if (!ret) req->flags |= REQ_F_NEED_CLEANUP; @@ -440,7 +435,11 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (req->ctx->compat) sr->msg_flags |= MSG_CMSG_COMPAT; #endif - return io_sendmsg_prep_setup(req, req->opcode == IORING_OP_SENDMSG); + if (unlikely(!io_msg_alloc_async(req))) + return -ENOMEM; + if (req->opcode != IORING_OP_SENDMSG) + return io_send_setup(req); + return io_sendmsg_setup(req); } static void io_req_msg_cleanup(struct io_kiocb *req, @@ -1278,7 +1277,11 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (req->ctx->compat) zc->msg_flags |= MSG_CMSG_COMPAT; #endif - return io_sendmsg_prep_setup(req, req->opcode == IORING_OP_SENDMSG_ZC); + if (unlikely(!io_msg_alloc_async(req))) + return -ENOMEM; + if (req->opcode != IORING_OP_SENDMSG_ZC) + return io_send_setup(req); + return io_sendmsg_setup(req); } static int io_sg_from_iter_iovec(struct sk_buff *skb, From ad438d070a3bf2a3ae45b59a885a5d7b0dbbc465 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 22 Oct 2024 15:43:13 +0100 Subject: [PATCH 25/79] io_uring/net: don't store send address ptr For non "msg" requests we copy the address at the prep stage and there is no need to store the address user pointer long term. Pass the SQE into io_send_setup(), let it parse it, and remove struct io_sr_msg addr addr_len fields. It saves some space and also less confusing. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/db3dce544e17ca9d4b17d2506fbbac1da8a87824.1729607201.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/net.c | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/io_uring/net.c b/io_uring/net.c index aa256aa46409..ad34c99930be 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -74,10 +74,8 @@ struct io_sr_msg { unsigned nr_multishot_loops; u16 flags; /* initialised and used only by !msg send variants */ - u16 addr_len; u16 buf_group; u16 buf_index; - void __user *addr; void __user *msg_control; /* used only for send zerocopy */ struct io_kiocb *notif; @@ -357,24 +355,31 @@ void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req) io_netmsg_iovec_free(io); } -static int io_send_setup(struct io_kiocb *req) +static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_async_msghdr *kmsg = req->async_data; + void __user *addr; + u16 addr_len; int ret; + if (READ_ONCE(sqe->__pad3[0])) + return -EINVAL; + kmsg->msg.msg_name = NULL; kmsg->msg.msg_namelen = 0; kmsg->msg.msg_control = NULL; kmsg->msg.msg_controllen = 0; kmsg->msg.msg_ubuf = NULL; - if (sr->addr) { - ret = move_addr_to_kernel(sr->addr, sr->addr_len, &kmsg->addr); + addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); + addr_len = READ_ONCE(sqe->addr_len); + if (addr) { + ret = move_addr_to_kernel(addr, addr_len, &kmsg->addr); if (unlikely(ret < 0)) return ret; kmsg->msg.msg_name = &kmsg->addr; - kmsg->msg.msg_namelen = sr->addr_len; + kmsg->msg.msg_namelen = addr_len; } if (!io_do_buffer_select(req)) { ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len, @@ -404,13 +409,9 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) sr->done_io = 0; - if (req->opcode == IORING_OP_SEND) { - if (READ_ONCE(sqe->__pad3[0])) + if (req->opcode != IORING_OP_SEND) { + if (sqe->addr2 || sqe->file_index) return -EINVAL; - sr->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); - sr->addr_len = READ_ONCE(sqe->addr_len); - } else if (sqe->addr2 || sqe->file_index) { - return -EINVAL; } sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); @@ -438,7 +439,7 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (unlikely(!io_msg_alloc_async(req))) return -ENOMEM; if (req->opcode != IORING_OP_SENDMSG) - return io_send_setup(req); + return io_send_setup(req, sqe); return io_sendmsg_setup(req); } @@ -1254,12 +1255,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) } } - if (req->opcode == IORING_OP_SEND_ZC) { - if (READ_ONCE(sqe->__pad3[0])) - return -EINVAL; - zc->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); - zc->addr_len = READ_ONCE(sqe->addr_len); - } else { + if (req->opcode != IORING_OP_SEND_ZC) { if (unlikely(sqe->addr2 || sqe->file_index)) return -EINVAL; if (unlikely(zc->flags & IORING_RECVSEND_FIXED_BUF)) @@ -1280,7 +1276,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (unlikely(!io_msg_alloc_async(req))) return -ENOMEM; if (req->opcode != IORING_OP_SENDMSG_ZC) - return io_send_setup(req); + return io_send_setup(req, sqe); return io_sendmsg_setup(req); } From 52838787350d4ea8132804940d5308d95ce5e035 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 22 Oct 2024 15:43:14 +0100 Subject: [PATCH 26/79] io_uring/net: don't alias send user pointer reads We keep user pointers in an union, which could be a user buffer or a user pointer to msghdr. What is confusing is that it potenitally reads and assigns sqe->addr as one type but then uses it as another via the union. Even more, it's not even consistent across copy and zerocopy versions. Make send and sendmsg setup helpers read sqe->addr and treat it as the right type from the beginning. The end goal would be to get rid of the use of struct io_sr_msg::umsg for send requests as we only need it at the prep side. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/685d788605f5d78af18802fcabf61ba65cfd8002.1729607201.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/net.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/io_uring/net.c b/io_uring/net.c index ad34c99930be..5e7263846243 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -363,6 +363,8 @@ static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe) u16 addr_len; int ret; + sr->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); + if (READ_ONCE(sqe->__pad3[0])) return -EINVAL; @@ -390,11 +392,14 @@ static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_sendmsg_setup(struct io_kiocb *req) +static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe) { + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_async_msghdr *kmsg = req->async_data; int ret; + sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); + ret = io_sendmsg_copy_hdr(req, kmsg); if (!ret) req->flags |= REQ_F_NEED_CLEANUP; @@ -414,7 +419,6 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EINVAL; } - sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); sr->len = READ_ONCE(sqe->len); sr->flags = READ_ONCE(sqe->ioprio); if (sr->flags & ~SENDMSG_FLAGS) @@ -440,7 +444,7 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -ENOMEM; if (req->opcode != IORING_OP_SENDMSG) return io_send_setup(req, sqe); - return io_sendmsg_setup(req); + return io_sendmsg_setup(req, sqe); } static void io_req_msg_cleanup(struct io_kiocb *req, @@ -1262,7 +1266,6 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EINVAL; } - zc->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); zc->len = READ_ONCE(sqe->len); zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL | MSG_ZEROCOPY; zc->buf_index = READ_ONCE(sqe->buf_index); @@ -1277,7 +1280,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -ENOMEM; if (req->opcode != IORING_OP_SENDMSG_ZC) return io_send_setup(req, sqe); - return io_sendmsg_setup(req); + return io_sendmsg_setup(req, sqe); } static int io_sg_from_iter_iovec(struct sk_buff *skb, From 882dec6c39c40c13dd03e418952c4af38d91bb38 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 22 Oct 2024 15:43:15 +0100 Subject: [PATCH 27/79] io_uring/net: clean up io_msg_copy_hdr Put sr->umsg into a local variable, so it doesn't repeat "sr->umsg->" for every field. It looks nicer, and likely without the patch it compiles into a bunch of umsg memory reads. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/26c2f30b491ea7998bfdb5bb290662572a61064d.1729607201.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/net.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/io_uring/net.c b/io_uring/net.c index 5e7263846243..2040195e33ab 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -262,6 +262,7 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, struct user_msghdr *msg, int ddir) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); + struct user_msghdr __user *umsg = sr->umsg; struct iovec *iov; int ret, nr_segs; @@ -273,16 +274,16 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, nr_segs = 1; } - if (!user_access_begin(sr->umsg, sizeof(*sr->umsg))) + if (!user_access_begin(umsg, sizeof(*umsg))) return -EFAULT; ret = -EFAULT; - unsafe_get_user(msg->msg_name, &sr->umsg->msg_name, ua_end); - unsafe_get_user(msg->msg_namelen, &sr->umsg->msg_namelen, ua_end); - unsafe_get_user(msg->msg_iov, &sr->umsg->msg_iov, ua_end); - unsafe_get_user(msg->msg_iovlen, &sr->umsg->msg_iovlen, ua_end); - unsafe_get_user(msg->msg_control, &sr->umsg->msg_control, ua_end); - unsafe_get_user(msg->msg_controllen, &sr->umsg->msg_controllen, ua_end); + unsafe_get_user(msg->msg_name, &umsg->msg_name, ua_end); + unsafe_get_user(msg->msg_namelen, &umsg->msg_namelen, ua_end); + unsafe_get_user(msg->msg_iov, &umsg->msg_iov, ua_end); + unsafe_get_user(msg->msg_iovlen, &umsg->msg_iovlen, ua_end); + unsafe_get_user(msg->msg_control, &umsg->msg_control, ua_end); + unsafe_get_user(msg->msg_controllen, &umsg->msg_controllen, ua_end); msg->msg_flags = 0; if (req->flags & REQ_F_BUFFER_SELECT) { From 09d0a8ea7facc8b1581c9bd85c3ea6f5aa62ab7d Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 21 Oct 2024 13:29:39 -0600 Subject: [PATCH 28/79] io_uring: move max entry definition and ring sizing into header In preparation for needing this somewhere else, move the definitions for the maximum CQ and SQ ring size into io_uring.h. Make the rings_size() helper available as well, and have it take just the setup flags argument rather than the fill ring pointer. That's all that is needed. Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 14 ++++++-------- io_uring/io_uring.h | 5 +++++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 58b401900b41..6dea5242d666 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -105,9 +105,6 @@ #include "alloc_cache.h" #include "eventfd.h" -#define IORING_MAX_ENTRIES 32768 -#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) - #define SQE_COMMON_FLAGS (IOSQE_FIXED_FILE | IOSQE_IO_LINK | \ IOSQE_IO_HARDLINK | IOSQE_ASYNC) @@ -2667,8 +2664,8 @@ static void io_rings_free(struct io_ring_ctx *ctx) ctx->sq_sqes = NULL; } -static unsigned long rings_size(struct io_ring_ctx *ctx, unsigned int sq_entries, - unsigned int cq_entries, size_t *sq_offset) +unsigned long rings_size(unsigned int flags, unsigned int sq_entries, + unsigned int cq_entries, size_t *sq_offset) { struct io_rings *rings; size_t off, sq_array_size; @@ -2676,7 +2673,7 @@ static unsigned long rings_size(struct io_ring_ctx *ctx, unsigned int sq_entries off = struct_size(rings, cqes, cq_entries); if (off == SIZE_MAX) return SIZE_MAX; - if (ctx->flags & IORING_SETUP_CQE32) { + if (flags & IORING_SETUP_CQE32) { if (check_shl_overflow(off, 1, &off)) return SIZE_MAX; } @@ -2687,7 +2684,7 @@ static unsigned long rings_size(struct io_ring_ctx *ctx, unsigned int sq_entries return SIZE_MAX; #endif - if (ctx->flags & IORING_SETUP_NO_SQARRAY) { + if (flags & IORING_SETUP_NO_SQARRAY) { *sq_offset = SIZE_MAX; return off; } @@ -3434,7 +3431,8 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, ctx->sq_entries = p->sq_entries; ctx->cq_entries = p->cq_entries; - size = rings_size(ctx, p->sq_entries, p->cq_entries, &sq_array_offset); + size = rings_size(ctx->flags, p->sq_entries, p->cq_entries, + &sq_array_offset); if (size == SIZE_MAX) return -EOVERFLOW; diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 9cd9a127e9ed..4a471a810f02 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -65,6 +65,11 @@ static inline bool io_should_wake(struct io_wait_queue *iowq) return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts; } +#define IORING_MAX_ENTRIES 32768 +#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) + +unsigned long rings_size(unsigned int flags, unsigned int sq_entries, + unsigned int cq_entries, size_t *sq_offset); bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow); int io_run_task_work_sig(struct io_ring_ctx *ctx); void io_req_defer_failed(struct io_kiocb *req, s32 res); From 81d8191eb99d95b32e55d09d74f682d40d3e74e9 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 21 Oct 2024 13:32:19 -0600 Subject: [PATCH 29/79] io_uring: abstract out a bit of the ring filling logic Abstract out a io_uring_fill_params() helper, which fills out the necessary bits of struct io_uring_params. Add it to io_uring.h as well, in preparation for having another internal user of it. Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 70 ++++++++++++++++++++++++++------------------- io_uring/io_uring.h | 1 + 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 6dea5242d666..b5974bdad48b 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3498,14 +3498,8 @@ static struct file *io_uring_get_file(struct io_ring_ctx *ctx) O_RDWR | O_CLOEXEC, NULL); } -static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, - struct io_uring_params __user *params) +int io_uring_fill_params(unsigned entries, struct io_uring_params *p) { - struct io_ring_ctx *ctx; - struct io_uring_task *tctx; - struct file *file; - int ret; - if (!entries) return -EINVAL; if (entries > IORING_MAX_ENTRIES) { @@ -3547,6 +3541,42 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, p->cq_entries = 2 * p->sq_entries; } + p->sq_off.head = offsetof(struct io_rings, sq.head); + p->sq_off.tail = offsetof(struct io_rings, sq.tail); + p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask); + p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries); + p->sq_off.flags = offsetof(struct io_rings, sq_flags); + p->sq_off.dropped = offsetof(struct io_rings, sq_dropped); + p->sq_off.resv1 = 0; + if (!(p->flags & IORING_SETUP_NO_MMAP)) + p->sq_off.user_addr = 0; + + p->cq_off.head = offsetof(struct io_rings, cq.head); + p->cq_off.tail = offsetof(struct io_rings, cq.tail); + p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask); + p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries); + p->cq_off.overflow = offsetof(struct io_rings, cq_overflow); + p->cq_off.cqes = offsetof(struct io_rings, cqes); + p->cq_off.flags = offsetof(struct io_rings, cq_flags); + p->cq_off.resv1 = 0; + if (!(p->flags & IORING_SETUP_NO_MMAP)) + p->cq_off.user_addr = 0; + + return 0; +} + +static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, + struct io_uring_params __user *params) +{ + struct io_ring_ctx *ctx; + struct io_uring_task *tctx; + struct file *file; + int ret; + + ret = io_uring_fill_params(entries, p); + if (unlikely(ret)) + return ret; + ctx = io_ring_ctx_alloc(p); if (!ctx) return -ENOMEM; @@ -3630,6 +3660,9 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, if (ret) goto err; + if (!(p->flags & IORING_SETUP_NO_SQARRAY)) + p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; + ret = io_sq_offload_create(ctx, p); if (ret) goto err; @@ -3638,29 +3671,6 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, if (ret) goto err; - p->sq_off.head = offsetof(struct io_rings, sq.head); - p->sq_off.tail = offsetof(struct io_rings, sq.tail); - p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask); - p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries); - p->sq_off.flags = offsetof(struct io_rings, sq_flags); - p->sq_off.dropped = offsetof(struct io_rings, sq_dropped); - if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) - p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; - p->sq_off.resv1 = 0; - if (!(ctx->flags & IORING_SETUP_NO_MMAP)) - p->sq_off.user_addr = 0; - - p->cq_off.head = offsetof(struct io_rings, cq.head); - p->cq_off.tail = offsetof(struct io_rings, cq.tail); - p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask); - p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries); - p->cq_off.overflow = offsetof(struct io_rings, cq_overflow); - p->cq_off.cqes = offsetof(struct io_rings, cqes); - p->cq_off.flags = offsetof(struct io_rings, cq_flags); - p->cq_off.resv1 = 0; - if (!(ctx->flags & IORING_SETUP_NO_MMAP)) - p->cq_off.user_addr = 0; - p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP | IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS | IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL | diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 4a471a810f02..e3e6cb14de5d 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -70,6 +70,7 @@ static inline bool io_should_wake(struct io_wait_queue *iowq) unsigned long rings_size(unsigned int flags, unsigned int sq_entries, unsigned int cq_entries, size_t *sq_offset); +int io_uring_fill_params(unsigned entries, struct io_uring_params *p); bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow); int io_run_task_work_sig(struct io_ring_ctx *ctx); void io_req_defer_failed(struct io_kiocb *req, s32 res); From d090bffab609762af06dec295a305ce270941b42 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 24 Oct 2024 10:52:02 -0600 Subject: [PATCH 30/79] io_uring/memmap: explicitly return -EFAULT for mmap on NULL rings The later mapping will actually check this too, but in terms of code clarify, explicitly check for whether or not the rings and sqes are valid during validation. That makes it explicit that if they are non-NULL, they are valid and can get mapped. Signed-off-by: Jens Axboe --- io_uring/memmap.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/io_uring/memmap.c b/io_uring/memmap.c index a0f32a255fd1..d614824e17bd 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -204,11 +204,15 @@ static void *io_uring_validate_mmap_request(struct file *file, loff_t pgoff, /* Don't allow mmap if the ring was setup without it */ if (ctx->flags & IORING_SETUP_NO_MMAP) return ERR_PTR(-EINVAL); + if (!ctx->rings) + return ERR_PTR(-EFAULT); return ctx->rings; case IORING_OFF_SQES: /* Don't allow mmap if the ring was setup without it */ if (ctx->flags & IORING_SETUP_NO_MMAP) return ERR_PTR(-EINVAL); + if (!ctx->sq_sqes) + return ERR_PTR(-EFAULT); return ctx->sq_sqes; case IORING_OFF_PBUF_RING: { struct io_buffer_list *bl; From 79cfe9e59c2a12c3b3faeeefe38d23f3d8030972 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 21 Oct 2024 13:34:10 -0600 Subject: [PATCH 31/79] io_uring/register: add IORING_REGISTER_RESIZE_RINGS Once a ring has been created, the size of the CQ and SQ rings are fixed. Usually this isn't a problem on the SQ ring side, as it merely controls the available number of requests that can be submitted in a single system call, and there's rarely a need to change that. For the CQ ring, it's a different story. For most efficient use of io_uring, it's important that the CQ ring never overflows. This means that applications must size it for the worst case scenario, which can be wasteful. Add IORING_REGISTER_RESIZE_RINGS, which allows an application to resize the existing rings. It takes a struct io_uring_params argument, the same one which is used to setup the ring initially, and resizes rings according to the sizes given. Certain properties are always inherited from the original ring setup, like SQE128/CQE32 and other setup options. The implementation only allows flag associated with how the CQ ring is sized and clamped. Existing unconsumed SQE and CQE entries are copied as part of the process. If either the SQ or CQ resized destination ring cannot hold the entries already present in the source rings, then the operation is failed with -EOVERFLOW. Any register op holds ->uring_lock, which prevents new submissions, and the internal mapping holds the completion lock as well across moving CQ ring state. To prevent races between mmap and ring resizing, add a mutex that's solely used to serialize ring resize and mmap. mmap_sem can't be used here, as as fork'ed process may be doing mmaps on the ring as well. The ctx->resize_lock is held across mmap operations, and the resize will grab it before swapping out the already mapped new data. Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 7 ++ include/uapi/linux/io_uring.h | 5 + io_uring/io_uring.c | 1 + io_uring/memmap.c | 8 ++ io_uring/register.c | 215 +++++++++++++++++++++++++++++++++ 5 files changed, 236 insertions(+) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 6d3ee71bd832..841579dcdae9 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -415,6 +415,13 @@ struct io_ring_ctx { /* protected by ->completion_lock */ unsigned evfd_last_cq_tail; + /* + * Protection for resize vs mmap races - both the mmap and resize + * side will need to grab this lock, to prevent either side from + * being run concurrently with the other. + */ + struct mutex resize_lock; + /* * If IORING_SETUP_NO_MMAP is used, then the below holds * the gup'ed pages for the two rings, and the sqes. diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 86cb385fe0b5..60b9c98595fa 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -615,6 +615,11 @@ enum io_uring_register_op { /* send MSG_RING without having a ring */ IORING_REGISTER_SEND_MSG_RING = 31, + /* 32 reserved for zc rx */ + + /* resize CQ ring */ + IORING_REGISTER_RESIZE_RINGS = 33, + /* this goes last */ IORING_REGISTER_LAST, diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index b5974bdad48b..140cd47fbdb3 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -353,6 +353,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_WQ_LIST(&ctx->submit_state.compl_reqs); INIT_HLIST_HEAD(&ctx->cancelable_uring_cmd); io_napi_init(ctx); + mutex_init(&ctx->resize_lock); return ctx; diff --git a/io_uring/memmap.c b/io_uring/memmap.c index d614824e17bd..85c66fa54956 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -251,6 +251,8 @@ __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma) unsigned int npages; void *ptr; + guard(mutex)(&ctx->resize_lock); + ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz); if (IS_ERR(ptr)) return PTR_ERR(ptr); @@ -274,6 +276,7 @@ unsigned long io_uring_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { + struct io_ring_ctx *ctx = filp->private_data; void *ptr; /* @@ -284,6 +287,8 @@ unsigned long io_uring_get_unmapped_area(struct file *filp, unsigned long addr, if (addr) return -EINVAL; + guard(mutex)(&ctx->resize_lock); + ptr = io_uring_validate_mmap_request(filp, pgoff, len); if (IS_ERR(ptr)) return -ENOMEM; @@ -329,8 +334,11 @@ unsigned long io_uring_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { + struct io_ring_ctx *ctx = file->private_data; void *ptr; + guard(mutex)(&ctx->resize_lock); + ptr = io_uring_validate_mmap_request(file, pgoff, len); if (IS_ERR(ptr)) return PTR_ERR(ptr); diff --git a/io_uring/register.c b/io_uring/register.c index 52b2f9b74af8..fc6c94d694b2 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -29,6 +29,7 @@ #include "napi.h" #include "eventfd.h" #include "msg_ring.h" +#include "memmap.h" #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ IORING_REGISTER_LAST + IORING_OP_LAST) @@ -361,6 +362,214 @@ static int io_register_clock(struct io_ring_ctx *ctx, return 0; } +/* + * State to maintain until we can swap. Both new and old state, used for + * either mapping or freeing. + */ +struct io_ring_ctx_rings { + unsigned short n_ring_pages; + unsigned short n_sqe_pages; + struct page **ring_pages; + struct page **sqe_pages; + struct io_uring_sqe *sq_sqes; + struct io_rings *rings; +}; + +static void io_register_free_rings(struct io_uring_params *p, + struct io_ring_ctx_rings *r) +{ + if (!(p->flags & IORING_SETUP_NO_MMAP)) { + io_pages_unmap(r->rings, &r->ring_pages, &r->n_ring_pages, + true); + io_pages_unmap(r->sq_sqes, &r->sqe_pages, &r->n_sqe_pages, + true); + } else { + io_pages_free(&r->ring_pages, r->n_ring_pages); + io_pages_free(&r->sqe_pages, r->n_sqe_pages); + vunmap(r->rings); + vunmap(r->sq_sqes); + } +} + +#define swap_old(ctx, o, n, field) \ + do { \ + (o).field = (ctx)->field; \ + (ctx)->field = (n).field; \ + } while (0) + +#define RESIZE_FLAGS (IORING_SETUP_CQSIZE | IORING_SETUP_CLAMP) +#define COPY_FLAGS (IORING_SETUP_NO_SQARRAY | IORING_SETUP_SQE128 | \ + IORING_SETUP_CQE32 | IORING_SETUP_NO_MMAP) + +static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) +{ + struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL; + size_t size, sq_array_offset; + struct io_uring_params p; + unsigned i, tail; + void *ptr; + int ret; + + /* for single issuer, must be owner resizing */ + if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && + current != ctx->submitter_task) + return -EEXIST; + if (copy_from_user(&p, arg, sizeof(p))) + return -EFAULT; + if (p.flags & ~RESIZE_FLAGS) + return -EINVAL; + + /* properties that are always inherited */ + p.flags |= (ctx->flags & COPY_FLAGS); + + ret = io_uring_fill_params(p.sq_entries, &p); + if (unlikely(ret)) + return ret; + + /* nothing to do, but copy params back */ + if (p.sq_entries == ctx->sq_entries && p.cq_entries == ctx->cq_entries) { + if (copy_to_user(arg, &p, sizeof(p))) + return -EFAULT; + return 0; + } + + size = rings_size(p.flags, p.sq_entries, p.cq_entries, + &sq_array_offset); + if (size == SIZE_MAX) + return -EOVERFLOW; + + if (!(p.flags & IORING_SETUP_NO_MMAP)) + n.rings = io_pages_map(&n.ring_pages, &n.n_ring_pages, size); + else + n.rings = __io_uaddr_map(&n.ring_pages, &n.n_ring_pages, + p.cq_off.user_addr, size); + if (IS_ERR(n.rings)) + return PTR_ERR(n.rings); + + n.rings->sq_ring_mask = p.sq_entries - 1; + n.rings->cq_ring_mask = p.cq_entries - 1; + n.rings->sq_ring_entries = p.sq_entries; + n.rings->cq_ring_entries = p.cq_entries; + + if (copy_to_user(arg, &p, sizeof(p))) { + io_register_free_rings(&p, &n); + return -EFAULT; + } + + if (p.flags & IORING_SETUP_SQE128) + size = array_size(2 * sizeof(struct io_uring_sqe), p.sq_entries); + else + size = array_size(sizeof(struct io_uring_sqe), p.sq_entries); + if (size == SIZE_MAX) { + io_register_free_rings(&p, &n); + return -EOVERFLOW; + } + + if (!(p.flags & IORING_SETUP_NO_MMAP)) + ptr = io_pages_map(&n.sqe_pages, &n.n_sqe_pages, size); + else + ptr = __io_uaddr_map(&n.sqe_pages, &n.n_sqe_pages, + p.sq_off.user_addr, + size); + if (IS_ERR(ptr)) { + io_register_free_rings(&p, &n); + return PTR_ERR(ptr); + } + + /* + * If using SQPOLL, park the thread + */ + if (ctx->sq_data) { + mutex_unlock(&ctx->uring_lock); + io_sq_thread_park(ctx->sq_data); + mutex_lock(&ctx->uring_lock); + } + + /* + * We'll do the swap. Grab the ctx->resize_lock, which will exclude + * any new mmap's on the ring fd. Clear out existing mappings to prevent + * mmap from seeing them, as we'll unmap them. Any attempt to mmap + * existing rings beyond this point will fail. Not that it could proceed + * at this point anyway, as the io_uring mmap side needs go grab the + * ctx->resize_lock as well. Likewise, hold the completion lock over the + * duration of the actual swap. + */ + mutex_lock(&ctx->resize_lock); + spin_lock(&ctx->completion_lock); + o.rings = ctx->rings; + ctx->rings = NULL; + o.sq_sqes = ctx->sq_sqes; + ctx->sq_sqes = NULL; + + /* + * Now copy SQ and CQ entries, if any. If either of the destination + * rings can't hold what is already there, then fail the operation. + */ + n.sq_sqes = ptr; + tail = o.rings->sq.tail; + if (tail - o.rings->sq.head > p.sq_entries) + goto overflow; + for (i = o.rings->sq.head; i < tail; i++) { + unsigned src_head = i & (ctx->sq_entries - 1); + unsigned dst_head = i & n.rings->sq_ring_mask; + + n.sq_sqes[dst_head] = o.sq_sqes[src_head]; + } + n.rings->sq.head = o.rings->sq.head; + n.rings->sq.tail = o.rings->sq.tail; + + tail = o.rings->cq.tail; + if (tail - o.rings->cq.head > p.cq_entries) { +overflow: + /* restore old rings, and return -EOVERFLOW via cleanup path */ + ctx->rings = o.rings; + ctx->sq_sqes = o.sq_sqes; + to_free = &n; + ret = -EOVERFLOW; + goto out; + } + for (i = o.rings->cq.head; i < tail; i++) { + unsigned src_head = i & (ctx->cq_entries - 1); + unsigned dst_head = i & n.rings->cq_ring_mask; + + n.rings->cqes[dst_head] = o.rings->cqes[src_head]; + } + n.rings->cq.head = o.rings->cq.head; + n.rings->cq.tail = o.rings->cq.tail; + /* invalidate cached cqe refill */ + ctx->cqe_cached = ctx->cqe_sentinel = NULL; + + n.rings->sq_dropped = o.rings->sq_dropped; + n.rings->sq_flags = o.rings->sq_flags; + n.rings->cq_flags = o.rings->cq_flags; + n.rings->cq_overflow = o.rings->cq_overflow; + + /* all done, store old pointers and assign new ones */ + if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) + ctx->sq_array = (u32 *)((char *)n.rings + sq_array_offset); + + ctx->sq_entries = p.sq_entries; + ctx->cq_entries = p.cq_entries; + + ctx->rings = n.rings; + ctx->sq_sqes = n.sq_sqes; + swap_old(ctx, o, n, n_ring_pages); + swap_old(ctx, o, n, n_sqe_pages); + swap_old(ctx, o, n, ring_pages); + swap_old(ctx, o, n, sqe_pages); + to_free = &o; + ret = 0; +out: + spin_unlock(&ctx->completion_lock); + mutex_unlock(&ctx->resize_lock); + io_register_free_rings(&p, to_free); + + if (ctx->sq_data) + io_sq_thread_unpark(ctx->sq_data); + + return ret; +} + static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, void __user *arg, unsigned nr_args) __releases(ctx->uring_lock) @@ -549,6 +758,12 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, break; ret = io_register_clone_buffers(ctx, arg); break; + case IORING_REGISTER_RESIZE_RINGS: + ret = -EINVAL; + if (!arg || nr_args != 1) + break; + ret = io_register_resize_rings(ctx, arg); + break; default: ret = -EINVAL; break; From b898b8c99ead1ce8bee95083bba296e4a86a6c05 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 28 Oct 2024 13:18:27 -0600 Subject: [PATCH 32/79] io_uring/sqpoll: wait on sqd->wait for thread parking io_sqd_handle_event() just does a mutex unlock/lock dance when it's supposed to park, somewhat relying on full ordering with the thread trying to park it which does a similar unlock/lock dance on sqd->lock. However, with adaptive spinning on mutexes, this can waste an awful lot of time. Normally this isn't very noticeable, as parking and unparking the thread isn't a common (or fast path) occurence. However, in testing ring resizing, it's testing exactly that, as each resize will require the SQPOLL to safely park and unpark. Have io_sq_thread_park() explicitly wait on sqd->park_pending being zero before attempting to grab the sqd->lock again. In a resize test, this brings the runtime of SQPOLL down from about 60 seconds to a few seconds, just like the !SQPOLL tests. And saves a ton of spinning time on the mutex, on both sides. Signed-off-by: Jens Axboe --- io_uring/sqpoll.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/io_uring/sqpoll.c b/io_uring/sqpoll.c index a26593979887..1f18b642fbd4 100644 --- a/io_uring/sqpoll.c +++ b/io_uring/sqpoll.c @@ -40,6 +40,7 @@ void io_sq_thread_unpark(struct io_sq_data *sqd) if (atomic_dec_return(&sqd->park_pending)) set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); mutex_unlock(&sqd->lock); + wake_up(&sqd->wait); } void io_sq_thread_park(struct io_sq_data *sqd) @@ -215,7 +216,7 @@ static bool io_sqd_handle_event(struct io_sq_data *sqd) mutex_unlock(&sqd->lock); if (signal_pending(current)) did_sig = get_signal(&ksig); - cond_resched(); + wait_event(sqd->wait, !atomic_read(&sqd->park_pending)); mutex_lock(&sqd->lock); sqd->sq_cpu = raw_smp_processor_id(); } From 0a54a7dd0a12b777721f5ca55c9d6331d2a46b01 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 22 Oct 2024 13:37:00 -0600 Subject: [PATCH 33/79] io_uring: switch struct ext_arg from __kernel_timespec to timespec64 This avoids intermediate storage for turning a __kernel_timespec user pointer into an on-stack struct timespec64, only then to turn it into a ktime_t. Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 140cd47fbdb3..8f0e0749a581 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2495,9 +2495,10 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, struct ext_arg { size_t argsz; - struct __kernel_timespec __user *ts; + struct timespec64 ts; const sigset_t __user *sig; ktime_t min_time; + bool ts_set; }; /* @@ -2535,13 +2536,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, iowq.timeout = KTIME_MAX; start_time = io_get_time(ctx); - if (ext_arg->ts) { - struct timespec64 ts; - - if (get_timespec64(&ts, ext_arg->ts)) - return -EFAULT; - - iowq.timeout = timespec64_to_ktime(ts); + if (ext_arg->ts_set) { + iowq.timeout = timespec64_to_ktime(ext_arg->ts); if (!(flags & IORING_ENTER_ABS_TIMER)) iowq.timeout = ktime_add(iowq.timeout, start_time); } @@ -3252,7 +3248,6 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp, */ if (!(flags & IORING_ENTER_EXT_ARG)) { ext_arg->sig = (const sigset_t __user *) argp; - ext_arg->ts = NULL; return 0; } @@ -3267,7 +3262,11 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp, ext_arg->min_time = arg.min_wait_usec * NSEC_PER_USEC; ext_arg->sig = u64_to_user_ptr(arg.sigmask); ext_arg->argsz = arg.sigmask_sz; - ext_arg->ts = u64_to_user_ptr(arg.ts); + if (arg.ts) { + if (get_timespec64(&ext_arg->ts, u64_to_user_ptr(arg.ts))) + return -EFAULT; + ext_arg->ts_set = true; + } return 0; } From 371b47da25e1f7a1a6323f84c776bd9fa079a490 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 22 Oct 2024 13:41:42 -0600 Subject: [PATCH 34/79] io_uring: change io_get_ext_arg() to use uaccess begin + end In scenarios where a high frequency of wait events are seen, the copy of the struct io_uring_getevents_arg is quite noticeable in the profiles in terms of time spent. It can be seen as up to 3.5-4.5%. Rewrite the copy-in logic, saving about 0.5% of the time. Reviewed-by: Keith Busch Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 8f0e0749a581..4cd0ee52710d 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3240,6 +3240,7 @@ static int io_validate_ext_arg(unsigned flags, const void __user *argp, size_t a static int io_get_ext_arg(unsigned flags, const void __user *argp, struct ext_arg *ext_arg) { + const struct io_uring_getevents_arg __user *uarg = argp; struct io_uring_getevents_arg arg; /* @@ -3257,8 +3258,18 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp, */ if (ext_arg->argsz != sizeof(arg)) return -EINVAL; - if (copy_from_user(&arg, argp, sizeof(arg))) +#ifdef CONFIG_64BIT + if (!user_access_begin(uarg, sizeof(*uarg))) return -EFAULT; + unsafe_get_user(arg.sigmask, &uarg->sigmask, uaccess_end); + unsafe_get_user(arg.sigmask_sz, &uarg->sigmask_sz, uaccess_end); + unsafe_get_user(arg.min_wait_usec, &uarg->min_wait_usec, uaccess_end); + unsafe_get_user(arg.ts, &uarg->ts, uaccess_end); + user_access_end(); +#else + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; +#endif ext_arg->min_time = arg.min_wait_usec * NSEC_PER_USEC; ext_arg->sig = u64_to_user_ptr(arg.sigmask); ext_arg->argsz = arg.sigmask_sz; @@ -3268,6 +3279,11 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp, ext_arg->ts_set = true; } return 0; +#ifdef CONFIG_64BIT +uaccess_end: + user_access_end(); + return -EFAULT; +#endif } SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, From aa00f67adc2c0d6439f81b5a81ff181377c47a7e Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 22 Oct 2024 13:47:00 -0600 Subject: [PATCH 35/79] io_uring: add support for fixed wait regions Generally applications have 1 or a few waits of waiting, yet they pass in a struct io_uring_getevents_arg every time. This needs to get copied and, in turn, the timeout value needs to get copied. Rather than do this for every invocation, allow the application to register a fixed set of wait regions that can simply be indexed when asking the kernel to wait on events. At ring setup time, the application can register a number of these wait regions and initialize region/index 0 upfront: struct io_uring_reg_wait *reg; reg = io_uring_setup_reg_wait(ring, nr_regions, &ret); /* set timeout and mark as set, sigmask/sigmask_sz as needed */ reg->ts.tv_sec = 0; reg->ts.tv_nsec = 100000; reg->flags = IORING_REG_WAIT_TS; where nr_regions >= 1 && nr_regions <= PAGE_SIZE / sizeof(*reg). The above initializes index 0, but 63 other regions can be initialized, if needed. Now, instead of doing: struct __kernel_timespec timeout = { .tv_nsec = 100000, }; io_uring_submit_and_wait_timeout(ring, &cqe, nr, &t, NULL); to wait for events for each submit_and_wait, or just wait, operation, it can just reference the above region at offset 0 and do: io_uring_submit_and_wait_reg(ring, &cqe, nr, 0); to achieve the same goal of waiting 100usec without needing to copy both struct io_uring_getevents_arg (24b) and struct __kernel_timeout (16b) for each invocation. Struct io_uring_reg_wait looks as follows: struct io_uring_reg_wait { struct __kernel_timespec ts; __u32 min_wait_usec; __u32 flags; __u64 sigmask; __u32 sigmask_sz; __u32 pad[3]; __u64 pad2[2]; }; embedding the timeout itself in the region, rather than passing it as a pointer as well. Note that the signal mask is still passed as a pointer, both for compatability reasons, but also because there doesn't seem to be a lot of high frequency waits scenarios that involve setting and resetting the signal mask for each wait. The application is free to modify any region before a wait call, or it can use keep multiple regions with different settings to avoid needing to modify the same one for wait calls. Up to a page size of regions is mapped by default, allowing PAGE_SIZE / 64 available regions for use. The registered region must fit within a page. On a 4kb page size system, that allows for 64 wait regions if a full page is used, as the size of struct io_uring_reg_wait is 64b. The region registered must be aligned to io_uring_reg_wait in size. It's valid to register less than 64 entries. In network performance testing with zero-copy, this reduced the time spent waiting on the TX side from 3.12% to 0.3% and the RX side from 4.4% to 0.3%. Wait regions are fixed for the lifetime of the ring - once registered, they are persistent until the ring is torn down. The regions support minimum wait timeout as well as the regular waits. Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 10 +++++ include/uapi/linux/io_uring.h | 41 +++++++++++++++++ io_uring/io_uring.c | 70 ++++++++++++++++++++++++----- io_uring/register.c | 82 ++++++++++++++++++++++++++++++++++ io_uring/register.h | 1 + 5 files changed, 192 insertions(+), 12 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 841579dcdae9..2f12828b22a4 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -327,6 +327,14 @@ struct io_ring_ctx { atomic_t cq_wait_nr; atomic_t cq_timeouts; struct wait_queue_head cq_wait; + + /* + * If registered with IORING_REGISTER_CQWAIT_REG, a single + * page holds N entries, mapped in cq_wait_arg. cq_wait_index + * is the maximum allowable index. + */ + struct io_uring_reg_wait *cq_wait_arg; + unsigned char cq_wait_index; } ____cacheline_aligned_in_smp; /* timeouts */ @@ -430,6 +438,8 @@ struct io_ring_ctx { unsigned short n_sqe_pages; struct page **ring_pages; struct page **sqe_pages; + + struct page **cq_wait_page; }; struct io_tw_state { diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 60b9c98595fa..65b7417c1b05 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -518,6 +518,7 @@ struct io_cqring_offsets { #define IORING_ENTER_EXT_ARG (1U << 3) #define IORING_ENTER_REGISTERED_RING (1U << 4) #define IORING_ENTER_ABS_TIMER (1U << 5) +#define IORING_ENTER_EXT_ARG_REG (1U << 6) /* * Passed in for io_uring_setup(2). Copied back with updated info on success @@ -620,6 +621,9 @@ enum io_uring_register_op { /* resize CQ ring */ IORING_REGISTER_RESIZE_RINGS = 33, + /* register fixed io_uring_reg_wait arguments */ + IORING_REGISTER_CQWAIT_REG = 34, + /* this goes last */ IORING_REGISTER_LAST, @@ -803,6 +807,43 @@ enum io_uring_register_restriction_op { IORING_RESTRICTION_LAST }; +enum { + IORING_REG_WAIT_TS = (1U << 0), +}; + +/* + * Argument for IORING_REGISTER_CQWAIT_REG, registering a region of + * struct io_uring_reg_wait that can be indexed when io_uring_enter(2) is + * called rather than pass in a wait argument structure separately. + */ +struct io_uring_cqwait_reg_arg { + __u32 flags; + __u32 struct_size; + __u32 nr_entries; + __u32 pad; + __u64 user_addr; + __u64 pad2[3]; +}; + +/* + * Argument for io_uring_enter(2) with + * IORING_GETEVENTS | IORING_ENTER_EXT_ARG_REG set, where the actual argument + * is an index into a previously registered fixed wait region described by + * the below structure. + */ +struct io_uring_reg_wait { + struct __kernel_timespec ts; + __u32 min_wait_usec; + __u32 flags; + __u64 sigmask; + __u32 sigmask_sz; + __u32 pad[3]; + __u64 pad2[2]; +}; + +/* + * Argument for io_uring_enter(2) with IORING_GETEVENTS | IORING_ENTER_EXT_ARG + */ struct io_uring_getevents_arg { __u64 sigmask; __u32 sigmask_sz; diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 4cd0ee52710d..2863b957e373 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2736,6 +2736,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) io_alloc_cache_free(&ctx->msg_cache, io_msg_cache_free); io_futex_cache_free(ctx); io_destroy_buffers(ctx); + io_unregister_cqwait_reg(ctx); mutex_unlock(&ctx->uring_lock); if (ctx->sq_creds) put_cred(ctx->sq_creds); @@ -3224,21 +3225,43 @@ void __io_uring_cancel(bool cancel_all) io_uring_cancel_generic(cancel_all, NULL); } -static int io_validate_ext_arg(unsigned flags, const void __user *argp, size_t argsz) +static struct io_uring_reg_wait *io_get_ext_arg_reg(struct io_ring_ctx *ctx, + const struct io_uring_getevents_arg __user *uarg) { - if (flags & IORING_ENTER_EXT_ARG) { - struct io_uring_getevents_arg arg; + struct io_uring_reg_wait *arg = READ_ONCE(ctx->cq_wait_arg); - if (argsz != sizeof(arg)) - return -EINVAL; - if (copy_from_user(&arg, argp, sizeof(arg))) - return -EFAULT; + if (arg) { + unsigned int index = (unsigned int) (uintptr_t) uarg; + + if (index <= ctx->cq_wait_index) + return arg + index; } + + return ERR_PTR(-EFAULT); +} + +static int io_validate_ext_arg(struct io_ring_ctx *ctx, unsigned flags, + const void __user *argp, size_t argsz) +{ + struct io_uring_getevents_arg arg; + + if (!(flags & IORING_ENTER_EXT_ARG)) + return 0; + + if (flags & IORING_ENTER_EXT_ARG_REG) { + if (argsz != sizeof(struct io_uring_reg_wait)) + return -EINVAL; + return PTR_ERR(io_get_ext_arg_reg(ctx, argp)); + } + if (argsz != sizeof(arg)) + return -EINVAL; + if (copy_from_user(&arg, argp, sizeof(arg))) + return -EFAULT; return 0; } -static int io_get_ext_arg(unsigned flags, const void __user *argp, - struct ext_arg *ext_arg) +static int io_get_ext_arg(struct io_ring_ctx *ctx, unsigned flags, + const void __user *argp, struct ext_arg *ext_arg) { const struct io_uring_getevents_arg __user *uarg = argp; struct io_uring_getevents_arg arg; @@ -3252,6 +3275,28 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp, return 0; } + if (flags & IORING_ENTER_EXT_ARG_REG) { + struct io_uring_reg_wait *w; + + if (ext_arg->argsz != sizeof(struct io_uring_reg_wait)) + return -EINVAL; + w = io_get_ext_arg_reg(ctx, argp); + if (IS_ERR(w)) + return PTR_ERR(w); + + if (w->flags & ~IORING_REG_WAIT_TS) + return -EINVAL; + ext_arg->min_time = READ_ONCE(w->min_wait_usec) * NSEC_PER_USEC; + ext_arg->sig = u64_to_user_ptr(READ_ONCE(w->sigmask)); + ext_arg->argsz = READ_ONCE(w->sigmask_sz); + if (w->flags & IORING_REG_WAIT_TS) { + ext_arg->ts.tv_sec = READ_ONCE(w->ts.tv_sec); + ext_arg->ts.tv_nsec = READ_ONCE(w->ts.tv_nsec); + ext_arg->ts_set = true; + } + return 0; + } + /* * EXT_ARG is set - ensure we agree on the size of it and copy in our * timespec and sigset_t pointers if good. @@ -3297,7 +3342,8 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP | IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG | IORING_ENTER_REGISTERED_RING | - IORING_ENTER_ABS_TIMER))) + IORING_ENTER_ABS_TIMER | + IORING_ENTER_EXT_ARG_REG))) return -EINVAL; /* @@ -3380,7 +3426,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, */ mutex_lock(&ctx->uring_lock); iopoll_locked: - ret2 = io_validate_ext_arg(flags, argp, argsz); + ret2 = io_validate_ext_arg(ctx, flags, argp, argsz); if (likely(!ret2)) { min_complete = min(min_complete, ctx->cq_entries); @@ -3390,7 +3436,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, } else { struct ext_arg ext_arg = { .argsz = argsz }; - ret2 = io_get_ext_arg(flags, argp, &ext_arg); + ret2 = io_get_ext_arg(ctx, flags, argp, &ext_arg); if (likely(!ret2)) { min_complete = min(min_complete, ctx->cq_entries); diff --git a/io_uring/register.c b/io_uring/register.c index fc6c94d694b2..1eb686eaa310 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -570,6 +570,82 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) return ret; } +void io_unregister_cqwait_reg(struct io_ring_ctx *ctx) +{ + unsigned short npages = 1; + + if (!ctx->cq_wait_page) + return; + + io_pages_unmap(ctx->cq_wait_arg, &ctx->cq_wait_page, &npages, true); + ctx->cq_wait_arg = NULL; + if (ctx->user) + __io_unaccount_mem(ctx->user, 1); +} + +/* + * Register a page holding N entries of struct io_uring_reg_wait, which can + * be used via io_uring_enter(2) if IORING_GETEVENTS_EXT_ARG_REG is set. + * If that is set with IORING_GETEVENTS_EXT_ARG, then instead of passing + * in a pointer for a struct io_uring_getevents_arg, an index into this + * registered array is passed, avoiding two (arg + timeout) copies per + * invocation. + */ +static int io_register_cqwait_reg(struct io_ring_ctx *ctx, void __user *uarg) +{ + struct io_uring_cqwait_reg_arg arg; + struct io_uring_reg_wait *reg; + struct page **pages; + unsigned long len; + int nr_pages, poff; + int ret; + + if (ctx->cq_wait_page || ctx->cq_wait_arg) + return -EBUSY; + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + if (!arg.nr_entries || arg.flags) + return -EINVAL; + if (arg.struct_size != sizeof(*reg)) + return -EINVAL; + if (check_mul_overflow(arg.struct_size, arg.nr_entries, &len)) + return -EOVERFLOW; + if (len > PAGE_SIZE) + return -EINVAL; + /* offset + len must fit within a page, and must be reg_wait aligned */ + poff = arg.user_addr & ~PAGE_MASK; + if (len + poff > PAGE_SIZE) + return -EINVAL; + if (poff % arg.struct_size) + return -EINVAL; + + pages = io_pin_pages(arg.user_addr, len, &nr_pages); + if (IS_ERR(pages)) + return PTR_ERR(pages); + ret = -EINVAL; + if (nr_pages != 1) + goto out_free; + if (ctx->user) { + ret = __io_account_mem(ctx->user, 1); + if (ret) + goto out_free; + } + + reg = vmap(pages, 1, VM_MAP, PAGE_KERNEL); + if (reg) { + ctx->cq_wait_index = arg.nr_entries - 1; + WRITE_ONCE(ctx->cq_wait_page, pages); + WRITE_ONCE(ctx->cq_wait_arg, (void *) reg + poff); + return 0; + } + ret = -ENOMEM; + if (ctx->user) + __io_unaccount_mem(ctx->user, 1); +out_free: + io_pages_free(&pages, nr_pages); + return ret; +} + static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, void __user *arg, unsigned nr_args) __releases(ctx->uring_lock) @@ -764,6 +840,12 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, break; ret = io_register_resize_rings(ctx, arg); break; + case IORING_REGISTER_CQWAIT_REG: + ret = -EINVAL; + if (!arg || nr_args != 1) + break; + ret = io_register_cqwait_reg(ctx, arg); + break; default: ret = -EINVAL; break; diff --git a/io_uring/register.h b/io_uring/register.h index a5f39d5ef9e0..3e935e8fa4b2 100644 --- a/io_uring/register.h +++ b/io_uring/register.h @@ -5,5 +5,6 @@ int io_eventfd_unregister(struct io_ring_ctx *ctx); int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id); struct file *io_uring_register_get_file(unsigned int fd, bool registered); +void io_unregister_cqwait_reg(struct io_ring_ctx *ctx); #endif From a85f31052bce52111b4e9d5a536003481d0421d0 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 27 Oct 2024 08:59:10 -0600 Subject: [PATCH 36/79] io_uring/nop: add support for testing registered files and buffers Useful for testing performance/efficiency impact of registered files and buffers, vs (particularly) non-registered files. Signed-off-by: Jens Axboe --- include/uapi/linux/io_uring.h | 3 +++ io_uring/nop.c | 49 +++++++++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 65b7417c1b05..024745283783 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -416,6 +416,9 @@ enum io_uring_msg_ring_flags { * IORING_NOP_INJECT_RESULT Inject result from sqe->result */ #define IORING_NOP_INJECT_RESULT (1U << 0) +#define IORING_NOP_FILE (1U << 1) +#define IORING_NOP_FIXED_FILE (1U << 2) +#define IORING_NOP_FIXED_BUFFER (1U << 3) /* * IO completion data structure (Completion Queue Entry) diff --git a/io_uring/nop.c b/io_uring/nop.c index a5bcf3d6984f..2c7a22ba4053 100644 --- a/io_uring/nop.c +++ b/io_uring/nop.c @@ -8,35 +8,74 @@ #include #include "io_uring.h" +#include "rsrc.h" #include "nop.h" struct io_nop { /* NOTE: kiocb has the file as the first member, so don't do it here */ struct file *file; int result; + int fd; + int buffer; + unsigned int flags; }; +#define NOP_FLAGS (IORING_NOP_INJECT_RESULT | IORING_NOP_FIXED_FILE | \ + IORING_NOP_FIXED_BUFFER | IORING_NOP_FILE) + int io_nop_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - unsigned int flags; struct io_nop *nop = io_kiocb_to_cmd(req, struct io_nop); - flags = READ_ONCE(sqe->nop_flags); - if (flags & ~IORING_NOP_INJECT_RESULT) + nop->flags = READ_ONCE(sqe->nop_flags); + if (nop->flags & ~NOP_FLAGS) return -EINVAL; - if (flags & IORING_NOP_INJECT_RESULT) + if (nop->flags & IORING_NOP_INJECT_RESULT) nop->result = READ_ONCE(sqe->len); else nop->result = 0; + if (nop->flags & IORING_NOP_FIXED_FILE) + nop->fd = READ_ONCE(sqe->fd); + if (nop->flags & IORING_NOP_FIXED_BUFFER) + nop->buffer = READ_ONCE(sqe->buf_index); return 0; } int io_nop(struct io_kiocb *req, unsigned int issue_flags) { struct io_nop *nop = io_kiocb_to_cmd(req, struct io_nop); + int ret = nop->result; - if (nop->result < 0) + if (nop->flags & IORING_NOP_FILE) { + if (nop->flags & IORING_NOP_FIXED_FILE) { + req->file = io_file_get_fixed(req, nop->fd, issue_flags); + req->flags |= REQ_F_FIXED_FILE; + } else { + req->file = io_file_get_normal(req, nop->fd); + } + if (!req->file) { + ret = -EBADF; + goto done; + } + } + if (nop->flags & IORING_NOP_FIXED_BUFFER) { + struct io_ring_ctx *ctx = req->ctx; + struct io_mapped_ubuf *imu; + int idx; + + ret = -EFAULT; + io_ring_submit_lock(ctx, issue_flags); + if (nop->buffer < ctx->nr_user_bufs) { + idx = array_index_nospec(nop->buffer, ctx->nr_user_bufs); + imu = READ_ONCE(ctx->user_bufs[idx]); + io_req_set_rsrc_node(req, ctx); + ret = 0; + } + io_ring_submit_unlock(ctx, issue_flags); + } +done: + if (ret < 0) req_set_fail(req); io_req_set_res(req, nop->result, 0); return IOU_OK; From ff1256b8f3c45f222bce19fbfc1e1bc498b31d03 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 25 Oct 2024 08:54:28 -0600 Subject: [PATCH 37/79] io_uring/rsrc: move struct io_fixed_file to rsrc.h header There's no need for this internal structure to be visible, move it to the private rsrc.h header instead. Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 5 ----- io_uring/filetable.h | 1 + io_uring/rsrc.h | 5 +++++ 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 2f12828b22a4..d4ba4ae480d6 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -55,11 +55,6 @@ struct io_wq_work { int cancel_seq; }; -struct io_fixed_file { - /* file * with additional FFS_* flags */ - unsigned long file_ptr; -}; - struct io_file_table { struct io_fixed_file *files; unsigned long *bitmap; diff --git a/io_uring/filetable.h b/io_uring/filetable.h index b2435c4dca1f..c027ed4ad68d 100644 --- a/io_uring/filetable.h +++ b/io_uring/filetable.h @@ -4,6 +4,7 @@ #include #include +#include "rsrc.h" bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files); void io_free_file_tables(struct io_file_table *table); diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index c50d4be4aa6d..e072fb3ee351 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -40,6 +40,11 @@ struct io_rsrc_node { struct io_rsrc_put item; }; +struct io_fixed_file { + /* file * with additional FFS_* flags */ + unsigned long file_ptr; +}; + struct io_mapped_ubuf { u64 ubuf; unsigned int len; From aaa736b186239b7dc7778ae94c75f26c96972796 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 15 Oct 2024 12:19:33 -0600 Subject: [PATCH 38/79] io_uring: specify freeptr usage for SLAB_TYPESAFE_BY_RCU io_kiocb cache Doesn't matter right now as there's still some bytes left for it, but let's prepare for the io_kiocb potentially growing and add a specific freeptr offset for it. Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 2863b957e373..a09c67b38c1b 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3846,6 +3846,8 @@ static int __init io_uring_init(void) struct kmem_cache_args kmem_args = { .useroffset = offsetof(struct io_kiocb, cmd.data), .usersize = sizeof_field(struct io_kiocb, cmd.data), + .freeptr_offset = offsetof(struct io_kiocb, work), + .use_freeptr_offset = true, }; #define __BUILD_BUG_VERIFY_OFFSET_SIZE(stype, eoffset, esize, ename) do { \ From 743fb58a35cde8fe27b07ee5a985ae76563845e3 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 28 Oct 2024 08:03:04 -0600 Subject: [PATCH 39/79] io_uring/splice: open code 2nd direct file assignment In preparation for not pinning the whole registered file table, open code the second potential direct file assignment. This will be handled by appropriate helpers in the future, for now just do it manually. Signed-off-by: Jens Axboe --- io_uring/opdef.c | 2 ++ io_uring/splice.c | 44 ++++++++++++++++++++++++++++++++++++-------- io_uring/splice.h | 1 + 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/io_uring/opdef.c b/io_uring/opdef.c index a2be3bbca5ff..3de75eca1c92 100644 --- a/io_uring/opdef.c +++ b/io_uring/opdef.c @@ -641,6 +641,7 @@ const struct io_cold_def io_cold_defs[] = { }, [IORING_OP_SPLICE] = { .name = "SPLICE", + .cleanup = io_splice_cleanup, }, [IORING_OP_PROVIDE_BUFFERS] = { .name = "PROVIDE_BUFFERS", @@ -650,6 +651,7 @@ const struct io_cold_def io_cold_defs[] = { }, [IORING_OP_TEE] = { .name = "TEE", + .cleanup = io_splice_cleanup, }, [IORING_OP_SHUTDOWN] = { .name = "SHUTDOWN", diff --git a/io_uring/splice.c b/io_uring/splice.c index 3b659cd23e9d..e62bc6497a94 100644 --- a/io_uring/splice.c +++ b/io_uring/splice.c @@ -21,6 +21,7 @@ struct io_splice { u64 len; int splice_fd_in; unsigned int flags; + struct io_rsrc_node *rsrc_node; }; static int __io_splice_prep(struct io_kiocb *req, @@ -34,6 +35,7 @@ static int __io_splice_prep(struct io_kiocb *req, if (unlikely(sp->flags & ~valid_flags)) return -EINVAL; sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in); + sp->rsrc_node = NULL; req->flags |= REQ_F_FORCE_ASYNC; return 0; } @@ -45,6 +47,38 @@ int io_tee_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return __io_splice_prep(req, sqe); } +void io_splice_cleanup(struct io_kiocb *req) +{ + struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice); + + io_put_rsrc_node(req->ctx, sp->rsrc_node); +} + +static struct file *io_splice_get_file(struct io_kiocb *req, + unsigned int issue_flags) +{ + struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice); + struct io_ring_ctx *ctx = req->ctx; + struct io_fixed_file *slot; + struct file *file = NULL; + + if (!(sp->flags & SPLICE_F_FD_IN_FIXED)) + return io_file_get_normal(req, sp->splice_fd_in); + + io_ring_submit_lock(ctx, issue_flags); + if (unlikely(sp->splice_fd_in >= ctx->nr_user_files)) + goto out; + sp->splice_fd_in = array_index_nospec(sp->splice_fd_in, ctx->nr_user_files); + slot = &ctx->file_table.files[sp->splice_fd_in]; + if (!req->rsrc_node) + __io_req_set_rsrc_node(req, ctx); + file = io_slot_file(slot); + req->flags |= REQ_F_NEED_CLEANUP; +out: + io_ring_submit_unlock(ctx, issue_flags); + return file; +} + int io_tee(struct io_kiocb *req, unsigned int issue_flags) { struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice); @@ -55,10 +89,7 @@ int io_tee(struct io_kiocb *req, unsigned int issue_flags) WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); - if (sp->flags & SPLICE_F_FD_IN_FIXED) - in = io_file_get_fixed(req, sp->splice_fd_in, issue_flags); - else - in = io_file_get_normal(req, sp->splice_fd_in); + in = io_splice_get_file(req, issue_flags); if (!in) { ret = -EBADF; goto done; @@ -96,10 +127,7 @@ int io_splice(struct io_kiocb *req, unsigned int issue_flags) WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); - if (sp->flags & SPLICE_F_FD_IN_FIXED) - in = io_file_get_fixed(req, sp->splice_fd_in, issue_flags); - else - in = io_file_get_normal(req, sp->splice_fd_in); + in = io_splice_get_file(req, issue_flags); if (!in) { ret = -EBADF; goto done; diff --git a/io_uring/splice.h b/io_uring/splice.h index 542f94168ad3..b9b2848327fb 100644 --- a/io_uring/splice.h +++ b/io_uring/splice.h @@ -3,5 +3,6 @@ int io_tee_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_tee(struct io_kiocb *req, unsigned int issue_flags); +void io_splice_cleanup(struct io_kiocb *req); int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_splice(struct io_kiocb *req, unsigned int issue_flags); From e410ffca588691e36d5449a5bf521a1a7b712911 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 28 Oct 2024 08:41:24 -0600 Subject: [PATCH 40/79] io_uring/rsrc: kill io_charge_rsrc_node() It's only used from __io_req_set_rsrc_node(), and it takes both the ctx and node itself, while never using the ctx. Just open-code the basic refs++ in __io_req_set_rsrc_node() instead. Signed-off-by: Jens Axboe --- io_uring/rsrc.h | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index e072fb3ee351..1589c9740083 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -97,18 +97,12 @@ static inline void io_put_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node io_rsrc_node_ref_zero(node); } -static inline void io_charge_rsrc_node(struct io_ring_ctx *ctx, - struct io_rsrc_node *node) -{ - node->refs++; -} - static inline void __io_req_set_rsrc_node(struct io_kiocb *req, struct io_ring_ctx *ctx) { lockdep_assert_held(&ctx->uring_lock); req->rsrc_node = ctx->rsrc_node; - io_charge_rsrc_node(ctx, ctx->rsrc_node); + ctx->rsrc_node->refs++; } static inline void io_req_set_rsrc_node(struct io_kiocb *req, From 7029acd8a950393ee3a3d8e1a7ee1a9b77808a3b Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 25 Oct 2024 19:27:39 -0600 Subject: [PATCH 41/79] io_uring/rsrc: get rid of per-ring io_rsrc_node list Work in progress, but get rid of the per-ring serialization of resource nodes, like registered buffers and files. Main issue here is that one node can otherwise hold up a bunch of other nodes from getting freed, which is especially a problem for file resource nodes and networked workloads where some descriptors may not see activity in a long time. As an example, instantiate an io_uring ring fd and create a sparse registered file table. Even 2 will do. Then create a socket and register it as fixed file 0, F0. The number of open files in the app is now 5, with 0/1/2 being the usual stdin/out/err, 3 being the ring fd, and 4 being the socket. Register this socket (eg "the listener") in slot 0 of the registered file table. Now add an operation on the socket that uses slot 0. Finally, loop N times, where each loop creates a new socket, registers said socket as a file, then unregisters the socket, and finally closes the socket. This is roughly similar to what a basic accept loop would look like. At the end of this loop, it's not unreasonable to expect that there would still be 5 open files. Each socket created and registered in the loop is also unregistered and closed. But since the listener socket registered first still has references to its resource node due to still being active, each subsequent socket unregistration is stuck behind it for reclaim. Hence 5 + N files are still open at that point, where N is awaiting the final put held up by the listener socket. Rewrite the io_rsrc_node handling to NOT rely on serialization. Struct io_kiocb now gets explicit resource nodes assigned, with each holding a reference to the parent node. A parent node is either of type FILE or BUFFER, which are the two types of nodes that exist. A request can have two nodes assigned, if it's using both registered files and buffers. Since request issue and task_work completion is both under the ring private lock, no atomics are needed to handle these references. It's a simple unlocked inc/dec. As before, the registered buffer or file table each hold a reference as well to the registered nodes. Final put of the node will remove the node and free the underlying resource, eg unmap the buffer or put the file. Outside of removing the stall in resource reclaim described above, it has the following advantages: 1) It's a lot simpler than the previous scheme, and easier to follow. No need to specific quiesce handling anymore. 2) There are no resource node allocations in the fast path, all of that happens at resource registration time. 3) The structs related to resource handling can all get simplified quite a bit, like io_rsrc_node and io_rsrc_data. io_rsrc_put can go away completely. 4) Handling of resource tags is much simpler, and doesn't require persistent storage as it can simply get assigned up front at registration time. Just copy them in one-by-one at registration time and assign to the resource node. The only real downside is that a request is now explicitly limited to pinning 2 resources, one file and one buffer, where before just assigning a resource node to a request would pin all of them. The upside is that it's easier to follow now, as an individual resource is explicitly referenced and assigned to the request. With this in place, the above mentioned example will be using exactly 5 files at the end of the loop, not N. Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 10 +- io_uring/fdinfo.c | 2 +- io_uring/filetable.c | 52 ++-- io_uring/filetable.h | 25 +- io_uring/io_uring.c | 38 +-- io_uring/net.c | 11 +- io_uring/nop.c | 6 +- io_uring/notif.c | 3 +- io_uring/rsrc.c | 485 ++++++++++++--------------------- io_uring/rsrc.h | 71 ++--- io_uring/rw.c | 8 +- io_uring/splice.c | 16 +- io_uring/uring_cmd.c | 12 +- 13 files changed, 272 insertions(+), 467 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index d4ba4ae480d6..42c5f2c992c4 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -56,7 +56,7 @@ struct io_wq_work { }; struct io_file_table { - struct io_fixed_file *files; + struct io_rsrc_node **nodes; unsigned long *bitmap; unsigned int alloc_hint; }; @@ -264,7 +264,6 @@ struct io_ring_ctx { * Fixed resources fast path, should be accessed only under * uring_lock, and updated through io_uring_register(2) */ - struct io_rsrc_node *rsrc_node; atomic_t cancel_seq; /* @@ -277,7 +276,7 @@ struct io_ring_ctx { struct io_wq_work_list iopoll_list; struct io_file_table file_table; - struct io_mapped_ubuf **user_bufs; + struct io_rsrc_node **user_bufs; unsigned nr_user_files; unsigned nr_user_bufs; @@ -372,10 +371,7 @@ struct io_ring_ctx { struct io_rsrc_data *buf_data; /* protected by ->uring_lock */ - struct list_head rsrc_ref_list; struct io_alloc_cache rsrc_node_cache; - struct wait_queue_head rsrc_quiesce_wq; - unsigned rsrc_quiesce; u32 pers_next; struct xarray personalities; @@ -642,7 +638,7 @@ struct io_kiocb { __poll_t apoll_events; }; - struct io_rsrc_node *rsrc_node; + struct io_rsrc_node *rsrc_nodes[2]; atomic_t refs; bool cancel_seq_set; diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index a6bac533edbe..064a79475c5f 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -176,7 +176,7 @@ __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *file) } seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs); for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) { - struct io_mapped_ubuf *buf = ctx->user_bufs[i]; + struct io_mapped_ubuf *buf = ctx->user_bufs[i]->buf; seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, buf->len); } diff --git a/io_uring/filetable.c b/io_uring/filetable.c index 997c56d32ee6..1b12a9a1cc16 100644 --- a/io_uring/filetable.c +++ b/io_uring/filetable.c @@ -38,14 +38,14 @@ static int io_file_bitmap_get(struct io_ring_ctx *ctx) bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files) { - table->files = kvcalloc(nr_files, sizeof(table->files[0]), - GFP_KERNEL_ACCOUNT); - if (unlikely(!table->files)) + table->nodes = kvmalloc_array(nr_files, sizeof(struct io_src_node *), + GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (unlikely(!table->nodes)) return false; table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT); if (unlikely(!table->bitmap)) { - kvfree(table->files); + kvfree(table->nodes); return false; } @@ -54,9 +54,9 @@ bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files) void io_free_file_tables(struct io_file_table *table) { - kvfree(table->files); + kvfree(table->nodes); bitmap_free(table->bitmap); - table->files = NULL; + table->nodes = NULL; table->bitmap = NULL; } @@ -64,8 +64,7 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file, u32 slot_index) __must_hold(&req->ctx->uring_lock) { - struct io_fixed_file *file_slot; - int ret; + struct io_rsrc_node *node; if (io_is_uring_fops(file)) return -EBADF; @@ -74,22 +73,18 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file, if (slot_index >= ctx->nr_user_files) return -EINVAL; + node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE); + if (!node) + return -ENOMEM; + slot_index = array_index_nospec(slot_index, ctx->nr_user_files); - file_slot = io_fixed_file_slot(&ctx->file_table, slot_index); - - if (file_slot->file_ptr) { - ret = io_queue_rsrc_removal(ctx->file_data, slot_index, - io_slot_file(file_slot)); - if (ret) - return ret; - - file_slot->file_ptr = 0; - } else { + if (ctx->file_table.nodes[slot_index]) + io_put_rsrc_node(ctx->file_table.nodes[slot_index]); + else io_file_bitmap_set(&ctx->file_table, slot_index); - } - *io_get_tag_slot(ctx->file_data, slot_index) = 0; - io_fixed_file_set(file_slot, file); + ctx->file_table.nodes[slot_index] = node; + io_fixed_file_set(node, file); return 0; } @@ -134,25 +129,16 @@ int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags, int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset) { - struct io_fixed_file *file_slot; - int ret; - if (unlikely(!ctx->file_data)) return -ENXIO; if (offset >= ctx->nr_user_files) return -EINVAL; offset = array_index_nospec(offset, ctx->nr_user_files); - file_slot = io_fixed_file_slot(&ctx->file_table, offset); - if (!file_slot->file_ptr) + if (!ctx->file_table.nodes[offset]) return -EBADF; - - ret = io_queue_rsrc_removal(ctx->file_data, offset, - io_slot_file(file_slot)); - if (ret) - return ret; - - file_slot->file_ptr = 0; + io_put_rsrc_node(ctx->file_table.nodes[offset]); + ctx->file_table.nodes[offset] = NULL; io_file_bitmap_clear(&ctx->file_table, offset); return 0; } diff --git a/io_uring/filetable.h b/io_uring/filetable.h index c027ed4ad68d..47616079abaa 100644 --- a/io_uring/filetable.h +++ b/io_uring/filetable.h @@ -34,36 +34,35 @@ static inline void io_file_bitmap_set(struct io_file_table *table, int bit) table->alloc_hint = bit + 1; } -static inline struct io_fixed_file * -io_fixed_file_slot(struct io_file_table *table, unsigned i) -{ - return &table->files[i]; -} - #define FFS_NOWAIT 0x1UL #define FFS_ISREG 0x2UL #define FFS_MASK ~(FFS_NOWAIT|FFS_ISREG) -static inline unsigned int io_slot_flags(struct io_fixed_file *slot) +static inline unsigned int io_slot_flags(struct io_rsrc_node *node) { - return (slot->file_ptr & ~FFS_MASK) << REQ_F_SUPPORT_NOWAIT_BIT; + + return (node->file_ptr & ~FFS_MASK) << REQ_F_SUPPORT_NOWAIT_BIT; } -static inline struct file *io_slot_file(struct io_fixed_file *slot) +static inline struct file *io_slot_file(struct io_rsrc_node *node) { - return (struct file *)(slot->file_ptr & FFS_MASK); + return (struct file *)(node->file_ptr & FFS_MASK); } static inline struct file *io_file_from_index(struct io_file_table *table, int index) { - return io_slot_file(io_fixed_file_slot(table, index)); + struct io_rsrc_node *node = table->nodes[index]; + + if (node) + return io_slot_file(node); + return NULL; } -static inline void io_fixed_file_set(struct io_fixed_file *file_slot, +static inline void io_fixed_file_set(struct io_rsrc_node *node, struct file *file) { - file_slot->file_ptr = (unsigned long)file | + node->file_ptr = (unsigned long)file | (io_file_get_flags(file) >> REQ_F_SUPPORT_NOWAIT_BIT); } diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index a09c67b38c1b..0876aa74c739 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -333,7 +333,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) mutex_init(&ctx->uring_lock); init_waitqueue_head(&ctx->cq_wait); init_waitqueue_head(&ctx->poll_wq); - init_waitqueue_head(&ctx->rsrc_quiesce_wq); spin_lock_init(&ctx->completion_lock); spin_lock_init(&ctx->timeout_lock); INIT_WQ_LIST(&ctx->iopoll_list); @@ -341,7 +340,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->defer_list); INIT_LIST_HEAD(&ctx->timeout_list); INIT_LIST_HEAD(&ctx->ltimeout_list); - INIT_LIST_HEAD(&ctx->rsrc_ref_list); init_llist_head(&ctx->work_llist); INIT_LIST_HEAD(&ctx->tctx_list); ctx->submit_state.free_list.next = NULL; @@ -1415,7 +1413,7 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, io_clean_op(req); } io_put_file(req); - io_put_rsrc_node(ctx, req->rsrc_node); + io_req_put_rsrc_nodes(req); io_put_task(req->task); node = req->comp_list.next; @@ -1878,7 +1876,7 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; - struct io_fixed_file *slot; + struct io_rsrc_node *node; struct file *file = NULL; io_ring_submit_lock(ctx, issue_flags); @@ -1886,11 +1884,12 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, if (unlikely((unsigned int)fd >= ctx->nr_user_files)) goto out; fd = array_index_nospec(fd, ctx->nr_user_files); - slot = io_fixed_file_slot(&ctx->file_table, fd); - if (!req->rsrc_node) - __io_req_set_rsrc_node(req, ctx); - req->flags |= io_slot_flags(slot); - file = io_slot_file(slot); + node = ctx->file_table.nodes[fd]; + if (node) { + io_req_assign_rsrc_node(req, node); + req->flags |= io_slot_flags(node); + file = io_slot_file(node); + } out: io_ring_submit_unlock(ctx, issue_flags); return file; @@ -2036,7 +2035,8 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->flags = (__force io_req_flags_t) sqe_flags; req->cqe.user_data = READ_ONCE(sqe->user_data); req->file = NULL; - req->rsrc_node = NULL; + req->rsrc_nodes[IORING_RSRC_FILE] = NULL; + req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL; req->task = current; req->cancel_seq_set = false; @@ -2718,15 +2718,10 @@ static void io_req_caches_free(struct io_ring_ctx *ctx) static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) { io_sq_thread_finish(ctx); - /* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */ - if (WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list))) - return; mutex_lock(&ctx->uring_lock); - if (ctx->buf_data) - __io_sqe_buffers_unregister(ctx); - if (ctx->file_data) - __io_sqe_files_unregister(ctx); + io_sqe_buffers_unregister(ctx); + io_sqe_files_unregister(ctx); io_cqring_overflow_kill(ctx); io_eventfd_unregister(ctx); io_alloc_cache_free(&ctx->apoll_cache, kfree); @@ -2743,11 +2738,6 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) if (ctx->submitter_task) put_task_struct(ctx->submitter_task); - /* there are no registered resources left, nobody uses it */ - if (ctx->rsrc_node) - io_rsrc_node_destroy(ctx, ctx->rsrc_node); - - WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list)); WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list)); io_alloc_cache_free(&ctx->rsrc_node_cache, kfree); @@ -3729,10 +3719,6 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, if (ret) goto err; - ret = io_rsrc_init(ctx); - if (ret) - goto err; - p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP | IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS | IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL | diff --git a/io_uring/net.c b/io_uring/net.c index 2040195e33ab..ce1156551d10 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -1342,15 +1342,15 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags) if (sr->flags & IORING_RECVSEND_FIXED_BUF) { struct io_ring_ctx *ctx = req->ctx; - struct io_mapped_ubuf *imu; + struct io_rsrc_node *node; int idx; ret = -EFAULT; io_ring_submit_lock(ctx, issue_flags); if (sr->buf_index < ctx->nr_user_bufs) { idx = array_index_nospec(sr->buf_index, ctx->nr_user_bufs); - imu = READ_ONCE(ctx->user_bufs[idx]); - io_req_set_rsrc_node(sr->notif, ctx); + node = ctx->user_bufs[idx]; + io_req_assign_rsrc_node(sr->notif, node); ret = 0; } io_ring_submit_unlock(ctx, issue_flags); @@ -1358,8 +1358,9 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags) if (unlikely(ret)) return ret; - ret = io_import_fixed(ITER_SOURCE, &kmsg->msg.msg_iter, imu, - (u64)(uintptr_t)sr->buf, sr->len); + ret = io_import_fixed(ITER_SOURCE, &kmsg->msg.msg_iter, + node->buf, (u64)(uintptr_t)sr->buf, + sr->len); if (unlikely(ret)) return ret; kmsg->msg.sg_from_iter = io_sg_from_iter; diff --git a/io_uring/nop.c b/io_uring/nop.c index 2c7a22ba4053..de91600a3bc6 100644 --- a/io_uring/nop.c +++ b/io_uring/nop.c @@ -61,15 +61,15 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags) } if (nop->flags & IORING_NOP_FIXED_BUFFER) { struct io_ring_ctx *ctx = req->ctx; - struct io_mapped_ubuf *imu; + struct io_rsrc_node *node; int idx; ret = -EFAULT; io_ring_submit_lock(ctx, issue_flags); if (nop->buffer < ctx->nr_user_bufs) { idx = array_index_nospec(nop->buffer, ctx->nr_user_bufs); - imu = READ_ONCE(ctx->user_bufs[idx]); - io_req_set_rsrc_node(req, ctx); + node = READ_ONCE(ctx->user_bufs[idx]); + io_req_assign_rsrc_node(req, node); ret = 0; } io_ring_submit_unlock(ctx, issue_flags); diff --git a/io_uring/notif.c b/io_uring/notif.c index 28859ae3ee6e..4f02e969cf08 100644 --- a/io_uring/notif.c +++ b/io_uring/notif.c @@ -117,7 +117,8 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) notif->file = NULL; notif->task = current; io_get_task_refs(1); - notif->rsrc_node = NULL; + notif->rsrc_nodes[IORING_RSRC_FILE] = NULL; + notif->rsrc_nodes[IORING_RSRC_BUFFER] = NULL; nd = io_notif_to_data(notif); nd->zc_report = false; diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index ca2ec8a018be..88d698efd75b 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -26,10 +26,8 @@ struct io_rsrc_update { u32 offset; }; -static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc); -static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, - struct io_mapped_ubuf **pimu, - struct page **last_hpage); +static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, + struct iovec *iov, struct page **last_hpage); /* only define max */ #define IORING_MAX_FIXED_FILES (1U << 20) @@ -110,13 +108,13 @@ static int io_buffer_validate(struct iovec *iov) return 0; } -static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot) +static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_rsrc_node *node) { - struct io_mapped_ubuf *imu = *slot; unsigned int i; - *slot = NULL; - if (imu != &dummy_ubuf) { + if (node->buf != &dummy_ubuf) { + struct io_mapped_ubuf *imu = node->buf; + if (!refcount_dec_and_test(&imu->refs)) return; for (i = 0; i < imu->nr_bvecs; i++) @@ -127,205 +125,55 @@ static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slo } } -static void io_rsrc_put_work(struct io_rsrc_node *node) +struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type) { - struct io_rsrc_put *prsrc = &node->item; + struct io_rsrc_node *node; - if (prsrc->tag) - io_post_aux_cqe(node->ctx, prsrc->tag, 0, 0); - - switch (node->type) { - case IORING_RSRC_FILE: - fput(prsrc->file); - break; - case IORING_RSRC_BUFFER: - io_rsrc_buf_put(node->ctx, prsrc); - break; - default: - WARN_ON_ONCE(1); - break; - } -} - -void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *node) -{ - if (!io_alloc_cache_put(&ctx->rsrc_node_cache, node)) - kfree(node); -} - -void io_rsrc_node_ref_zero(struct io_rsrc_node *node) - __must_hold(&node->ctx->uring_lock) -{ - struct io_ring_ctx *ctx = node->ctx; - - while (!list_empty(&ctx->rsrc_ref_list)) { - node = list_first_entry(&ctx->rsrc_ref_list, - struct io_rsrc_node, node); - /* recycle ref nodes in order */ - if (node->refs) - break; - list_del(&node->node); - - if (likely(!node->empty)) - io_rsrc_put_work(node); - io_rsrc_node_destroy(ctx, node); - } - if (list_empty(&ctx->rsrc_ref_list) && unlikely(ctx->rsrc_quiesce)) - wake_up_all(&ctx->rsrc_quiesce_wq); -} - -struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx) -{ - struct io_rsrc_node *ref_node; - - ref_node = io_alloc_cache_get(&ctx->rsrc_node_cache); - if (!ref_node) { - ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL); - if (!ref_node) + node = io_alloc_cache_get(&ctx->rsrc_node_cache); + if (!node) { + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) return NULL; } - ref_node->ctx = ctx; - ref_node->empty = 0; - ref_node->refs = 1; - return ref_node; -} - -__cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, - struct io_ring_ctx *ctx) -{ - struct io_rsrc_node *backup; - DEFINE_WAIT(we); - int ret; - - /* As We may drop ->uring_lock, other task may have started quiesce */ - if (data->quiesce) - return -ENXIO; - - backup = io_rsrc_node_alloc(ctx); - if (!backup) - return -ENOMEM; - ctx->rsrc_node->empty = true; - ctx->rsrc_node->type = -1; - list_add_tail(&ctx->rsrc_node->node, &ctx->rsrc_ref_list); - io_put_rsrc_node(ctx, ctx->rsrc_node); - ctx->rsrc_node = backup; - - if (list_empty(&ctx->rsrc_ref_list)) - return 0; - - if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) { - atomic_set(&ctx->cq_wait_nr, 1); - smp_mb(); - } - - ctx->rsrc_quiesce++; - data->quiesce = true; - do { - prepare_to_wait(&ctx->rsrc_quiesce_wq, &we, TASK_INTERRUPTIBLE); - mutex_unlock(&ctx->uring_lock); - - ret = io_run_task_work_sig(ctx); - if (ret < 0) { - finish_wait(&ctx->rsrc_quiesce_wq, &we); - mutex_lock(&ctx->uring_lock); - if (list_empty(&ctx->rsrc_ref_list)) - ret = 0; - break; - } - - schedule(); - mutex_lock(&ctx->uring_lock); - ret = 0; - } while (!list_empty(&ctx->rsrc_ref_list)); - - finish_wait(&ctx->rsrc_quiesce_wq, &we); - data->quiesce = false; - ctx->rsrc_quiesce--; - - if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) { - atomic_set(&ctx->cq_wait_nr, 0); - smp_mb(); - } - return ret; -} - -static void io_free_page_table(void **table, size_t size) -{ - unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE); - - for (i = 0; i < nr_tables; i++) - kfree(table[i]); - kfree(table); + node->ctx = ctx; + node->refs = 1; + node->type = type; + return node; } static void io_rsrc_data_free(struct io_rsrc_data *data) { - size_t size = data->nr * sizeof(data->tags[0][0]); + int i; - if (data->tags) - io_free_page_table((void **)data->tags, size); + for (i = 0; i < data->nr; i++) { + struct io_rsrc_node *node = data->nodes[i]; + + io_put_rsrc_node(node); + } + kvfree(data->nodes); kfree(data); } -static __cold void **io_alloc_page_table(size_t size) -{ - unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE); - size_t init_size = size; - void **table; - - table = kcalloc(nr_tables, sizeof(*table), GFP_KERNEL_ACCOUNT); - if (!table) - return NULL; - - for (i = 0; i < nr_tables; i++) { - unsigned int this_size = min_t(size_t, size, PAGE_SIZE); - - table[i] = kzalloc(this_size, GFP_KERNEL_ACCOUNT); - if (!table[i]) { - io_free_page_table(table, init_size); - return NULL; - } - size -= this_size; - } - return table; -} - -__cold static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, int type, - u64 __user *utags, - unsigned nr, struct io_rsrc_data **pdata) +__cold static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, unsigned nr, + struct io_rsrc_data **pdata) { struct io_rsrc_data *data; - int ret = 0; - unsigned i; data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; - data->tags = (u64 **)io_alloc_page_table(nr * sizeof(data->tags[0][0])); - if (!data->tags) { + + data->nodes = kvmalloc_array(nr, sizeof(struct io_rsrc_node *), + GFP_KERNEL | __GFP_ZERO); + if (!data->nodes) { kfree(data); return -ENOMEM; } data->nr = nr; - data->ctx = ctx; - data->rsrc_type = type; - if (utags) { - ret = -EFAULT; - for (i = 0; i < nr; i++) { - u64 *tag_slot = io_get_tag_slot(data, i); - - if (copy_from_user(tag_slot, &utags[i], - sizeof(*tag_slot))) - goto fail; - } - } *pdata = data; return 0; -fail: - io_rsrc_data_free(data); - return ret; } static int __io_sqe_files_update(struct io_ring_ctx *ctx, @@ -334,8 +182,6 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, { u64 __user *tags = u64_to_user_ptr(up->tags); __s32 __user *fds = u64_to_user_ptr(up->data); - struct io_rsrc_data *data = ctx->file_data; - struct io_fixed_file *file_slot; int fd, i, err = 0; unsigned int done; @@ -360,18 +206,14 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, continue; i = array_index_nospec(up->offset + done, ctx->nr_user_files); - file_slot = io_fixed_file_slot(&ctx->file_table, i); - - if (file_slot->file_ptr) { - err = io_queue_rsrc_removal(data, i, - io_slot_file(file_slot)); - if (err) - break; - file_slot->file_ptr = 0; + if (ctx->file_table.nodes[i]) { + io_put_rsrc_node(ctx->file_table.nodes[i]); + ctx->file_table.nodes[i] = NULL; io_file_bitmap_clear(&ctx->file_table, i); } if (fd != -1) { struct file *file = fget(fd); + struct io_rsrc_node *node; if (!file) { err = -EBADF; @@ -385,8 +227,15 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, err = -EBADF; break; } - *io_get_tag_slot(data, i) = tag; - io_fixed_file_set(file_slot, file); + node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE); + if (!node) { + err = -ENOMEM; + fput(file); + break; + } + ctx->file_table.nodes[i] = node; + node->tag = tag; + io_fixed_file_set(node, file); io_file_bitmap_set(&ctx->file_table, i); } } @@ -411,7 +260,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, return -EINVAL; for (done = 0; done < nr_args; done++) { - struct io_mapped_ubuf *imu; + struct io_rsrc_node *node; u64 tag = 0; uvec = u64_to_user_ptr(user_data); @@ -431,23 +280,16 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, err = -EINVAL; break; } - err = io_sqe_buffer_register(ctx, iov, &imu, &last_hpage); - if (err) - break; - i = array_index_nospec(up->offset + done, ctx->nr_user_bufs); - if (ctx->user_bufs[i] != &dummy_ubuf) { - err = io_queue_rsrc_removal(ctx->buf_data, i, - ctx->user_bufs[i]); - if (unlikely(err)) { - io_buffer_unmap(ctx, &imu); - break; - } - ctx->user_bufs[i] = (struct io_mapped_ubuf *)&dummy_ubuf; + node = io_sqe_buffer_register(ctx, iov, &last_hpage); + if (IS_ERR(node)) { + err = PTR_ERR(node); + break; } + io_put_rsrc_node(ctx->user_bufs[i]); - ctx->user_bufs[i] = imu; - *io_get_tag_slot(ctx->buf_data, i) = tag; + ctx->user_bufs[i] = node; + node->tag = tag; if (ctx->compat) user_data += sizeof(struct compat_iovec); else @@ -622,38 +464,47 @@ int io_files_update(struct io_kiocb *req, unsigned int issue_flags) return IOU_OK; } -int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, void *rsrc) +void io_free_rsrc_node(struct io_rsrc_node *node) { - struct io_ring_ctx *ctx = data->ctx; - struct io_rsrc_node *node = ctx->rsrc_node; - u64 *tag_slot = io_get_tag_slot(data, idx); + struct io_ring_ctx *ctx = node->ctx; - ctx->rsrc_node = io_rsrc_node_alloc(ctx); - if (unlikely(!ctx->rsrc_node)) { - ctx->rsrc_node = node; - return -ENOMEM; + lockdep_assert_held(&ctx->uring_lock); + + if (node->tag) + io_post_aux_cqe(node->ctx, node->tag, 0, 0); + + switch (node->type) { + case IORING_RSRC_FILE: + if (io_slot_file(node)) + fput(io_slot_file(node)); + break; + case IORING_RSRC_BUFFER: + if (node->buf) + io_buffer_unmap(node->ctx, node); + break; + default: + WARN_ON_ONCE(1); + break; } - node->item.rsrc = rsrc; - node->type = data->rsrc_type; - node->item.tag = *tag_slot; - *tag_slot = 0; - list_add_tail(&node->node, &ctx->rsrc_ref_list); - io_put_rsrc_node(ctx, node); - return 0; + if (!io_alloc_cache_put(&ctx->rsrc_node_cache, node)) + kfree(node); } -void __io_sqe_files_unregister(struct io_ring_ctx *ctx) +static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) { int i; - for (i = 0; i < ctx->nr_user_files; i++) { - struct file *file = io_file_from_index(&ctx->file_table, i); + lockdep_assert_held(&ctx->uring_lock); - if (!file) - continue; - io_file_bitmap_clear(&ctx->file_table, i); - fput(file); + for (i = 0; i < ctx->nr_user_files; i++) { + struct io_rsrc_node *node = ctx->file_table.nodes[i]; + + if (node) { + io_put_rsrc_node(node); + io_file_bitmap_clear(&ctx->file_table, i); + ctx->file_table.nodes[i] = NULL; + } } io_free_file_tables(&ctx->file_table); @@ -665,22 +516,11 @@ void __io_sqe_files_unregister(struct io_ring_ctx *ctx) int io_sqe_files_unregister(struct io_ring_ctx *ctx) { - unsigned nr = ctx->nr_user_files; - int ret; - if (!ctx->file_data) return -ENXIO; - /* - * Quiesce may unlock ->uring_lock, and while it's not held - * prevent new requests using the table. - */ - ctx->nr_user_files = 0; - ret = io_rsrc_ref_quiesce(ctx->file_data, ctx); - ctx->nr_user_files = nr; - if (!ret) - __io_sqe_files_unregister(ctx); - return ret; + __io_sqe_files_unregister(ctx); + return 0; } int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, @@ -699,8 +539,7 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, return -EMFILE; if (nr_args > rlimit(RLIMIT_NOFILE)) return -EMFILE; - ret = io_rsrc_data_alloc(ctx, IORING_RSRC_FILE, tags, nr_args, - &ctx->file_data); + ret = io_rsrc_data_alloc(ctx, nr_args, &ctx->file_data); if (ret) return ret; @@ -711,16 +550,18 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, } for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { - struct io_fixed_file *file_slot; + struct io_rsrc_node *node; + u64 tag = 0; - if (fds && copy_from_user(&fd, &fds[i], sizeof(fd))) { - ret = -EFAULT; + ret = -EFAULT; + if (tags && copy_from_user(&tag, &tags[i], sizeof(tag))) + goto fail; + if (fds && copy_from_user(&fd, &fds[i], sizeof(fd))) goto fail; - } /* allow sparse sets */ if (!fds || fd == -1) { ret = -EINVAL; - if (unlikely(*io_get_tag_slot(ctx->file_data, i))) + if (tag) goto fail; continue; } @@ -737,8 +578,16 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, fput(file); goto fail; } - file_slot = io_fixed_file_slot(&ctx->file_table, i); - io_fixed_file_set(file_slot, file); + ret = -ENOMEM; + node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE); + if (!node) { + fput(file); + goto fail; + } + if (tag) + node->tag = tag; + ctx->file_table.nodes[i] = node; + io_fixed_file_set(node, file); io_file_bitmap_set(&ctx->file_table, i); } @@ -750,43 +599,30 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, return ret; } -static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc) -{ - io_buffer_unmap(ctx, &prsrc->buf); - prsrc->buf = NULL; -} - -void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx) +static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx) { unsigned int i; - for (i = 0; i < ctx->nr_user_bufs; i++) - io_buffer_unmap(ctx, &ctx->user_bufs[i]); - kfree(ctx->user_bufs); - io_rsrc_data_free(ctx->buf_data); + lockdep_assert_held(&ctx->uring_lock); + + for (i = 0; i < ctx->nr_user_bufs; i++) { + io_put_rsrc_node(ctx->user_bufs[i]); + ctx->user_bufs[i] = NULL; + } + kvfree(ctx->user_bufs); ctx->user_bufs = NULL; + io_rsrc_data_free(ctx->buf_data); ctx->buf_data = NULL; ctx->nr_user_bufs = 0; } int io_sqe_buffers_unregister(struct io_ring_ctx *ctx) { - unsigned nr = ctx->nr_user_bufs; - int ret; - if (!ctx->buf_data) return -ENXIO; - /* - * Quiesce may unlock ->uring_lock, and while it's not held - * prevent new requests using the table. - */ - ctx->nr_user_bufs = 0; - ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx); - ctx->nr_user_bufs = nr; - if (!ret) - __io_sqe_buffers_unregister(ctx); - return ret; + __io_sqe_buffers_unregister(ctx); + return 0; } /* @@ -813,7 +649,8 @@ static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages, /* check previously registered pages */ for (i = 0; i < ctx->nr_user_bufs; i++) { - struct io_mapped_ubuf *imu = ctx->user_bufs[i]; + struct io_rsrc_node *node = ctx->user_bufs[i]; + struct io_mapped_ubuf *imu = node->buf; for (j = 0; j < imu->nr_bvecs; j++) { if (!PageCompound(imu->bvec[j].bv_page)) @@ -950,21 +787,28 @@ static bool io_try_coalesce_buffer(struct page ***pages, int *nr_pages, return io_do_coalesce_buffer(pages, nr_pages, data, nr_folios); } -static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, - struct io_mapped_ubuf **pimu, - struct page **last_hpage) +static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, + struct iovec *iov, + struct page **last_hpage) { struct io_mapped_ubuf *imu = NULL; struct page **pages = NULL; + struct io_rsrc_node *node; unsigned long off; size_t size; int ret, nr_pages, i; struct io_imu_folio_data data; bool coalesced; - *pimu = (struct io_mapped_ubuf *)&dummy_ubuf; - if (!iov->iov_base) - return 0; + node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); + if (!node) + return ERR_PTR(-ENOMEM); + node->buf = NULL; + + if (!iov->iov_base) { + node->buf = (struct io_mapped_ubuf *) &dummy_ubuf; + return node; + } ret = -ENOMEM; pages = io_pin_pages((unsigned long) iov->iov_base, iov->iov_len, @@ -998,7 +842,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, imu->folio_shift = data.folio_shift; refcount_set(&imu->refs, 1); off = (unsigned long) iov->iov_base & ((1UL << imu->folio_shift) - 1); - *pimu = imu; + node->buf = imu; ret = 0; for (i = 0; i < nr_pages; i++) { @@ -1010,10 +854,14 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, size -= vec_len; } done: - if (ret) + if (ret) { kvfree(imu); + if (node) + io_put_rsrc_node(node); + node = ERR_PTR(ret); + } kvfree(pages); - return ret; + return node; } static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args) @@ -1037,7 +885,7 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, return -EBUSY; if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS) return -EINVAL; - ret = io_rsrc_data_alloc(ctx, IORING_RSRC_BUFFER, tags, nr_args, &data); + ret = io_rsrc_data_alloc(ctx, nr_args, &data); if (ret) return ret; ret = io_buffers_map_alloc(ctx, nr_args); @@ -1050,6 +898,9 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, memset(iov, 0, sizeof(*iov)); for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) { + struct io_rsrc_node *node; + u64 tag = 0; + if (arg) { uvec = (struct iovec __user *) arg; iov = iovec_from_user(uvec, 1, 1, &fast_iov, ctx->compat); @@ -1066,15 +917,24 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, arg += sizeof(struct iovec); } - if (!iov->iov_base && *io_get_tag_slot(data, i)) { - ret = -EINVAL; - break; + if (tags) { + if (copy_from_user(&tag, &tags[i], sizeof(tag))) { + ret = -EFAULT; + break; + } + if (tag && !iov->iov_base) { + ret = -EINVAL; + break; + } } - ret = io_sqe_buffer_register(ctx, iov, &ctx->user_bufs[i], - &last_hpage); - if (ret) + node = io_sqe_buffer_register(ctx, iov, &last_hpage); + if (IS_ERR(node)) { + ret = PTR_ERR(node); break; + } + node->tag = tag; + ctx->user_bufs[i] = node; } WARN_ON_ONCE(ctx->buf_data); @@ -1148,7 +1008,7 @@ int io_import_fixed(int ddir, struct iov_iter *iter, static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx) { - struct io_mapped_ubuf **user_bufs; + struct io_rsrc_node **user_bufs; struct io_rsrc_data *data; int i, ret, nbufs; @@ -1163,21 +1023,31 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx nbufs = src_ctx->nr_user_bufs; if (!nbufs) goto out_unlock; - ret = io_rsrc_data_alloc(ctx, IORING_RSRC_BUFFER, NULL, nbufs, &data); + ret = io_rsrc_data_alloc(ctx, nbufs, &data); if (ret) goto out_unlock; ret = -ENOMEM; - user_bufs = kcalloc(nbufs, sizeof(*ctx->user_bufs), GFP_KERNEL); + user_bufs = kvmalloc_array(nbufs, sizeof(struct io_rsrc_node *), + GFP_KERNEL | __GFP_ZERO); if (!user_bufs) goto out_free_data; for (i = 0; i < nbufs; i++) { - struct io_mapped_ubuf *src = src_ctx->user_bufs[i]; + struct io_mapped_ubuf *imu = src_ctx->user_bufs[i]->buf; + struct io_rsrc_node *dst_node; - if (src != &dummy_ubuf) - refcount_inc(&src->refs); - user_bufs[i] = src; + dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); + if (!dst_node) + goto out_put_free; + + if (imu == &dummy_ubuf) { + dst_node->buf = (struct io_mapped_ubuf *) &dummy_ubuf; + } else { + refcount_inc(&imu->refs); + dst_node->buf = imu; + } + user_bufs[i] = dst_node; } /* Have a ref on the bufs now, drop src lock and re-grab our own lock */ @@ -1190,12 +1060,17 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx return 0; } + mutex_unlock(&ctx->uring_lock); + mutex_lock(&src_ctx->uring_lock); /* someone raced setting up buffers, dump ours */ - for (i = 0; i < nbufs; i++) - io_buffer_unmap(ctx, &user_bufs[i]); - io_rsrc_data_free(data); - kfree(user_bufs); - return -EBUSY; + ret = -EBUSY; + i = nbufs; +out_put_free: + while (i--) { + io_buffer_unmap(src_ctx, user_bufs[i]); + kfree(user_bufs[i]); + } + kvfree(user_bufs); out_free_data: io_rsrc_data_free(data); out_unlock: diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 1589c9740083..20a316854238 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -13,36 +13,21 @@ enum { IORING_RSRC_BUFFER = 1, }; -struct io_rsrc_put { - u64 tag; - union { - void *rsrc; - struct file *file; - struct io_mapped_ubuf *buf; - }; -}; - struct io_rsrc_data { - struct io_ring_ctx *ctx; - - u64 **tags; unsigned int nr; - u16 rsrc_type; - bool quiesce; + struct io_rsrc_node **nodes; }; struct io_rsrc_node { struct io_ring_ctx *ctx; int refs; - bool empty; u16 type; - struct list_head node; - struct io_rsrc_put item; -}; -struct io_fixed_file { - /* file * with additional FFS_* flags */ - unsigned long file_ptr; + u64 tag; + union { + unsigned long file_ptr; + struct io_mapped_ubuf *buf; + }; }; struct io_mapped_ubuf { @@ -63,21 +48,17 @@ struct io_imu_folio_data { unsigned int folio_shift; }; -void io_rsrc_node_ref_zero(struct io_rsrc_node *node); -void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *ref_node); -struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx); -int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, void *rsrc); +struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type); +void io_free_rsrc_node(struct io_rsrc_node *node); int io_import_fixed(int ddir, struct iov_iter *iter, struct io_mapped_ubuf *imu, u64 buf_addr, size_t len); int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg); -void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx); int io_sqe_buffers_unregister(struct io_ring_ctx *ctx); int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, unsigned int nr_args, u64 __user *tags); -void __io_sqe_files_unregister(struct io_ring_ctx *ctx); int io_sqe_files_unregister(struct io_ring_ctx *ctx); int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args, u64 __user *tags); @@ -89,41 +70,23 @@ int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg, int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, unsigned int size, unsigned int type); -static inline void io_put_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node) +static inline void io_put_rsrc_node(struct io_rsrc_node *node) { - lockdep_assert_held(&ctx->uring_lock); - if (node && !--node->refs) - io_rsrc_node_ref_zero(node); + io_free_rsrc_node(node); } -static inline void __io_req_set_rsrc_node(struct io_kiocb *req, - struct io_ring_ctx *ctx) +static inline void io_req_put_rsrc_nodes(struct io_kiocb *req) { - lockdep_assert_held(&ctx->uring_lock); - req->rsrc_node = ctx->rsrc_node; - ctx->rsrc_node->refs++; + io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_FILE]); + io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_BUFFER]); } -static inline void io_req_set_rsrc_node(struct io_kiocb *req, - struct io_ring_ctx *ctx) +static inline void io_req_assign_rsrc_node(struct io_kiocb *req, + struct io_rsrc_node *node) { - if (!req->rsrc_node) - __io_req_set_rsrc_node(req, ctx); -} - -static inline u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx) -{ - unsigned int off = idx & IO_RSRC_TAG_TABLE_MASK; - unsigned int table_idx = idx >> IO_RSRC_TAG_TABLE_SHIFT; - - return &data->tags[table_idx][off]; -} - -static inline int io_rsrc_init(struct io_ring_ctx *ctx) -{ - ctx->rsrc_node = io_rsrc_node_alloc(ctx); - return ctx->rsrc_node ? 0 : -ENOMEM; + node->refs++; + req->rsrc_nodes[node->type] = node; } int io_files_update(struct io_kiocb *req, unsigned int issue_flags); diff --git a/io_uring/rw.c b/io_uring/rw.c index 8080ffd6d571..65491f4f2c7e 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -330,7 +330,7 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct io_ring_ctx *ctx = req->ctx; - struct io_mapped_ubuf *imu; + struct io_rsrc_node *node; struct io_async_rw *io; u16 index; int ret; @@ -342,11 +342,11 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe if (unlikely(req->buf_index >= ctx->nr_user_bufs)) return -EFAULT; index = array_index_nospec(req->buf_index, ctx->nr_user_bufs); - imu = ctx->user_bufs[index]; - io_req_set_rsrc_node(req, ctx); + node = ctx->user_bufs[index]; + io_req_assign_rsrc_node(req, node); io = req->async_data; - ret = io_import_fixed(ddir, &io->iter, imu, rw->addr, rw->len); + ret = io_import_fixed(ddir, &io->iter, node->buf, rw->addr, rw->len); iov_iter_save_state(&io->iter, &io->iter_state); return ret; } diff --git a/io_uring/splice.c b/io_uring/splice.c index e62bc6497a94..a0b4e0435b8b 100644 --- a/io_uring/splice.c +++ b/io_uring/splice.c @@ -51,7 +51,7 @@ void io_splice_cleanup(struct io_kiocb *req) { struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice); - io_put_rsrc_node(req->ctx, sp->rsrc_node); + io_put_rsrc_node(sp->rsrc_node); } static struct file *io_splice_get_file(struct io_kiocb *req, @@ -59,7 +59,7 @@ static struct file *io_splice_get_file(struct io_kiocb *req, { struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice); struct io_ring_ctx *ctx = req->ctx; - struct io_fixed_file *slot; + struct io_rsrc_node *node; struct file *file = NULL; if (!(sp->flags & SPLICE_F_FD_IN_FIXED)) @@ -69,11 +69,13 @@ static struct file *io_splice_get_file(struct io_kiocb *req, if (unlikely(sp->splice_fd_in >= ctx->nr_user_files)) goto out; sp->splice_fd_in = array_index_nospec(sp->splice_fd_in, ctx->nr_user_files); - slot = &ctx->file_table.files[sp->splice_fd_in]; - if (!req->rsrc_node) - __io_req_set_rsrc_node(req, ctx); - file = io_slot_file(slot); - req->flags |= REQ_F_NEED_CLEANUP; + node = ctx->file_table.nodes[sp->splice_fd_in]; + if (node) { + node->refs++; + sp->rsrc_node = node; + file = io_slot_file(node); + req->flags |= REQ_F_NEED_CLEANUP; + } out: io_ring_submit_unlock(ctx, issue_flags); return file; diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 6994f60d7ec7..0899c71008ae 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -220,7 +220,7 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) * being called. This prevents destruction of the mapped buffer * we'll need at actual import time. */ - io_req_set_rsrc_node(req, ctx); + io_req_assign_rsrc_node(req, ctx->user_bufs[req->buf_index]); } ioucmd->cmd_op = READ_ONCE(sqe->cmd_op); @@ -276,15 +276,11 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, struct iov_iter *iter, void *ioucmd) { struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); - struct io_ring_ctx *ctx = req->ctx; + struct io_rsrc_node *node = req->rsrc_nodes[IORING_RSRC_BUFFER]; /* Must have had rsrc_node assigned at prep time */ - if (req->rsrc_node) { - struct io_mapped_ubuf *imu; - - imu = READ_ONCE(ctx->user_bufs[req->buf_index]); - return io_import_fixed(rw, iter, imu, ubuf, len); - } + if (node) + return io_import_fixed(rw, iter, node->buf, ubuf, len); return -EFAULT; } From fbbb8e991d86bb7539de6161746b6c747f93f533 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 26 Oct 2024 06:43:44 -0600 Subject: [PATCH 42/79] io_uring/rsrc: get rid of io_rsrc_node allocation cache It's not going to be needed in the fast path going forward, so kill it off. Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 3 --- io_uring/io_uring.c | 6 +----- io_uring/rsrc.c | 18 ++++++------------ 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 42c5f2c992c4..696f2a05a98b 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -370,9 +370,6 @@ struct io_ring_ctx { struct io_rsrc_data *file_data; struct io_rsrc_data *buf_data; - /* protected by ->uring_lock */ - struct io_alloc_cache rsrc_node_cache; - u32 pers_next; struct xarray personalities; diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 0876aa74c739..094788cca47f 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -312,9 +312,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->sqd_list); INIT_LIST_HEAD(&ctx->cq_overflow_list); INIT_LIST_HEAD(&ctx->io_buffers_cache); - ret = io_alloc_cache_init(&ctx->rsrc_node_cache, IO_NODE_ALLOC_CACHE_MAX, - sizeof(struct io_rsrc_node)); - ret |= io_alloc_cache_init(&ctx->apoll_cache, IO_POLL_ALLOC_CACHE_MAX, + ret = io_alloc_cache_init(&ctx->apoll_cache, IO_POLL_ALLOC_CACHE_MAX, sizeof(struct async_poll)); ret |= io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX, sizeof(struct io_async_msghdr)); @@ -358,7 +356,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) free_ref: percpu_ref_exit(&ctx->refs); err: - io_alloc_cache_free(&ctx->rsrc_node_cache, kfree); io_alloc_cache_free(&ctx->apoll_cache, kfree); io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free); io_alloc_cache_free(&ctx->rw_cache, io_rw_cache_free); @@ -2740,7 +2737,6 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list)); - io_alloc_cache_free(&ctx->rsrc_node_cache, kfree); if (ctx->mm_account) { mmdrop(ctx->mm_account); ctx->mm_account = NULL; diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 88d698efd75b..8f8147dd714c 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -13,7 +13,6 @@ #include #include "io_uring.h" -#include "alloc_cache.h" #include "openclose.h" #include "rsrc.h" #include "memmap.h" @@ -129,16 +128,12 @@ struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type) { struct io_rsrc_node *node; - node = io_alloc_cache_get(&ctx->rsrc_node_cache); - if (!node) { - node = kzalloc(sizeof(*node), GFP_KERNEL); - if (!node) - return NULL; + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (node) { + node->ctx = ctx; + node->refs = 1; + node->type = type; } - - node->ctx = ctx; - node->refs = 1; - node->type = type; return node; } @@ -487,8 +482,7 @@ void io_free_rsrc_node(struct io_rsrc_node *node) break; } - if (!io_alloc_cache_put(&ctx->rsrc_node_cache, node)) - kfree(node); + kfree(node); } static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) From 0701db7439208951c8a7d8600668e5cfdd5f63d2 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 26 Oct 2024 10:41:51 -0600 Subject: [PATCH 43/79] io_uring/rsrc: add an empty io_rsrc_node for sparse buffer entries Rather than allocate an io_rsrc_node for an empty/sparse buffer entry, add a const entry that can be used for that. This just needs checking for writing the tag, and the put check needs to check for that sparse node rather than NULL for validity. This avoids allocating rsrc nodes for sparse buffer entries. Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 4 ++-- io_uring/notif.c | 4 ++-- io_uring/rsrc.c | 49 ++++++++++++++++++++++++++------------------- io_uring/rsrc.h | 11 +++++++--- io_uring/splice.c | 2 +- 5 files changed, 41 insertions(+), 29 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 094788cca47f..9282d5fa45d3 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2032,8 +2032,8 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->flags = (__force io_req_flags_t) sqe_flags; req->cqe.user_data = READ_ONCE(sqe->user_data); req->file = NULL; - req->rsrc_nodes[IORING_RSRC_FILE] = NULL; - req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL; + req->rsrc_nodes[IORING_RSRC_FILE] = rsrc_empty_node; + req->rsrc_nodes[IORING_RSRC_BUFFER] = rsrc_empty_node; req->task = current; req->cancel_seq_set = false; diff --git a/io_uring/notif.c b/io_uring/notif.c index 4f02e969cf08..44bf21c0f810 100644 --- a/io_uring/notif.c +++ b/io_uring/notif.c @@ -117,8 +117,8 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) notif->file = NULL; notif->task = current; io_get_task_refs(1); - notif->rsrc_nodes[IORING_RSRC_FILE] = NULL; - notif->rsrc_nodes[IORING_RSRC_BUFFER] = NULL; + notif->rsrc_nodes[IORING_RSRC_FILE] = rsrc_empty_node; + notif->rsrc_nodes[IORING_RSRC_BUFFER] = rsrc_empty_node; nd = io_notif_to_data(notif); nd->zc_report = false; diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 8f8147dd714c..69a9cd82460d 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -38,6 +38,11 @@ static const struct io_mapped_ubuf dummy_ubuf = { .len = UINT_MAX, }; +const struct io_rsrc_node empty_node = { + .type = IORING_RSRC_BUFFER, + .buf = (struct io_mapped_ubuf *) &dummy_ubuf, +}; + int __io_account_mem(struct user_struct *user, unsigned long nr_pages) { unsigned long page_limit, cur_pages, new_pages; @@ -144,7 +149,8 @@ static void io_rsrc_data_free(struct io_rsrc_data *data) for (i = 0; i < data->nr; i++) { struct io_rsrc_node *node = data->nodes[i]; - io_put_rsrc_node(node); + if (node) + io_put_rsrc_node(node); } kvfree(data->nodes); kfree(data); @@ -229,7 +235,8 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, break; } ctx->file_table.nodes[i] = node; - node->tag = tag; + if (tag) + node->tag = tag; io_fixed_file_set(node, file); io_file_bitmap_set(&ctx->file_table, i); } @@ -281,10 +288,12 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, err = PTR_ERR(node); break; } - io_put_rsrc_node(ctx->user_bufs[i]); + if (ctx->user_bufs[i]) + io_put_rsrc_node(ctx->user_bufs[i]); ctx->user_bufs[i] = node; - node->tag = tag; + if (tag) + node->tag = tag; if (ctx->compat) user_data += sizeof(struct compat_iovec); else @@ -600,8 +609,10 @@ static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx) lockdep_assert_held(&ctx->uring_lock); for (i = 0; i < ctx->nr_user_bufs; i++) { - io_put_rsrc_node(ctx->user_bufs[i]); - ctx->user_bufs[i] = NULL; + if (ctx->user_bufs[i]) { + io_put_rsrc_node(ctx->user_bufs[i]); + ctx->user_bufs[i] = NULL; + } } kvfree(ctx->user_bufs); ctx->user_bufs = NULL; @@ -799,11 +810,6 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, return ERR_PTR(-ENOMEM); node->buf = NULL; - if (!iov->iov_base) { - node->buf = (struct io_mapped_ubuf *) &dummy_ubuf; - return node; - } - ret = -ENOMEM; pages = io_pin_pages((unsigned long) iov->iov_base, iov->iov_len, &nr_pages); @@ -927,7 +933,8 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, ret = PTR_ERR(node); break; } - node->tag = tag; + if (tag) + node->tag = tag; ctx->user_bufs[i] = node; } @@ -1028,18 +1035,18 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx goto out_free_data; for (i = 0; i < nbufs; i++) { - struct io_mapped_ubuf *imu = src_ctx->user_bufs[i]->buf; + struct io_rsrc_node *src_node = src_ctx->user_bufs[i]; struct io_rsrc_node *dst_node; - dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); - if (!dst_node) - goto out_put_free; - - if (imu == &dummy_ubuf) { - dst_node->buf = (struct io_mapped_ubuf *) &dummy_ubuf; + if (src_node == rsrc_empty_node) { + dst_node = rsrc_empty_node; } else { - refcount_inc(&imu->refs); - dst_node->buf = imu; + dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); + if (!dst_node) + goto out_put_free; + + refcount_inc(&src_node->buf->refs); + dst_node->buf = src_node->buf; } user_bufs[i] = dst_node; } diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 20a316854238..323c3e78b864 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -70,9 +70,12 @@ int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg, int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, unsigned int size, unsigned int type); +extern const struct io_rsrc_node empty_node; +#define rsrc_empty_node (struct io_rsrc_node *) &empty_node + static inline void io_put_rsrc_node(struct io_rsrc_node *node) { - if (node && !--node->refs) + if (node != rsrc_empty_node && !--node->refs) io_free_rsrc_node(node); } @@ -85,8 +88,10 @@ static inline void io_req_put_rsrc_nodes(struct io_kiocb *req) static inline void io_req_assign_rsrc_node(struct io_kiocb *req, struct io_rsrc_node *node) { - node->refs++; - req->rsrc_nodes[node->type] = node; + if (node != rsrc_empty_node) { + node->refs++; + req->rsrc_nodes[node->type] = node; + } } int io_files_update(struct io_kiocb *req, unsigned int issue_flags); diff --git a/io_uring/splice.c b/io_uring/splice.c index a0b4e0435b8b..f78afb575ae6 100644 --- a/io_uring/splice.c +++ b/io_uring/splice.c @@ -35,7 +35,7 @@ static int __io_splice_prep(struct io_kiocb *req, if (unlikely(sp->flags & ~valid_flags)) return -EINVAL; sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in); - sp->rsrc_node = NULL; + sp->rsrc_node = rsrc_empty_node; req->flags |= REQ_F_FORCE_ASYNC; return 0; } From f38f2847646f8be29a8fcb722e8b1dc8c8cb3924 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 26 Oct 2024 10:46:10 -0600 Subject: [PATCH 44/79] io_uring: only initialize io_kiocb rsrc_nodes when needed Add the empty node initializing to the preinit part of the io_kiocb allocation, and reset them if they have been used. Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 4 ++-- io_uring/rsrc.h | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 9282d5fa45d3..60c947114fa3 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -947,6 +947,8 @@ void io_req_defer_failed(struct io_kiocb *req, s32 res) static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx) { req->ctx = ctx; + req->rsrc_nodes[IORING_RSRC_FILE] = rsrc_empty_node; + req->rsrc_nodes[IORING_RSRC_BUFFER] = rsrc_empty_node; req->link = NULL; req->async_data = NULL; /* not necessary, but safer to zero */ @@ -2032,8 +2034,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->flags = (__force io_req_flags_t) sqe_flags; req->cqe.user_data = READ_ONCE(sqe->user_data); req->file = NULL; - req->rsrc_nodes[IORING_RSRC_FILE] = rsrc_empty_node; - req->rsrc_nodes[IORING_RSRC_BUFFER] = rsrc_empty_node; req->task = current; req->cancel_seq_set = false; diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 323c3e78b864..48f712488f6b 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -81,8 +81,14 @@ static inline void io_put_rsrc_node(struct io_rsrc_node *node) static inline void io_req_put_rsrc_nodes(struct io_kiocb *req) { - io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_FILE]); - io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_BUFFER]); + if (req->rsrc_nodes[IORING_RSRC_FILE] != rsrc_empty_node) { + io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_FILE]); + req->rsrc_nodes[IORING_RSRC_FILE] = rsrc_empty_node; + } + if (req->rsrc_nodes[IORING_RSRC_BUFFER] != rsrc_empty_node) { + io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_BUFFER]); + req->rsrc_nodes[IORING_RSRC_BUFFER] = rsrc_empty_node; + } } static inline void io_req_assign_rsrc_node(struct io_kiocb *req, From 3597f2786b687a7f26361ce00a805ea0af41b65f Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 26 Oct 2024 14:50:13 -0600 Subject: [PATCH 45/79] io_uring/rsrc: unify file and buffer resource tables For files, there's nr_user_files/file_table/file_data, and buffers have nr_user_bufs/user_bufs/buf_data. There's no reason why file_table and file_data can't be the same thing, and ditto for the buffer side. That gets rid of more io_ring_ctx state that's in two spots rather than just being in one spot, as it should be. Put all the registered file data in one locations, and ditto on the buffer front. This also avoids having both io_rsrc_data->nodes being an allocated array, and ->user_bufs[] or ->file_table.nodes. There's no reason to have this information duplicated. Keep it in one spot, io_rsrc_data, along with how many resources are available. Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 15 ++- io_uring/cancel.c | 4 +- io_uring/fdinfo.c | 10 +- io_uring/filetable.c | 44 +++---- io_uring/filetable.h | 2 +- io_uring/io_uring.c | 7 +- io_uring/msg_ring.c | 4 +- io_uring/net.c | 6 +- io_uring/nop.c | 6 +- io_uring/register.c | 3 +- io_uring/rsrc.c | 209 ++++++++++----------------------- io_uring/rsrc.h | 7 +- io_uring/rw.c | 6 +- io_uring/splice.c | 6 +- io_uring/uring_cmd.c | 6 +- 15 files changed, 123 insertions(+), 212 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 696f2a05a98b..77fd508d043a 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -55,8 +55,13 @@ struct io_wq_work { int cancel_seq; }; +struct io_rsrc_data { + unsigned int nr; + struct io_rsrc_node **nodes; +}; + struct io_file_table { - struct io_rsrc_node **nodes; + struct io_rsrc_data data; unsigned long *bitmap; unsigned int alloc_hint; }; @@ -276,9 +281,7 @@ struct io_ring_ctx { struct io_wq_work_list iopoll_list; struct io_file_table file_table; - struct io_rsrc_node **user_bufs; - unsigned nr_user_files; - unsigned nr_user_bufs; + struct io_rsrc_data buf_table; struct io_submit_state submit_state; @@ -366,10 +369,6 @@ struct io_ring_ctx { struct wait_queue_head poll_wq; struct io_restriction restrictions; - /* slow path rsrc auxilary data, used by update/register */ - struct io_rsrc_data *file_data; - struct io_rsrc_data *buf_data; - u32 pers_next; struct xarray personalities; diff --git a/io_uring/cancel.c b/io_uring/cancel.c index cc3475b22ae5..3a2996307025 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -240,9 +240,9 @@ static int __io_sync_cancel(struct io_uring_task *tctx, /* fixed must be grabbed every time since we drop the uring_lock */ if ((cd->flags & IORING_ASYNC_CANCEL_FD) && (cd->flags & IORING_ASYNC_CANCEL_FD_FIXED)) { - if (unlikely(fd >= ctx->nr_user_files)) + if (unlikely(fd >= ctx->file_table.data.nr)) return -EBADF; - fd = array_index_nospec(fd, ctx->nr_user_files); + fd = array_index_nospec(fd, ctx->file_table.data.nr); cd->file = io_file_from_index(&ctx->file_table, fd); if (!cd->file) return -EBADF; diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index 064a79475c5f..e3f5e9fe5562 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -165,8 +165,8 @@ __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *file) seq_printf(m, "SqThreadCpu:\t%d\n", sq_cpu); seq_printf(m, "SqTotalTime:\t%llu\n", sq_total_time); seq_printf(m, "SqWorkTime:\t%llu\n", sq_work_time); - seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files); - for (i = 0; has_lock && i < ctx->nr_user_files; i++) { + seq_printf(m, "UserFiles:\t%u\n", ctx->file_table.data.nr); + for (i = 0; has_lock && i < ctx->file_table.data.nr; i++) { struct file *f = io_file_from_index(&ctx->file_table, i); if (f) @@ -174,9 +174,9 @@ __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *file) else seq_printf(m, "%5u: \n", i); } - seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs); - for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) { - struct io_mapped_ubuf *buf = ctx->user_bufs[i]->buf; + seq_printf(m, "UserBufs:\t%u\n", ctx->buf_table.nr); + for (i = 0; has_lock && i < ctx->buf_table.nr; i++) { + struct io_mapped_ubuf *buf = ctx->buf_table.nodes[i]->buf; seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, buf->len); } diff --git a/io_uring/filetable.c b/io_uring/filetable.c index 1b12a9a1cc16..c1f9f9550446 100644 --- a/io_uring/filetable.c +++ b/io_uring/filetable.c @@ -38,25 +38,19 @@ static int io_file_bitmap_get(struct io_ring_ctx *ctx) bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files) { - table->nodes = kvmalloc_array(nr_files, sizeof(struct io_src_node *), - GFP_KERNEL_ACCOUNT | __GFP_ZERO); - if (unlikely(!table->nodes)) + if (io_rsrc_data_alloc(&table->data, nr_files)) return false; - table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT); - if (unlikely(!table->bitmap)) { - kvfree(table->nodes); - return false; - } - - return true; + if (table->bitmap) + return true; + io_rsrc_data_free(&table->data); + return false; } void io_free_file_tables(struct io_file_table *table) { - kvfree(table->nodes); + io_rsrc_data_free(&table->data); bitmap_free(table->bitmap); - table->nodes = NULL; table->bitmap = NULL; } @@ -68,22 +62,22 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file, if (io_is_uring_fops(file)) return -EBADF; - if (!ctx->file_data) + if (!ctx->file_table.data.nr) return -ENXIO; - if (slot_index >= ctx->nr_user_files) + if (slot_index >= ctx->file_table.data.nr) return -EINVAL; node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE); if (!node) return -ENOMEM; - slot_index = array_index_nospec(slot_index, ctx->nr_user_files); - if (ctx->file_table.nodes[slot_index]) - io_put_rsrc_node(ctx->file_table.nodes[slot_index]); + slot_index = array_index_nospec(slot_index, ctx->file_table.data.nr); + if (ctx->file_table.data.nodes[slot_index]) + io_put_rsrc_node(ctx->file_table.data.nodes[slot_index]); else io_file_bitmap_set(&ctx->file_table, slot_index); - ctx->file_table.nodes[slot_index] = node; + ctx->file_table.data.nodes[slot_index] = node; io_fixed_file_set(node, file); return 0; } @@ -129,16 +123,16 @@ int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags, int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset) { - if (unlikely(!ctx->file_data)) + if (unlikely(!ctx->file_table.data.nr)) return -ENXIO; - if (offset >= ctx->nr_user_files) + if (offset >= ctx->file_table.data.nr) return -EINVAL; - offset = array_index_nospec(offset, ctx->nr_user_files); - if (!ctx->file_table.nodes[offset]) + offset = array_index_nospec(offset, ctx->file_table.data.nr); + if (!ctx->file_table.data.nodes[offset]) return -EBADF; - io_put_rsrc_node(ctx->file_table.nodes[offset]); - ctx->file_table.nodes[offset] = NULL; + io_put_rsrc_node(ctx->file_table.data.nodes[offset]); + ctx->file_table.data.nodes[offset] = NULL; io_file_bitmap_clear(&ctx->file_table, offset); return 0; } @@ -153,7 +147,7 @@ int io_register_file_alloc_range(struct io_ring_ctx *ctx, return -EFAULT; if (check_add_overflow(range.off, range.len, &end)) return -EOVERFLOW; - if (range.resv || end > ctx->nr_user_files) + if (range.resv || end > ctx->file_table.data.nr) return -EINVAL; io_file_table_set_alloc_range(ctx, range.off, range.len); diff --git a/io_uring/filetable.h b/io_uring/filetable.h index 47616079abaa..664c31502dbb 100644 --- a/io_uring/filetable.h +++ b/io_uring/filetable.h @@ -52,7 +52,7 @@ static inline struct file *io_slot_file(struct io_rsrc_node *node) static inline struct file *io_file_from_index(struct io_file_table *table, int index) { - struct io_rsrc_node *node = table->nodes[index]; + struct io_rsrc_node *node = table->data.nodes[index]; if (node) return io_slot_file(node); diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 60c947114fa3..78df515fb3a7 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1879,11 +1879,10 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, struct file *file = NULL; io_ring_submit_lock(ctx, issue_flags); - - if (unlikely((unsigned int)fd >= ctx->nr_user_files)) + if (unlikely((unsigned int)fd >= ctx->file_table.data.nr)) goto out; - fd = array_index_nospec(fd, ctx->nr_user_files); - node = ctx->file_table.nodes[fd]; + fd = array_index_nospec(fd, ctx->file_table.data.nr); + node = ctx->file_table.data.nodes[fd]; if (node) { io_req_assign_rsrc_node(req, node); req->flags |= io_slot_flags(node); diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index edea1ffd501c..b90ab3b8f5e0 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -180,8 +180,8 @@ static struct file *io_msg_grab_file(struct io_kiocb *req, unsigned int issue_fl int idx = msg->src_fd; io_ring_submit_lock(ctx, issue_flags); - if (likely(idx < ctx->nr_user_files)) { - idx = array_index_nospec(idx, ctx->nr_user_files); + if (likely(idx < ctx->file_table.data.nr)) { + idx = array_index_nospec(idx, ctx->file_table.data.nr); file = io_file_from_index(&ctx->file_table, idx); if (file) get_file(file); diff --git a/io_uring/net.c b/io_uring/net.c index ce1156551d10..3e1f31574abb 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -1347,9 +1347,9 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags) ret = -EFAULT; io_ring_submit_lock(ctx, issue_flags); - if (sr->buf_index < ctx->nr_user_bufs) { - idx = array_index_nospec(sr->buf_index, ctx->nr_user_bufs); - node = ctx->user_bufs[idx]; + if (sr->buf_index < ctx->buf_table.nr) { + idx = array_index_nospec(sr->buf_index, ctx->buf_table.nr); + node = ctx->buf_table.nodes[idx]; io_req_assign_rsrc_node(sr->notif, node); ret = 0; } diff --git a/io_uring/nop.c b/io_uring/nop.c index de91600a3bc6..0dac01127de5 100644 --- a/io_uring/nop.c +++ b/io_uring/nop.c @@ -66,9 +66,9 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags) ret = -EFAULT; io_ring_submit_lock(ctx, issue_flags); - if (nop->buffer < ctx->nr_user_bufs) { - idx = array_index_nospec(nop->buffer, ctx->nr_user_bufs); - node = READ_ONCE(ctx->user_bufs[idx]); + if (nop->buffer < ctx->buf_table.nr) { + idx = array_index_nospec(nop->buffer, ctx->buf_table.nr); + node = READ_ONCE(ctx->buf_table.nodes[idx]); io_req_assign_rsrc_node(req, node); ret = 0; } diff --git a/io_uring/register.c b/io_uring/register.c index 1eb686eaa310..45edfc57963a 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -937,7 +937,8 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, mutex_lock(&ctx->uring_lock); ret = __io_uring_register(ctx, opcode, arg, nr_args); mutex_unlock(&ctx->uring_lock); - trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs, ret); + trace_io_uring_register(ctx, opcode, ctx->file_table.data.nr, + ctx->buf_table.nr, ret); if (!use_registered_ring) fput(file); return ret; diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 69a9cd82460d..49a6ab5f3ae9 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -142,39 +142,28 @@ struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type) return node; } -static void io_rsrc_data_free(struct io_rsrc_data *data) +__cold void io_rsrc_data_free(struct io_rsrc_data *data) { - int i; - - for (i = 0; i < data->nr; i++) { - struct io_rsrc_node *node = data->nodes[i]; - - if (node) - io_put_rsrc_node(node); + if (!data->nr) + return; + while (data->nr--) { + if (data->nodes[data->nr]) + io_put_rsrc_node(data->nodes[data->nr]); } kvfree(data->nodes); - kfree(data); + data->nodes = NULL; + data->nr = 0; } -__cold static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, unsigned nr, - struct io_rsrc_data **pdata) +__cold int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr) { - struct io_rsrc_data *data; - - data = kzalloc(sizeof(*data), GFP_KERNEL); - if (!data) - return -ENOMEM; - data->nodes = kvmalloc_array(nr, sizeof(struct io_rsrc_node *), - GFP_KERNEL | __GFP_ZERO); - if (!data->nodes) { - kfree(data); - return -ENOMEM; + GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (data->nodes) { + data->nr = nr; + return 0; } - - data->nr = nr; - *pdata = data; - return 0; + return -ENOMEM; } static int __io_sqe_files_update(struct io_ring_ctx *ctx, @@ -186,9 +175,9 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, int fd, i, err = 0; unsigned int done; - if (!ctx->file_data) + if (!ctx->file_table.data.nr) return -ENXIO; - if (up->offset + nr_args > ctx->nr_user_files) + if (up->offset + nr_args > ctx->file_table.data.nr) return -EINVAL; for (done = 0; done < nr_args; done++) { @@ -206,10 +195,10 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, if (fd == IORING_REGISTER_FILES_SKIP) continue; - i = array_index_nospec(up->offset + done, ctx->nr_user_files); - if (ctx->file_table.nodes[i]) { - io_put_rsrc_node(ctx->file_table.nodes[i]); - ctx->file_table.nodes[i] = NULL; + i = array_index_nospec(up->offset + done, ctx->file_table.data.nr); + if (ctx->file_table.data.nodes[i]) { + io_put_rsrc_node(ctx->file_table.data.nodes[i]); + ctx->file_table.data.nodes[i] = NULL; io_file_bitmap_clear(&ctx->file_table, i); } if (fd != -1) { @@ -234,7 +223,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, fput(file); break; } - ctx->file_table.nodes[i] = node; + ctx->file_table.data.nodes[i] = node; if (tag) node->tag = tag; io_fixed_file_set(node, file); @@ -256,9 +245,9 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, __u32 done; int i, err; - if (!ctx->buf_data) + if (!ctx->buf_table.nr) return -ENXIO; - if (up->offset + nr_args > ctx->nr_user_bufs) + if (up->offset + nr_args > ctx->buf_table.nr) return -EINVAL; for (done = 0; done < nr_args; done++) { @@ -282,16 +271,16 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, err = -EINVAL; break; } - i = array_index_nospec(up->offset + done, ctx->nr_user_bufs); node = io_sqe_buffer_register(ctx, iov, &last_hpage); if (IS_ERR(node)) { err = PTR_ERR(node); break; } - if (ctx->user_bufs[i]) - io_put_rsrc_node(ctx->user_bufs[i]); + i = array_index_nospec(up->offset + done, ctx->buf_table.nr); + if (ctx->buf_table.nodes[i]) + io_put_rsrc_node(ctx->buf_table.nodes[i]); - ctx->user_bufs[i] = node; + ctx->buf_table.nodes[i] = node; if (tag) node->tag = tag; if (ctx->compat) @@ -409,7 +398,7 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req, struct file *file; int ret, fd; - if (!req->ctx->file_data) + if (!req->ctx->file_table.data.nr) return -ENXIO; for (done = 0; done < up->nr_args; done++) { @@ -494,35 +483,13 @@ void io_free_rsrc_node(struct io_rsrc_node *node) kfree(node); } -static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) +int io_sqe_files_unregister(struct io_ring_ctx *ctx) { - int i; - - lockdep_assert_held(&ctx->uring_lock); - - for (i = 0; i < ctx->nr_user_files; i++) { - struct io_rsrc_node *node = ctx->file_table.nodes[i]; - - if (node) { - io_put_rsrc_node(node); - io_file_bitmap_clear(&ctx->file_table, i); - ctx->file_table.nodes[i] = NULL; - } - } + if (!ctx->file_table.data.nr) + return -ENXIO; io_free_file_tables(&ctx->file_table); io_file_table_set_alloc_range(ctx, 0, 0); - io_rsrc_data_free(ctx->file_data); - ctx->file_data = NULL; - ctx->nr_user_files = 0; -} - -int io_sqe_files_unregister(struct io_ring_ctx *ctx) -{ - if (!ctx->file_data) - return -ENXIO; - - __io_sqe_files_unregister(ctx); return 0; } @@ -534,7 +501,7 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, int fd, ret; unsigned i; - if (ctx->file_data) + if (ctx->file_table.data.nr) return -EBUSY; if (!nr_args) return -EINVAL; @@ -542,17 +509,10 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, return -EMFILE; if (nr_args > rlimit(RLIMIT_NOFILE)) return -EMFILE; - ret = io_rsrc_data_alloc(ctx, nr_args, &ctx->file_data); - if (ret) - return ret; - - if (!io_alloc_file_tables(&ctx->file_table, nr_args)) { - io_rsrc_data_free(ctx->file_data); - ctx->file_data = NULL; + if (!io_alloc_file_tables(&ctx->file_table, nr_args)) return -ENOMEM; - } - for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { + for (i = 0; i < nr_args; i++) { struct io_rsrc_node *node; u64 tag = 0; @@ -589,44 +549,24 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, } if (tag) node->tag = tag; - ctx->file_table.nodes[i] = node; + ctx->file_table.data.nodes[i] = node; io_fixed_file_set(node, file); io_file_bitmap_set(&ctx->file_table, i); } /* default it to the whole table */ - io_file_table_set_alloc_range(ctx, 0, ctx->nr_user_files); + io_file_table_set_alloc_range(ctx, 0, ctx->file_table.data.nr); return 0; fail: - __io_sqe_files_unregister(ctx); + io_sqe_files_unregister(ctx); return ret; } -static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx) -{ - unsigned int i; - - lockdep_assert_held(&ctx->uring_lock); - - for (i = 0; i < ctx->nr_user_bufs; i++) { - if (ctx->user_bufs[i]) { - io_put_rsrc_node(ctx->user_bufs[i]); - ctx->user_bufs[i] = NULL; - } - } - kvfree(ctx->user_bufs); - ctx->user_bufs = NULL; - io_rsrc_data_free(ctx->buf_data); - ctx->buf_data = NULL; - ctx->nr_user_bufs = 0; -} - int io_sqe_buffers_unregister(struct io_ring_ctx *ctx) { - if (!ctx->buf_data) + if (!ctx->buf_table.nr) return -ENXIO; - - __io_sqe_buffers_unregister(ctx); + io_rsrc_data_free(&ctx->buf_table); return 0; } @@ -653,8 +593,8 @@ static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages, } /* check previously registered pages */ - for (i = 0; i < ctx->nr_user_bufs; i++) { - struct io_rsrc_node *node = ctx->user_bufs[i]; + for (i = 0; i < ctx->buf_table.nr; i++) { + struct io_rsrc_node *node = ctx->buf_table.nodes[i]; struct io_mapped_ubuf *imu = node->buf; for (j = 0; j < imu->nr_bvecs; j++) { @@ -805,6 +745,9 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, struct io_imu_folio_data data; bool coalesced; + if (!iov->iov_base) + return rsrc_empty_node; + node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); if (!node) return ERR_PTR(-ENOMEM); @@ -864,40 +807,29 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, return node; } -static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args) -{ - ctx->user_bufs = kcalloc(nr_args, sizeof(*ctx->user_bufs), GFP_KERNEL); - return ctx->user_bufs ? 0 : -ENOMEM; -} - int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, unsigned int nr_args, u64 __user *tags) { struct page *last_hpage = NULL; - struct io_rsrc_data *data; + struct io_rsrc_data data; struct iovec fast_iov, *iov = &fast_iov; const struct iovec __user *uvec; int i, ret; BUILD_BUG_ON(IORING_MAX_REG_BUFFERS >= (1u << 16)); - if (ctx->user_bufs) + if (ctx->buf_table.nr) return -EBUSY; if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS) return -EINVAL; - ret = io_rsrc_data_alloc(ctx, nr_args, &data); + ret = io_rsrc_data_alloc(&data, nr_args); if (ret) return ret; - ret = io_buffers_map_alloc(ctx, nr_args); - if (ret) { - io_rsrc_data_free(data); - return ret; - } if (!arg) memset(iov, 0, sizeof(*iov)); - for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) { + for (i = 0; i < nr_args; i++) { struct io_rsrc_node *node; u64 tag = 0; @@ -935,14 +867,12 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, } if (tag) node->tag = tag; - ctx->user_bufs[i] = node; + data.nodes[i] = node; } - WARN_ON_ONCE(ctx->buf_data); - - ctx->buf_data = data; + ctx->buf_table = data; if (ret) - __io_sqe_buffers_unregister(ctx); + io_sqe_buffers_unregister(ctx); return ret; } @@ -1009,8 +939,7 @@ int io_import_fixed(int ddir, struct iov_iter *iter, static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx) { - struct io_rsrc_node **user_bufs; - struct io_rsrc_data *data; + struct io_rsrc_data data; int i, ret, nbufs; /* @@ -1021,43 +950,37 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx mutex_lock(&src_ctx->uring_lock); ret = -ENXIO; - nbufs = src_ctx->nr_user_bufs; + nbufs = src_ctx->buf_table.nr; if (!nbufs) goto out_unlock; - ret = io_rsrc_data_alloc(ctx, nbufs, &data); + ret = io_rsrc_data_alloc(&data, nbufs); if (ret) goto out_unlock; - ret = -ENOMEM; - user_bufs = kvmalloc_array(nbufs, sizeof(struct io_rsrc_node *), - GFP_KERNEL | __GFP_ZERO); - if (!user_bufs) - goto out_free_data; - for (i = 0; i < nbufs; i++) { - struct io_rsrc_node *src_node = src_ctx->user_bufs[i]; + struct io_rsrc_node *src_node = src_ctx->buf_table.nodes[i]; struct io_rsrc_node *dst_node; if (src_node == rsrc_empty_node) { dst_node = rsrc_empty_node; } else { dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); - if (!dst_node) + if (!dst_node) { + ret = -ENOMEM; goto out_put_free; + } refcount_inc(&src_node->buf->refs); dst_node->buf = src_node->buf; } - user_bufs[i] = dst_node; + data.nodes[i] = dst_node; } /* Have a ref on the bufs now, drop src lock and re-grab our own lock */ mutex_unlock(&src_ctx->uring_lock); mutex_lock(&ctx->uring_lock); - if (!ctx->user_bufs) { - ctx->user_bufs = user_bufs; - ctx->buf_data = data; - ctx->nr_user_bufs = nbufs; + if (!ctx->buf_table.nr) { + ctx->buf_table = data; return 0; } @@ -1068,12 +991,10 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx i = nbufs; out_put_free: while (i--) { - io_buffer_unmap(src_ctx, user_bufs[i]); - kfree(user_bufs[i]); + io_buffer_unmap(src_ctx, data.nodes[i]); + kfree(data.nodes[i]); } - kvfree(user_bufs); -out_free_data: - io_rsrc_data_free(data); + io_rsrc_data_free(&data); out_unlock: mutex_unlock(&src_ctx->uring_lock); mutex_lock(&ctx->uring_lock); @@ -1094,7 +1015,7 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) struct file *file; int ret; - if (ctx->user_bufs || ctx->nr_user_bufs) + if (ctx->buf_table.nr) return -EBUSY; if (copy_from_user(&buf, arg, sizeof(buf))) return -EFAULT; diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 48f712488f6b..569ea9ce1405 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -13,11 +13,6 @@ enum { IORING_RSRC_BUFFER = 1, }; -struct io_rsrc_data { - unsigned int nr; - struct io_rsrc_node **nodes; -}; - struct io_rsrc_node { struct io_ring_ctx *ctx; int refs; @@ -50,6 +45,8 @@ struct io_imu_folio_data { struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type); void io_free_rsrc_node(struct io_rsrc_node *node); +void io_rsrc_data_free(struct io_rsrc_data *data); +int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr); int io_import_fixed(int ddir, struct iov_iter *iter, struct io_mapped_ubuf *imu, diff --git a/io_uring/rw.c b/io_uring/rw.c index 65491f4f2c7e..28fff18ebb19 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -339,10 +339,10 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe if (unlikely(ret)) return ret; - if (unlikely(req->buf_index >= ctx->nr_user_bufs)) + if (unlikely(req->buf_index >= ctx->buf_table.nr)) return -EFAULT; - index = array_index_nospec(req->buf_index, ctx->nr_user_bufs); - node = ctx->user_bufs[index]; + index = array_index_nospec(req->buf_index, ctx->buf_table.nr); + node = ctx->buf_table.nodes[index]; io_req_assign_rsrc_node(req, node); io = req->async_data; diff --git a/io_uring/splice.c b/io_uring/splice.c index f78afb575ae6..aaaddb66e90a 100644 --- a/io_uring/splice.c +++ b/io_uring/splice.c @@ -66,10 +66,10 @@ static struct file *io_splice_get_file(struct io_kiocb *req, return io_file_get_normal(req, sp->splice_fd_in); io_ring_submit_lock(ctx, issue_flags); - if (unlikely(sp->splice_fd_in >= ctx->nr_user_files)) + if (unlikely(sp->splice_fd_in >= ctx->file_table.data.nr)) goto out; - sp->splice_fd_in = array_index_nospec(sp->splice_fd_in, ctx->nr_user_files); - node = ctx->file_table.nodes[sp->splice_fd_in]; + sp->splice_fd_in = array_index_nospec(sp->splice_fd_in, ctx->file_table.data.nr); + node = ctx->file_table.data.nodes[sp->splice_fd_in]; if (node) { node->refs++; sp->rsrc_node = node; diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 0899c71008ae..17d5f5004702 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -212,15 +212,15 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) u16 index; index = READ_ONCE(sqe->buf_index); - if (unlikely(index >= ctx->nr_user_bufs)) + if (unlikely(index >= ctx->buf_table.nr)) return -EFAULT; - req->buf_index = array_index_nospec(index, ctx->nr_user_bufs); + req->buf_index = array_index_nospec(index, ctx->buf_table.nr); /* * Pi node upfront, prior to io_uring_cmd_import_fixed() * being called. This prevents destruction of the mapped buffer * we'll need at actual import time. */ - io_req_assign_rsrc_node(req, ctx->user_bufs[req->buf_index]); + io_req_assign_rsrc_node(req, ctx->buf_table.nodes[req->buf_index]); } ioucmd->cmd_op = READ_ONCE(sqe->cmd_op); From b54a14041ee6444692d95ff38c8b3d1af682aa17 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 27 Oct 2024 09:08:31 -0600 Subject: [PATCH 46/79] io_uring/rsrc: add io_rsrc_node_lookup() helper There are lots of spots open-coding this functionality, add a generic helper that does the node lookup in a speculation safe way. Signed-off-by: Jens Axboe --- io_uring/cancel.c | 8 +++++--- io_uring/filetable.c | 16 +++++++++------- io_uring/filetable.h | 2 +- io_uring/io_uring.c | 6 +----- io_uring/msg_ring.c | 31 +++++++++++++++---------------- io_uring/net.c | 6 ++---- io_uring/nop.c | 6 ++---- io_uring/rsrc.c | 12 +++++++----- io_uring/rsrc.h | 8 ++++++++ io_uring/rw.c | 6 ++---- io_uring/splice.c | 6 +----- io_uring/uring_cmd.c | 9 ++++----- 12 files changed, 57 insertions(+), 59 deletions(-) diff --git a/io_uring/cancel.c b/io_uring/cancel.c index 3a2996307025..bbca5cb69cb5 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -240,10 +240,12 @@ static int __io_sync_cancel(struct io_uring_task *tctx, /* fixed must be grabbed every time since we drop the uring_lock */ if ((cd->flags & IORING_ASYNC_CANCEL_FD) && (cd->flags & IORING_ASYNC_CANCEL_FD_FIXED)) { - if (unlikely(fd >= ctx->file_table.data.nr)) + struct io_rsrc_node *node; + + node = io_rsrc_node_lookup(&ctx->file_table.data, fd); + if (unlikely(!node)) return -EBADF; - fd = array_index_nospec(fd, ctx->file_table.data.nr); - cd->file = io_file_from_index(&ctx->file_table, fd); + cd->file = io_slot_file(node); if (!cd->file) return -EBADF; } diff --git a/io_uring/filetable.c b/io_uring/filetable.c index c1f9f9550446..7a9de6718b8a 100644 --- a/io_uring/filetable.c +++ b/io_uring/filetable.c @@ -58,7 +58,7 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file, u32 slot_index) __must_hold(&req->ctx->uring_lock) { - struct io_rsrc_node *node; + struct io_rsrc_node *node, *old_node; if (io_is_uring_fops(file)) return -EBADF; @@ -71,9 +71,9 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file, if (!node) return -ENOMEM; - slot_index = array_index_nospec(slot_index, ctx->file_table.data.nr); - if (ctx->file_table.data.nodes[slot_index]) - io_put_rsrc_node(ctx->file_table.data.nodes[slot_index]); + old_node = io_rsrc_node_lookup(&ctx->file_table.data, slot_index); + if (old_node) + io_put_rsrc_node(old_node); else io_file_bitmap_set(&ctx->file_table, slot_index); @@ -123,15 +123,17 @@ int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags, int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset) { + struct io_rsrc_node *node; + if (unlikely(!ctx->file_table.data.nr)) return -ENXIO; if (offset >= ctx->file_table.data.nr) return -EINVAL; - offset = array_index_nospec(offset, ctx->file_table.data.nr); - if (!ctx->file_table.data.nodes[offset]) + node = io_rsrc_node_lookup(&ctx->file_table.data, offset); + if (!node) return -EBADF; - io_put_rsrc_node(ctx->file_table.data.nodes[offset]); + io_put_rsrc_node(node); ctx->file_table.data.nodes[offset] = NULL; io_file_bitmap_clear(&ctx->file_table, offset); return 0; diff --git a/io_uring/filetable.h b/io_uring/filetable.h index 664c31502dbb..29edda0caa65 100644 --- a/io_uring/filetable.h +++ b/io_uring/filetable.h @@ -52,7 +52,7 @@ static inline struct file *io_slot_file(struct io_rsrc_node *node) static inline struct file *io_file_from_index(struct io_file_table *table, int index) { - struct io_rsrc_node *node = table->data.nodes[index]; + struct io_rsrc_node *node = io_rsrc_node_lookup(&table->data, index); if (node) return io_slot_file(node); diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 78df515fb3a7..3a535e9e8ac3 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1879,16 +1879,12 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, struct file *file = NULL; io_ring_submit_lock(ctx, issue_flags); - if (unlikely((unsigned int)fd >= ctx->file_table.data.nr)) - goto out; - fd = array_index_nospec(fd, ctx->file_table.data.nr); - node = ctx->file_table.data.nodes[fd]; + node = io_rsrc_node_lookup(&ctx->file_table.data, fd); if (node) { io_req_assign_rsrc_node(req, node); req->flags |= io_slot_flags(node); file = io_slot_file(node); } -out: io_ring_submit_unlock(ctx, issue_flags); return file; } diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index b90ab3b8f5e0..99af39e1d0fb 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -172,22 +172,24 @@ static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags) return __io_msg_ring_data(target_ctx, msg, issue_flags); } -static struct file *io_msg_grab_file(struct io_kiocb *req, unsigned int issue_flags) +static int io_msg_grab_file(struct io_kiocb *req, unsigned int issue_flags) { struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); struct io_ring_ctx *ctx = req->ctx; - struct file *file = NULL; - int idx = msg->src_fd; + struct io_rsrc_node *node; + int ret = -EBADF; io_ring_submit_lock(ctx, issue_flags); - if (likely(idx < ctx->file_table.data.nr)) { - idx = array_index_nospec(idx, ctx->file_table.data.nr); - file = io_file_from_index(&ctx->file_table, idx); - if (file) - get_file(file); + node = io_rsrc_node_lookup(&ctx->file_table.data, msg->src_fd); + if (node) { + msg->src_file = io_slot_file(node); + if (msg->src_file) + get_file(msg->src_file); + req->flags |= REQ_F_NEED_CLEANUP; + ret = 0; } io_ring_submit_unlock(ctx, issue_flags); - return file; + return ret; } static int io_msg_install_complete(struct io_kiocb *req, unsigned int issue_flags) @@ -256,7 +258,6 @@ static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags) struct io_ring_ctx *target_ctx = req->file->private_data; struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); struct io_ring_ctx *ctx = req->ctx; - struct file *src_file = msg->src_file; if (msg->len) return -EINVAL; @@ -264,12 +265,10 @@ static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags) return -EINVAL; if (target_ctx->flags & IORING_SETUP_R_DISABLED) return -EBADFD; - if (!src_file) { - src_file = io_msg_grab_file(req, issue_flags); - if (!src_file) - return -EBADF; - msg->src_file = src_file; - req->flags |= REQ_F_NEED_CLEANUP; + if (!msg->src_file) { + int ret = io_msg_grab_file(req, issue_flags); + if (unlikely(ret)) + return ret; } if (io_msg_need_remote(target_ctx)) diff --git a/io_uring/net.c b/io_uring/net.c index 3e1f31574abb..2f7b334ed708 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -1343,13 +1343,11 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags) if (sr->flags & IORING_RECVSEND_FIXED_BUF) { struct io_ring_ctx *ctx = req->ctx; struct io_rsrc_node *node; - int idx; ret = -EFAULT; io_ring_submit_lock(ctx, issue_flags); - if (sr->buf_index < ctx->buf_table.nr) { - idx = array_index_nospec(sr->buf_index, ctx->buf_table.nr); - node = ctx->buf_table.nodes[idx]; + node = io_rsrc_node_lookup(&ctx->buf_table, sr->buf_index); + if (node) { io_req_assign_rsrc_node(sr->notif, node); ret = 0; } diff --git a/io_uring/nop.c b/io_uring/nop.c index 0dac01127de5..149dbdc53607 100644 --- a/io_uring/nop.c +++ b/io_uring/nop.c @@ -62,13 +62,11 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags) if (nop->flags & IORING_NOP_FIXED_BUFFER) { struct io_ring_ctx *ctx = req->ctx; struct io_rsrc_node *node; - int idx; ret = -EFAULT; io_ring_submit_lock(ctx, issue_flags); - if (nop->buffer < ctx->buf_table.nr) { - idx = array_index_nospec(nop->buffer, ctx->buf_table.nr); - node = READ_ONCE(ctx->buf_table.nodes[idx]); + node = io_rsrc_node_lookup(&ctx->buf_table, nop->buffer); + if (node) { io_req_assign_rsrc_node(req, node); ret = 0; } diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 49a6ab5f3ae9..0380b2f4ed8d 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -181,6 +181,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, return -EINVAL; for (done = 0; done < nr_args; done++) { + struct io_rsrc_node *node; u64 tag = 0; if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) || @@ -195,9 +196,10 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, if (fd == IORING_REGISTER_FILES_SKIP) continue; - i = array_index_nospec(up->offset + done, ctx->file_table.data.nr); - if (ctx->file_table.data.nodes[i]) { - io_put_rsrc_node(ctx->file_table.data.nodes[i]); + i = up->offset + done; + node = io_rsrc_node_lookup(&ctx->file_table.data, i); + if (node) { + io_put_rsrc_node(node); ctx->file_table.data.nodes[i] = NULL; io_file_bitmap_clear(&ctx->file_table, i); } @@ -958,9 +960,9 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx goto out_unlock; for (i = 0; i < nbufs; i++) { - struct io_rsrc_node *src_node = src_ctx->buf_table.nodes[i]; - struct io_rsrc_node *dst_node; + struct io_rsrc_node *dst_node, *src_node; + src_node = io_rsrc_node_lookup(&src_ctx->buf_table, i); if (src_node == rsrc_empty_node) { dst_node = rsrc_empty_node; } else { diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 569ea9ce1405..82e08cbba02e 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -70,6 +70,14 @@ int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, extern const struct io_rsrc_node empty_node; #define rsrc_empty_node (struct io_rsrc_node *) &empty_node +static inline struct io_rsrc_node *io_rsrc_node_lookup(struct io_rsrc_data *data, + int index) +{ + if (index < data->nr) + return data->nodes[array_index_nospec(index, data->nr)]; + return NULL; +} + static inline void io_put_rsrc_node(struct io_rsrc_node *node) { if (node != rsrc_empty_node && !--node->refs) diff --git a/io_uring/rw.c b/io_uring/rw.c index 28fff18ebb19..30448f343c7f 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -332,17 +332,15 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe struct io_ring_ctx *ctx = req->ctx; struct io_rsrc_node *node; struct io_async_rw *io; - u16 index; int ret; ret = io_prep_rw(req, sqe, ddir, false); if (unlikely(ret)) return ret; - if (unlikely(req->buf_index >= ctx->buf_table.nr)) + node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index); + if (!node) return -EFAULT; - index = array_index_nospec(req->buf_index, ctx->buf_table.nr); - node = ctx->buf_table.nodes[index]; io_req_assign_rsrc_node(req, node); io = req->async_data; diff --git a/io_uring/splice.c b/io_uring/splice.c index aaaddb66e90a..deeb8bb18651 100644 --- a/io_uring/splice.c +++ b/io_uring/splice.c @@ -66,17 +66,13 @@ static struct file *io_splice_get_file(struct io_kiocb *req, return io_file_get_normal(req, sp->splice_fd_in); io_ring_submit_lock(ctx, issue_flags); - if (unlikely(sp->splice_fd_in >= ctx->file_table.data.nr)) - goto out; - sp->splice_fd_in = array_index_nospec(sp->splice_fd_in, ctx->file_table.data.nr); - node = ctx->file_table.data.nodes[sp->splice_fd_in]; + node = io_rsrc_node_lookup(&ctx->file_table.data, sp->splice_fd_in); if (node) { node->refs++; sp->rsrc_node = node; file = io_slot_file(node); req->flags |= REQ_F_NEED_CLEANUP; } -out: io_ring_submit_unlock(ctx, issue_flags); return file; } diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 17d5f5004702..535909a38e76 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -209,18 +209,17 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (ioucmd->flags & IORING_URING_CMD_FIXED) { struct io_ring_ctx *ctx = req->ctx; - u16 index; + struct io_rsrc_node *node; - index = READ_ONCE(sqe->buf_index); - if (unlikely(index >= ctx->buf_table.nr)) + node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index); + if (unlikely(!node)) return -EFAULT; - req->buf_index = array_index_nospec(index, ctx->buf_table.nr); /* * Pi node upfront, prior to io_uring_cmd_import_fixed() * being called. This prevents destruction of the mapped buffer * we'll need at actual import time. */ - io_req_assign_rsrc_node(req, ctx->buf_table.nodes[req->buf_index]); + io_req_assign_rsrc_node(req, node); } ioucmd->cmd_op = READ_ONCE(sqe->cmd_op); From cb1717a7cd0fc8a063bd7fe3b4eb6fd81defb11c Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 28 Oct 2024 08:35:05 -0600 Subject: [PATCH 47/79] io_uring/filetable: remove io_file_from_index() helper It's only used in fdinfo, nothing really gained from having this helper. Signed-off-by: Jens Axboe --- io_uring/fdinfo.c | 4 +++- io_uring/filetable.h | 10 ---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index e3f5e9fe5562..9d96481e2eb6 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -167,8 +167,10 @@ __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *file) seq_printf(m, "SqWorkTime:\t%llu\n", sq_work_time); seq_printf(m, "UserFiles:\t%u\n", ctx->file_table.data.nr); for (i = 0; has_lock && i < ctx->file_table.data.nr; i++) { - struct file *f = io_file_from_index(&ctx->file_table, i); + struct file *f = NULL; + if (ctx->file_table.data.nodes[i]) + f = io_slot_file(ctx->file_table.data.nodes[i]); if (f) seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname); else diff --git a/io_uring/filetable.h b/io_uring/filetable.h index 29edda0caa65..6c0c9642f6e9 100644 --- a/io_uring/filetable.h +++ b/io_uring/filetable.h @@ -49,16 +49,6 @@ static inline struct file *io_slot_file(struct io_rsrc_node *node) return (struct file *)(node->file_ptr & FFS_MASK); } -static inline struct file *io_file_from_index(struct io_file_table *table, - int index) -{ - struct io_rsrc_node *node = io_rsrc_node_lookup(&table->data, index); - - if (node) - return io_slot_file(node); - return NULL; -} - static inline void io_fixed_file_set(struct io_rsrc_node *node, struct file *file) { From 5f3829fdd69d746f36a5e87df21ce58470b8e9fa Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 28 Oct 2024 08:36:12 -0600 Subject: [PATCH 48/79] io_uring/filetable: kill io_reset_alloc_hint() helper It's only used internally, and in one spot, just open-code ti. Signed-off-by: Jens Axboe --- io_uring/filetable.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/io_uring/filetable.h b/io_uring/filetable.h index 6c0c9642f6e9..bfacadb8d089 100644 --- a/io_uring/filetable.h +++ b/io_uring/filetable.h @@ -56,17 +56,12 @@ static inline void io_fixed_file_set(struct io_rsrc_node *node, (io_file_get_flags(file) >> REQ_F_SUPPORT_NOWAIT_BIT); } -static inline void io_reset_alloc_hint(struct io_ring_ctx *ctx) -{ - ctx->file_table.alloc_hint = ctx->file_alloc_start; -} - static inline void io_file_table_set_alloc_range(struct io_ring_ctx *ctx, unsigned off, unsigned len) { ctx->file_alloc_start = off; ctx->file_alloc_end = off + len; - io_reset_alloc_hint(ctx); + ctx->file_table.alloc_hint = ctx->file_alloc_start; } #endif From 4007c3d8c22a2025367953f4ee36ae106a69d855 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 29 Oct 2024 09:02:38 -0600 Subject: [PATCH 49/79] io_uring/rsrc: add io_reset_rsrc_node() helper Puts and reset an existing node in a slot, if one exists. Returns true if a node was there, false if not. This helps cleanup some of the code that does a lookup just to clear an existing node. Signed-off-by: Jens Axboe --- io_uring/filetable.c | 10 +++------- io_uring/rsrc.c | 12 +++--------- io_uring/rsrc.h | 11 +++++++++++ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/io_uring/filetable.c b/io_uring/filetable.c index 7a9de6718b8a..45f005f5db42 100644 --- a/io_uring/filetable.c +++ b/io_uring/filetable.c @@ -58,7 +58,7 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file, u32 slot_index) __must_hold(&req->ctx->uring_lock) { - struct io_rsrc_node *node, *old_node; + struct io_rsrc_node *node; if (io_is_uring_fops(file)) return -EBADF; @@ -71,10 +71,7 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file, if (!node) return -ENOMEM; - old_node = io_rsrc_node_lookup(&ctx->file_table.data, slot_index); - if (old_node) - io_put_rsrc_node(old_node); - else + if (!io_reset_rsrc_node(&ctx->file_table.data, slot_index)) io_file_bitmap_set(&ctx->file_table, slot_index); ctx->file_table.data.nodes[slot_index] = node; @@ -133,8 +130,7 @@ int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset) node = io_rsrc_node_lookup(&ctx->file_table.data, offset); if (!node) return -EBADF; - io_put_rsrc_node(node); - ctx->file_table.data.nodes[offset] = NULL; + io_reset_rsrc_node(&ctx->file_table.data, offset); io_file_bitmap_clear(&ctx->file_table, offset); return 0; } diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 0380b2f4ed8d..378f33746457 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -181,7 +181,6 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, return -EINVAL; for (done = 0; done < nr_args; done++) { - struct io_rsrc_node *node; u64 tag = 0; if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) || @@ -197,12 +196,9 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, continue; i = up->offset + done; - node = io_rsrc_node_lookup(&ctx->file_table.data, i); - if (node) { - io_put_rsrc_node(node); - ctx->file_table.data.nodes[i] = NULL; + if (io_reset_rsrc_node(&ctx->file_table.data, i)) io_file_bitmap_clear(&ctx->file_table, i); - } + if (fd != -1) { struct file *file = fget(fd); struct io_rsrc_node *node; @@ -279,9 +275,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, break; } i = array_index_nospec(up->offset + done, ctx->buf_table.nr); - if (ctx->buf_table.nodes[i]) - io_put_rsrc_node(ctx->buf_table.nodes[i]); - + io_reset_rsrc_node(&ctx->buf_table, i); ctx->buf_table.nodes[i] = node; if (tag) node->tag = tag; diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 82e08cbba02e..43b19e516f5f 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -84,6 +84,17 @@ static inline void io_put_rsrc_node(struct io_rsrc_node *node) io_free_rsrc_node(node); } +static inline bool io_reset_rsrc_node(struct io_rsrc_data *data, int index) +{ + struct io_rsrc_node *node = data->nodes[index]; + + if (!node) + return false; + io_put_rsrc_node(node); + data->nodes[index] = NULL; + return true; +} + static inline void io_req_put_rsrc_nodes(struct io_kiocb *req) { if (req->rsrc_nodes[IORING_RSRC_FILE] != rsrc_empty_node) { From d50f94d761a5d9a34e03a86e512e19d88cbeaf06 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 30 Oct 2024 09:51:58 -0600 Subject: [PATCH 50/79] io_uring/rsrc: get rid of the empty node and dummy_ubuf The empty node was used as a placeholder for a sparse entry, but it didn't really solve any issues. The caller still has to check for whether it's the empty node or not, it may as well just check for a NULL return instead. The dummy_ubuf was used for a sparse buffer entry, but NULL will serve the same purpose there of ensuring an -EFAULT on attempted import. Just use NULL for a sparse node, regardless of whether or not it's a file or buffer resource. Signed-off-by: Jens Axboe --- io_uring/fdinfo.c | 9 +++++++-- io_uring/io_uring.c | 4 ++-- io_uring/notif.c | 4 ++-- io_uring/rsrc.c | 48 ++++++++++++++++++++------------------------- io_uring/rsrc.h | 23 +++++++--------------- io_uring/splice.c | 2 +- 6 files changed, 40 insertions(+), 50 deletions(-) diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index 9d96481e2eb6..8da0d9e4533a 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -178,9 +178,14 @@ __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *file) } seq_printf(m, "UserBufs:\t%u\n", ctx->buf_table.nr); for (i = 0; has_lock && i < ctx->buf_table.nr; i++) { - struct io_mapped_ubuf *buf = ctx->buf_table.nodes[i]->buf; + struct io_mapped_ubuf *buf = NULL; - seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, buf->len); + if (ctx->buf_table.nodes[i]) + buf = ctx->buf_table.nodes[i]->buf; + if (buf) + seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, buf->len); + else + seq_printf(m, "%5u: \n", i); } if (has_lock && !xa_empty(&ctx->personalities)) { unsigned long index; diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 3a535e9e8ac3..44a772013c09 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -947,8 +947,8 @@ void io_req_defer_failed(struct io_kiocb *req, s32 res) static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx) { req->ctx = ctx; - req->rsrc_nodes[IORING_RSRC_FILE] = rsrc_empty_node; - req->rsrc_nodes[IORING_RSRC_BUFFER] = rsrc_empty_node; + req->rsrc_nodes[IORING_RSRC_FILE] = NULL; + req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL; req->link = NULL; req->async_data = NULL; /* not necessary, but safer to zero */ diff --git a/io_uring/notif.c b/io_uring/notif.c index 44bf21c0f810..4f02e969cf08 100644 --- a/io_uring/notif.c +++ b/io_uring/notif.c @@ -117,8 +117,8 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) notif->file = NULL; notif->task = current; io_get_task_refs(1); - notif->rsrc_nodes[IORING_RSRC_FILE] = rsrc_empty_node; - notif->rsrc_nodes[IORING_RSRC_BUFFER] = rsrc_empty_node; + notif->rsrc_nodes[IORING_RSRC_FILE] = NULL; + notif->rsrc_nodes[IORING_RSRC_BUFFER] = NULL; nd = io_notif_to_data(notif); nd->zc_report = false; diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 378f33746457..7ad91f180566 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -32,17 +32,6 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, #define IORING_MAX_FIXED_FILES (1U << 20) #define IORING_MAX_REG_BUFFERS (1U << 14) -static const struct io_mapped_ubuf dummy_ubuf = { - /* set invalid range, so io_import_fixed() fails meeting it */ - .ubuf = -1UL, - .len = UINT_MAX, -}; - -const struct io_rsrc_node empty_node = { - .type = IORING_RSRC_BUFFER, - .buf = (struct io_mapped_ubuf *) &dummy_ubuf, -}; - int __io_account_mem(struct user_struct *user, unsigned long nr_pages) { unsigned long page_limit, cur_pages, new_pages; @@ -116,7 +105,7 @@ static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_rsrc_node *node) { unsigned int i; - if (node->buf != &dummy_ubuf) { + if (node->buf) { struct io_mapped_ubuf *imu = node->buf; if (!refcount_dec_and_test(&imu->refs)) @@ -265,20 +254,21 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, err = io_buffer_validate(iov); if (err) break; - if (!iov->iov_base && tag) { - err = -EINVAL; - break; - } node = io_sqe_buffer_register(ctx, iov, &last_hpage); if (IS_ERR(node)) { err = PTR_ERR(node); break; } + if (tag) { + if (!node) { + err = -EINVAL; + break; + } + node->tag = tag; + } i = array_index_nospec(up->offset + done, ctx->buf_table.nr); io_reset_rsrc_node(&ctx->buf_table, i); ctx->buf_table.nodes[i] = node; - if (tag) - node->tag = tag; if (ctx->compat) user_data += sizeof(struct compat_iovec); else @@ -591,8 +581,11 @@ static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages, /* check previously registered pages */ for (i = 0; i < ctx->buf_table.nr; i++) { struct io_rsrc_node *node = ctx->buf_table.nodes[i]; - struct io_mapped_ubuf *imu = node->buf; + struct io_mapped_ubuf *imu; + if (!node) + continue; + imu = node->buf; for (j = 0; j < imu->nr_bvecs; j++) { if (!PageCompound(imu->bvec[j].bv_page)) continue; @@ -742,7 +735,7 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, bool coalesced; if (!iov->iov_base) - return rsrc_empty_node; + return NULL; node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); if (!node) @@ -850,10 +843,6 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, ret = -EFAULT; break; } - if (tag && !iov->iov_base) { - ret = -EINVAL; - break; - } } node = io_sqe_buffer_register(ctx, iov, &last_hpage); @@ -861,8 +850,13 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, ret = PTR_ERR(node); break; } - if (tag) + if (tag) { + if (!node) { + ret = -EINVAL; + break; + } node->tag = tag; + } data.nodes[i] = node; } @@ -957,8 +951,8 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx struct io_rsrc_node *dst_node, *src_node; src_node = io_rsrc_node_lookup(&src_ctx->buf_table, i); - if (src_node == rsrc_empty_node) { - dst_node = rsrc_empty_node; + if (!src_node) { + dst_node = NULL; } else { dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); if (!dst_node) { diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 43b19e516f5f..a40fad783a69 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -67,9 +67,6 @@ int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg, int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, unsigned int size, unsigned int type); -extern const struct io_rsrc_node empty_node; -#define rsrc_empty_node (struct io_rsrc_node *) &empty_node - static inline struct io_rsrc_node *io_rsrc_node_lookup(struct io_rsrc_data *data, int index) { @@ -80,7 +77,7 @@ static inline struct io_rsrc_node *io_rsrc_node_lookup(struct io_rsrc_data *data static inline void io_put_rsrc_node(struct io_rsrc_node *node) { - if (node != rsrc_empty_node && !--node->refs) + if (node && !--node->refs) io_free_rsrc_node(node); } @@ -97,23 +94,17 @@ static inline bool io_reset_rsrc_node(struct io_rsrc_data *data, int index) static inline void io_req_put_rsrc_nodes(struct io_kiocb *req) { - if (req->rsrc_nodes[IORING_RSRC_FILE] != rsrc_empty_node) { - io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_FILE]); - req->rsrc_nodes[IORING_RSRC_FILE] = rsrc_empty_node; - } - if (req->rsrc_nodes[IORING_RSRC_BUFFER] != rsrc_empty_node) { - io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_BUFFER]); - req->rsrc_nodes[IORING_RSRC_BUFFER] = rsrc_empty_node; - } + io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_FILE]); + io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_BUFFER]); + req->rsrc_nodes[IORING_RSRC_FILE] = NULL; + req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL; } static inline void io_req_assign_rsrc_node(struct io_kiocb *req, struct io_rsrc_node *node) { - if (node != rsrc_empty_node) { - node->refs++; - req->rsrc_nodes[node->type] = node; - } + node->refs++; + req->rsrc_nodes[node->type] = node; } int io_files_update(struct io_kiocb *req, unsigned int issue_flags); diff --git a/io_uring/splice.c b/io_uring/splice.c index deeb8bb18651..e8ed15f4ea1a 100644 --- a/io_uring/splice.c +++ b/io_uring/splice.c @@ -35,7 +35,7 @@ static int __io_splice_prep(struct io_kiocb *req, if (unlikely(sp->flags & ~valid_flags)) return -EINVAL; sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in); - sp->rsrc_node = rsrc_empty_node; + sp->rsrc_node = NULL; req->flags |= REQ_F_FORCE_ASYNC; return 0; } From b16e920a1909da6799c43000db730d8fcdcae907 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 28 Oct 2024 18:43:13 -0600 Subject: [PATCH 51/79] io_uring/rsrc: allow cloning at an offset Right now buffer cloning is an all-or-nothing kind of thing - either the whole table is cloned from a source to a destination ring, or nothing at all. However, it's not always desired to clone the whole thing. Allow for the application to specify a source and destination offset, and a number of buffers to clone. If the destination offset is non-zero, then allocate sparse nodes upfront. Signed-off-by: Jens Axboe --- include/uapi/linux/io_uring.h | 5 ++++- io_uring/rsrc.c | 32 ++++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 024745283783..cc8dbe78c126 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -719,7 +719,10 @@ enum { struct io_uring_clone_buffers { __u32 src_fd; __u32 flags; - __u32 pad[6]; + __u32 src_off; + __u32 dst_off; + __u32 nr; + __u32 pad[3]; }; struct io_uring_buf { diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 7ad91f180566..289866315ecf 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -927,10 +927,11 @@ int io_import_fixed(int ddir, struct iov_iter *iter, return 0; } -static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx) +static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx, + struct io_uring_clone_buffers *arg) { + int i, ret, nbufs, off, nr; struct io_rsrc_data data; - int i, ret, nbufs; /* * Drop our own lock here. We'll setup the data we need and reference @@ -943,11 +944,29 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx nbufs = src_ctx->buf_table.nr; if (!nbufs) goto out_unlock; - ret = io_rsrc_data_alloc(&data, nbufs); + ret = -EINVAL; + if (!arg->nr) + arg->nr = nbufs; + else if (arg->nr > nbufs) + goto out_unlock; + ret = -EOVERFLOW; + if (check_add_overflow(arg->nr, arg->src_off, &off)) + goto out_unlock; + if (off > nbufs) + goto out_unlock; + if (check_add_overflow(arg->nr, arg->dst_off, &off)) + goto out_unlock; + ret = -EINVAL; + if (off > IORING_MAX_REG_BUFFERS) + goto out_unlock; + ret = io_rsrc_data_alloc(&data, off); if (ret) goto out_unlock; - for (i = 0; i < nbufs; i++) { + off = arg->dst_off; + i = arg->src_off; + nr = arg->nr; + while (nr--) { struct io_rsrc_node *dst_node, *src_node; src_node = io_rsrc_node_lookup(&src_ctx->buf_table, i); @@ -963,7 +982,8 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx refcount_inc(&src_node->buf->refs); dst_node->buf = src_node->buf; } - data.nodes[i] = dst_node; + data.nodes[off++] = dst_node; + i++; } /* Have a ref on the bufs now, drop src lock and re-grab our own lock */ @@ -1018,7 +1038,7 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) file = io_uring_register_get_file(buf.src_fd, registered_src); if (IS_ERR(file)) return PTR_ERR(file); - ret = io_clone_buffers(ctx, file->private_data); + ret = io_clone_buffers(ctx, file->private_data, &buf); if (!registered_src) fput(file); return ret; From c1329532d5aabecf79788924941afb8a7b7c1024 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 29 Oct 2024 07:50:56 -0600 Subject: [PATCH 52/79] io_uring/rsrc: allow cloning with node replacements Currently cloning a buffer table will fail if the destination already has a table. But it should be possible to use it to replace existing elements. Add a IORING_REGISTER_DST_REPLACE cloning flag, which if set, will allow the destination to already having a buffer table. If that is the case, then entries designated by offset + nr buffers will be replaced if they already exist. Note that it's allowed to use IORING_REGISTER_DST_REPLACE and not have an existing table, in which case it'll work just like not having the flag set and an empty table - it'll just assign the newly created table for that case. Signed-off-by: Jens Axboe --- include/uapi/linux/io_uring.h | 3 +- io_uring/rsrc.c | 66 +++++++++++++++++++++++++++-------- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index cc8dbe78c126..ce58c4590de6 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -713,7 +713,8 @@ struct io_uring_clock_register { }; enum { - IORING_REGISTER_SRC_REGISTERED = 1, + IORING_REGISTER_SRC_REGISTERED = (1U << 0), + IORING_REGISTER_DST_REPLACE = (1U << 1), }; struct io_uring_clone_buffers { diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 289866315ecf..60fa857985cb 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -930,8 +930,40 @@ int io_import_fixed(int ddir, struct iov_iter *iter, static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx, struct io_uring_clone_buffers *arg) { - int i, ret, nbufs, off, nr; struct io_rsrc_data data; + int i, ret, off, nr; + unsigned int nbufs; + + /* if offsets are given, must have nr specified too */ + if (!arg->nr && (arg->dst_off || arg->src_off)) + return -EINVAL; + /* not allowed unless REPLACE is set */ + if (ctx->buf_table.nr && !(arg->flags & IORING_REGISTER_DST_REPLACE)) + return -EBUSY; + + nbufs = READ_ONCE(src_ctx->buf_table.nr); + if (!arg->nr) + arg->nr = nbufs; + else if (arg->nr > nbufs) + return -EINVAL; + else if (arg->nr > IORING_MAX_REG_BUFFERS) + return -EINVAL; + if (check_add_overflow(arg->nr, arg->dst_off, &nbufs)) + return -EOVERFLOW; + + ret = io_rsrc_data_alloc(&data, max(nbufs, ctx->buf_table.nr)); + if (ret) + return ret; + + /* Fill entries in data from dst that won't overlap with src */ + for (i = 0; i < min(arg->dst_off, ctx->buf_table.nr); i++) { + struct io_rsrc_node *src_node = ctx->buf_table.nodes[i]; + + if (src_node) { + data.nodes[i] = src_node; + src_node->refs++; + } + } /* * Drop our own lock here. We'll setup the data we need and reference @@ -954,14 +986,6 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx goto out_unlock; if (off > nbufs) goto out_unlock; - if (check_add_overflow(arg->nr, arg->dst_off, &off)) - goto out_unlock; - ret = -EINVAL; - if (off > IORING_MAX_REG_BUFFERS) - goto out_unlock; - ret = io_rsrc_data_alloc(&data, off); - if (ret) - goto out_unlock; off = arg->dst_off; i = arg->src_off; @@ -989,6 +1013,20 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx /* Have a ref on the bufs now, drop src lock and re-grab our own lock */ mutex_unlock(&src_ctx->uring_lock); mutex_lock(&ctx->uring_lock); + + /* + * If asked for replace, put the old table. data->nodes[] holds both + * old and new nodes at this point. + */ + if (arg->flags & IORING_REGISTER_DST_REPLACE) + io_rsrc_data_free(&ctx->buf_table); + + /* + * ctx->buf_table should be empty now - either the contents are being + * replaced and we just freed the table, or someone raced setting up + * a buffer table while the clone was happening. If not empty, fall + * through to failure handling. + */ if (!ctx->buf_table.nr) { ctx->buf_table = data; return 0; @@ -998,14 +1036,14 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx mutex_lock(&src_ctx->uring_lock); /* someone raced setting up buffers, dump ours */ ret = -EBUSY; - i = nbufs; out_put_free: + i = data.nr; while (i--) { io_buffer_unmap(src_ctx, data.nodes[i]); kfree(data.nodes[i]); } - io_rsrc_data_free(&data); out_unlock: + io_rsrc_data_free(&data); mutex_unlock(&src_ctx->uring_lock); mutex_lock(&ctx->uring_lock); return ret; @@ -1025,12 +1063,12 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) struct file *file; int ret; - if (ctx->buf_table.nr) - return -EBUSY; if (copy_from_user(&buf, arg, sizeof(buf))) return -EFAULT; - if (buf.flags & ~IORING_REGISTER_SRC_REGISTERED) + if (buf.flags & ~(IORING_REGISTER_SRC_REGISTERED|IORING_REGISTER_DST_REPLACE)) return -EINVAL; + if (!(buf.flags & IORING_REGISTER_DST_REPLACE) && ctx->buf_table.nr) + return -EBUSY; if (memchr_inv(buf.pad, 0, sizeof(buf.pad))) return -EINVAL; From 01ee194d1aba1202f0926d5047a2a4cf84d0e45d Mon Sep 17 00:00:00 2001 From: hexue Date: Fri, 1 Nov 2024 17:19:57 +0800 Subject: [PATCH 53/79] io_uring: add support for hybrid IOPOLL A new hybrid poll is implemented on the io_uring layer. Once an IO is issued, it will not poll immediately, but rather block first and re-run before IO complete, then poll to reap IO. While this poll method could be a suboptimal solution when running on a single thread, it offers performance lower than regular polling but higher than IRQ, and CPU utilization is also lower than polling. To use hybrid polling, the ring must be setup with both the IORING_SETUP_IOPOLL and IORING_SETUP_HYBRID)IOPOLL flags set. Hybrid polling has the same restrictions as IOPOLL, in that commands must explicitly support it. Signed-off-by: hexue Link: https://lore.kernel.org/r/20241101091957.564220-2-xue01.he@samsung.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 19 ++++++- include/uapi/linux/io_uring.h | 3 ++ io_uring/io_uring.c | 8 ++- io_uring/rw.c | 92 ++++++++++++++++++++++++++++++---- 4 files changed, 108 insertions(+), 14 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 77fd508d043a..d52fec533c51 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -298,6 +298,11 @@ struct io_ring_ctx { * ->uring_cmd() by io_uring_cmd_insert_cancelable() */ struct hlist_head cancelable_uring_cmd; + /* + * For Hybrid IOPOLL, runtime in hybrid polling, without + * scheduling time + */ + u64 hybrid_poll_time; } ____cacheline_aligned_in_smp; struct { @@ -449,6 +454,7 @@ enum { REQ_F_LINK_TIMEOUT_BIT, REQ_F_NEED_CLEANUP_BIT, REQ_F_POLLED_BIT, + REQ_F_HYBRID_IOPOLL_STATE_BIT, REQ_F_BUFFER_SELECTED_BIT, REQ_F_BUFFER_RING_BIT, REQ_F_REISSUE_BIT, @@ -507,6 +513,8 @@ enum { REQ_F_NEED_CLEANUP = IO_REQ_FLAG(REQ_F_NEED_CLEANUP_BIT), /* already went through poll handler */ REQ_F_POLLED = IO_REQ_FLAG(REQ_F_POLLED_BIT), + /* every req only blocks once in hybrid poll */ + REQ_F_IOPOLL_STATE = IO_REQ_FLAG(REQ_F_HYBRID_IOPOLL_STATE_BIT), /* buffer already selected */ REQ_F_BUFFER_SELECTED = IO_REQ_FLAG(REQ_F_BUFFER_SELECTED_BIT), /* buffer selected from ring, needs commit */ @@ -639,8 +647,15 @@ struct io_kiocb { atomic_t refs; bool cancel_seq_set; struct io_task_work io_task_work; - /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */ - struct hlist_node hash_node; + union { + /* + * for polled requests, i.e. IORING_OP_POLL_ADD and async armed + * poll + */ + struct hlist_node hash_node; + /* For IOPOLL setup queues, with hybrid polling */ + u64 iopoll_start; + }; /* internal polling, see IORING_FEAT_FAST_POLL */ struct async_poll *apoll; /* opcode allocated if it needs to store data for async defer */ diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index ce58c4590de6..47977a5c65f5 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -200,6 +200,9 @@ enum io_uring_sqe_flags_bit { */ #define IORING_SETUP_NO_SQARRAY (1U << 16) +/* Use hybrid poll in iopoll process */ +#define IORING_SETUP_HYBRID_IOPOLL (1U << 17) + enum io_uring_op { IORING_OP_NOP, IORING_OP_READV, diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 44a772013c09..f08ea7fd5998 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -307,6 +307,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) goto err; ctx->flags = p->flags; + ctx->hybrid_poll_time = LLONG_MAX; atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT); init_waitqueue_head(&ctx->sqo_sq_wait); INIT_LIST_HEAD(&ctx->sqd_list); @@ -3630,6 +3631,11 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) static_branch_inc(&io_key_has_sqarray); + /* HYBRID_IOPOLL only valid with IOPOLL */ + if ((ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_HYBRID_IOPOLL)) == + IORING_SETUP_HYBRID_IOPOLL) + return -EINVAL; + if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) && !(ctx->flags & IORING_SETUP_IOPOLL) && !(ctx->flags & IORING_SETUP_SQPOLL)) @@ -3785,7 +3791,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) IORING_SETUP_SQE128 | IORING_SETUP_CQE32 | IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_NO_MMAP | IORING_SETUP_REGISTERED_FD_ONLY | - IORING_SETUP_NO_SQARRAY)) + IORING_SETUP_NO_SQARRAY | IORING_SETUP_HYBRID_IOPOLL)) return -EINVAL; return io_uring_create(entries, &p, params); diff --git a/io_uring/rw.c b/io_uring/rw.c index 30448f343c7f..1ea6be2ccc90 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -817,6 +817,11 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode, int rw_type) kiocb->ki_flags |= IOCB_HIPRI; kiocb->ki_complete = io_complete_rw_iopoll; req->iopoll_completed = 0; + if (ctx->flags & IORING_SETUP_HYBRID_IOPOLL) { + /* make sure every req only blocks once*/ + req->flags &= ~REQ_F_IOPOLL_STATE; + req->iopoll_start = ktime_get_ns(); + } } else { if (kiocb->ki_flags & IOCB_HIPRI) return -EINVAL; @@ -1115,6 +1120,78 @@ void io_rw_fail(struct io_kiocb *req) io_req_set_res(req, res, req->cqe.flags); } +static int io_uring_classic_poll(struct io_kiocb *req, struct io_comp_batch *iob, + unsigned int poll_flags) +{ + struct file *file = req->file; + + if (req->opcode == IORING_OP_URING_CMD) { + struct io_uring_cmd *ioucmd; + + ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); + return file->f_op->uring_cmd_iopoll(ioucmd, iob, poll_flags); + } else { + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); + + return file->f_op->iopoll(&rw->kiocb, iob, poll_flags); + } +} + +static u64 io_hybrid_iopoll_delay(struct io_ring_ctx *ctx, struct io_kiocb *req) +{ + struct hrtimer_sleeper timer; + enum hrtimer_mode mode; + ktime_t kt; + u64 sleep_time; + + if (req->flags & REQ_F_IOPOLL_STATE) + return 0; + + if (ctx->hybrid_poll_time == LLONG_MAX) + return 0; + + /* Using half the running time to do schedule */ + sleep_time = ctx->hybrid_poll_time / 2; + + kt = ktime_set(0, sleep_time); + req->flags |= REQ_F_IOPOLL_STATE; + + mode = HRTIMER_MODE_REL; + hrtimer_init_sleeper_on_stack(&timer, CLOCK_MONOTONIC, mode); + hrtimer_set_expires(&timer.timer, kt); + set_current_state(TASK_INTERRUPTIBLE); + hrtimer_sleeper_start_expires(&timer, mode); + + if (timer.task) + io_schedule(); + + hrtimer_cancel(&timer.timer); + __set_current_state(TASK_RUNNING); + destroy_hrtimer_on_stack(&timer.timer); + return sleep_time; +} + +static int io_uring_hybrid_poll(struct io_kiocb *req, + struct io_comp_batch *iob, unsigned int poll_flags) +{ + struct io_ring_ctx *ctx = req->ctx; + u64 runtime, sleep_time; + int ret; + + sleep_time = io_hybrid_iopoll_delay(ctx, req); + ret = io_uring_classic_poll(req, iob, poll_flags); + runtime = ktime_get_ns() - req->iopoll_start - sleep_time; + + /* + * Use minimum sleep time if we're polling devices with different + * latencies. We could get more completions from the faster ones. + */ + if (ctx->hybrid_poll_time > runtime) + ctx->hybrid_poll_time = runtime; + + return ret; +} + int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) { struct io_wq_work_node *pos, *start, *prev; @@ -1131,7 +1208,6 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) wq_list_for_each(pos, start, &ctx->iopoll_list) { struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list); - struct file *file = req->file; int ret; /* @@ -1142,17 +1218,11 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) if (READ_ONCE(req->iopoll_completed)) break; - if (req->opcode == IORING_OP_URING_CMD) { - struct io_uring_cmd *ioucmd; + if (ctx->flags & IORING_SETUP_HYBRID_IOPOLL) + ret = io_uring_hybrid_poll(req, &iob, poll_flags); + else + ret = io_uring_classic_poll(req, &iob, poll_flags); - ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); - ret = file->f_op->uring_cmd_iopoll(ioucmd, &iob, - poll_flags); - } else { - struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); - - ret = file->f_op->iopoll(&rw->kiocb, &iob, poll_flags); - } if (unlikely(ret < 0)) return ret; else if (ret) From 6af82f7614a2e31e7ef23e5e160697aef31e8edd Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 3 Nov 2024 08:17:28 -0700 Subject: [PATCH 54/79] io_uring/rsrc: encode node type and ctx together Rather than keep the type field separate rom ctx, use the fact that we can encode up to 4 types of nodes in the LSB of the ctx pointer. Doesn't reclaim any space right now on 64-bit archs, but it leaves a full int for future use. Signed-off-by: Jens Axboe --- io_uring/rsrc.c | 11 +++++------ io_uring/rsrc.h | 17 ++++++++++++++--- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 60fa857985cb..2fb1791d7255 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -124,9 +124,8 @@ struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type) node = kzalloc(sizeof(*node), GFP_KERNEL); if (node) { - node->ctx = ctx; + node->ctx_ptr = (unsigned long) ctx | type; node->refs = 1; - node->type = type; } return node; } @@ -445,21 +444,21 @@ int io_files_update(struct io_kiocb *req, unsigned int issue_flags) void io_free_rsrc_node(struct io_rsrc_node *node) { - struct io_ring_ctx *ctx = node->ctx; + struct io_ring_ctx *ctx = io_rsrc_node_ctx(node); lockdep_assert_held(&ctx->uring_lock); if (node->tag) - io_post_aux_cqe(node->ctx, node->tag, 0, 0); + io_post_aux_cqe(ctx, node->tag, 0, 0); - switch (node->type) { + switch (io_rsrc_node_type(node)) { case IORING_RSRC_FILE: if (io_slot_file(node)) fput(io_slot_file(node)); break; case IORING_RSRC_BUFFER: if (node->buf) - io_buffer_unmap(node->ctx, node); + io_buffer_unmap(ctx, node); break; default: WARN_ON_ONCE(1); diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index a40fad783a69..9a8fac31fa49 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -11,12 +11,13 @@ enum { IORING_RSRC_FILE = 0, IORING_RSRC_BUFFER = 1, + + IORING_RSRC_TYPE_MASK = 0x3UL, }; struct io_rsrc_node { - struct io_ring_ctx *ctx; + unsigned long ctx_ptr; int refs; - u16 type; u64 tag; union { @@ -100,11 +101,21 @@ static inline void io_req_put_rsrc_nodes(struct io_kiocb *req) req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL; } +static inline struct io_ring_ctx *io_rsrc_node_ctx(struct io_rsrc_node *node) +{ + return (struct io_ring_ctx *) (node->ctx_ptr & ~IORING_RSRC_TYPE_MASK); +} + +static inline int io_rsrc_node_type(struct io_rsrc_node *node) +{ + return node->ctx_ptr & IORING_RSRC_TYPE_MASK; +} + static inline void io_req_assign_rsrc_node(struct io_kiocb *req, struct io_rsrc_node *node) { node->refs++; - req->rsrc_nodes[node->type] = node; + req->rsrc_nodes[io_rsrc_node_type(node)] = node; } int io_files_update(struct io_kiocb *req, unsigned int issue_flags); From 6f94cbc29adacc15007c5a16295052e674099282 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 3 Nov 2024 08:46:07 -0700 Subject: [PATCH 55/79] io_uring/rsrc: split io_kiocb node type assignments Currently the io_rsrc_node assignment in io_kiocb is an array of two pointers, as two nodes may be assigned to a request - one file node, and one buffer node. However, the buffer node can co-exist with the provided buffers, as currently it's not supported to use both provided and registered buffers at the same time. This crucially brings struct io_kiocb down to 4 cache lines again, as before it spilled into the 5th cacheline. Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 7 ++++++- io_uring/io_uring.c | 6 +++--- io_uring/net.c | 3 ++- io_uring/nop.c | 3 ++- io_uring/notif.c | 4 ++-- io_uring/rsrc.h | 16 ++++++++++------ io_uring/rw.c | 3 ++- io_uring/uring_cmd.c | 4 ++-- 8 files changed, 29 insertions(+), 17 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index d52fec533c51..01e7fb9fcfe2 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -475,6 +475,7 @@ enum { REQ_F_BL_EMPTY_BIT, REQ_F_BL_NO_RECYCLE_BIT, REQ_F_BUFFERS_COMMIT_BIT, + REQ_F_BUF_NODE_BIT, /* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT, @@ -553,6 +554,8 @@ enum { REQ_F_BL_NO_RECYCLE = IO_REQ_FLAG(REQ_F_BL_NO_RECYCLE_BIT), /* buffer ring head needs incrementing on put */ REQ_F_BUFFERS_COMMIT = IO_REQ_FLAG(REQ_F_BUFFERS_COMMIT_BIT), + /* buf node is valid */ + REQ_F_BUF_NODE = IO_REQ_FLAG(REQ_F_BUF_NODE_BIT), }; typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts); @@ -633,6 +636,8 @@ struct io_kiocb { * REQ_F_BUFFER_RING is set. */ struct io_buffer_list *buf_list; + + struct io_rsrc_node *buf_node; }; union { @@ -642,7 +647,7 @@ struct io_kiocb { __poll_t apoll_events; }; - struct io_rsrc_node *rsrc_nodes[2]; + struct io_rsrc_node *file_node; atomic_t refs; bool cancel_seq_set; diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index f08ea7fd5998..5bab8a3b0456 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -948,8 +948,8 @@ void io_req_defer_failed(struct io_kiocb *req, s32 res) static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx) { req->ctx = ctx; - req->rsrc_nodes[IORING_RSRC_FILE] = NULL; - req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL; + req->buf_node = NULL; + req->file_node = NULL; req->link = NULL; req->async_data = NULL; /* not necessary, but safer to zero */ @@ -1882,7 +1882,7 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, io_ring_submit_lock(ctx, issue_flags); node = io_rsrc_node_lookup(&ctx->file_table.data, fd); if (node) { - io_req_assign_rsrc_node(req, node); + io_req_assign_rsrc_node(&req->file_node, node); req->flags |= io_slot_flags(node); file = io_slot_file(node); } diff --git a/io_uring/net.c b/io_uring/net.c index 2f7b334ed708..2ccc2b409431 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -1348,7 +1348,8 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags) io_ring_submit_lock(ctx, issue_flags); node = io_rsrc_node_lookup(&ctx->buf_table, sr->buf_index); if (node) { - io_req_assign_rsrc_node(sr->notif, node); + io_req_assign_rsrc_node(&sr->notif->buf_node, node); + sr->notif->flags |= REQ_F_BUF_NODE; ret = 0; } io_ring_submit_unlock(ctx, issue_flags); diff --git a/io_uring/nop.c b/io_uring/nop.c index 149dbdc53607..bc22bcc739f3 100644 --- a/io_uring/nop.c +++ b/io_uring/nop.c @@ -67,7 +67,8 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags) io_ring_submit_lock(ctx, issue_flags); node = io_rsrc_node_lookup(&ctx->buf_table, nop->buffer); if (node) { - io_req_assign_rsrc_node(req, node); + io_req_assign_rsrc_node(&req->buf_node, node); + req->flags |= REQ_F_BUF_NODE; ret = 0; } io_ring_submit_unlock(ctx, issue_flags); diff --git a/io_uring/notif.c b/io_uring/notif.c index 4f02e969cf08..8dfbb0bd8e4d 100644 --- a/io_uring/notif.c +++ b/io_uring/notif.c @@ -117,8 +117,8 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) notif->file = NULL; notif->task = current; io_get_task_refs(1); - notif->rsrc_nodes[IORING_RSRC_FILE] = NULL; - notif->rsrc_nodes[IORING_RSRC_BUFFER] = NULL; + notif->file_node = NULL; + notif->buf_node = NULL; nd = io_notif_to_data(notif); nd->zc_report = false; diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 9a8fac31fa49..bc3a863b14bb 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -95,10 +95,14 @@ static inline bool io_reset_rsrc_node(struct io_rsrc_data *data, int index) static inline void io_req_put_rsrc_nodes(struct io_kiocb *req) { - io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_FILE]); - io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_BUFFER]); - req->rsrc_nodes[IORING_RSRC_FILE] = NULL; - req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL; + if (req->file_node) { + io_put_rsrc_node(req->file_node); + req->file_node = NULL; + } + if (req->flags & REQ_F_BUF_NODE) { + io_put_rsrc_node(req->buf_node); + req->buf_node = NULL; + } } static inline struct io_ring_ctx *io_rsrc_node_ctx(struct io_rsrc_node *node) @@ -111,11 +115,11 @@ static inline int io_rsrc_node_type(struct io_rsrc_node *node) return node->ctx_ptr & IORING_RSRC_TYPE_MASK; } -static inline void io_req_assign_rsrc_node(struct io_kiocb *req, +static inline void io_req_assign_rsrc_node(struct io_rsrc_node **dst_node, struct io_rsrc_node *node) { node->refs++; - req->rsrc_nodes[io_rsrc_node_type(node)] = node; + *dst_node = node; } int io_files_update(struct io_kiocb *req, unsigned int issue_flags); diff --git a/io_uring/rw.c b/io_uring/rw.c index 1ea6be2ccc90..144730344c0f 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -341,7 +341,8 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index); if (!node) return -EFAULT; - io_req_assign_rsrc_node(req, node); + io_req_assign_rsrc_node(&req->buf_node, node); + req->flags |= REQ_F_BUF_NODE; io = req->async_data; ret = io_import_fixed(ddir, &io->iter, node->buf, rw->addr, rw->len); diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 535909a38e76..88a73d21fc0b 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -219,7 +219,7 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) * being called. This prevents destruction of the mapped buffer * we'll need at actual import time. */ - io_req_assign_rsrc_node(req, node); + io_req_assign_rsrc_node(&req->buf_node, node); } ioucmd->cmd_op = READ_ONCE(sqe->cmd_op); @@ -275,7 +275,7 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, struct iov_iter *iter, void *ioucmd) { struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); - struct io_rsrc_node *node = req->rsrc_nodes[IORING_RSRC_BUFFER]; + struct io_rsrc_node *node = req->buf_node; /* Must have had rsrc_node assigned at prep time */ if (node) From f03baece08188f2e239c0ca0c098c14c71739ffb Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 3 Nov 2024 10:22:43 -0700 Subject: [PATCH 56/79] io_uring: move cancelations to be io_uring_task based Right now the task_struct pointer is used as the key to match a task, but in preparation for some io_kiocb changes, move it to using struct io_uring_task instead. No functional changes intended in this patch. Signed-off-by: Jens Axboe --- io_uring/futex.c | 4 ++-- io_uring/futex.h | 4 ++-- io_uring/io_uring.c | 42 +++++++++++++++++++++--------------------- io_uring/io_uring.h | 2 +- io_uring/poll.c | 4 ++-- io_uring/poll.h | 2 +- io_uring/timeout.c | 8 ++++---- io_uring/timeout.h | 2 +- io_uring/uring_cmd.c | 4 ++-- io_uring/uring_cmd.h | 2 +- io_uring/waitid.c | 4 ++-- io_uring/waitid.h | 2 +- 12 files changed, 40 insertions(+), 40 deletions(-) diff --git a/io_uring/futex.c b/io_uring/futex.c index 914848f46beb..e29662f039e1 100644 --- a/io_uring/futex.c +++ b/io_uring/futex.c @@ -141,7 +141,7 @@ int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, return -ENOENT; } -bool io_futex_remove_all(struct io_ring_ctx *ctx, struct task_struct *task, +bool io_futex_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, bool cancel_all) { struct hlist_node *tmp; @@ -151,7 +151,7 @@ bool io_futex_remove_all(struct io_ring_ctx *ctx, struct task_struct *task, lockdep_assert_held(&ctx->uring_lock); hlist_for_each_entry_safe(req, tmp, &ctx->futex_list, hash_node) { - if (!io_match_task_safe(req, task, cancel_all)) + if (!io_match_task_safe(req, tctx, cancel_all)) continue; hlist_del_init(&req->hash_node); __io_futex_cancel(ctx, req); diff --git a/io_uring/futex.h b/io_uring/futex.h index b8bb09873d57..d789fcf715e3 100644 --- a/io_uring/futex.h +++ b/io_uring/futex.h @@ -11,7 +11,7 @@ int io_futex_wake(struct io_kiocb *req, unsigned int issue_flags); #if defined(CONFIG_FUTEX) int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, unsigned int issue_flags); -bool io_futex_remove_all(struct io_ring_ctx *ctx, struct task_struct *task, +bool io_futex_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, bool cancel_all); bool io_futex_cache_init(struct io_ring_ctx *ctx); void io_futex_cache_free(struct io_ring_ctx *ctx); @@ -23,7 +23,7 @@ static inline int io_futex_cancel(struct io_ring_ctx *ctx, return 0; } static inline bool io_futex_remove_all(struct io_ring_ctx *ctx, - struct task_struct *task, bool cancel_all) + struct io_uring_task *tctx, bool cancel_all) { return false; } diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 5bab8a3b0456..4a2282c85464 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -142,7 +142,7 @@ struct io_defer_entry { #define IO_CQ_WAKE_FORCE (IO_CQ_WAKE_INIT >> 1) static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, - struct task_struct *task, + struct io_uring_task *tctx, bool cancel_all); static void io_queue_sqe(struct io_kiocb *req); @@ -201,12 +201,12 @@ static bool io_match_linked(struct io_kiocb *head) * As io_match_task() but protected against racing with linked timeouts. * User must not hold timeout_lock. */ -bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task, +bool io_match_task_safe(struct io_kiocb *head, struct io_uring_task *tctx, bool cancel_all) { bool matched; - if (task && head->task != task) + if (tctx && head->task->io_uring != tctx) return false; if (cancel_all) return true; @@ -2987,7 +2987,7 @@ static int io_uring_release(struct inode *inode, struct file *file) } struct io_task_cancel { - struct task_struct *task; + struct io_uring_task *tctx; bool all; }; @@ -2996,11 +2996,11 @@ static bool io_cancel_task_cb(struct io_wq_work *work, void *data) struct io_kiocb *req = container_of(work, struct io_kiocb, work); struct io_task_cancel *cancel = data; - return io_match_task_safe(req, cancel->task, cancel->all); + return io_match_task_safe(req, cancel->tctx, cancel->all); } static __cold bool io_cancel_defer_files(struct io_ring_ctx *ctx, - struct task_struct *task, + struct io_uring_task *tctx, bool cancel_all) { struct io_defer_entry *de; @@ -3008,7 +3008,7 @@ static __cold bool io_cancel_defer_files(struct io_ring_ctx *ctx, spin_lock(&ctx->completion_lock); list_for_each_entry_reverse(de, &ctx->defer_list, list) { - if (io_match_task_safe(de->req, task, cancel_all)) { + if (io_match_task_safe(de->req, tctx, cancel_all)) { list_cut_position(&list, &ctx->defer_list, &de->list); break; } @@ -3051,11 +3051,10 @@ static __cold bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx) } static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, - struct task_struct *task, + struct io_uring_task *tctx, bool cancel_all) { - struct io_task_cancel cancel = { .task = task, .all = cancel_all, }; - struct io_uring_task *tctx = task ? task->io_uring : NULL; + struct io_task_cancel cancel = { .tctx = tctx, .all = cancel_all, }; enum io_wq_cancel cret; bool ret = false; @@ -3069,9 +3068,9 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, if (!ctx->rings) return false; - if (!task) { + if (!tctx) { ret |= io_uring_try_cancel_iowq(ctx); - } else if (tctx && tctx->io_wq) { + } else if (tctx->io_wq) { /* * Cancels requests of all rings, not only @ctx, but * it's fine as the task is in exit/exec. @@ -3094,15 +3093,15 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) && io_allowed_defer_tw_run(ctx)) ret |= io_run_local_work(ctx, INT_MAX) > 0; - ret |= io_cancel_defer_files(ctx, task, cancel_all); + ret |= io_cancel_defer_files(ctx, tctx, cancel_all); mutex_lock(&ctx->uring_lock); - ret |= io_poll_remove_all(ctx, task, cancel_all); - ret |= io_waitid_remove_all(ctx, task, cancel_all); - ret |= io_futex_remove_all(ctx, task, cancel_all); - ret |= io_uring_try_cancel_uring_cmd(ctx, task, cancel_all); + ret |= io_poll_remove_all(ctx, tctx, cancel_all); + ret |= io_waitid_remove_all(ctx, tctx, cancel_all); + ret |= io_futex_remove_all(ctx, tctx, cancel_all); + ret |= io_uring_try_cancel_uring_cmd(ctx, tctx, cancel_all); mutex_unlock(&ctx->uring_lock); - ret |= io_kill_timeouts(ctx, task, cancel_all); - if (task) + ret |= io_kill_timeouts(ctx, tctx, cancel_all); + if (tctx) ret |= io_run_task_work() > 0; else ret |= flush_delayed_work(&ctx->fallback_work); @@ -3155,12 +3154,13 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd) if (node->ctx->sq_data) continue; loop |= io_uring_try_cancel_requests(node->ctx, - current, cancel_all); + current->io_uring, + cancel_all); } } else { list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) loop |= io_uring_try_cancel_requests(ctx, - current, + current->io_uring, cancel_all); } diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index e3e6cb14de5d..17ffdb1e41c5 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -115,7 +115,7 @@ void io_queue_next(struct io_kiocb *req); void io_task_refs_refill(struct io_uring_task *tctx); bool __io_alloc_req_refill(struct io_ring_ctx *ctx); -bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task, +bool io_match_task_safe(struct io_kiocb *head, struct io_uring_task *tctx, bool cancel_all); void io_activate_pollwq(struct io_ring_ctx *ctx); diff --git a/io_uring/poll.c b/io_uring/poll.c index 2d6698fb7400..7db3010b5733 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -714,7 +714,7 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) /* * Returns true if we found and killed one or more poll requests */ -__cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, +__cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, bool cancel_all) { unsigned nr_buckets = 1U << ctx->cancel_table.hash_bits; @@ -729,7 +729,7 @@ __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i]; hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) { - if (io_match_task_safe(req, tsk, cancel_all)) { + if (io_match_task_safe(req, tctx, cancel_all)) { hlist_del_init(&req->hash_node); io_poll_cancel_req(req); found = true; diff --git a/io_uring/poll.h b/io_uring/poll.h index b0e3745f5a29..04ede93113dc 100644 --- a/io_uring/poll.h +++ b/io_uring/poll.h @@ -40,7 +40,7 @@ struct io_cancel_data; int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, unsigned issue_flags); int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags); -bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, +bool io_poll_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, bool cancel_all); void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts); diff --git a/io_uring/timeout.c b/io_uring/timeout.c index 9973876d91b0..18286cb53a69 100644 --- a/io_uring/timeout.c +++ b/io_uring/timeout.c @@ -637,13 +637,13 @@ void io_queue_linked_timeout(struct io_kiocb *req) io_put_req(req); } -static bool io_match_task(struct io_kiocb *head, struct task_struct *task, +static bool io_match_task(struct io_kiocb *head, struct io_uring_task *tctx, bool cancel_all) __must_hold(&head->ctx->timeout_lock) { struct io_kiocb *req; - if (task && head->task != task) + if (tctx && head->task->io_uring != tctx) return false; if (cancel_all) return true; @@ -656,7 +656,7 @@ static bool io_match_task(struct io_kiocb *head, struct task_struct *task, } /* Returns true if we found and killed one or more timeouts */ -__cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, +__cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct io_uring_task *tctx, bool cancel_all) { struct io_timeout *timeout, *tmp; @@ -671,7 +671,7 @@ __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { struct io_kiocb *req = cmd_to_io_kiocb(timeout); - if (io_match_task(req, tsk, cancel_all) && + if (io_match_task(req, tctx, cancel_all) && io_kill_timeout(req, -ECANCELED)) canceled++; } diff --git a/io_uring/timeout.h b/io_uring/timeout.h index a6939f18313e..e91b32448dcf 100644 --- a/io_uring/timeout.h +++ b/io_uring/timeout.h @@ -24,7 +24,7 @@ static inline struct io_kiocb *io_disarm_linked_timeout(struct io_kiocb *req) __cold void io_flush_timeouts(struct io_ring_ctx *ctx); struct io_cancel_data; int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd); -__cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, +__cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct io_uring_task *tctx, bool cancel_all); void io_queue_linked_timeout(struct io_kiocb *req); void io_disarm_next(struct io_kiocb *req); diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 88a73d21fc0b..f88fbc9869d0 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -47,7 +47,7 @@ static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags) } bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx, - struct task_struct *task, bool cancel_all) + struct io_uring_task *tctx, bool cancel_all) { struct hlist_node *tmp; struct io_kiocb *req; @@ -61,7 +61,7 @@ bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx, struct io_uring_cmd); struct file *file = req->file; - if (!cancel_all && req->task != task) + if (!cancel_all && req->task->io_uring != tctx) continue; if (cmd->flags & IORING_URING_CMD_CANCELABLE) { diff --git a/io_uring/uring_cmd.h b/io_uring/uring_cmd.h index a361f98664d2..7dba0f1efc58 100644 --- a/io_uring/uring_cmd.h +++ b/io_uring/uring_cmd.h @@ -8,4 +8,4 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags); int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx, - struct task_struct *task, bool cancel_all); + struct io_uring_task *tctx, bool cancel_all); diff --git a/io_uring/waitid.c b/io_uring/waitid.c index 6362ec20abc0..9b7c23f96c47 100644 --- a/io_uring/waitid.c +++ b/io_uring/waitid.c @@ -184,7 +184,7 @@ int io_waitid_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, return -ENOENT; } -bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct task_struct *task, +bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, bool cancel_all) { struct hlist_node *tmp; @@ -194,7 +194,7 @@ bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct task_struct *task, lockdep_assert_held(&ctx->uring_lock); hlist_for_each_entry_safe(req, tmp, &ctx->waitid_list, hash_node) { - if (!io_match_task_safe(req, task, cancel_all)) + if (!io_match_task_safe(req, tctx, cancel_all)) continue; hlist_del_init(&req->hash_node); __io_waitid_cancel(ctx, req); diff --git a/io_uring/waitid.h b/io_uring/waitid.h index 956a8adafe8c..d5544aaf302a 100644 --- a/io_uring/waitid.h +++ b/io_uring/waitid.h @@ -11,5 +11,5 @@ int io_waitid_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_waitid(struct io_kiocb *req, unsigned int issue_flags); int io_waitid_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, unsigned int issue_flags); -bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct task_struct *task, +bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, bool cancel_all); From 6ed368cc5d5d255ffffad33cfa02ecf2b77b7c44 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 2 Nov 2024 21:26:16 -0600 Subject: [PATCH 57/79] io_uring: remove task ref helpers They are only used right where they are defined, just open-code them inside io_put_task(). Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 4a2282c85464..43afd9da7d07 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -677,30 +677,19 @@ static void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx) mutex_unlock(&ctx->uring_lock); } -/* can be called by any task */ -static void io_put_task_remote(struct task_struct *task) -{ - struct io_uring_task *tctx = task->io_uring; - - percpu_counter_sub(&tctx->inflight, 1); - if (unlikely(atomic_read(&tctx->in_cancel))) - wake_up(&tctx->wait); - put_task_struct(task); -} - -/* used by a task to put its own references */ -static void io_put_task_local(struct task_struct *task) -{ - task->io_uring->cached_refs++; -} - /* must to be called somewhat shortly after putting a request */ static inline void io_put_task(struct task_struct *task) { - if (likely(task == current)) - io_put_task_local(task); - else - io_put_task_remote(task); + struct io_uring_task *tctx = task->io_uring; + + if (likely(task == current)) { + tctx->cached_refs++; + } else { + percpu_counter_sub(&tctx->inflight, 1); + if (unlikely(atomic_read(&tctx->in_cancel))) + wake_up(&tctx->wait); + put_task_struct(task); + } } void io_task_refs_refill(struct io_uring_task *tctx) From b6f58a3f4aa8dba424356c7a69388a81f4459300 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 3 Nov 2024 10:23:38 -0700 Subject: [PATCH 58/79] io_uring: move struct io_kiocb from task_struct to io_uring_task Rather than store the task_struct itself in struct io_kiocb, store the io_uring specific task_struct. The life times are the same in terms of io_uring, and this avoids doing some dereferences through the task_struct. For the hot path of putting local task references, we can deref req->tctx instead, which we'll need anyway in that function regardless of whether it's local or remote references. This is mostly straight forward, except the original task PF_EXITING check needs a bit of tweaking. task_work is _always_ run from the originating task, except in the fallback case, where it's run from a kernel thread. Replace the potentially racy (in case of fallback work) checks for req->task->flags with current->flags. It's either the still the original task, in which case PF_EXITING will be sane, or it has PF_KTHREAD set, in which case it's fallback work. Both cases should prevent moving forward with the given request. Signed-off-by: Jens Axboe --- include/linux/io_uring/cmd.h | 2 +- include/linux/io_uring_types.h | 3 ++- io_uring/cancel.c | 2 +- io_uring/fdinfo.c | 2 +- io_uring/io_uring.c | 34 +++++++++++++++------------------- io_uring/io_uring.h | 13 +++++++++++++ io_uring/msg_ring.c | 4 ++-- io_uring/notif.c | 4 ++-- io_uring/poll.c | 3 +-- io_uring/rw.c | 2 +- io_uring/tctx.c | 1 + io_uring/timeout.c | 10 ++++++---- io_uring/uring_cmd.c | 2 +- io_uring/waitid.c | 2 +- 14 files changed, 48 insertions(+), 36 deletions(-) diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h index c189d36ad55e..578a3fdf5c71 100644 --- a/include/linux/io_uring/cmd.h +++ b/include/linux/io_uring/cmd.h @@ -110,7 +110,7 @@ static inline void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd, static inline struct task_struct *io_uring_cmd_get_task(struct io_uring_cmd *cmd) { - return cmd_to_io_kiocb(cmd)->task; + return cmd_to_io_kiocb(cmd)->tctx->task; } #endif /* _LINUX_IO_URING_CMD_H */ diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 01e7fb9fcfe2..fba2988accc3 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -84,6 +84,7 @@ struct io_uring_task { /* submission side */ int cached_refs; const struct io_ring_ctx *last; + struct task_struct *task; struct io_wq *io_wq; struct file *registered_rings[IO_RINGFD_REG_MAX]; @@ -625,7 +626,7 @@ struct io_kiocb { struct io_cqe cqe; struct io_ring_ctx *ctx; - struct task_struct *task; + struct io_uring_task *tctx; union { /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */ diff --git a/io_uring/cancel.c b/io_uring/cancel.c index bbca5cb69cb5..484193567839 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -205,7 +205,7 @@ int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) .opcode = cancel->opcode, .seq = atomic_inc_return(&req->ctx->cancel_seq), }; - struct io_uring_task *tctx = req->task->io_uring; + struct io_uring_task *tctx = req->tctx; int ret; if (cd.flags & IORING_ASYNC_CANCEL_FD) { diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index 8da0d9e4533a..efbec34ccb18 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -203,7 +203,7 @@ __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *file) hlist_for_each_entry(req, &hb->list, hash_node) seq_printf(m, " op=%d, task_works=%d\n", req->opcode, - task_work_pending(req->task)); + task_work_pending(req->tctx->task)); } if (has_lock) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 43afd9da7d07..7c1ca36b117b 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -206,7 +206,7 @@ bool io_match_task_safe(struct io_kiocb *head, struct io_uring_task *tctx, { bool matched; - if (tctx && head->task->io_uring != tctx) + if (tctx && head->tctx != tctx) return false; if (cancel_all) return true; @@ -407,11 +407,8 @@ static void io_clean_op(struct io_kiocb *req) kfree(req->apoll); req->apoll = NULL; } - if (req->flags & REQ_F_INFLIGHT) { - struct io_uring_task *tctx = req->task->io_uring; - - atomic_dec(&tctx->inflight_tracked); - } + if (req->flags & REQ_F_INFLIGHT) + atomic_dec(&req->tctx->inflight_tracked); if (req->flags & REQ_F_CREDS) put_cred(req->creds); if (req->flags & REQ_F_ASYNC_DATA) { @@ -425,7 +422,7 @@ static inline void io_req_track_inflight(struct io_kiocb *req) { if (!(req->flags & REQ_F_INFLIGHT)) { req->flags |= REQ_F_INFLIGHT; - atomic_inc(&req->task->io_uring->inflight_tracked); + atomic_inc(&req->tctx->inflight_tracked); } } @@ -514,7 +511,7 @@ static void io_prep_async_link(struct io_kiocb *req) static void io_queue_iowq(struct io_kiocb *req) { struct io_kiocb *link = io_prep_linked_timeout(req); - struct io_uring_task *tctx = req->task->io_uring; + struct io_uring_task *tctx = req->tctx; BUG_ON(!tctx); BUG_ON(!tctx->io_wq); @@ -529,7 +526,7 @@ static void io_queue_iowq(struct io_kiocb *req) * procedure rather than attempt to run this request (or create a new * worker for it). */ - if (WARN_ON_ONCE(!same_thread_group(req->task, current))) + if (WARN_ON_ONCE(!same_thread_group(tctx->task, current))) atomic_or(IO_WQ_WORK_CANCEL, &req->work.flags); trace_io_uring_queue_async_work(req, io_wq_is_hashed(&req->work)); @@ -678,17 +675,17 @@ static void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx) } /* must to be called somewhat shortly after putting a request */ -static inline void io_put_task(struct task_struct *task) +static inline void io_put_task(struct io_kiocb *req) { - struct io_uring_task *tctx = task->io_uring; + struct io_uring_task *tctx = req->tctx; - if (likely(task == current)) { + if (likely(tctx->task == current)) { tctx->cached_refs++; } else { percpu_counter_sub(&tctx->inflight, 1); if (unlikely(atomic_read(&tctx->in_cancel))) wake_up(&tctx->wait); - put_task_struct(task); + put_task_struct(tctx->task); } } @@ -1207,7 +1204,7 @@ static inline void io_req_local_work_add(struct io_kiocb *req, static void io_req_normal_work_add(struct io_kiocb *req) { - struct io_uring_task *tctx = req->task->io_uring; + struct io_uring_task *tctx = req->tctx; struct io_ring_ctx *ctx = req->ctx; /* task_work already pending, we're done */ @@ -1226,7 +1223,7 @@ static void io_req_normal_work_add(struct io_kiocb *req) return; } - if (likely(!task_work_add(req->task, &tctx->task_work, ctx->notify_method))) + if (likely(!task_work_add(tctx->task, &tctx->task_work, ctx->notify_method))) return; io_fallback_tw(tctx, false); @@ -1343,8 +1340,7 @@ static void io_req_task_cancel(struct io_kiocb *req, struct io_tw_state *ts) void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts) { io_tw_lock(req->ctx, ts); - /* req->task == current here, checking PF_EXITING is safe */ - if (unlikely(req->task->flags & PF_EXITING)) + if (unlikely(io_should_terminate_tw())) io_req_defer_failed(req, -EFAULT); else if (req->flags & REQ_F_FORCE_ASYNC) io_queue_iowq(req); @@ -1403,7 +1399,7 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, } io_put_file(req); io_req_put_rsrc_nodes(req); - io_put_task(req->task); + io_put_task(req); node = req->comp_list.next; io_req_add_to_cache(req, ctx); @@ -2019,7 +2015,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->flags = (__force io_req_flags_t) sqe_flags; req->cqe.user_data = READ_ONCE(sqe->user_data); req->file = NULL; - req->task = current; + req->tctx = current->io_uring; req->cancel_seq_set = false; if (unlikely(opcode >= IORING_OP_LAST)) { diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 17ffdb1e41c5..702c8e987430 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -426,6 +426,19 @@ static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx) ctx->submitter_task == current); } +/* + * Terminate the request if either of these conditions are true: + * + * 1) It's being executed by the original task, but that task is marked + * with PF_EXITING as it's exiting. + * 2) PF_KTHREAD is set, in which case the invoker of the task_work is + * our fallback task_work. + */ +static inline bool io_should_terminate_tw(void) +{ + return current->flags & (PF_KTHREAD | PF_EXITING); +} + static inline void io_req_queue_tw_complete(struct io_kiocb *req, s32 res) { io_req_set_res(req, res, 0); diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index 99af39e1d0fb..e63af34004b7 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -89,8 +89,8 @@ static void io_msg_tw_complete(struct io_kiocb *req, struct io_tw_state *ts) static int io_msg_remote_post(struct io_ring_ctx *ctx, struct io_kiocb *req, int res, u32 cflags, u64 user_data) { - req->task = READ_ONCE(ctx->submitter_task); - if (!req->task) { + req->tctx = READ_ONCE(ctx->submitter_task->io_uring); + if (!req->tctx) { kmem_cache_free(req_cachep, req); return -EOWNERDEAD; } diff --git a/io_uring/notif.c b/io_uring/notif.c index 8dfbb0bd8e4d..ee3a33510b3c 100644 --- a/io_uring/notif.c +++ b/io_uring/notif.c @@ -89,7 +89,7 @@ static int io_link_skb(struct sk_buff *skb, struct ubuf_info *uarg) /* make sure all noifications can be finished in the same task_work */ if (unlikely(notif->ctx != prev_notif->ctx || - notif->task != prev_notif->task)) + notif->tctx != prev_notif->tctx)) return -EEXIST; nd->head = prev_nd->head; @@ -115,7 +115,7 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) notif->opcode = IORING_OP_NOP; notif->flags = 0; notif->file = NULL; - notif->task = current; + notif->tctx = current->io_uring; io_get_task_refs(1); notif->file_node = NULL; notif->buf_node = NULL; diff --git a/io_uring/poll.c b/io_uring/poll.c index 7db3010b5733..bced9edd5233 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -224,8 +224,7 @@ static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts) { int v; - /* req->task == current here, checking PF_EXITING is safe */ - if (unlikely(req->task->flags & PF_EXITING)) + if (unlikely(io_should_terminate_tw())) return -ECANCELED; do { diff --git a/io_uring/rw.c b/io_uring/rw.c index 144730344c0f..e368b9afde03 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -435,7 +435,7 @@ static bool io_rw_should_reissue(struct io_kiocb *req) * Play it safe and assume not safe to re-import and reissue if we're * not in the original thread group (or in task context). */ - if (!same_thread_group(req->task, current) || !in_task()) + if (!same_thread_group(req->tctx->task, current) || !in_task()) return false; return true; } diff --git a/io_uring/tctx.c b/io_uring/tctx.c index c043fe93a3f2..503f3ff8bc4f 100644 --- a/io_uring/tctx.c +++ b/io_uring/tctx.c @@ -81,6 +81,7 @@ __cold int io_uring_alloc_task_context(struct task_struct *task, return ret; } + tctx->task = task; xa_init(&tctx->xa); init_waitqueue_head(&tctx->wait); atomic_set(&tctx->in_cancel, 0); diff --git a/io_uring/timeout.c b/io_uring/timeout.c index 18286cb53a69..5b12bd6a804c 100644 --- a/io_uring/timeout.c +++ b/io_uring/timeout.c @@ -300,16 +300,18 @@ static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *t { struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_kiocb *prev = timeout->prev; - int ret = -ENOENT; + int ret; if (prev) { - if (!(req->task->flags & PF_EXITING)) { + if (!io_should_terminate_tw()) { struct io_cancel_data cd = { .ctx = req->ctx, .data = prev->cqe.user_data, }; - ret = io_try_cancel(req->task->io_uring, &cd, 0); + ret = io_try_cancel(req->tctx, &cd, 0); + } else { + ret = -ECANCELED; } io_req_set_res(req, ret ?: -ETIME, 0); io_req_task_complete(req, ts); @@ -643,7 +645,7 @@ static bool io_match_task(struct io_kiocb *head, struct io_uring_task *tctx, { struct io_kiocb *req; - if (tctx && head->task->io_uring != tctx) + if (tctx && head->tctx != tctx) return false; if (cancel_all) return true; diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index f88fbc9869d0..40b8b777ba12 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -61,7 +61,7 @@ bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx, struct io_uring_cmd); struct file *file = req->file; - if (!cancel_all && req->task->io_uring != tctx) + if (!cancel_all && req->tctx != tctx) continue; if (cmd->flags & IORING_URING_CMD_CANCELABLE) { diff --git a/io_uring/waitid.c b/io_uring/waitid.c index 9b7c23f96c47..daef5dd644f0 100644 --- a/io_uring/waitid.c +++ b/io_uring/waitid.c @@ -331,7 +331,7 @@ int io_waitid(struct io_kiocb *req, unsigned int issue_flags) hlist_add_head(&req->hash_node, &ctx->waitid_list); init_waitqueue_func_entry(&iwa->wo.child_wait, io_waitid_wait); - iwa->wo.child_wait.private = req->task; + iwa->wo.child_wait.private = req->tctx->task; iw->head = ¤t->signal->wait_chldexit; add_wait_queue(iw->head, &iwa->wo.child_wait); From 483242714fcc853f3f5ef728116f5ec168468bca Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Nov 2024 12:02:47 +0000 Subject: [PATCH 59/79] io_uring: prevent speculating sq_array indexing The SQ index array consists of user provided indexes, which io_uring then uses to index the SQ, and so it's susceptible to speculation. For all other queues io_uring tracks heads and tails in kernel, and they shouldn't need any special care. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/c6c7a25962924a55869e317e4fdb682dfdc6b279.1730687889.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 1 + 1 file changed, 1 insertion(+) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 7c1ca36b117b..751b9e19da6e 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2245,6 +2245,7 @@ static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) READ_ONCE(ctx->rings->sq_dropped) + 1); return false; } + head = array_index_nospec(head, ctx->sq_entries); } /* From 2f3cc8e441c9f657ff036c56baaab7dddbd0b350 Mon Sep 17 00:00:00 2001 From: Olivier Langlois Date: Sun, 13 Oct 2024 14:28:24 -0400 Subject: [PATCH 60/79] io_uring/napi: protect concurrent io_napi_entry timeout accesses io_napi_entry timeout value can be updated while accessed from the poll functions. Its concurrent accesses are wrapped with READ_ONCE()/WRITE_ONCE() macros to avoid incorrect compiler optimizations. Signed-off-by: Olivier Langlois Link: https://lore.kernel.org/r/3de3087563cf98f75266fd9f85fdba063a8720db.1728828877.git.olivier@trillion01.com Signed-off-by: Jens Axboe --- io_uring/napi.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/io_uring/napi.c b/io_uring/napi.c index d0cf694d0172..dda2e083fb5d 100644 --- a/io_uring/napi.c +++ b/io_uring/napi.c @@ -60,7 +60,7 @@ void __io_napi_add(struct io_ring_ctx *ctx, struct socket *sock) rcu_read_lock(); e = io_napi_hash_find(hash_list, napi_id); if (e) { - e->timeout = jiffies + NAPI_TIMEOUT; + WRITE_ONCE(e->timeout, jiffies + NAPI_TIMEOUT); rcu_read_unlock(); return; } @@ -92,7 +92,7 @@ static void __io_napi_remove_stale(struct io_ring_ctx *ctx) spin_lock(&ctx->napi_lock); hash_for_each(ctx->napi_ht, i, e, node) { - if (time_after(jiffies, e->timeout)) { + if (time_after(jiffies, READ_ONCE(e->timeout))) { list_del(&e->list); hash_del_rcu(&e->node); kfree_rcu(e, rcu); @@ -150,7 +150,7 @@ static bool __io_napi_do_busy_loop(struct io_ring_ctx *ctx, napi_busy_loop_rcu(e->napi_id, loop_end, loop_end_arg, ctx->napi_prefer_busy_poll, BUSY_POLL_BUDGET); - if (time_after(jiffies, e->timeout)) + if (time_after(jiffies, READ_ONCE(e->timeout))) is_stale = true; } From 45b3941d09d13b3503309be1f023b83deaf69b4d Mon Sep 17 00:00:00 2001 From: Olivier Langlois Date: Sun, 13 Oct 2024 14:28:38 -0400 Subject: [PATCH 61/79] io_uring/napi: fix io_napi_entry RCU accesses correct 3 RCU structures modifications that were not using the RCU functions to make their update. Signed-off-by: Olivier Langlois Link: https://lore.kernel.org/r/9f53b5169afa8c7bf3665a0b19dc2f7061173530.1728828877.git.olivier@trillion01.com Signed-off-by: Jens Axboe --- io_uring/napi.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/io_uring/napi.c b/io_uring/napi.c index dda2e083fb5d..921de9de8d75 100644 --- a/io_uring/napi.c +++ b/io_uring/napi.c @@ -81,19 +81,24 @@ void __io_napi_add(struct io_ring_ctx *ctx, struct socket *sock) } hlist_add_tail_rcu(&e->node, hash_list); - list_add_tail(&e->list, &ctx->napi_list); + list_add_tail_rcu(&e->list, &ctx->napi_list); spin_unlock(&ctx->napi_lock); } static void __io_napi_remove_stale(struct io_ring_ctx *ctx) { struct io_napi_entry *e; - unsigned int i; spin_lock(&ctx->napi_lock); - hash_for_each(ctx->napi_ht, i, e, node) { + /* + * list_for_each_entry_safe() is not required as long as: + * 1. list_del_rcu() does not reset the deleted node next pointer + * 2. kfree_rcu() delays the memory freeing until the next quiescent + * state + */ + list_for_each_entry(e, &ctx->napi_list, list) { if (time_after(jiffies, READ_ONCE(e->timeout))) { - list_del(&e->list); + list_del_rcu(&e->list); hash_del_rcu(&e->node); kfree_rcu(e, rcu); } @@ -204,13 +209,13 @@ void io_napi_init(struct io_ring_ctx *ctx) void io_napi_free(struct io_ring_ctx *ctx) { struct io_napi_entry *e; - unsigned int i; spin_lock(&ctx->napi_lock); - hash_for_each(ctx->napi_ht, i, e, node) { + list_for_each_entry(e, &ctx->napi_list, list) { hash_del_rcu(&e->node); kfree_rcu(e, rcu); } + INIT_LIST_HEAD_RCU(&ctx->napi_list); spin_unlock(&ctx->napi_lock); } From a5e26f49fef9485bc4ae24666d984a6de11e058c Mon Sep 17 00:00:00 2001 From: Olivier Langlois Date: Sun, 13 Oct 2024 14:28:50 -0400 Subject: [PATCH 62/79] io_uring/napi: improve __io_napi_add 1. move the sock->sk pointer validity test outside the function to avoid the function call overhead and to make the function more more reusable 2. change its name to __io_napi_add_id to be more precise about it is doing 3. return an error code to report errors Signed-off-by: Olivier Langlois Link: https://lore.kernel.org/r/d637fa3b437d753c0f4e44ff6a7b5bf2c2611270.1728828877.git.olivier@trillion01.com Signed-off-by: Jens Axboe --- io_uring/napi.c | 19 ++++++------------- io_uring/napi.h | 6 +++--- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/io_uring/napi.c b/io_uring/napi.c index 921de9de8d75..5e2299e7ff8e 100644 --- a/io_uring/napi.c +++ b/io_uring/napi.c @@ -38,22 +38,14 @@ static inline ktime_t net_to_ktime(unsigned long t) return ns_to_ktime(t << 10); } -void __io_napi_add(struct io_ring_ctx *ctx, struct socket *sock) +int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id) { struct hlist_head *hash_list; - unsigned int napi_id; - struct sock *sk; struct io_napi_entry *e; - sk = sock->sk; - if (!sk) - return; - - napi_id = READ_ONCE(sk->sk_napi_id); - /* Non-NAPI IDs can be rejected. */ if (napi_id < MIN_NAPI_ID) - return; + return -EINVAL; hash_list = &ctx->napi_ht[hash_min(napi_id, HASH_BITS(ctx->napi_ht))]; @@ -62,13 +54,13 @@ void __io_napi_add(struct io_ring_ctx *ctx, struct socket *sock) if (e) { WRITE_ONCE(e->timeout, jiffies + NAPI_TIMEOUT); rcu_read_unlock(); - return; + return -EEXIST; } rcu_read_unlock(); e = kmalloc(sizeof(*e), GFP_NOWAIT); if (!e) - return; + return -ENOMEM; e->napi_id = napi_id; e->timeout = jiffies + NAPI_TIMEOUT; @@ -77,12 +69,13 @@ void __io_napi_add(struct io_ring_ctx *ctx, struct socket *sock) if (unlikely(io_napi_hash_find(hash_list, napi_id))) { spin_unlock(&ctx->napi_lock); kfree(e); - return; + return -EEXIST; } hlist_add_tail_rcu(&e->node, hash_list); list_add_tail_rcu(&e->list, &ctx->napi_list); spin_unlock(&ctx->napi_lock); + return 0; } static void __io_napi_remove_stale(struct io_ring_ctx *ctx) diff --git a/io_uring/napi.h b/io_uring/napi.h index fd275ef0456d..4ae622f37b30 100644 --- a/io_uring/napi.h +++ b/io_uring/napi.h @@ -15,7 +15,7 @@ void io_napi_free(struct io_ring_ctx *ctx); int io_register_napi(struct io_ring_ctx *ctx, void __user *arg); int io_unregister_napi(struct io_ring_ctx *ctx, void __user *arg); -void __io_napi_add(struct io_ring_ctx *ctx, struct socket *sock); +int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id); void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq); int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx); @@ -48,8 +48,8 @@ static inline void io_napi_add(struct io_kiocb *req) return; sock = sock_from_file(req->file); - if (sock) - __io_napi_add(ctx, sock); + if (sock && sock->sk) + __io_napi_add_id(ctx, READ_ONCE(sock->sk->sk_napi_id)); } #else From db1e1adf6f993b1c2cef605d86eff709a8db5052 Mon Sep 17 00:00:00 2001 From: Olivier Langlois Date: Sun, 13 Oct 2024 14:29:02 -0400 Subject: [PATCH 63/79] io_uring/napi: Use lock guards Convert napi locks to use the shiny new Scope-Based Resource Management machinery. Signed-off-by: Olivier Langlois Link: https://lore.kernel.org/r/2680ca47ee183cfdb89d1a40c84d349edeb620ab.1728828877.git.olivier@trillion01.com Signed-off-by: Jens Axboe --- io_uring/napi.c | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/io_uring/napi.c b/io_uring/napi.c index 5e2299e7ff8e..6d5fdd397f2f 100644 --- a/io_uring/napi.c +++ b/io_uring/napi.c @@ -49,14 +49,13 @@ int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id) hash_list = &ctx->napi_ht[hash_min(napi_id, HASH_BITS(ctx->napi_ht))]; - rcu_read_lock(); - e = io_napi_hash_find(hash_list, napi_id); - if (e) { - WRITE_ONCE(e->timeout, jiffies + NAPI_TIMEOUT); - rcu_read_unlock(); - return -EEXIST; + scoped_guard(rcu) { + e = io_napi_hash_find(hash_list, napi_id); + if (e) { + WRITE_ONCE(e->timeout, jiffies + NAPI_TIMEOUT); + return -EEXIST; + } } - rcu_read_unlock(); e = kmalloc(sizeof(*e), GFP_NOWAIT); if (!e) @@ -65,6 +64,10 @@ int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id) e->napi_id = napi_id; e->timeout = jiffies + NAPI_TIMEOUT; + /* + * guard(spinlock) is not used to manually unlock it before calling + * kfree() + */ spin_lock(&ctx->napi_lock); if (unlikely(io_napi_hash_find(hash_list, napi_id))) { spin_unlock(&ctx->napi_lock); @@ -82,7 +85,7 @@ static void __io_napi_remove_stale(struct io_ring_ctx *ctx) { struct io_napi_entry *e; - spin_lock(&ctx->napi_lock); + guard(spinlock)(&ctx->napi_lock); /* * list_for_each_entry_safe() is not required as long as: * 1. list_del_rcu() does not reset the deleted node next pointer @@ -96,7 +99,6 @@ static void __io_napi_remove_stale(struct io_ring_ctx *ctx) kfree_rcu(e, rcu); } } - spin_unlock(&ctx->napi_lock); } static inline void io_napi_remove_stale(struct io_ring_ctx *ctx, bool is_stale) @@ -168,11 +170,12 @@ static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx, if (list_is_singular(&ctx->napi_list)) loop_end_arg = iowq; - rcu_read_lock(); - do { - is_stale = __io_napi_do_busy_loop(ctx, loop_end_arg); - } while (!io_napi_busy_loop_should_end(iowq, start_time) && !loop_end_arg); - rcu_read_unlock(); + scoped_guard(rcu) { + do { + is_stale = __io_napi_do_busy_loop(ctx, loop_end_arg); + } while (!io_napi_busy_loop_should_end(iowq, start_time) && + !loop_end_arg); + } io_napi_remove_stale(ctx, is_stale); } @@ -203,13 +206,12 @@ void io_napi_free(struct io_ring_ctx *ctx) { struct io_napi_entry *e; - spin_lock(&ctx->napi_lock); + guard(spinlock)(&ctx->napi_lock); list_for_each_entry(e, &ctx->napi_list, list) { hash_del_rcu(&e->node); kfree_rcu(e, rcu); } INIT_LIST_HEAD_RCU(&ctx->napi_list); - spin_unlock(&ctx->napi_lock); } /* @@ -305,9 +307,9 @@ int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx) if (list_empty_careful(&ctx->napi_list)) return 0; - rcu_read_lock(); - is_stale = __io_napi_do_busy_loop(ctx, NULL); - rcu_read_unlock(); + scoped_guard(rcu) { + is_stale = __io_napi_do_busy_loop(ctx, NULL); + } io_napi_remove_stale(ctx, is_stale); return 1; From 71afd926f292bb8f3ca86e6c3c40123037f109c6 Mon Sep 17 00:00:00 2001 From: Olivier Langlois Date: Sun, 13 Oct 2024 14:29:12 -0400 Subject: [PATCH 64/79] io_uring/napi: clean up __io_napi_do_busy_loop __io_napi_do_busy_loop now requires to have loop_end in its parameters. This makes the code cleaner and also has the benefit of removing a branch since the only caller not passing NULL for loop_end_arg is also setting the value conditionally. Signed-off-by: Olivier Langlois Link: https://lore.kernel.org/r/d5b9bb91b1a08fff50525e1c18d7b4709b9ca100.1728828877.git.olivier@trillion01.com Signed-off-by: Jens Axboe --- io_uring/napi.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/io_uring/napi.c b/io_uring/napi.c index 6d5fdd397f2f..1de1543d8034 100644 --- a/io_uring/napi.c +++ b/io_uring/napi.c @@ -137,15 +137,12 @@ static bool io_napi_busy_loop_should_end(void *data, } static bool __io_napi_do_busy_loop(struct io_ring_ctx *ctx, + bool (*loop_end)(void *, unsigned long), void *loop_end_arg) { struct io_napi_entry *e; - bool (*loop_end)(void *, unsigned long) = NULL; bool is_stale = false; - if (loop_end_arg) - loop_end = io_napi_busy_loop_should_end; - list_for_each_entry_rcu(e, &ctx->napi_list, list) { napi_busy_loop_rcu(e->napi_id, loop_end, loop_end_arg, ctx->napi_prefer_busy_poll, BUSY_POLL_BUDGET); @@ -161,18 +158,22 @@ static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq) { unsigned long start_time = busy_loop_current_time(); + bool (*loop_end)(void *, unsigned long) = NULL; void *loop_end_arg = NULL; bool is_stale = false; /* Singular lists use a different napi loop end check function and are * only executed once. */ - if (list_is_singular(&ctx->napi_list)) + if (list_is_singular(&ctx->napi_list)) { + loop_end = io_napi_busy_loop_should_end; loop_end_arg = iowq; + } scoped_guard(rcu) { do { - is_stale = __io_napi_do_busy_loop(ctx, loop_end_arg); + is_stale = __io_napi_do_busy_loop(ctx, loop_end, + loop_end_arg); } while (!io_napi_busy_loop_should_end(iowq, start_time) && !loop_end_arg); } @@ -308,7 +309,7 @@ int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx) return 0; scoped_guard(rcu) { - is_stale = __io_napi_do_busy_loop(ctx, NULL); + is_stale = __io_napi_do_busy_loop(ctx, NULL, NULL); } io_napi_remove_stale(ctx, is_stale); From 6bf90bd8c58a305994948eb3409d91a7d8f2edae Mon Sep 17 00:00:00 2001 From: Olivier Langlois Date: Sun, 13 Oct 2024 14:29:24 -0400 Subject: [PATCH 65/79] io_uring/napi: add static napi tracking strategy Add the static napi tracking strategy. That allows the user to manually manage the napi ids list for busy polling, and eliminate the overhead of dynamically updating the list from the fast path. Signed-off-by: Olivier Langlois Link: https://lore.kernel.org/r/96943de14968c35a5c599352259ad98f3c0770ba.1728828877.git.olivier@trillion01.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 2 +- include/uapi/linux/io_uring.h | 32 ++++++++++- io_uring/fdinfo.c | 54 ++++++++++++++----- io_uring/napi.c | 97 ++++++++++++++++++++++++++++++---- io_uring/napi.h | 2 +- 5 files changed, 160 insertions(+), 27 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index fba2988accc3..072e65e93105 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -408,7 +408,7 @@ struct io_ring_ctx { /* napi busy poll default timeout */ ktime_t napi_busy_poll_dt; bool napi_prefer_busy_poll; - bool napi_enabled; + u8 napi_track_mode; DECLARE_HASHTABLE(napi_ht, 4); #endif diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 47977a5c65f5..5d08435b95a8 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -790,12 +790,40 @@ struct io_uring_buf_status { __u32 resv[8]; }; +enum io_uring_napi_op { + /* register/ungister backward compatible opcode */ + IO_URING_NAPI_REGISTER_OP = 0, + + /* opcodes to update napi_list when static tracking is used */ + IO_URING_NAPI_STATIC_ADD_ID = 1, + IO_URING_NAPI_STATIC_DEL_ID = 2 +}; + +enum io_uring_napi_tracking_strategy { + /* value must be 0 for backward compatibility */ + IO_URING_NAPI_TRACKING_DYNAMIC = 0, + IO_URING_NAPI_TRACKING_STATIC = 1, + IO_URING_NAPI_TRACKING_INACTIVE = 255 +}; + /* argument for IORING_(UN)REGISTER_NAPI */ struct io_uring_napi { __u32 busy_poll_to; __u8 prefer_busy_poll; - __u8 pad[3]; - __u64 resv; + + /* a io_uring_napi_op value */ + __u8 opcode; + __u8 pad[2]; + + /* + * for IO_URING_NAPI_REGISTER_OP, it is a + * io_uring_napi_tracking_strategy value. + * + * for IO_URING_NAPI_STATIC_ADD_ID/IO_URING_NAPI_STATIC_DEL_ID + * it is the napi id to add/del from napi_list. + */ + __u32 op_param; + __u32 resv; }; /* diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index efbec34ccb18..b214e5a407b5 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -46,6 +46,46 @@ static __cold int io_uring_show_cred(struct seq_file *m, unsigned int id, return 0; } +#ifdef CONFIG_NET_RX_BUSY_POLL +static __cold void common_tracking_show_fdinfo(struct io_ring_ctx *ctx, + struct seq_file *m, + const char *tracking_strategy) +{ + seq_puts(m, "NAPI:\tenabled\n"); + seq_printf(m, "napi tracking:\t%s\n", tracking_strategy); + seq_printf(m, "napi_busy_poll_dt:\t%llu\n", ctx->napi_busy_poll_dt); + if (ctx->napi_prefer_busy_poll) + seq_puts(m, "napi_prefer_busy_poll:\ttrue\n"); + else + seq_puts(m, "napi_prefer_busy_poll:\tfalse\n"); +} + +static __cold void napi_show_fdinfo(struct io_ring_ctx *ctx, + struct seq_file *m) +{ + unsigned int mode = READ_ONCE(ctx->napi_track_mode); + + switch (mode) { + case IO_URING_NAPI_TRACKING_INACTIVE: + seq_puts(m, "NAPI:\tdisabled\n"); + break; + case IO_URING_NAPI_TRACKING_DYNAMIC: + common_tracking_show_fdinfo(ctx, m, "dynamic"); + break; + case IO_URING_NAPI_TRACKING_STATIC: + common_tracking_show_fdinfo(ctx, m, "static"); + break; + default: + seq_printf(m, "NAPI:\tunknown mode (%u)\n", mode); + } +} +#else +static inline void napi_show_fdinfo(struct io_ring_ctx *ctx, + struct seq_file *m) +{ +} +#endif + /* * Caller holds a reference to the file already, we don't need to do * anything else to get an extra reference. @@ -219,18 +259,6 @@ __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *file) } spin_unlock(&ctx->completion_lock); - -#ifdef CONFIG_NET_RX_BUSY_POLL - if (ctx->napi_enabled) { - seq_puts(m, "NAPI:\tenabled\n"); - seq_printf(m, "napi_busy_poll_dt:\t%llu\n", ctx->napi_busy_poll_dt); - if (ctx->napi_prefer_busy_poll) - seq_puts(m, "napi_prefer_busy_poll:\ttrue\n"); - else - seq_puts(m, "napi_prefer_busy_poll:\tfalse\n"); - } else { - seq_puts(m, "NAPI:\tdisabled\n"); - } -#endif + napi_show_fdinfo(ctx, m); } #endif diff --git a/io_uring/napi.c b/io_uring/napi.c index 1de1543d8034..b1ade3fda30f 100644 --- a/io_uring/napi.c +++ b/io_uring/napi.c @@ -81,6 +81,27 @@ int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id) return 0; } +static int __io_napi_del_id(struct io_ring_ctx *ctx, unsigned int napi_id) +{ + struct hlist_head *hash_list; + struct io_napi_entry *e; + + /* Non-NAPI IDs can be rejected. */ + if (napi_id < MIN_NAPI_ID) + return -EINVAL; + + hash_list = &ctx->napi_ht[hash_min(napi_id, HASH_BITS(ctx->napi_ht))]; + guard(spinlock)(&ctx->napi_lock); + e = io_napi_hash_find(hash_list, napi_id); + if (!e) + return -ENOENT; + + list_del_rcu(&e->list); + hash_del_rcu(&e->node); + kfree_rcu(e, rcu); + return 0; +} + static void __io_napi_remove_stale(struct io_ring_ctx *ctx) { struct io_napi_entry *e; @@ -136,9 +157,25 @@ static bool io_napi_busy_loop_should_end(void *data, return false; } -static bool __io_napi_do_busy_loop(struct io_ring_ctx *ctx, - bool (*loop_end)(void *, unsigned long), - void *loop_end_arg) +/* + * never report stale entries + */ +static bool static_tracking_do_busy_loop(struct io_ring_ctx *ctx, + bool (*loop_end)(void *, unsigned long), + void *loop_end_arg) +{ + struct io_napi_entry *e; + + list_for_each_entry_rcu(e, &ctx->napi_list, list) + napi_busy_loop_rcu(e->napi_id, loop_end, loop_end_arg, + ctx->napi_prefer_busy_poll, BUSY_POLL_BUDGET); + return false; +} + +static bool +dynamic_tracking_do_busy_loop(struct io_ring_ctx *ctx, + bool (*loop_end)(void *, unsigned long), + void *loop_end_arg) { struct io_napi_entry *e; bool is_stale = false; @@ -154,6 +191,16 @@ static bool __io_napi_do_busy_loop(struct io_ring_ctx *ctx, return is_stale; } +static inline bool +__io_napi_do_busy_loop(struct io_ring_ctx *ctx, + bool (*loop_end)(void *, unsigned long), + void *loop_end_arg) +{ + if (READ_ONCE(ctx->napi_track_mode) == IO_URING_NAPI_TRACKING_STATIC) + return static_tracking_do_busy_loop(ctx, loop_end, loop_end_arg); + return dynamic_tracking_do_busy_loop(ctx, loop_end, loop_end_arg); +} + static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq) { @@ -195,6 +242,7 @@ void io_napi_init(struct io_ring_ctx *ctx) spin_lock_init(&ctx->napi_lock); ctx->napi_prefer_busy_poll = false; ctx->napi_busy_poll_dt = ns_to_ktime(sys_dt); + ctx->napi_track_mode = IO_URING_NAPI_TRACKING_INACTIVE; } /* @@ -215,6 +263,24 @@ void io_napi_free(struct io_ring_ctx *ctx) INIT_LIST_HEAD_RCU(&ctx->napi_list); } +static int io_napi_register_napi(struct io_ring_ctx *ctx, + struct io_uring_napi *napi) +{ + switch (napi->op_param) { + case IO_URING_NAPI_TRACKING_DYNAMIC: + case IO_URING_NAPI_TRACKING_STATIC: + break; + default: + return -EINVAL; + } + /* clean the napi list for new settings */ + io_napi_free(ctx); + WRITE_ONCE(ctx->napi_track_mode, napi->op_param); + WRITE_ONCE(ctx->napi_busy_poll_dt, napi->busy_poll_to * NSEC_PER_USEC); + WRITE_ONCE(ctx->napi_prefer_busy_poll, !!napi->prefer_busy_poll); + return 0; +} + /* * io_napi_register() - Register napi with io-uring * @ctx: pointer to io-uring context structure @@ -226,7 +292,8 @@ int io_register_napi(struct io_ring_ctx *ctx, void __user *arg) { const struct io_uring_napi curr = { .busy_poll_to = ktime_to_us(ctx->napi_busy_poll_dt), - .prefer_busy_poll = ctx->napi_prefer_busy_poll + .prefer_busy_poll = ctx->napi_prefer_busy_poll, + .op_param = ctx->napi_track_mode }; struct io_uring_napi napi; @@ -234,16 +301,26 @@ int io_register_napi(struct io_ring_ctx *ctx, void __user *arg) return -EINVAL; if (copy_from_user(&napi, arg, sizeof(napi))) return -EFAULT; - if (napi.pad[0] || napi.pad[1] || napi.pad[2] || napi.resv) + if (napi.pad[0] || napi.pad[1] || napi.resv) return -EINVAL; if (copy_to_user(arg, &curr, sizeof(curr))) return -EFAULT; - WRITE_ONCE(ctx->napi_busy_poll_dt, napi.busy_poll_to * NSEC_PER_USEC); - WRITE_ONCE(ctx->napi_prefer_busy_poll, !!napi.prefer_busy_poll); - WRITE_ONCE(ctx->napi_enabled, true); - return 0; + switch (napi.opcode) { + case IO_URING_NAPI_REGISTER_OP: + return io_napi_register_napi(ctx, &napi); + case IO_URING_NAPI_STATIC_ADD_ID: + if (curr.op_param != IO_URING_NAPI_TRACKING_STATIC) + return -EINVAL; + return __io_napi_add_id(ctx, napi.op_param); + case IO_URING_NAPI_STATIC_DEL_ID: + if (curr.op_param != IO_URING_NAPI_TRACKING_STATIC) + return -EINVAL; + return __io_napi_del_id(ctx, napi.op_param); + default: + return -EINVAL; + } } /* @@ -266,7 +343,7 @@ int io_unregister_napi(struct io_ring_ctx *ctx, void __user *arg) WRITE_ONCE(ctx->napi_busy_poll_dt, 0); WRITE_ONCE(ctx->napi_prefer_busy_poll, false); - WRITE_ONCE(ctx->napi_enabled, false); + WRITE_ONCE(ctx->napi_track_mode, IO_URING_NAPI_TRACKING_INACTIVE); return 0; } diff --git a/io_uring/napi.h b/io_uring/napi.h index 4ae622f37b30..fa742f42e09b 100644 --- a/io_uring/napi.h +++ b/io_uring/napi.h @@ -44,7 +44,7 @@ static inline void io_napi_add(struct io_kiocb *req) struct io_ring_ctx *ctx = req->ctx; struct socket *sock; - if (!READ_ONCE(ctx->napi_enabled)) + if (READ_ONCE(ctx->napi_track_mode) != IO_URING_NAPI_TRACKING_DYNAMIC) return; sock = sock_from_file(req->file); From af0a2ffef0e6d23412dd55df29f5caef8f3583f2 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 5 Nov 2024 02:12:33 +0000 Subject: [PATCH 66/79] io_uring: avoid normal tw intermediate fallback When a DEFER_TASKRUN io_uring is terminating it requeues deferred task work items as normal tw, which can further fallback to kthread execution. Avoid this extra step and always push them to the fallback kthread. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/d1cd472cec2230c66bd1c8d412a5833f0af75384.1730772720.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 21 ++++++++++----------- io_uring/io_uring.h | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 751b9e19da6e..042a65d38d0c 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1066,9 +1066,8 @@ struct llist_node *io_handle_tw_list(struct llist_node *node, return node; } -static __cold void io_fallback_tw(struct io_uring_task *tctx, bool sync) +static __cold void __io_fallback_tw(struct llist_node *node, bool sync) { - struct llist_node *node = llist_del_all(&tctx->task_list); struct io_ring_ctx *last_ctx = NULL; struct io_kiocb *req; @@ -1094,6 +1093,13 @@ static __cold void io_fallback_tw(struct io_uring_task *tctx, bool sync) } } +static void io_fallback_tw(struct io_uring_task *tctx, bool sync) +{ + struct llist_node *node = llist_del_all(&tctx->task_list); + + __io_fallback_tw(node, sync); +} + struct llist_node *tctx_task_work_run(struct io_uring_task *tctx, unsigned int max_entries, unsigned int *count) @@ -1247,16 +1253,9 @@ void io_req_task_work_add_remote(struct io_kiocb *req, struct io_ring_ctx *ctx, static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) { - struct llist_node *node; + struct llist_node *node = llist_del_all(&ctx->work_llist); - node = llist_del_all(&ctx->work_llist); - while (node) { - struct io_kiocb *req = container_of(node, struct io_kiocb, - io_task_work.node); - - node = node->next; - io_req_normal_work_add(req); - } + __io_fallback_tw(node, false); } static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events, diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 702c8e987430..4070d4c8ef97 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -136,7 +136,7 @@ static inline void io_lockdep_assert_cq_locked(struct io_ring_ctx *ctx) * Not from an SQE, as those cannot be submitted, but via * updating tagged resources. */ - if (ctx->submitter_task->flags & PF_EXITING) + if (percpu_ref_is_dying(&ctx->refs)) lockdep_assert(current_work()); else lockdep_assert(current == ctx->submitter_task); From 0d98c509086837a8cf5a32f82f2a58f39a539192 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 7 Nov 2024 19:01:34 +0800 Subject: [PATCH 67/79] io_uring/rsrc: pass 'struct io_ring_ctx' reference to rsrc helpers `io_rsrc_node` instance won't be shared among different io_uring ctxs, and its allocation 'ctx' is always same with the user's 'ctx', so it is safe to pass user 'ctx' reference to rsrc helpers. Even in io_clone_buffers(), `io_rsrc_node` instance is allocated actually for destination io_uring_ctx. Then io_rsrc_node_ctx() can be removed, and the 8 bytes `ctx` pointer will be removed from `io_rsrc_node` in the following patch. Signed-off-by: Ming Lei Link: https://lore.kernel.org/r/20241107110149.890530-2-ming.lei@redhat.com Signed-off-by: Jens Axboe --- io_uring/filetable.c | 13 +++++++------ io_uring/filetable.h | 4 ++-- io_uring/rsrc.c | 24 +++++++++++------------- io_uring/rsrc.h | 22 +++++++++------------- io_uring/splice.c | 2 +- 5 files changed, 30 insertions(+), 35 deletions(-) diff --git a/io_uring/filetable.c b/io_uring/filetable.c index 45f005f5db42..a21660e3145a 100644 --- a/io_uring/filetable.c +++ b/io_uring/filetable.c @@ -36,20 +36,21 @@ static int io_file_bitmap_get(struct io_ring_ctx *ctx) return -ENFILE; } -bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files) +bool io_alloc_file_tables(struct io_ring_ctx *ctx, struct io_file_table *table, + unsigned nr_files) { if (io_rsrc_data_alloc(&table->data, nr_files)) return false; table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT); if (table->bitmap) return true; - io_rsrc_data_free(&table->data); + io_rsrc_data_free(ctx, &table->data); return false; } -void io_free_file_tables(struct io_file_table *table) +void io_free_file_tables(struct io_ring_ctx *ctx, struct io_file_table *table) { - io_rsrc_data_free(&table->data); + io_rsrc_data_free(ctx, &table->data); bitmap_free(table->bitmap); table->bitmap = NULL; } @@ -71,7 +72,7 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file, if (!node) return -ENOMEM; - if (!io_reset_rsrc_node(&ctx->file_table.data, slot_index)) + if (!io_reset_rsrc_node(ctx, &ctx->file_table.data, slot_index)) io_file_bitmap_set(&ctx->file_table, slot_index); ctx->file_table.data.nodes[slot_index] = node; @@ -130,7 +131,7 @@ int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset) node = io_rsrc_node_lookup(&ctx->file_table.data, offset); if (!node) return -EBADF; - io_reset_rsrc_node(&ctx->file_table.data, offset); + io_reset_rsrc_node(ctx, &ctx->file_table.data, offset); io_file_bitmap_clear(&ctx->file_table, offset); return 0; } diff --git a/io_uring/filetable.h b/io_uring/filetable.h index bfacadb8d089..7717ea9efd0e 100644 --- a/io_uring/filetable.h +++ b/io_uring/filetable.h @@ -6,8 +6,8 @@ #include #include "rsrc.h" -bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files); -void io_free_file_tables(struct io_file_table *table); +bool io_alloc_file_tables(struct io_ring_ctx *ctx, struct io_file_table *table, unsigned nr_files); +void io_free_file_tables(struct io_ring_ctx *ctx, struct io_file_table *table); int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags, struct file *file, unsigned int file_slot); diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 2fb1791d7255..d7db36a2c66e 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -130,13 +130,13 @@ struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type) return node; } -__cold void io_rsrc_data_free(struct io_rsrc_data *data) +__cold void io_rsrc_data_free(struct io_ring_ctx *ctx, struct io_rsrc_data *data) { if (!data->nr) return; while (data->nr--) { if (data->nodes[data->nr]) - io_put_rsrc_node(data->nodes[data->nr]); + io_put_rsrc_node(ctx, data->nodes[data->nr]); } kvfree(data->nodes); data->nodes = NULL; @@ -184,7 +184,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, continue; i = up->offset + done; - if (io_reset_rsrc_node(&ctx->file_table.data, i)) + if (io_reset_rsrc_node(ctx, &ctx->file_table.data, i)) io_file_bitmap_clear(&ctx->file_table, i); if (fd != -1) { @@ -266,7 +266,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, node->tag = tag; } i = array_index_nospec(up->offset + done, ctx->buf_table.nr); - io_reset_rsrc_node(&ctx->buf_table, i); + io_reset_rsrc_node(ctx, &ctx->buf_table, i); ctx->buf_table.nodes[i] = node; if (ctx->compat) user_data += sizeof(struct compat_iovec); @@ -442,10 +442,8 @@ int io_files_update(struct io_kiocb *req, unsigned int issue_flags) return IOU_OK; } -void io_free_rsrc_node(struct io_rsrc_node *node) +void io_free_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node) { - struct io_ring_ctx *ctx = io_rsrc_node_ctx(node); - lockdep_assert_held(&ctx->uring_lock); if (node->tag) @@ -473,7 +471,7 @@ int io_sqe_files_unregister(struct io_ring_ctx *ctx) if (!ctx->file_table.data.nr) return -ENXIO; - io_free_file_tables(&ctx->file_table); + io_free_file_tables(ctx, &ctx->file_table); io_file_table_set_alloc_range(ctx, 0, 0); return 0; } @@ -494,7 +492,7 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, return -EMFILE; if (nr_args > rlimit(RLIMIT_NOFILE)) return -EMFILE; - if (!io_alloc_file_tables(&ctx->file_table, nr_args)) + if (!io_alloc_file_tables(ctx, &ctx->file_table, nr_args)) return -ENOMEM; for (i = 0; i < nr_args; i++) { @@ -551,7 +549,7 @@ int io_sqe_buffers_unregister(struct io_ring_ctx *ctx) { if (!ctx->buf_table.nr) return -ENXIO; - io_rsrc_data_free(&ctx->buf_table); + io_rsrc_data_free(ctx, &ctx->buf_table); return 0; } @@ -788,7 +786,7 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, if (ret) { kvfree(imu); if (node) - io_put_rsrc_node(node); + io_put_rsrc_node(ctx, node); node = ERR_PTR(ret); } kvfree(pages); @@ -1018,7 +1016,7 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx * old and new nodes at this point. */ if (arg->flags & IORING_REGISTER_DST_REPLACE) - io_rsrc_data_free(&ctx->buf_table); + io_rsrc_data_free(ctx, &ctx->buf_table); /* * ctx->buf_table should be empty now - either the contents are being @@ -1042,7 +1040,7 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx kfree(data.nodes[i]); } out_unlock: - io_rsrc_data_free(&data); + io_rsrc_data_free(ctx, &data); mutex_unlock(&src_ctx->uring_lock); mutex_lock(&ctx->uring_lock); return ret; diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index bc3a863b14bb..c9057f7a06f5 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -45,8 +45,8 @@ struct io_imu_folio_data { }; struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type); -void io_free_rsrc_node(struct io_rsrc_node *node); -void io_rsrc_data_free(struct io_rsrc_data *data); +void io_free_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node); +void io_rsrc_data_free(struct io_ring_ctx *ctx, struct io_rsrc_data *data); int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr); int io_import_fixed(int ddir, struct iov_iter *iter, @@ -76,19 +76,20 @@ static inline struct io_rsrc_node *io_rsrc_node_lookup(struct io_rsrc_data *data return NULL; } -static inline void io_put_rsrc_node(struct io_rsrc_node *node) +static inline void io_put_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node) { if (node && !--node->refs) - io_free_rsrc_node(node); + io_free_rsrc_node(ctx, node); } -static inline bool io_reset_rsrc_node(struct io_rsrc_data *data, int index) +static inline bool io_reset_rsrc_node(struct io_ring_ctx *ctx, + struct io_rsrc_data *data, int index) { struct io_rsrc_node *node = data->nodes[index]; if (!node) return false; - io_put_rsrc_node(node); + io_put_rsrc_node(ctx, node); data->nodes[index] = NULL; return true; } @@ -96,20 +97,15 @@ static inline bool io_reset_rsrc_node(struct io_rsrc_data *data, int index) static inline void io_req_put_rsrc_nodes(struct io_kiocb *req) { if (req->file_node) { - io_put_rsrc_node(req->file_node); + io_put_rsrc_node(req->ctx, req->file_node); req->file_node = NULL; } if (req->flags & REQ_F_BUF_NODE) { - io_put_rsrc_node(req->buf_node); + io_put_rsrc_node(req->ctx, req->buf_node); req->buf_node = NULL; } } -static inline struct io_ring_ctx *io_rsrc_node_ctx(struct io_rsrc_node *node) -{ - return (struct io_ring_ctx *) (node->ctx_ptr & ~IORING_RSRC_TYPE_MASK); -} - static inline int io_rsrc_node_type(struct io_rsrc_node *node) { return node->ctx_ptr & IORING_RSRC_TYPE_MASK; diff --git a/io_uring/splice.c b/io_uring/splice.c index e8ed15f4ea1a..5b84f1630611 100644 --- a/io_uring/splice.c +++ b/io_uring/splice.c @@ -51,7 +51,7 @@ void io_splice_cleanup(struct io_kiocb *req) { struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice); - io_put_rsrc_node(sp->rsrc_node); + io_put_rsrc_node(req->ctx, sp->rsrc_node); } static struct file *io_splice_get_file(struct io_kiocb *req, From 4f219fcce5e4366cc121fc98270beb1fbbb3df2b Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 7 Nov 2024 19:01:35 +0800 Subject: [PATCH 68/79] io_uring/rsrc: remove '->ctx_ptr' of 'struct io_rsrc_node' Remove '->ctx_ptr' of 'struct io_rsrc_node', and add 'type' field, meantime remove io_rsrc_node_type(). Signed-off-by: Ming Lei Link: https://lore.kernel.org/r/20241107110149.890530-3-ming.lei@redhat.com Signed-off-by: Jens Axboe --- io_uring/rsrc.c | 4 ++-- io_uring/rsrc.h | 9 +-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index d7db36a2c66e..adaae8630932 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -124,7 +124,7 @@ struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type) node = kzalloc(sizeof(*node), GFP_KERNEL); if (node) { - node->ctx_ptr = (unsigned long) ctx | type; + node->type = type; node->refs = 1; } return node; @@ -449,7 +449,7 @@ void io_free_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node) if (node->tag) io_post_aux_cqe(ctx, node->tag, 0, 0); - switch (io_rsrc_node_type(node)) { + switch (node->type) { case IORING_RSRC_FILE: if (io_slot_file(node)) fput(io_slot_file(node)); diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index c9057f7a06f5..c8a64a9ed5b9 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -11,12 +11,10 @@ enum { IORING_RSRC_FILE = 0, IORING_RSRC_BUFFER = 1, - - IORING_RSRC_TYPE_MASK = 0x3UL, }; struct io_rsrc_node { - unsigned long ctx_ptr; + unsigned char type; int refs; u64 tag; @@ -106,11 +104,6 @@ static inline void io_req_put_rsrc_nodes(struct io_kiocb *req) } } -static inline int io_rsrc_node_type(struct io_rsrc_node *node) -{ - return node->ctx_ptr & IORING_RSRC_TYPE_MASK; -} - static inline void io_req_assign_rsrc_node(struct io_rsrc_node **dst_node, struct io_rsrc_node *node) { From 039c878db7add23c1c9ea18424c442cce76670f9 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 7 Nov 2024 19:01:36 +0800 Subject: [PATCH 69/79] io_uring/rsrc: add & apply io_req_assign_buf_node() The following pattern becomes more and more: + io_req_assign_rsrc_node(&req->buf_node, node); + req->flags |= REQ_F_BUF_NODE; so make it a helper, which is less fragile to use than above code, for example, the BUF_NODE flag is even missed in current io_uring_cmd_prep(). Signed-off-by: Ming Lei Link: https://lore.kernel.org/r/20241107110149.890530-4-ming.lei@redhat.com Signed-off-by: Jens Axboe --- io_uring/net.c | 3 +-- io_uring/nop.c | 3 +-- io_uring/rsrc.h | 7 +++++++ io_uring/rw.c | 3 +-- io_uring/uring_cmd.c | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/io_uring/net.c b/io_uring/net.c index 2ccc2b409431..df1f7dc6f1c8 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -1348,8 +1348,7 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags) io_ring_submit_lock(ctx, issue_flags); node = io_rsrc_node_lookup(&ctx->buf_table, sr->buf_index); if (node) { - io_req_assign_rsrc_node(&sr->notif->buf_node, node); - sr->notif->flags |= REQ_F_BUF_NODE; + io_req_assign_buf_node(sr->notif, node); ret = 0; } io_ring_submit_unlock(ctx, issue_flags); diff --git a/io_uring/nop.c b/io_uring/nop.c index bc22bcc739f3..6d470d4251ee 100644 --- a/io_uring/nop.c +++ b/io_uring/nop.c @@ -67,8 +67,7 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags) io_ring_submit_lock(ctx, issue_flags); node = io_rsrc_node_lookup(&ctx->buf_table, nop->buffer); if (node) { - io_req_assign_rsrc_node(&req->buf_node, node); - req->flags |= REQ_F_BUF_NODE; + io_req_assign_buf_node(req, node); ret = 0; } io_ring_submit_unlock(ctx, issue_flags); diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index c8a64a9ed5b9..7a4668deaa1a 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -111,6 +111,13 @@ static inline void io_req_assign_rsrc_node(struct io_rsrc_node **dst_node, *dst_node = node; } +static inline void io_req_assign_buf_node(struct io_kiocb *req, + struct io_rsrc_node *node) +{ + io_req_assign_rsrc_node(&req->buf_node, node); + req->flags |= REQ_F_BUF_NODE; +} + int io_files_update(struct io_kiocb *req, unsigned int issue_flags); int io_files_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); diff --git a/io_uring/rw.c b/io_uring/rw.c index e368b9afde03..b62cdb5fc936 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -341,8 +341,7 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index); if (!node) return -EFAULT; - io_req_assign_rsrc_node(&req->buf_node, node); - req->flags |= REQ_F_BUF_NODE; + io_req_assign_buf_node(req, node); io = req->async_data; ret = io_import_fixed(ddir, &io->iter, node->buf, rw->addr, rw->len); diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 40b8b777ba12..b62965f58f30 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -219,7 +219,7 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) * being called. This prevents destruction of the mapped buffer * we'll need at actual import time. */ - io_req_assign_rsrc_node(&req->buf_node, node); + io_req_assign_buf_node(req, node); } ioucmd->cmd_op = READ_ONCE(sqe->cmd_op); From a43e236fb9aef4528f2bd24095d1f348030f5d9d Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Mon, 11 Nov 2024 18:13:18 +0800 Subject: [PATCH 70/79] io_uring/uring_cmd: fix buffer index retrieval Add back buffer index retrieval for IORING_URING_CMD_FIXED. Reported-by: Guangwu Zhang Cc: Jeff Moyer Fixes: b54a14041ee6 ("io_uring/rsrc: add io_rsrc_node_lookup() helper") Signed-off-by: Ming Lei Reviewed-by: Kanchan Joshi Reviewed-by: Anuj Gupta Tested-by: Guangwu Zhang Link: https://lore.kernel.org/r/20241111101318.1387557-1-ming.lei@redhat.com Signed-off-by: Jens Axboe --- io_uring/uring_cmd.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index b62965f58f30..e9d99d3ecc34 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -210,8 +210,9 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (ioucmd->flags & IORING_URING_CMD_FIXED) { struct io_ring_ctx *ctx = req->ctx; struct io_rsrc_node *node; + u16 index = READ_ONCE(sqe->buf_index); - node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index); + node = io_rsrc_node_lookup(&ctx->buf_table, index); if (unlikely(!node)) return -EFAULT; /* From b9d69371e8fa90fa3ab100f4fcb4815b13b3673a Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 13 Nov 2024 03:26:01 +0000 Subject: [PATCH 71/79] io_uring: fix invalid hybrid polling ctx leaks It has already allocated the ctx by the point where it checks the hybrid poll configuration, plain return leaks the memory. Fixes: 01ee194d1aba1 ("io_uring: add support for hybrid IOPOLL") Signed-off-by: Pavel Begunkov Reviewed-by: Anuj Gupta Link: https://lore.kernel.org/r/b57f2608088020501d352fcdeebdb949e281d65b.1731468230.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 042a65d38d0c..bd71782057de 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3616,11 +3616,6 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) static_branch_inc(&io_key_has_sqarray); - /* HYBRID_IOPOLL only valid with IOPOLL */ - if ((ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_HYBRID_IOPOLL)) == - IORING_SETUP_HYBRID_IOPOLL) - return -EINVAL; - if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) && !(ctx->flags & IORING_SETUP_IOPOLL) && !(ctx->flags & IORING_SETUP_SQPOLL)) @@ -3671,6 +3666,11 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, ctx->notify_method = TWA_SIGNAL; } + /* HYBRID_IOPOLL only valid with IOPOLL */ + if ((ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_HYBRID_IOPOLL)) == + IORING_SETUP_HYBRID_IOPOLL) + goto err; + /* * For DEFER_TASKRUN we require the completion task to be the same as the * submission task. This implies that there is only one submitter, so enforce From 56cec28dc4da396d6032c59ae9614c5a6ae7d7a8 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Fri, 15 Nov 2024 03:49:02 +0000 Subject: [PATCH 72/79] switch io_msg_ring() to CLASS(fd) Use CLASS(fd) to get the file for sync message ring requests, rather than open-code the file retrieval dance. Signed-off-by: Al Viro Link: https://lore.kernel.org/r/20241115034902.GP3387508@ZenIV [axboe: make a more coherent commit message] Signed-off-by: Jens Axboe --- io_uring/msg_ring.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index e63af34004b7..333c220d322a 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -333,7 +333,6 @@ int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags) int io_uring_sync_msg_ring(struct io_uring_sqe *sqe) { struct io_msg io_msg = { }; - struct fd f; int ret; ret = __io_msg_ring_prep(&io_msg, sqe); @@ -347,16 +346,13 @@ int io_uring_sync_msg_ring(struct io_uring_sqe *sqe) if (io_msg.cmd != IORING_MSG_DATA) return -EINVAL; - ret = -EBADF; - f = fdget(sqe->fd); - if (fd_file(f)) { - ret = -EBADFD; - if (io_is_uring_fops(fd_file(f))) - ret = __io_msg_ring_data(fd_file(f)->private_data, - &io_msg, IO_URING_F_UNLOCKED); - fdput(f); - } - return ret; + CLASS(fd, f)(sqe->fd); + if (fd_empty(f)) + return -EBADF; + if (!io_is_uring_fops(fd_file(f))) + return -EBADFD; + return __io_msg_ring_data(fd_file(f)->private_data, + &io_msg, IO_URING_F_UNLOCKED); } void io_msg_cache_free(const void *entry) From 68685fa20edc5307fc893a06473c19661c236f29 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 15 Nov 2024 16:54:38 +0000 Subject: [PATCH 73/79] io_uring: fortify io_pin_pages with a warning We're a bit too frivolous with types of nr_pages arguments, converting it to long and back to int, passing an unsigned int pointer as an int pointer and so on. Shouldn't cause any problem but should be carefully reviewed, but until then let's add a WARN_ON_ONCE check to be more confident callers don't pass poorely checked arguents. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/d48e0c097cbd90fb47acaddb6c247596510d8cfc.1731689588.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/memmap.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/io_uring/memmap.c b/io_uring/memmap.c index 85c66fa54956..6ab59c60dfd0 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -140,6 +140,8 @@ struct page **io_pin_pages(unsigned long uaddr, unsigned long len, int *npages) nr_pages = end - start; if (WARN_ON_ONCE(!nr_pages)) return ERR_PTR(-EINVAL); + if (WARN_ON_ONCE(nr_pages > INT_MAX)) + return ERR_PTR(-EOVERFLOW); pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL); if (!pages) From 3730aebbdac8770f64ab66eb5e7129bc8dae731d Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 15 Nov 2024 16:54:39 +0000 Subject: [PATCH 74/79] io_uring: disable ENTER_EXT_ARG_REG for IOPOLL IOPOLL doesn't use the extended arguments, no need for it to support IORING_ENTER_EXT_ARG_REG. Let's disable it for IOPOLL, if anything it leaves more space for future extensions. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/a35ecd919dbdc17bd5b7932273e317832c531b45.1731689588.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index bd71782057de..464a70bde7e6 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3214,12 +3214,8 @@ static int io_validate_ext_arg(struct io_ring_ctx *ctx, unsigned flags, if (!(flags & IORING_ENTER_EXT_ARG)) return 0; - - if (flags & IORING_ENTER_EXT_ARG_REG) { - if (argsz != sizeof(struct io_uring_reg_wait)) - return -EINVAL; - return PTR_ERR(io_get_ext_arg_reg(ctx, argp)); - } + if (flags & IORING_ENTER_EXT_ARG_REG) + return -EINVAL; if (argsz != sizeof(arg)) return -EINVAL; if (copy_from_user(&arg, argp, sizeof(arg))) From 83e041522eb9c45479f4490b212687cf1e7e9999 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 15 Nov 2024 16:54:40 +0000 Subject: [PATCH 75/79] io_uring: temporarily disable registered waits Disable wait argument registration as it'll be replaced with a more generic feature. We'll still need IORING_ENTER_EXT_ARG_REG parsing in a few commits so leave it be. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/70b1d1d218c41ba77a76d1789c8641dab0b0563e.1731689588.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 10 ----- include/uapi/linux/io_uring.h | 3 -- io_uring/io_uring.c | 10 ----- io_uring/register.c | 82 ---------------------------------- io_uring/register.h | 1 - 5 files changed, 106 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 072e65e93105..52a5da99a205 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -330,14 +330,6 @@ struct io_ring_ctx { atomic_t cq_wait_nr; atomic_t cq_timeouts; struct wait_queue_head cq_wait; - - /* - * If registered with IORING_REGISTER_CQWAIT_REG, a single - * page holds N entries, mapped in cq_wait_arg. cq_wait_index - * is the maximum allowable index. - */ - struct io_uring_reg_wait *cq_wait_arg; - unsigned char cq_wait_index; } ____cacheline_aligned_in_smp; /* timeouts */ @@ -431,8 +423,6 @@ struct io_ring_ctx { unsigned short n_sqe_pages; struct page **ring_pages; struct page **sqe_pages; - - struct page **cq_wait_page; }; struct io_tw_state { diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 5d08435b95a8..132f5db3d4e8 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -627,9 +627,6 @@ enum io_uring_register_op { /* resize CQ ring */ IORING_REGISTER_RESIZE_RINGS = 33, - /* register fixed io_uring_reg_wait arguments */ - IORING_REGISTER_CQWAIT_REG = 34, - /* this goes last */ IORING_REGISTER_LAST, diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 464a70bde7e6..286b7bb73978 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2709,7 +2709,6 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) io_alloc_cache_free(&ctx->msg_cache, io_msg_cache_free); io_futex_cache_free(ctx); io_destroy_buffers(ctx); - io_unregister_cqwait_reg(ctx); mutex_unlock(&ctx->uring_lock); if (ctx->sq_creds) put_cred(ctx->sq_creds); @@ -3195,15 +3194,6 @@ void __io_uring_cancel(bool cancel_all) static struct io_uring_reg_wait *io_get_ext_arg_reg(struct io_ring_ctx *ctx, const struct io_uring_getevents_arg __user *uarg) { - struct io_uring_reg_wait *arg = READ_ONCE(ctx->cq_wait_arg); - - if (arg) { - unsigned int index = (unsigned int) (uintptr_t) uarg; - - if (index <= ctx->cq_wait_index) - return arg + index; - } - return ERR_PTR(-EFAULT); } diff --git a/io_uring/register.c b/io_uring/register.c index 45edfc57963a..3c5a3cfb186b 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -570,82 +570,6 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) return ret; } -void io_unregister_cqwait_reg(struct io_ring_ctx *ctx) -{ - unsigned short npages = 1; - - if (!ctx->cq_wait_page) - return; - - io_pages_unmap(ctx->cq_wait_arg, &ctx->cq_wait_page, &npages, true); - ctx->cq_wait_arg = NULL; - if (ctx->user) - __io_unaccount_mem(ctx->user, 1); -} - -/* - * Register a page holding N entries of struct io_uring_reg_wait, which can - * be used via io_uring_enter(2) if IORING_GETEVENTS_EXT_ARG_REG is set. - * If that is set with IORING_GETEVENTS_EXT_ARG, then instead of passing - * in a pointer for a struct io_uring_getevents_arg, an index into this - * registered array is passed, avoiding two (arg + timeout) copies per - * invocation. - */ -static int io_register_cqwait_reg(struct io_ring_ctx *ctx, void __user *uarg) -{ - struct io_uring_cqwait_reg_arg arg; - struct io_uring_reg_wait *reg; - struct page **pages; - unsigned long len; - int nr_pages, poff; - int ret; - - if (ctx->cq_wait_page || ctx->cq_wait_arg) - return -EBUSY; - if (copy_from_user(&arg, uarg, sizeof(arg))) - return -EFAULT; - if (!arg.nr_entries || arg.flags) - return -EINVAL; - if (arg.struct_size != sizeof(*reg)) - return -EINVAL; - if (check_mul_overflow(arg.struct_size, arg.nr_entries, &len)) - return -EOVERFLOW; - if (len > PAGE_SIZE) - return -EINVAL; - /* offset + len must fit within a page, and must be reg_wait aligned */ - poff = arg.user_addr & ~PAGE_MASK; - if (len + poff > PAGE_SIZE) - return -EINVAL; - if (poff % arg.struct_size) - return -EINVAL; - - pages = io_pin_pages(arg.user_addr, len, &nr_pages); - if (IS_ERR(pages)) - return PTR_ERR(pages); - ret = -EINVAL; - if (nr_pages != 1) - goto out_free; - if (ctx->user) { - ret = __io_account_mem(ctx->user, 1); - if (ret) - goto out_free; - } - - reg = vmap(pages, 1, VM_MAP, PAGE_KERNEL); - if (reg) { - ctx->cq_wait_index = arg.nr_entries - 1; - WRITE_ONCE(ctx->cq_wait_page, pages); - WRITE_ONCE(ctx->cq_wait_arg, (void *) reg + poff); - return 0; - } - ret = -ENOMEM; - if (ctx->user) - __io_unaccount_mem(ctx->user, 1); -out_free: - io_pages_free(&pages, nr_pages); - return ret; -} - static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, void __user *arg, unsigned nr_args) __releases(ctx->uring_lock) @@ -840,12 +764,6 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, break; ret = io_register_resize_rings(ctx, arg); break; - case IORING_REGISTER_CQWAIT_REG: - ret = -EINVAL; - if (!arg || nr_args != 1) - break; - ret = io_register_cqwait_reg(ctx, arg); - break; default: ret = -EINVAL; break; diff --git a/io_uring/register.h b/io_uring/register.h index 3e935e8fa4b2..a5f39d5ef9e0 100644 --- a/io_uring/register.h +++ b/io_uring/register.h @@ -5,6 +5,5 @@ int io_eventfd_unregister(struct io_ring_ctx *ctx); int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id); struct file *io_uring_register_get_file(unsigned int fd, bool registered); -void io_unregister_cqwait_reg(struct io_ring_ctx *ctx); #endif From dfbbfbf191878e8dd422768ce009858d8b5b761e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 15 Nov 2024 16:54:41 +0000 Subject: [PATCH 76/79] io_uring: introduce concept of memory regions We've got a good number of mappings we share with the userspace, that includes the main rings, provided buffer rings, upcoming rings for zerocopy rx and more. All of them duplicate user argument parsing and some internal details as well (page pinnning, huge page optimisations, mmap'ing, etc.) Introduce a notion of regions. For userspace for now it's just a new structure called struct io_uring_region_desc which is supposed to parameterise all such mapping / queue creations. A region either represents a user provided chunk of memory, in which case the user_addr field should point to it, or a request for the kernel to allocate the memory, in which case the user would need to mmap it after using the offset returned in the mmap_offset field. With a uniform userspace API we can avoid additional boiler plate code and apply future optimisation to all of them at once. Internally, there is a new structure struct io_mapped_region holding all relevant runtime information and some helpers to work with it. This patch limits it to user provided regions. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/0e6fe25818dfbaebd1bd90b870a6cac503fe1a24.1731689588.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 6 +++ include/uapi/linux/io_uring.h | 14 +++++++ io_uring/memmap.c | 67 ++++++++++++++++++++++++++++++++++ io_uring/memmap.h | 14 +++++++ 4 files changed, 101 insertions(+) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 52a5da99a205..1d3a37234ace 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -75,6 +75,12 @@ struct io_hash_table { unsigned hash_bits; }; +struct io_mapped_region { + struct page **pages; + void *vmap_ptr; + size_t nr_pages; +}; + /* * Arbitrary limit, can be raised if need be */ diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 132f5db3d4e8..5cbfd330c688 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -647,6 +647,20 @@ struct io_uring_files_update { __aligned_u64 /* __s32 * */ fds; }; +enum { + /* initialise with user provided memory pointed by user_addr */ + IORING_MEM_REGION_TYPE_USER = 1, +}; + +struct io_uring_region_desc { + __u64 user_addr; + __u64 size; + __u32 flags; + __u32 id; + __u64 mmap_offset; + __u64 __resv[4]; +}; + /* * Register a fully sparse file space, rather than pass in an array of all * -1 file descriptors. diff --git a/io_uring/memmap.c b/io_uring/memmap.c index 6ab59c60dfd0..bbd9569a0120 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -12,6 +12,7 @@ #include "memmap.h" #include "kbuf.h" +#include "rsrc.h" static void *io_mem_alloc_compound(struct page **pages, int nr_pages, size_t size, gfp_t gfp) @@ -194,6 +195,72 @@ void *__io_uaddr_map(struct page ***pages, unsigned short *npages, return ERR_PTR(-ENOMEM); } +void io_free_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr) +{ + if (mr->pages) { + unpin_user_pages(mr->pages, mr->nr_pages); + kvfree(mr->pages); + } + if (mr->vmap_ptr) + vunmap(mr->vmap_ptr); + if (mr->nr_pages && ctx->user) + __io_unaccount_mem(ctx->user, mr->nr_pages); + + memset(mr, 0, sizeof(*mr)); +} + +int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, + struct io_uring_region_desc *reg) +{ + int pages_accounted = 0; + struct page **pages; + int nr_pages, ret; + void *vptr; + u64 end; + + if (WARN_ON_ONCE(mr->pages || mr->vmap_ptr || mr->nr_pages)) + return -EFAULT; + if (memchr_inv(®->__resv, 0, sizeof(reg->__resv))) + return -EINVAL; + if (reg->flags != IORING_MEM_REGION_TYPE_USER) + return -EINVAL; + if (!reg->user_addr) + return -EFAULT; + if (!reg->size || reg->mmap_offset || reg->id) + return -EINVAL; + if ((reg->size >> PAGE_SHIFT) > INT_MAX) + return E2BIG; + if ((reg->user_addr | reg->size) & ~PAGE_MASK) + return -EINVAL; + if (check_add_overflow(reg->user_addr, reg->size, &end)) + return -EOVERFLOW; + + pages = io_pin_pages(reg->user_addr, reg->size, &nr_pages); + if (IS_ERR(pages)) + return PTR_ERR(pages); + + if (ctx->user) { + ret = __io_account_mem(ctx->user, nr_pages); + if (ret) + goto out_free; + pages_accounted = nr_pages; + } + + vptr = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL); + if (!vptr) + goto out_free; + + mr->pages = pages; + mr->vmap_ptr = vptr; + mr->nr_pages = nr_pages; + return 0; +out_free: + if (pages_accounted) + __io_unaccount_mem(ctx->user, pages_accounted); + io_pages_free(&pages, nr_pages); + return ret; +} + static void *io_uring_validate_mmap_request(struct file *file, loff_t pgoff, size_t sz) { diff --git a/io_uring/memmap.h b/io_uring/memmap.h index 5cec5b7ac49a..f361a635b6c7 100644 --- a/io_uring/memmap.h +++ b/io_uring/memmap.h @@ -22,4 +22,18 @@ unsigned long io_uring_get_unmapped_area(struct file *file, unsigned long addr, unsigned long flags); int io_uring_mmap(struct file *file, struct vm_area_struct *vma); +void io_free_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr); +int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, + struct io_uring_region_desc *reg); + +static inline void *io_region_get_ptr(struct io_mapped_region *mr) +{ + return mr->vmap_ptr; +} + +static inline bool io_region_is_set(struct io_mapped_region *mr) +{ + return !!mr->nr_pages; +} + #endif From 93238e66185524aad925acefb2312203b9e26d63 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 15 Nov 2024 16:54:42 +0000 Subject: [PATCH 77/79] io_uring: add memory region registration Regions will serve multiple purposes. First, with it we can decouple ring/etc. object creation from registration / mapping of the memory they will be placed in. We already have hacks that allow to put both SQ and CQ into the same huge page, in the future we should be able to: region = create_region(io_ring); create_pbuf_ring(io_uring, region, offset=0); create_pbuf_ring(io_uring, region, offset=N); The second use case is efficiently passing parameters. The following patch enables back on top of regions IORING_ENTER_EXT_ARG_REG, which optimises wait arguments. It'll also be useful for request arguments replacing iovecs, msghdr, etc. pointers. Eventually it would also be handy for BPF as well if it comes to fruition. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/0798cf3a14fad19cfc96fc9feca5f3e11481691d.1731689588.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 3 +++ include/uapi/linux/io_uring.h | 8 ++++++++ io_uring/io_uring.c | 1 + io_uring/register.c | 37 ++++++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 1d3a37234ace..e1d69123e164 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -429,6 +429,9 @@ struct io_ring_ctx { unsigned short n_sqe_pages; struct page **ring_pages; struct page **sqe_pages; + + /* used for optimised request parameter and wait argument passing */ + struct io_mapped_region param_region; }; struct io_tw_state { diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 5cbfd330c688..1ee35890125b 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -627,6 +627,8 @@ enum io_uring_register_op { /* resize CQ ring */ IORING_REGISTER_RESIZE_RINGS = 33, + IORING_REGISTER_MEM_REGION = 34, + /* this goes last */ IORING_REGISTER_LAST, @@ -661,6 +663,12 @@ struct io_uring_region_desc { __u64 __resv[4]; }; +struct io_uring_mem_region_reg { + __u64 region_uptr; /* struct io_uring_region_desc * */ + __u64 flags; + __u64 __resv[2]; +}; + /* * Register a fully sparse file space, rather than pass in an array of all * -1 file descriptors. diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 286b7bb73978..c640b8a4ceee 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2709,6 +2709,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) io_alloc_cache_free(&ctx->msg_cache, io_msg_cache_free); io_futex_cache_free(ctx); io_destroy_buffers(ctx); + io_free_region(ctx, &ctx->param_region); mutex_unlock(&ctx->uring_lock); if (ctx->sq_creds) put_cred(ctx->sq_creds); diff --git a/io_uring/register.c b/io_uring/register.c index 3c5a3cfb186b..2cbac3d9b288 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -570,6 +570,37 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) return ret; } +static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg) +{ + struct io_uring_mem_region_reg __user *reg_uptr = uarg; + struct io_uring_mem_region_reg reg; + struct io_uring_region_desc __user *rd_uptr; + struct io_uring_region_desc rd; + int ret; + + if (io_region_is_set(&ctx->param_region)) + return -EBUSY; + if (copy_from_user(®, reg_uptr, sizeof(reg))) + return -EFAULT; + rd_uptr = u64_to_user_ptr(reg.region_uptr); + if (copy_from_user(&rd, rd_uptr, sizeof(rd))) + return -EFAULT; + + if (memchr_inv(®.__resv, 0, sizeof(reg.__resv))) + return -EINVAL; + if (reg.flags) + return -EINVAL; + + ret = io_create_region(ctx, &ctx->param_region, &rd); + if (ret) + return ret; + if (copy_to_user(rd_uptr, &rd, sizeof(rd))) { + io_free_region(ctx, &ctx->param_region); + return -EFAULT; + } + return 0; +} + static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, void __user *arg, unsigned nr_args) __releases(ctx->uring_lock) @@ -764,6 +795,12 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, break; ret = io_register_resize_rings(ctx, arg); break; + case IORING_REGISTER_MEM_REGION: + ret = -EINVAL; + if (!arg || nr_args != 1) + break; + ret = io_register_mem_region(ctx, arg); + break; default: ret = -EINVAL; break; From d617b3147d54c42351eac63b5398d4ddf4f4011b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 15 Nov 2024 16:54:43 +0000 Subject: [PATCH 78/79] io_uring: restore back registered wait arguments Now we've got a more generic region registration API, place IORING_ENTER_EXT_ARG_REG and re-enable it. First, the user has to register a region with the IORING_MEM_REGION_REG_WAIT_ARG flag set. It can only be done for a ring in a disabled state, aka IORING_SETUP_R_DISABLED, to avoid races with already running waiters. With that we should have stable constant values for ctx->cq_wait_{size,arg} in io_get_ext_arg_reg() and hence no READ_ONCE required. The other API difference is that we're now passing byte offsets instead of indexes. The user _must_ align all offsets / pointers to the native word size, failing to do so might but not necessarily has to lead to a failure usually returned as -EFAULT. liburing will be hiding this details from users. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/81822c1b4ffbe8ad391b4f9ad1564def0d26d990.1731689588.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 3 +++ include/uapi/linux/io_uring.h | 5 +++++ io_uring/io_uring.c | 14 +++++++++++++- io_uring/register.c | 16 +++++++++++++++- 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index e1d69123e164..aa5f5ea98076 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -324,6 +324,9 @@ struct io_ring_ctx { unsigned cq_entries; struct io_ev_fd __rcu *io_ev_fd; unsigned cq_extra; + + void *cq_wait_arg; + size_t cq_wait_size; } ____cacheline_aligned_in_smp; /* diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 1ee35890125b..4418d0192959 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -663,6 +663,11 @@ struct io_uring_region_desc { __u64 __resv[4]; }; +enum { + /* expose the region as registered wait arguments */ + IORING_MEM_REGION_REG_WAIT_ARG = 1, +}; + struct io_uring_mem_region_reg { __u64 region_uptr; /* struct io_uring_region_desc * */ __u64 flags; diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index c640b8a4ceee..da8fd460977b 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3195,7 +3195,19 @@ void __io_uring_cancel(bool cancel_all) static struct io_uring_reg_wait *io_get_ext_arg_reg(struct io_ring_ctx *ctx, const struct io_uring_getevents_arg __user *uarg) { - return ERR_PTR(-EFAULT); + unsigned long size = sizeof(struct io_uring_reg_wait); + unsigned long offset = (uintptr_t)uarg; + unsigned long end; + + if (unlikely(offset % sizeof(long))) + return ERR_PTR(-EFAULT); + + /* also protects from NULL ->cq_wait_arg as the size would be 0 */ + if (unlikely(check_add_overflow(offset, size, &end) || + end > ctx->cq_wait_size)) + return ERR_PTR(-EFAULT); + + return ctx->cq_wait_arg + offset; } static int io_validate_ext_arg(struct io_ring_ctx *ctx, unsigned flags, diff --git a/io_uring/register.c b/io_uring/register.c index 2cbac3d9b288..1a60f4916649 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -588,7 +588,16 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg) if (memchr_inv(®.__resv, 0, sizeof(reg.__resv))) return -EINVAL; - if (reg.flags) + if (reg.flags & ~IORING_MEM_REGION_REG_WAIT_ARG) + return -EINVAL; + + /* + * This ensures there are no waiters. Waiters are unlocked and it's + * hard to synchronise with them, especially if we need to initialise + * the region. + */ + if ((reg.flags & IORING_MEM_REGION_REG_WAIT_ARG) && + !(ctx->flags & IORING_SETUP_R_DISABLED)) return -EINVAL; ret = io_create_region(ctx, &ctx->param_region, &rd); @@ -598,6 +607,11 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg) io_free_region(ctx, &ctx->param_region); return -EFAULT; } + + if (reg.flags & IORING_MEM_REGION_REG_WAIT_ARG) { + ctx->cq_wait_arg = io_region_get_ptr(&ctx->param_region); + ctx->cq_wait_size = rd.size; + } return 0; } From a652958888fb1ada3e4f6b548576c2d2c1b60d66 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 17 Nov 2024 00:38:33 +0000 Subject: [PATCH 79/79] io_uring/region: fix error codes after failed vmap io_create_region() jumps after a vmap failure without setting the return code, it could be 0 or just uninitialised. Fixes: dfbbfbf191878 ("io_uring: introduce concept of memory regions") Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/0abac19dbf81c061cffaa9534a2471ed5460ad3e.1731803848.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/memmap.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/io_uring/memmap.c b/io_uring/memmap.c index bbd9569a0120..6e6ee79ba94f 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -247,8 +247,10 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, } vptr = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL); - if (!vptr) + if (!vptr) { + ret = -ENOMEM; goto out_free; + } mr->pages = pages; mr->vmap_ptr = vptr;