mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2024-11-21 19:46:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

io_uring/rsrc: split io_kiocb node type assignments

Browse Source 
Currently the io_rsrc_node assignment in io_kiocb is an array of two
pointers, as two nodes may be assigned to a request - one file node,
and one buffer node. However, the buffer node can co-exist with the
provided buffers, as currently it's not supported to use both provided
and registered buffers at the same time.

This crucially brings struct io_kiocb down to 4 cache lines again, as
before it spilled into the 5th cacheline.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe 2024-11-03 08:46:07 -07:00
parent 6af82f7614
commit 6f94cbc29a
8 changed files with 29 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
include/linux/io_uring_types.h
View File

@ -475,6 +475,7 @@ enum {
REQ_F_BL_EMPTY_BIT,
REQ_F_BL_NO_RECYCLE_BIT,
REQ_F_BUFFERS_COMMIT_BIT,
REQ_F_BUF_NODE_BIT,
/* not a real bit, just to check we're not overflowing the space */
__REQ_F_LAST_BIT,
@ -553,6 +554,8 @@ enum {
REQ_F_BL_NO_RECYCLE = IO_REQ_FLAG(REQ_F_BL_NO_RECYCLE_BIT),
/* buffer ring head needs incrementing on put */
REQ_F_BUFFERS_COMMIT = IO_REQ_FLAG(REQ_F_BUFFERS_COMMIT_BIT),
/* buf node is valid */
REQ_F_BUF_NODE = IO_REQ_FLAG(REQ_F_BUF_NODE_BIT),
};
typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);
@ -633,6 +636,8 @@ struct io_kiocb {
* REQ_F_BUFFER_RING is set.
*/
struct io_buffer_list *buf_list;
struct io_rsrc_node *buf_node;
};
union {
@ -642,7 +647,7 @@ struct io_kiocb {
__poll_t apoll_events;
};
struct io_rsrc_node *rsrc_nodes[2];
struct io_rsrc_node *file_node;
atomic_t refs;
bool cancel_seq_set;

6
io_uring/io_uring.c
View File

@ -948,8 +948,8 @@ void io_req_defer_failed(struct io_kiocb *req, s32 res)
static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)
{
req->ctx = ctx;
req->rsrc_nodes[IORING_RSRC_FILE] = NULL;
req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL;
req->buf_node = NULL;
req->file_node = NULL;
req->link = NULL;
req->async_data = NULL;
/* not necessary, but safer to zero */
@ -1882,7 +1882,7 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
io_ring_submit_lock(ctx, issue_flags);
node = io_rsrc_node_lookup(&ctx->file_table.data, fd);
if (node) {
io_req_assign_rsrc_node(req, node);
io_req_assign_rsrc_node(&req->file_node, node);
req->flags |= io_slot_flags(node);
file = io_slot_file(node);
}

3
io_uring/net.c
View File

@ -1348,7 +1348,8 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags)
io_ring_submit_lock(ctx, issue_flags);
node = io_rsrc_node_lookup(&ctx->buf_table, sr->buf_index);
if (node) {
io_req_assign_rsrc_node(sr->notif, node);
io_req_assign_rsrc_node(&sr->notif->buf_node, node);
sr->notif->flags |= REQ_F_BUF_NODE;
ret = 0;
}
io_ring_submit_unlock(ctx, issue_flags);

3
io_uring/nop.c
View File

@ -67,7 +67,8 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags)
io_ring_submit_lock(ctx, issue_flags);
node = io_rsrc_node_lookup(&ctx->buf_table, nop->buffer);
if (node) {
io_req_assign_rsrc_node(req, node);
io_req_assign_rsrc_node(&req->buf_node, node);
req->flags |= REQ_F_BUF_NODE;
ret = 0;
}
io_ring_submit_unlock(ctx, issue_flags);

4
io_uring/notif.c
View File

@ -117,8 +117,8 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
notif->file = NULL;
notif->task = current;
io_get_task_refs(1);
notif->rsrc_nodes[IORING_RSRC_FILE] = NULL;
notif->rsrc_nodes[IORING_RSRC_BUFFER] = NULL;
notif->file_node = NULL;
notif->buf_node = NULL;
nd = io_notif_to_data(notif);
nd->zc_report = false;

16
io_uring/rsrc.h
View File

@ -95,10 +95,14 @@ static inline bool io_reset_rsrc_node(struct io_rsrc_data *data, int index)
static inline void io_req_put_rsrc_nodes(struct io_kiocb *req)
{
io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_FILE]);
io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_BUFFER]);
req->rsrc_nodes[IORING_RSRC_FILE] = NULL;
req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL;
if (req->file_node) {
io_put_rsrc_node(req->file_node);
req->file_node = NULL;
}
if (req->flags & REQ_F_BUF_NODE) {
io_put_rsrc_node(req->buf_node);
req->buf_node = NULL;
}
}
static inline struct io_ring_ctx *io_rsrc_node_ctx(struct io_rsrc_node *node)
@ -111,11 +115,11 @@ static inline int io_rsrc_node_type(struct io_rsrc_node *node)
return node->ctx_ptr & IORING_RSRC_TYPE_MASK;
}
static inline void io_req_assign_rsrc_node(struct io_kiocb *req,
static inline void io_req_assign_rsrc_node(struct io_rsrc_node **dst_node,
struct io_rsrc_node *node)
{
node->refs++;
req->rsrc_nodes[io_rsrc_node_type(node)] = node;
*dst_node = node;
}
int io_files_update(struct io_kiocb *req, unsigned int issue_flags);

3
io_uring/rw.c
View File

@ -341,7 +341,8 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe
node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index);
if (!node)
return -EFAULT;
io_req_assign_rsrc_node(req, node);
io_req_assign_rsrc_node(&req->buf_node, node);
req->flags |= REQ_F_BUF_NODE;
io = req->async_data;
ret = io_import_fixed(ddir, &io->iter, node->buf, rw->addr, rw->len);

4
io_uring/uring_cmd.c
View File

@ -219,7 +219,7 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
* being called. This prevents destruction of the mapped buffer
* we'll need at actual import time.
*/
io_req_assign_rsrc_node(req, node);
io_req_assign_rsrc_node(&req->buf_node, node);
}
ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
@ -275,7 +275,7 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
struct iov_iter *iter, void *ioucmd)
{
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
struct io_rsrc_node *node = req->rsrc_nodes[IORING_RSRC_BUFFER];
struct io_rsrc_node *node = req->buf_node;
/* Must have had rsrc_node assigned at prep time */
if (node)