nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

io_uring: do msg_ring in target task via tw

Browse source 
While executing in a context of one io_uring instance msg_ring
manipulates another ring. We're trying to keep CQEs posting contained in
the context of the ring-owner task, use task_work to send the request to
the target ring's task when we're modifying its CQ or trying to install
a file. Note, we can't safely use io_uring task_work infra and have to
use task_work directly.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/4d76c7b28ed5d71b520de4482fbb7f660f21cd80.1670384893.git.asml.silence@gmail.com
[axboe: use TWA_SIGNAL_NO_IPI]
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2022-12-07 03:53:36 +00:00 committed by Jens Axboe
parent 1721131016
commit 6d043ee116
1 changed files with 53 additions and 3 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

56
io_uring/msg_ring.c
View file

@ -16,6 +16,7 @@
struct io_msg { struct io_msg {
struct file *file; struct file *file;
struct file *src_file; struct file *src_file;
struct callback_head tw;
u64 user_data; u64 user_data;
u32 len; u32 len;
u32 cmd; u32 cmd;
@ -35,6 +36,23 @@ void io_msg_ring_cleanup(struct io_kiocb *req)
msg->src_file = NULL; msg->src_file = NULL;
} }
static void io_msg_tw_complete(struct callback_head *head)
{
struct io_msg *msg = container_of(head, struct io_msg, tw);
struct io_kiocb *req = cmd_to_io_kiocb(msg);
struct io_ring_ctx *target_ctx = req->file->private_data;
int ret = 0;
if (current->flags & PF_EXITING)
ret = -EOWNERDEAD;
else if (!io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0))
ret = -EOVERFLOW;
if (ret < 0)
req_set_fail(req);
io_req_queue_tw_complete(req, ret);
}
static int io_msg_ring_data(struct io_kiocb *req) static int io_msg_ring_data(struct io_kiocb *req)
{ {
struct io_ring_ctx *target_ctx = req->file->private_data; struct io_ring_ctx *target_ctx = req->file->private_data;
@ -43,6 +61,15 @@ static int io_msg_ring_data(struct io_kiocb *req)
if (msg->src_fd || msg->dst_fd || msg->flags) if (msg->src_fd || msg->dst_fd || msg->flags)
return -EINVAL; return -EINVAL;
if (target_ctx->task_complete && current != target_ctx->submitter_task) {
init_task_work(&msg->tw, io_msg_tw_complete);
if (task_work_add(target_ctx->submitter_task, &msg->tw,
TWA_SIGNAL_NO_IPI))
return -EOWNERDEAD;
return IOU_ISSUE_SKIP_COMPLETE;
}
if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0)) if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0))
return 0; return 0;
@ -124,6 +151,19 @@ static int io_msg_install_complete(struct io_kiocb *req, unsigned int issue_flag
return ret; return ret;
} }
static void io_msg_tw_fd_complete(struct callback_head *head)
{
struct io_msg *msg = container_of(head, struct io_msg, tw);
struct io_kiocb *req = cmd_to_io_kiocb(msg);
int ret = -EOWNERDEAD;
if (!(current->flags & PF_EXITING))
ret = io_msg_install_complete(req, IO_URING_F_UNLOCKED);
if (ret < 0)
req_set_fail(req);
io_req_queue_tw_complete(req, ret);
}
static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags) static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags)
{ {
struct io_ring_ctx *target_ctx = req->file->private_data; struct io_ring_ctx *target_ctx = req->file->private_data;
@ -140,6 +180,15 @@ static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags)
msg->src_file = src_file; msg->src_file = src_file;
req->flags |= REQ_F_NEED_CLEANUP; req->flags |= REQ_F_NEED_CLEANUP;
} }
if (target_ctx->task_complete && current != target_ctx->submitter_task) {
init_task_work(&msg->tw, io_msg_tw_fd_complete);
if (task_work_add(target_ctx->submitter_task, &msg->tw,
TWA_SIGNAL))
return -EOWNERDEAD;
return IOU_ISSUE_SKIP_COMPLETE;
}
return io_msg_install_complete(req, issue_flags); return io_msg_install_complete(req, issue_flags);
} }
@ -185,10 +234,11 @@ int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags)
} }
done: done:
if (ret == -EAGAIN) if (ret < 0) {
return -EAGAIN; if (ret == -EAGAIN || ret == IOU_ISSUE_SKIP_COMPLETE)
if (ret < 0) return ret;
req_set_fail(req); req_set_fail(req);
}
io_req_set_res(req, ret, 0); io_req_set_res(req, ret, 0);
return IOU_OK; return IOU_OK;
} }