lkml.org 
[lkml]   [2022]   [Jun]   [28]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
Patch in this message
/
From
Subject[RFC net-next v3 17/29] io_uring: complete notifiers in tw
Date
We need a task context to post CQEs but using wq is too expensive.
Try to complete notifiers using task_work and fall back to wq if fails.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
fs/io_uring.c | 21 ++++++++++++++++++++-
1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 422ff835bf36..9ade0ea8552b 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -384,6 +384,8 @@ struct io_notif {
/* hook into ctx->notif_list and ctx->notif_list_locked */
struct list_head cache_node;

+ /* complete via tw if ->task is non-NULL, fallback to wq otherwise */
+ struct task_struct *task;
union {
struct callback_head task_work;
struct work_struct commit_work;
@@ -2802,6 +2804,11 @@ static void __io_notif_complete_tw(struct callback_head *cb)
struct io_notif *notif = container_of(cb, struct io_notif, task_work);
struct io_ring_ctx *ctx = notif->ctx;

+ if (likely(notif->task)) {
+ io_put_task(notif->task, 1);
+ notif->task = NULL;
+ }
+
spin_lock(&ctx->completion_lock);
io_fill_cqe_aux(ctx, notif->tag, 0, notif->seq);

@@ -2835,6 +2842,14 @@ static void io_uring_tx_zerocopy_callback(struct sk_buff *skb,

if (!refcount_dec_and_test(&uarg->refcnt))
return;
+
+ if (likely(notif->task)) {
+ init_task_work(&notif->task_work, __io_notif_complete_tw);
+ if (likely(!task_work_add(notif->task, &notif->task_work,
+ TWA_SIGNAL)))
+ return;
+ }
+
INIT_WORK(&notif->commit_work, io_notif_complete_wq);
queue_work(system_unbound_wq, &notif->commit_work);
}
@@ -2946,8 +2961,12 @@ static __cold int io_notif_unregister(struct io_ring_ctx *ctx)
for (i = 0; i < ctx->nr_notif_slots; i++) {
struct io_notif_slot *slot = &ctx->notif_slots[i];

- if (slot->notif)
+ if (slot->notif) {
+ WARN_ON_ONCE(slot->notif->task);
+
+ slot->notif->task = NULL;
io_notif_slot_flush(slot);
+ }
}

kvfree(ctx->notif_slots);
--
2.36.1
\
 
 \ /
  Last update: 2022-06-28 21:04    [W:0.223 / U:0.096 seconds]
©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site