lkml.org 
[lkml]   [2022]   [Jul]   [12]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH net-next v5 16/27] io_uring: complete notifiers in tw
    Date
    We need a task context to post CQEs but using wq is too expensive.
    Try to complete notifiers using task_work and fall back to wq if fails.

    Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
    ---
    io_uring/notif.c | 22 +++++++++++++++++++---
    io_uring/notif.h | 3 +++
    2 files changed, 22 insertions(+), 3 deletions(-)

    diff --git a/io_uring/notif.c b/io_uring/notif.c
    index b257db2120b4..aec74f88fc33 100644
    --- a/io_uring/notif.c
    +++ b/io_uring/notif.c
    @@ -13,6 +13,11 @@ static void __io_notif_complete_tw(struct callback_head *cb)
    struct io_notif *notif = container_of(cb, struct io_notif, task_work);
    struct io_ring_ctx *ctx = notif->ctx;

    + if (likely(notif->task)) {
    + io_put_task(notif->task, 1);
    + notif->task = NULL;
    + }
    +
    io_cq_lock(ctx);
    io_fill_cqe_aux(ctx, notif->tag, 0, notif->seq, true);

    @@ -43,6 +48,14 @@ static void io_uring_tx_zerocopy_callback(struct sk_buff *skb,

    if (!refcount_dec_and_test(&uarg->refcnt))
    return;
    +
    + if (likely(notif->task)) {
    + init_task_work(&notif->task_work, __io_notif_complete_tw);
    + if (likely(!task_work_add(notif->task, &notif->task_work,
    + TWA_SIGNAL)))
    + return;
    + }
    +
    INIT_WORK(&notif->commit_work, io_notif_complete_wq);
    queue_work(system_unbound_wq, &notif->commit_work);
    }
    @@ -134,12 +147,15 @@ __cold int io_notif_unregister(struct io_ring_ctx *ctx)
    for (i = 0; i < ctx->nr_notif_slots; i++) {
    struct io_notif_slot *slot = &ctx->notif_slots[i];

    - if (slot->notif)
    - io_notif_slot_flush(slot);
    + if (!slot->notif)
    + continue;
    + if (WARN_ON_ONCE(slot->notif->task))
    + slot->notif->task = NULL;
    + io_notif_slot_flush(slot);
    }

    kvfree(ctx->notif_slots);
    ctx->notif_slots = NULL;
    ctx->nr_notif_slots = 0;
    return 0;
    -}
    \ No newline at end of file
    +}
    diff --git a/io_uring/notif.h b/io_uring/notif.h
    index b23c9c0515bb..23ca7620fff9 100644
    --- a/io_uring/notif.h
    +++ b/io_uring/notif.h
    @@ -11,6 +11,9 @@ struct io_notif {
    struct ubuf_info uarg;
    struct io_ring_ctx *ctx;

    + /* complete via tw if ->task is non-NULL, fallback to wq otherwise */
    + struct task_struct *task;
    +
    /* cqe->user_data, io_notif_slot::tag if not overridden */
    u64 tag;
    /* see struct io_notif_slot::seq */
    --
    2.37.0
    \
     
     \ /
      Last update: 2022-07-12 22:56    [W:3.443 / U:0.016 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site