lkml.org 
[lkml]   [2022]   [Jun]   [13]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH 5.17 179/298] i40e: xsk: Move tmp desc array from driver to pool
    Date
    From: Magnus Karlsson <magnus.karlsson@intel.com>

    [ Upstream commit d1bc532e99becf104635ed4da6fefa306f452321 ]

    Move desc_array from the driver to the pool. The reason behind this is
    that we can then reuse this array as a temporary storage for descriptors
    in all zero-copy drivers that use the batched interface. This will make
    it easier to add batching to more drivers.

    i40e is the only driver that has a batched Tx zero-copy
    implementation, so no need to touch any other driver.

    Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
    Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
    Reviewed-by: Alexander Lobakin <alexandr.lobakin@intel.com>
    Link: https://lore.kernel.org/bpf/20220125160446.78976-6-maciej.fijalkowski@intel.com
    Signed-off-by: Sasha Levin <sashal@kernel.org>
    ---
    drivers/net/ethernet/intel/i40e/i40e_txrx.c | 11 -----------
    drivers/net/ethernet/intel/i40e/i40e_txrx.h | 1 -
    drivers/net/ethernet/intel/i40e/i40e_xsk.c | 4 ++--
    include/net/xdp_sock_drv.h | 5 ++---
    include/net/xsk_buff_pool.h | 1 +
    net/xdp/xsk.c | 13 ++++++-------
    net/xdp/xsk_buff_pool.c | 7 +++++++
    net/xdp/xsk_queue.h | 12 ++++++------
    8 files changed, 24 insertions(+), 30 deletions(-)

    diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
    index 66cc79500c10..af9c88e71452 100644
    --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c
    +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
    @@ -830,8 +830,6 @@ void i40e_free_tx_resources(struct i40e_ring *tx_ring)
    i40e_clean_tx_ring(tx_ring);
    kfree(tx_ring->tx_bi);
    tx_ring->tx_bi = NULL;
    - kfree(tx_ring->xsk_descs);
    - tx_ring->xsk_descs = NULL;

    if (tx_ring->desc) {
    dma_free_coherent(tx_ring->dev, tx_ring->size,
    @@ -1433,13 +1431,6 @@ int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
    if (!tx_ring->tx_bi)
    goto err;

    - if (ring_is_xdp(tx_ring)) {
    - tx_ring->xsk_descs = kcalloc(I40E_MAX_NUM_DESCRIPTORS, sizeof(*tx_ring->xsk_descs),
    - GFP_KERNEL);
    - if (!tx_ring->xsk_descs)
    - goto err;
    - }
    -
    u64_stats_init(&tx_ring->syncp);

    /* round up to nearest 4K */
    @@ -1463,8 +1454,6 @@ int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
    return 0;

    err:
    - kfree(tx_ring->xsk_descs);
    - tx_ring->xsk_descs = NULL;
    kfree(tx_ring->tx_bi);
    tx_ring->tx_bi = NULL;
    return -ENOMEM;
    diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
    index bfc2845c99d1..f6d91fa1562e 100644
    --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h
    +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
    @@ -390,7 +390,6 @@ struct i40e_ring {
    u16 rx_offset;
    struct xdp_rxq_info xdp_rxq;
    struct xsk_buff_pool *xsk_pool;
    - struct xdp_desc *xsk_descs; /* For storing descriptors in the AF_XDP ZC path */
    } ____cacheline_internodealigned_in_smp;

    static inline bool ring_uses_build_skb(struct i40e_ring *ring)
    diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
    index e5e72b5bb619..c1d25b0b0ca2 100644
    --- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c
    +++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
    @@ -470,11 +470,11 @@ static void i40e_set_rs_bit(struct i40e_ring *xdp_ring)
    **/
    static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget)
    {
    - struct xdp_desc *descs = xdp_ring->xsk_descs;
    + struct xdp_desc *descs = xdp_ring->xsk_pool->tx_descs;
    u32 nb_pkts, nb_processed = 0;
    unsigned int total_bytes = 0;

    - nb_pkts = xsk_tx_peek_release_desc_batch(xdp_ring->xsk_pool, descs, budget);
    + nb_pkts = xsk_tx_peek_release_desc_batch(xdp_ring->xsk_pool, budget);
    if (!nb_pkts)
    return true;

    diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h
    index 443d45951564..4aa031849668 100644
    --- a/include/net/xdp_sock_drv.h
    +++ b/include/net/xdp_sock_drv.h
    @@ -13,7 +13,7 @@

    void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries);
    bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc);
    -u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *desc, u32 max);
    +u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max);
    void xsk_tx_release(struct xsk_buff_pool *pool);
    struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev,
    u16 queue_id);
    @@ -142,8 +142,7 @@ static inline bool xsk_tx_peek_desc(struct xsk_buff_pool *pool,
    return false;
    }

    -static inline u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *desc,
    - u32 max)
    +static inline u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max)
    {
    return 0;
    }
    diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h
    index ddeefc4a1040..5554ee75e7da 100644
    --- a/include/net/xsk_buff_pool.h
    +++ b/include/net/xsk_buff_pool.h
    @@ -60,6 +60,7 @@ struct xsk_buff_pool {
    */
    dma_addr_t *dma_pages;
    struct xdp_buff_xsk *heads;
    + struct xdp_desc *tx_descs;
    u64 chunk_mask;
    u64 addrs_cnt;
    u32 free_list_cnt;
    diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
    index 39a82bfb5caa..7d3a00cb24ec 100644
    --- a/net/xdp/xsk.c
    +++ b/net/xdp/xsk.c
    @@ -343,9 +343,9 @@ bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc)
    }
    EXPORT_SYMBOL(xsk_tx_peek_desc);

    -static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, struct xdp_desc *descs,
    - u32 max_entries)
    +static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, u32 max_entries)
    {
    + struct xdp_desc *descs = pool->tx_descs;
    u32 nb_pkts = 0;

    while (nb_pkts < max_entries && xsk_tx_peek_desc(pool, &descs[nb_pkts]))
    @@ -355,8 +355,7 @@ static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, struct xdp_d
    return nb_pkts;
    }

    -u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *descs,
    - u32 max_entries)
    +u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max_entries)
    {
    struct xdp_sock *xs;
    u32 nb_pkts;
    @@ -365,7 +364,7 @@ u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *
    if (!list_is_singular(&pool->xsk_tx_list)) {
    /* Fallback to the non-batched version */
    rcu_read_unlock();
    - return xsk_tx_peek_release_fallback(pool, descs, max_entries);
    + return xsk_tx_peek_release_fallback(pool, max_entries);
    }

    xs = list_first_or_null_rcu(&pool->xsk_tx_list, struct xdp_sock, tx_list);
    @@ -374,7 +373,7 @@ u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *
    goto out;
    }

    - nb_pkts = xskq_cons_peek_desc_batch(xs->tx, descs, pool, max_entries);
    + nb_pkts = xskq_cons_peek_desc_batch(xs->tx, pool, max_entries);
    if (!nb_pkts) {
    xs->tx->queue_empty_descs++;
    goto out;
    @@ -386,7 +385,7 @@ u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *
    * packets. This avoids having to implement any buffering in
    * the Tx path.
    */
    - nb_pkts = xskq_prod_reserve_addr_batch(pool->cq, descs, nb_pkts);
    + nb_pkts = xskq_prod_reserve_addr_batch(pool->cq, pool->tx_descs, nb_pkts);
    if (!nb_pkts)
    goto out;

    diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c
    index 0202a90b65e3..af040ffa14ff 100644
    --- a/net/xdp/xsk_buff_pool.c
    +++ b/net/xdp/xsk_buff_pool.c
    @@ -37,6 +37,7 @@ void xp_destroy(struct xsk_buff_pool *pool)
    if (!pool)
    return;

    + kvfree(pool->tx_descs);
    kvfree(pool->heads);
    kvfree(pool);
    }
    @@ -58,6 +59,12 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs,
    if (!pool->heads)
    goto out;

    + if (xs->tx) {
    + pool->tx_descs = kcalloc(xs->tx->nentries, sizeof(*pool->tx_descs), GFP_KERNEL);
    + if (!pool->tx_descs)
    + goto out;
    + }
    +
    pool->chunk_mask = ~((u64)umem->chunk_size - 1);
    pool->addrs_cnt = umem->size;
    pool->heads_cnt = umem->chunks;
    diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h
    index e9aa2c236356..638138fbe475 100644
    --- a/net/xdp/xsk_queue.h
    +++ b/net/xdp/xsk_queue.h
    @@ -205,11 +205,11 @@ static inline bool xskq_cons_read_desc(struct xsk_queue *q,
    return false;
    }

    -static inline u32 xskq_cons_read_desc_batch(struct xsk_queue *q,
    - struct xdp_desc *descs,
    - struct xsk_buff_pool *pool, u32 max)
    +static inline u32 xskq_cons_read_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool,
    + u32 max)
    {
    u32 cached_cons = q->cached_cons, nb_entries = 0;
    + struct xdp_desc *descs = pool->tx_descs;

    while (cached_cons != q->cached_prod && nb_entries < max) {
    struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;
    @@ -282,12 +282,12 @@ static inline bool xskq_cons_peek_desc(struct xsk_queue *q,
    return xskq_cons_read_desc(q, desc, pool);
    }

    -static inline u32 xskq_cons_peek_desc_batch(struct xsk_queue *q, struct xdp_desc *descs,
    - struct xsk_buff_pool *pool, u32 max)
    +static inline u32 xskq_cons_peek_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool,
    + u32 max)
    {
    u32 entries = xskq_cons_nb_entries(q, max);

    - return xskq_cons_read_desc_batch(q, descs, pool, entries);
    + return xskq_cons_read_desc_batch(q, pool, entries);
    }

    /* To improve performance in the xskq_cons_release functions, only update local state here.
    --
    2.35.1


    \
     
     \ /
      Last update: 2022-06-13 16:35    [W:4.272 / U:0.012 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site