lkml.org 
[lkml]   [2021]   [Nov]   [15]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH 5.15 418/917] gve: Recover from queue stall due to missed IRQ
    Date
    From: John Fraker <jfraker@google.com>

    [ Upstream commit 87a7f321bb6a45e54b7d6c90d032ee5636a6ad97 ]

    Don't always reset the driver on a TX timeout. Attempt to
    recover by kicking the queue in case an IRQ was missed.

    Fixes: 9e5f7d26a4c08 ("gve: Add workqueue and reset support")
    Signed-off-by: John Fraker <jfraker@google.com>
    Signed-off-by: David Awogbemila <awogbemila@google.com>
    Signed-off-by: David S. Miller <davem@davemloft.net>
    Signed-off-by: Sasha Levin <sashal@kernel.org>
    ---
    drivers/net/ethernet/google/gve/gve.h | 4 +-
    drivers/net/ethernet/google/gve/gve_adminq.h | 1 +
    drivers/net/ethernet/google/gve/gve_main.c | 48 +++++++++++++++++++-
    3 files changed, 51 insertions(+), 2 deletions(-)

    diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
    index 2f93ed4705905..c1d4042671f9f 100644
    --- a/drivers/net/ethernet/google/gve/gve.h
    +++ b/drivers/net/ethernet/google/gve/gve.h
    @@ -30,7 +30,7 @@
    #define GVE_MIN_MSIX 3

    /* Numbers of gve tx/rx stats in stats report. */
    -#define GVE_TX_STATS_REPORT_NUM 5
    +#define GVE_TX_STATS_REPORT_NUM 6
    #define GVE_RX_STATS_REPORT_NUM 2

    /* Interval to schedule a stats report update, 20000ms. */
    @@ -413,7 +413,9 @@ struct gve_tx_ring {
    u32 q_num ____cacheline_aligned; /* queue idx */
    u32 stop_queue; /* count of queue stops */
    u32 wake_queue; /* count of queue wakes */
    + u32 queue_timeout; /* count of queue timeouts */
    u32 ntfy_id; /* notification block index */
    + u32 last_kick_msec; /* Last time the queue was kicked */
    dma_addr_t bus; /* dma address of the descr ring */
    dma_addr_t q_resources_bus; /* dma address of the queue resources */
    dma_addr_t complq_bus_dqo; /* dma address of the dqo.compl_ring */
    diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
    index 47c3d8f313fcf..3953f6f7a4273 100644
    --- a/drivers/net/ethernet/google/gve/gve_adminq.h
    +++ b/drivers/net/ethernet/google/gve/gve_adminq.h
    @@ -270,6 +270,7 @@ enum gve_stat_names {
    TX_LAST_COMPLETION_PROCESSED = 5,
    RX_NEXT_EXPECTED_SEQUENCE = 6,
    RX_BUFFERS_POSTED = 7,
    + TX_TIMEOUT_CNT = 8,
    // stats from NIC
    RX_QUEUE_DROP_CNT = 65,
    RX_NO_BUFFERS_POSTED = 66,
    diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
    index bf8a4a7c43f78..8c996e72748d2 100644
    --- a/drivers/net/ethernet/google/gve/gve_main.c
    +++ b/drivers/net/ethernet/google/gve/gve_main.c
    @@ -24,6 +24,9 @@
    #define GVE_VERSION "1.0.0"
    #define GVE_VERSION_PREFIX "GVE-"

    +// Minimum amount of time between queue kicks in msec (10 seconds)
    +#define MIN_TX_TIMEOUT_GAP (1000 * 10)
    +
    const char gve_version_str[] = GVE_VERSION;
    static const char gve_version_prefix[] = GVE_VERSION_PREFIX;

    @@ -1116,9 +1119,47 @@ static void gve_turnup(struct gve_priv *priv)

    static void gve_tx_timeout(struct net_device *dev, unsigned int txqueue)
    {
    - struct gve_priv *priv = netdev_priv(dev);
    + struct gve_notify_block *block;
    + struct gve_tx_ring *tx = NULL;
    + struct gve_priv *priv;
    + u32 last_nic_done;
    + u32 current_time;
    + u32 ntfy_idx;
    +
    + netdev_info(dev, "Timeout on tx queue, %d", txqueue);
    + priv = netdev_priv(dev);
    + if (txqueue > priv->tx_cfg.num_queues)
    + goto reset;
    +
    + ntfy_idx = gve_tx_idx_to_ntfy(priv, txqueue);
    + if (ntfy_idx > priv->num_ntfy_blks)
    + goto reset;
    +
    + block = &priv->ntfy_blocks[ntfy_idx];
    + tx = block->tx;

    + current_time = jiffies_to_msecs(jiffies);
    + if (tx->last_kick_msec + MIN_TX_TIMEOUT_GAP > current_time)
    + goto reset;
    +
    + /* Check to see if there are missed completions, which will allow us to
    + * kick the queue.
    + */
    + last_nic_done = gve_tx_load_event_counter(priv, tx);
    + if (last_nic_done - tx->done) {
    + netdev_info(dev, "Kicking queue %d", txqueue);
    + iowrite32be(GVE_IRQ_MASK, gve_irq_doorbell(priv, block));
    + napi_schedule(&block->napi);
    + tx->last_kick_msec = current_time;
    + goto out;
    + } // Else reset.
    +
    +reset:
    gve_schedule_reset(priv);
    +
    +out:
    + if (tx)
    + tx->queue_timeout++;
    priv->tx_timeo_cnt++;
    }

    @@ -1247,6 +1288,11 @@ void gve_handle_report_stats(struct gve_priv *priv)
    .value = cpu_to_be64(last_completion),
    .queue_id = cpu_to_be32(idx),
    };
    + stats[stats_idx++] = (struct stats) {
    + .stat_name = cpu_to_be32(TX_TIMEOUT_CNT),
    + .value = cpu_to_be64(priv->tx[idx].queue_timeout),
    + .queue_id = cpu_to_be32(idx),
    + };
    }
    }
    /* rx stats */
    --
    2.33.0


    \
     
     \ /
      Last update: 2021-11-16 00:39    [W:2.187 / U:0.316 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site