lkml.org 
[lkml]   [2022]   [May]   [23]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH 5.17 037/158] nvme-multipath: fix hang when disk goes live over reconnect
    Date
    From: Anton Eidelman <anton.eidelman@gmail.com>

    [ Upstream commit a4a6f3c8f61c3cfbda4998ad94596059ad7e4332 ]

    nvme_mpath_init_identify() invoked from nvme_init_identify() fetches a
    fresh ANA log from the ctrl. This is essential to have an up to date
    path states for both existing namespaces and for those scan_work may
    discover once the ctrl is up.

    This happens in the following cases:
    1) A new ctrl is being connected.
    2) An existing ctrl is successfully reconnected.
    3) An existing ctrl is being reset.

    While in (1) ctrl->namespaces is empty, (2 & 3) may have namespaces, and
    nvme_read_ana_log() may call nvme_update_ns_ana_state().

    This result in a hang when the ANA state of an existing namespace changes
    and makes the disk live: nvme_mpath_set_live() issues IO to the namespace
    through the ctrl, which does NOT have IO queues yet.

    See sample hang below.

    Solution:
    - nvme_update_ns_ana_state() to call set_live only if ctrl is live
    - nvme_read_ana_log() call from nvme_mpath_init_identify()
    therefore only fetches and parses the ANA log;
    any erros in this process will fail the ctrl setup as appropriate;
    - a separate function nvme_mpath_update()
    is called in nvme_start_ctrl();
    this parses the ANA log without fetching it.
    At this point the ctrl is live,
    therefore, disks can be set live normally.

    Sample failure:
    nvme nvme0: starting error recovery
    nvme nvme0: Reconnecting in 10 seconds...
    block nvme0n6: no usable path - requeuing I/O
    INFO: task kworker/u8:3:312 blocked for more than 122 seconds.
    Tainted: G E 5.14.5-1.el7.elrepo.x86_64 #1
    Workqueue: nvme-wq nvme_tcp_reconnect_ctrl_work [nvme_tcp]
    Call Trace:
    __schedule+0x2a2/0x7e0
    schedule+0x4e/0xb0
    io_schedule+0x16/0x40
    wait_on_page_bit_common+0x15c/0x3e0
    do_read_cache_page+0x1e0/0x410
    read_cache_page+0x12/0x20
    read_part_sector+0x46/0x100
    read_lba+0x121/0x240
    efi_partition+0x1d2/0x6a0
    bdev_disk_changed.part.0+0x1df/0x430
    bdev_disk_changed+0x18/0x20
    blkdev_get_whole+0x77/0xe0
    blkdev_get_by_dev+0xd2/0x3a0
    __device_add_disk+0x1ed/0x310
    device_add_disk+0x13/0x20
    nvme_mpath_set_live+0x138/0x1b0 [nvme_core]
    nvme_update_ns_ana_state+0x2b/0x30 [nvme_core]
    nvme_update_ana_state+0xca/0xe0 [nvme_core]
    nvme_parse_ana_log+0xac/0x170 [nvme_core]
    nvme_read_ana_log+0x7d/0xe0 [nvme_core]
    nvme_mpath_init_identify+0x105/0x150 [nvme_core]
    nvme_init_identify+0x2df/0x4d0 [nvme_core]
    nvme_init_ctrl_finish+0x8d/0x3b0 [nvme_core]
    nvme_tcp_setup_ctrl+0x337/0x390 [nvme_tcp]
    nvme_tcp_reconnect_ctrl_work+0x24/0x40 [nvme_tcp]
    process_one_work+0x1bd/0x360
    worker_thread+0x50/0x3d0

    Signed-off-by: Anton Eidelman <anton@lightbitslabs.com>
    Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
    Signed-off-by: Christoph Hellwig <hch@lst.de>
    Signed-off-by: Sasha Levin <sashal@kernel.org>
    ---
    drivers/nvme/host/core.c | 1 +
    drivers/nvme/host/multipath.c | 25 +++++++++++++++++++++++--
    drivers/nvme/host/nvme.h | 4 ++++
    3 files changed, 28 insertions(+), 2 deletions(-)

    diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
    index 10f7c79caac2..0abd772c57f0 100644
    --- a/drivers/nvme/host/core.c
    +++ b/drivers/nvme/host/core.c
    @@ -4422,6 +4422,7 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl)
    if (ctrl->queue_count > 1) {
    nvme_queue_scan(ctrl);
    nvme_start_queues(ctrl);
    + nvme_mpath_update(ctrl);
    }
    }
    EXPORT_SYMBOL_GPL(nvme_start_ctrl);
    diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
    index a703f1f5fb64..189175fff7e4 100644
    --- a/drivers/nvme/host/multipath.c
    +++ b/drivers/nvme/host/multipath.c
    @@ -635,8 +635,17 @@ static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
    ns->ana_grpid = le32_to_cpu(desc->grpid);
    ns->ana_state = desc->state;
    clear_bit(NVME_NS_ANA_PENDING, &ns->flags);
    -
    - if (nvme_state_is_live(ns->ana_state))
    + /*
    + * nvme_mpath_set_live() will trigger I/O to the multipath path device
    + * and in turn to this path device. However we cannot accept this I/O
    + * if the controller is not live. This may deadlock if called from
    + * nvme_mpath_init_identify() and the ctrl will never complete
    + * initialization, preventing I/O from completing. For this case we
    + * will reprocess the ANA log page in nvme_mpath_update() once the
    + * controller is ready.
    + */
    + if (nvme_state_is_live(ns->ana_state) &&
    + ns->ctrl->state == NVME_CTRL_LIVE)
    nvme_mpath_set_live(ns);
    }

    @@ -723,6 +732,18 @@ static void nvme_ana_work(struct work_struct *work)
    nvme_read_ana_log(ctrl);
    }

    +void nvme_mpath_update(struct nvme_ctrl *ctrl)
    +{
    + u32 nr_change_groups = 0;
    +
    + if (!ctrl->ana_log_buf)
    + return;
    +
    + mutex_lock(&ctrl->ana_lock);
    + nvme_parse_ana_log(ctrl, &nr_change_groups, nvme_update_ana_state);
    + mutex_unlock(&ctrl->ana_lock);
    +}
    +
    static void nvme_anatt_timeout(struct timer_list *t)
    {
    struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer);
    diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
    index 68c42e831117..85f3f55c71c5 100644
    --- a/drivers/nvme/host/nvme.h
    +++ b/drivers/nvme/host/nvme.h
    @@ -800,6 +800,7 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id);
    void nvme_mpath_remove_disk(struct nvme_ns_head *head);
    int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id);
    void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl);
    +void nvme_mpath_update(struct nvme_ctrl *ctrl);
    void nvme_mpath_uninit(struct nvme_ctrl *ctrl);
    void nvme_mpath_stop(struct nvme_ctrl *ctrl);
    bool nvme_mpath_clear_current_path(struct nvme_ns *ns);
    @@ -874,6 +875,9 @@ static inline int nvme_mpath_init_identify(struct nvme_ctrl *ctrl,
    "Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n");
    return 0;
    }
    +static inline void nvme_mpath_update(struct nvme_ctrl *ctrl)
    +{
    +}
    static inline void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
    {
    }
    --
    2.35.1


    \
     
     \ /
      Last update: 2022-05-23 20:00    [W:2.272 / U:0.184 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site