lkml.org 
[lkml]   [2023]   [May]   [18]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    Subject[PATCH v4 6/6] KVM: arm64: Use TLBI range-based intructions for unmap
    From
    The current implementation of the stage-2 unmap walker traverses
    the given range and, as a part of break-before-make, performs
    TLB invalidations with a DSB for every PTE. A multitude of this
    combination could cause a performance bottleneck.

    Hence, if the system supports FEAT_TLBIRANGE, defer the TLB
    invalidations until the entire walk is finished, and then
    use range-based instructions to invalidate the TLBs in one go.
    Condition this upon S2FWB in order to avoid walking the page-table
    again to perform the CMOs after issuing the TLBI.

    Rename stage2_put_pte() to stage2_unmap_put_pte() as the function
    now serves the stage-2 unmap walker specifically, rather than
    acting generic.

    Signed-off-by: Raghavendra Rao Ananta <rananta@google.com>
    ---
    arch/arm64/kvm/hyp/pgtable.c | 35 ++++++++++++++++++++++++++++++-----
    1 file changed, 30 insertions(+), 5 deletions(-)

    diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
    index b8f0dbd12f773..5832ee3418fb0 100644
    --- a/arch/arm64/kvm/hyp/pgtable.c
    +++ b/arch/arm64/kvm/hyp/pgtable.c
    @@ -771,16 +771,34 @@ static void stage2_make_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t n
    smp_store_release(ctx->ptep, new);
    }

    -static void stage2_put_pte(const struct kvm_pgtable_visit_ctx *ctx, struct kvm_s2_mmu *mmu,
    - struct kvm_pgtable_mm_ops *mm_ops)
    +static bool stage2_unmap_defer_tlb_flush(struct kvm_pgtable *pgt)
    {
    + /*
    + * If FEAT_TLBIRANGE is implemented, defer the individial PTE
    + * TLB invalidations until the entire walk is finished, and
    + * then use the range-based TLBI instructions to do the
    + * invalidations. Condition this upon S2FWB in order to avoid
    + * a page-table walk again to perform the CMOs after TLBI.
    + */
    + return system_supports_tlb_range() && stage2_has_fwb(pgt);
    +}
    +
    +static void stage2_unmap_put_pte(const struct kvm_pgtable_visit_ctx *ctx,
    + struct kvm_s2_mmu *mmu,
    + struct kvm_pgtable_mm_ops *mm_ops)
    +{
    + struct kvm_pgtable *pgt = ctx->arg;
    +
    /*
    * Clear the existing PTE, and perform break-before-make with
    * TLB maintenance if it was valid.
    */
    if (kvm_pte_valid(ctx->old)) {
    kvm_clear_pte(ctx->ptep);
    - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr, ctx->level);
    +
    + if (!stage2_unmap_defer_tlb_flush(pgt))
    + kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu,
    + ctx->addr, ctx->level);
    }

    mm_ops->put_page(ctx->ptep);
    @@ -1015,7 +1033,7 @@ static int stage2_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx,
    * block entry and rely on the remaining portions being faulted
    * back lazily.
    */
    - stage2_put_pte(ctx, mmu, mm_ops);
    + stage2_unmap_put_pte(ctx, mmu, mm_ops);

    if (need_flush && mm_ops->dcache_clean_inval_poc)
    mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),
    @@ -1029,13 +1047,20 @@ static int stage2_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx,

    int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)
    {
    + int ret;
    struct kvm_pgtable_walker walker = {
    .cb = stage2_unmap_walker,
    .arg = pgt,
    .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
    };

    - return kvm_pgtable_walk(pgt, addr, size, &walker);
    + ret = kvm_pgtable_walk(pgt, addr, size, &walker);
    + if (stage2_unmap_defer_tlb_flush(pgt))
    + /* Perform the deferred TLB invalidations */
    + kvm_call_hyp(__kvm_tlb_flush_vmid_range, pgt->mmu,
    + addr, addr + size);
    +
    + return ret;
    }

    struct stage2_attr_data {
    --
    2.40.1.698.g37aff9b760-goog
    \
     
     \ /
      Last update: 2023-05-19 02:53    [W:4.047 / U:0.104 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site