lkml.org 
[lkml]   [2022]   [Jul]   [16]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH v11 22/69] mm/mmap: change do_brk_flags() to expand existing VMA and add do_brk_munmap()
    Date
    From: "Liam R. Howlett" <Liam.Howlett@Oracle.com>

    Avoid allocating a new VMA when it a vma modification can occur. When a
    brk() can expand or contract a VMA, then the single store operation will
    only modify one index of the maple tree instead of causing a node to split
    or coalesce. This avoids unnecessary allocations/frees of maple tree
    nodes and VMAs.

    Move some limit & flag verifications out of the do_brk_flags() function to
    use only relevant checks in the code path of bkr() and vm_brk_flags().

    Set the vma to check if it can expand in vm_brk_flags() if extra criteria
    are met.

    Drop userfaultfd from do_brk_flags() path and only use it in
    vm_brk_flags() path since that is the only place a munmap will happen.

    Link: https://lkml.kernel.org/r/20220504011345.662299-7-Liam.Howlett@oracle.com
    Link: https://lkml.kernel.org/r/20220621204632.3370049-23-Liam.Howlett@oracle.com
    Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
    Cc: Catalin Marinas <catalin.marinas@arm.com>
    Cc: David Howells <dhowells@redhat.com>
    Cc: "Matthew Wilcox (Oracle)" <willy@infradead.org>
    Cc: SeongJae Park <sj@kernel.org>
    Cc: Vlastimil Babka <vbabka@suse.cz>
    Cc: Will Deacon <will@kernel.org>
    Cc: Davidlohr Bueso <dave@stgolabs.net>
    Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
    ---
    mm/mmap.c | 292 +++++++++++++++++++++++++++++++++++++++++++-----------
    1 file changed, 234 insertions(+), 58 deletions(-)

    diff --git a/mm/mmap.c b/mm/mmap.c
    index 973512b14d55..7570e008ef52 100644
    --- a/mm/mmap.c
    +++ b/mm/mmap.c
    @@ -194,17 +194,40 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
    return next;
    }

    -static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long flags,
    - struct list_head *uf);
    +/*
    + * check_brk_limits() - Use platform specific check of range & verify mlock
    + * limits.
    + * @addr: The address to check
    + * @len: The size of increase.
    + *
    + * Return: 0 on success.
    + */
    +static int check_brk_limits(unsigned long addr, unsigned long len)
    +{
    + unsigned long mapped_addr;
    +
    + mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
    + if (IS_ERR_VALUE(mapped_addr))
    + return mapped_addr;
    +
    + return mlock_future_check(current->mm, current->mm->def_flags, len);
    +}
    +static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma,
    + unsigned long newbrk, unsigned long oldbrk,
    + struct list_head *uf);
    +static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *brkvma,
    + unsigned long addr, unsigned long request,
    + unsigned long flags);
    SYSCALL_DEFINE1(brk, unsigned long, brk)
    {
    unsigned long newbrk, oldbrk, origbrk;
    struct mm_struct *mm = current->mm;
    - struct vm_area_struct *next;
    + struct vm_area_struct *brkvma, *next = NULL;
    unsigned long min_brk;
    bool populate;
    bool downgraded = false;
    LIST_HEAD(uf);
    + MA_STATE(mas, &mm->mm_mt, 0, 0);

    if (mmap_write_lock_killable(mm))
    return -EINTR;
    @@ -246,35 +269,52 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)

    /*
    * Always allow shrinking brk.
    - * __do_munmap() may downgrade mmap_lock to read.
    + * do_brk_munmap() may downgrade mmap_lock to read.
    */
    if (brk <= mm->brk) {
    int ret;

    + /* Search one past newbrk */
    + mas_set(&mas, newbrk);
    + brkvma = mas_find(&mas, oldbrk);
    + BUG_ON(brkvma == NULL);
    + if (brkvma->vm_start >= oldbrk)
    + goto out; /* mapping intersects with an existing non-brk vma. */
    /*
    - * mm->brk must to be protected by write mmap_lock so update it
    - * before downgrading mmap_lock. When __do_munmap() fails,
    - * mm->brk will be restored from origbrk.
    + * mm->brk must be protected by write mmap_lock.
    + * do_brk_munmap() may downgrade the lock, so update it
    + * before calling do_brk_munmap().
    */
    mm->brk = brk;
    - ret = __do_munmap(mm, newbrk, oldbrk-newbrk, &uf, true);
    - if (ret < 0) {
    - mm->brk = origbrk;
    - goto out;
    - } else if (ret == 1) {
    + mas.last = oldbrk - 1;
    + ret = do_brk_munmap(&mas, brkvma, newbrk, oldbrk, &uf);
    + if (ret == 1) {
    downgraded = true;
    - }
    - goto success;
    + goto success;
    + } else if (!ret)
    + goto success;
    +
    + mm->brk = origbrk;
    + goto out;
    }

    - /* Check against existing mmap mappings. */
    - next = find_vma(mm, oldbrk);
    + if (check_brk_limits(oldbrk, newbrk - oldbrk))
    + goto out;
    +
    + /*
    + * Only check if the next VMA is within the stack_guard_gap of the
    + * expansion area
    + */
    + mas_set(&mas, oldbrk);
    + next = mas_find(&mas, newbrk - 1 + PAGE_SIZE + stack_guard_gap);
    if (next && newbrk + PAGE_SIZE > vm_start_gap(next))
    goto out;

    + brkvma = mas_prev(&mas, mm->start_brk);
    /* Ok, looks good - let it rip. */
    - if (do_brk_flags(oldbrk, newbrk-oldbrk, 0, &uf) < 0)
    + if (do_brk_flags(&mas, brkvma, oldbrk, newbrk - oldbrk, 0) < 0)
    goto out;
    +
    mm->brk = brk;

    success:
    @@ -2809,38 +2849,112 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
    }

    /*
    - * this is really a simplified "do_mmap". it only handles
    - * anonymous maps. eventually we may be able to do some
    - * brk-specific accounting here.
    + * brk_munmap() - Unmap a parital vma.
    + * @mas: The maple tree state.
    + * @vma: The vma to be modified
    + * @newbrk: the start of the address to unmap
    + * @oldbrk: The end of the address to unmap
    + * @uf: The userfaultfd list_head
    + *
    + * Returns: 1 on success.
    + * unmaps a partial VMA mapping. Does not handle alignment, downgrades lock if
    + * possible.
    */
    -static int do_brk_flags(unsigned long addr, unsigned long len,
    - unsigned long flags, struct list_head *uf)
    +static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma,
    + unsigned long newbrk, unsigned long oldbrk,
    + struct list_head *uf)
    {
    - struct mm_struct *mm = current->mm;
    - struct vm_area_struct *vma, *prev;
    - pgoff_t pgoff = addr >> PAGE_SHIFT;
    - int error;
    - unsigned long mapped_addr;
    - validate_mm_mt(mm);
    + struct mm_struct *mm = vma->vm_mm;
    + struct vm_area_struct unmap;
    + unsigned long unmap_pages;
    + int ret = 1;
    +
    + arch_unmap(mm, newbrk, oldbrk);
    +
    + if (likely((vma->vm_end < oldbrk) ||
    + ((vma->vm_start == newbrk) && (vma->vm_end == oldbrk)))) {
    + /* remove entire mapping(s) */
    + mas_set(mas, newbrk);
    + if (vma->vm_start != newbrk)
    + mas_reset(mas); /* cause a re-walk for the first overlap. */
    + ret = __do_munmap(mm, newbrk, oldbrk - newbrk, uf, true);
    + goto munmap_full_vma;
    + }
    +
    + vma_init(&unmap, mm);
    + unmap.vm_start = newbrk;
    + unmap.vm_end = oldbrk;
    + ret = userfaultfd_unmap_prep(&unmap, newbrk, oldbrk, uf);
    + if (ret)
    + return ret;
    + ret = 1;

    - /* Until we need other flags, refuse anything except VM_EXEC. */
    - if ((flags & (~VM_EXEC)) != 0)
    - return -EINVAL;
    - flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
    + /* Change the oldbrk of vma to the newbrk of the munmap area */
    + vma_adjust_trans_huge(vma, vma->vm_start, newbrk, 0);
    + if (mas_preallocate(mas, vma, GFP_KERNEL))
    + return -ENOMEM;

    - mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
    - if (IS_ERR_VALUE(mapped_addr))
    - return mapped_addr;
    + if (vma->anon_vma) {
    + anon_vma_lock_write(vma->anon_vma);
    + anon_vma_interval_tree_pre_update_vma(vma);
    + }

    - error = mlock_future_check(mm, mm->def_flags, len);
    - if (error)
    - return error;
    + vma->vm_end = newbrk;
    + vma_init(&unmap, mm);
    + unmap.vm_start = newbrk;
    + unmap.vm_end = oldbrk;
    + if (vma->anon_vma)
    + vma_set_anonymous(&unmap);

    - /* Clear old maps, set up prev and uf */
    - if (munmap_vma_range(mm, addr, len, &prev, uf))
    - return -ENOMEM;
    + vma_mas_remove(&unmap, mas);
    +
    + vmacache_invalidate(vma->vm_mm);
    + if (vma->anon_vma) {
    + anon_vma_interval_tree_post_update_vma(vma);
    + anon_vma_unlock_write(vma->anon_vma);
    + }
    +
    + unmap_pages = vma_pages(&unmap);
    + if (vma->vm_flags & VM_LOCKED)
    + mm->locked_vm -= unmap_pages;

    - /* Check against address space limits *after* clearing old maps... */
    + mmap_write_downgrade(mm);
    + unmap_region(mm, &unmap, vma, newbrk, oldbrk);
    + /* Statistics */
    + vm_stat_account(mm, vma->vm_flags, -unmap_pages);
    + if (vma->vm_flags & VM_ACCOUNT)
    + vm_unacct_memory(unmap_pages);
    +
    +munmap_full_vma:
    + validate_mm_mt(mm);
    + return ret;
    +}
    +
    +/*
    + * do_brk_flags() - Increase the brk vma if the flags match.
    + * @mas: The maple tree state.
    + * @addr: The start address
    + * @len: The length of the increase
    + * @vma: The vma,
    + * @flags: The VMA Flags
    + *
    + * Extend the brk VMA from addr to addr + len. If the VMA is NULL or the flags
    + * do not match then create a new anonymous VMA. Eventually we may be able to
    + * do some brk-specific accounting here.
    + */
    +static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *vma,
    + unsigned long addr, unsigned long len,
    + unsigned long flags)
    +{
    + struct mm_struct *mm = current->mm;
    + struct vm_area_struct *prev = NULL;
    +
    + validate_mm_mt(mm);
    + /*
    + * Check against address space limits by the changed size
    + * Note: This happens *after* clearing old mappings in some code paths.
    + */
    + flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
    if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT))
    return -ENOMEM;

    @@ -2850,30 +2964,56 @@ static int do_brk_flags(unsigned long addr, unsigned long len,
    if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
    return -ENOMEM;

    - /* Can we just expand an old private anonymous mapping? */
    - vma = vma_merge(mm, prev, addr, addr + len, flags,
    - NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX, NULL);
    - if (vma)
    - goto out;
    -
    /*
    - * create a vma struct for an anonymous mapping
    + * Expand the existing vma if possible; Note that singular lists do not
    + * occur after forking, so the expand will only happen on new VMAs.
    */
    - vma = vm_area_alloc(mm);
    - if (!vma) {
    - vm_unacct_memory(len >> PAGE_SHIFT);
    - return -ENOMEM;
    + if (vma &&
    + (!vma->anon_vma || list_is_singular(&vma->anon_vma_chain)) &&
    + ((vma->vm_flags & ~VM_SOFTDIRTY) == flags)) {
    + mas->index = vma->vm_start;
    + mas->last = addr + len - 1;
    + vma_adjust_trans_huge(vma, addr, addr + len, 0);
    + if (vma->anon_vma) {
    + anon_vma_lock_write(vma->anon_vma);
    + anon_vma_interval_tree_pre_update_vma(vma);
    + }
    + vma->vm_end = addr + len;
    + vma->vm_flags |= VM_SOFTDIRTY;
    + if (mas_store_gfp(mas, vma, GFP_KERNEL))
    + goto mas_expand_failed;
    +
    + if (vma->anon_vma) {
    + anon_vma_interval_tree_post_update_vma(vma);
    + anon_vma_unlock_write(vma->anon_vma);
    + }
    + khugepaged_enter_vma(vma, flags);
    + goto out;
    }
    + prev = vma;
    +
    + /* create a vma struct for an anonymous mapping */
    + vma = vm_area_alloc(mm);
    + if (!vma)
    + goto vma_alloc_fail;

    vma_set_anonymous(vma);
    vma->vm_start = addr;
    vma->vm_end = addr + len;
    - vma->vm_pgoff = pgoff;
    + vma->vm_pgoff = addr >> PAGE_SHIFT;
    vma->vm_flags = flags;
    vma->vm_page_prot = vm_get_page_prot(flags);
    - if (vma_link(mm, vma, prev))
    - goto no_vma_link;
    + mas_set_range(mas, vma->vm_start, addr + len - 1);
    + if (mas_store_gfp(mas, vma, GFP_KERNEL))
    + goto mas_store_fail;

    + mm->map_count++;
    +
    + if (!prev)
    + prev = mas_prev(mas, 0);
    +
    + __vma_link_list(mm, vma, prev);
    + mm->map_count++;
    out:
    perf_event_mmap(vma);
    mm->total_vm += len >> PAGE_SHIFT;
    @@ -2884,18 +3024,29 @@ static int do_brk_flags(unsigned long addr, unsigned long len,
    validate_mm_mt(mm);
    return 0;

    -no_vma_link:
    +mas_store_fail:
    vm_area_free(vma);
    +vma_alloc_fail:
    + vm_unacct_memory(len >> PAGE_SHIFT);
    + return -ENOMEM;
    +
    +mas_expand_failed:
    + if (vma->anon_vma) {
    + anon_vma_interval_tree_post_update_vma(vma);
    + anon_vma_unlock_write(vma->anon_vma);
    + }
    return -ENOMEM;
    }

    int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags)
    {
    struct mm_struct *mm = current->mm;
    + struct vm_area_struct *vma = NULL;
    unsigned long len;
    int ret;
    bool populate;
    LIST_HEAD(uf);
    + MA_STATE(mas, &mm->mm_mt, addr, addr);

    len = PAGE_ALIGN(request);
    if (len < request)
    @@ -2906,13 +3057,38 @@ int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags)
    if (mmap_write_lock_killable(mm))
    return -EINTR;

    - ret = do_brk_flags(addr, len, flags, &uf);
    + /* Until we need other flags, refuse anything except VM_EXEC. */
    + if ((flags & (~VM_EXEC)) != 0)
    + return -EINVAL;
    +
    + ret = check_brk_limits(addr, len);
    + if (ret)
    + goto limits_failed;
    +
    + if (find_vma_intersection(mm, addr, addr + len))
    + ret = do_munmap(mm, addr, len, &uf);
    +
    + if (ret)
    + goto munmap_failed;
    +
    + vma = mas_prev(&mas, 0);
    + if (!vma || vma->vm_end != addr || vma_policy(vma) ||
    + !can_vma_merge_after(vma, flags, NULL, NULL,
    + addr >> PAGE_SHIFT, NULL_VM_UFFD_CTX, NULL))
    + vma = NULL;
    +
    + ret = do_brk_flags(&mas, vma, addr, len, flags);
    populate = ((mm->def_flags & VM_LOCKED) != 0);
    mmap_write_unlock(mm);
    userfaultfd_unmap_complete(mm, &uf);
    if (populate && !ret)
    mm_populate(addr, len);
    return ret;
    +
    +munmap_failed:
    +limits_failed:
    + mmap_write_unlock(mm);
    + return ret;
    }
    EXPORT_SYMBOL(vm_brk_flags);

    --
    2.35.1
    \
     
     \ /
      Last update: 2022-07-17 04:48    [W:4.702 / U:0.320 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site