lkml.org 
[lkml]   [2022]   [Apr]   [4]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH v7 57/70] mm/mempolicy: Use vma iterator & maple state instead of vma linked list
    Date
    From: "Liam R. Howlett" <Liam.Howlett@Oracle.com>

    Reworked the way mbind_range() finds the first VMA to reuse the maple
    state and limit the number of tree walks needed.

    Note, this drops the VM_BUG_ON(!vma) call, which would catch a start
    address higher than the last VMA. The code was written in a way that
    allowed no VMA updates to occur and still return success. There should
    be no functional change to this scenario with the new code.

    Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
    Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
    ---
    mm/mempolicy.c | 55 ++++++++++++++++++++++++++++----------------------
    1 file changed, 31 insertions(+), 24 deletions(-)

    diff --git a/mm/mempolicy.c b/mm/mempolicy.c
    index a2516d31db6c..a8f07b60dc6d 100644
    --- a/mm/mempolicy.c
    +++ b/mm/mempolicy.c
    @@ -380,9 +380,10 @@ void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
    void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
    {
    struct vm_area_struct *vma;
    + VMA_ITERATOR(vmi, mm, 0);

    mmap_write_lock(mm);
    - for (vma = mm->mmap; vma; vma = vma->vm_next)
    + for_each_vma(vmi, vma)
    mpol_rebind_policy(vma->vm_policy, new);
    mmap_write_unlock(mm);
    }
    @@ -655,7 +656,7 @@ static unsigned long change_prot_numa(struct vm_area_struct *vma,
    static int queue_pages_test_walk(unsigned long start, unsigned long end,
    struct mm_walk *walk)
    {
    - struct vm_area_struct *vma = walk->vma;
    + struct vm_area_struct *next, *vma = walk->vma;
    struct queue_pages *qp = walk->private;
    unsigned long endvma = vma->vm_end;
    unsigned long flags = qp->flags;
    @@ -670,9 +671,10 @@ static int queue_pages_test_walk(unsigned long start, unsigned long end,
    /* hole at head side of range */
    return -EFAULT;
    }
    + next = find_vma(vma->vm_mm, vma->vm_end);
    if (!(flags & MPOL_MF_DISCONTIG_OK) &&
    ((vma->vm_end < qp->end) &&
    - (!vma->vm_next || vma->vm_end < vma->vm_next->vm_start)))
    + (!next || vma->vm_end < next->vm_start)))
    /* hole at middle or tail of range */
    return -EFAULT;

    @@ -786,26 +788,24 @@ static int vma_replace_policy(struct vm_area_struct *vma,
    static int mbind_range(struct mm_struct *mm, unsigned long start,
    unsigned long end, struct mempolicy *new_pol)
    {
    + MA_STATE(mas, &mm->mm_mt, start - 1, start - 1);
    struct vm_area_struct *prev;
    struct vm_area_struct *vma;
    int err = 0;
    pgoff_t pgoff;
    - unsigned long vmstart;
    - unsigned long vmend;
    -
    - vma = find_vma(mm, start);
    - VM_BUG_ON(!vma);

    - prev = vma->vm_prev;
    - if (start > vma->vm_start)
    - prev = vma;
    + prev = mas_find_rev(&mas, 0);
    + if (prev && (start < prev->vm_end))
    + vma = prev;
    + else
    + vma = mas_next(&mas, end - 1);

    - for (; vma && vma->vm_start < end; prev = vma, vma = vma->vm_next) {
    - vmstart = max(start, vma->vm_start);
    - vmend = min(end, vma->vm_end);
    + for (; vma; vma = mas_next(&mas, end - 1)) {
    + unsigned long vmstart = max(start, vma->vm_start);
    + unsigned long vmend = min(end, vma->vm_end);

    if (mpol_equal(vma_policy(vma), new_pol))
    - continue;
    + goto next;

    pgoff = vma->vm_pgoff +
    ((vmstart - vma->vm_start) >> PAGE_SHIFT);
    @@ -814,6 +814,8 @@ static int mbind_range(struct mm_struct *mm, unsigned long start,
    new_pol, vma->vm_userfaultfd_ctx,
    anon_vma_name(vma));
    if (prev) {
    + /* vma_merge() invalidated the mas */
    + mas_pause(&mas);
    vma = prev;
    goto replace;
    }
    @@ -821,19 +823,24 @@ static int mbind_range(struct mm_struct *mm, unsigned long start,
    err = split_vma(vma->vm_mm, vma, vmstart, 1);
    if (err)
    goto out;
    + /* split_vma() invalidated the mas */
    + mas_pause(&mas);
    }
    if (vma->vm_end != vmend) {
    err = split_vma(vma->vm_mm, vma, vmend, 0);
    if (err)
    goto out;
    + /* mas_pause() unnecessary as the loop is ending */
    }
    - replace:
    +replace:
    err = vma_replace_policy(vma, new_pol);
    if (err)
    goto out;
    +next:
    + prev = vma;
    }

    - out:
    +out:
    return err;
    }

    @@ -1048,6 +1055,7 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
    int flags)
    {
    nodemask_t nmask;
    + struct vm_area_struct *vma;
    LIST_HEAD(pagelist);
    int err = 0;
    struct migration_target_control mtc = {
    @@ -1063,8 +1071,9 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
    * need migration. Between passing in the full user address
    * space range and MPOL_MF_DISCONTIG_OK, this call can not fail.
    */
    + vma = find_vma(mm, 0);
    VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
    - queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
    + queue_pages_range(mm, vma->vm_start, mm->task_size, &nmask,
    flags | MPOL_MF_DISCONTIG_OK, &pagelist);

    if (!list_empty(&pagelist)) {
    @@ -1193,13 +1202,12 @@ static struct page *new_page(struct page *page, unsigned long start)
    {
    struct vm_area_struct *vma;
    unsigned long address;
    + VMA_ITERATOR(vmi, current->mm, start);

    - vma = find_vma(current->mm, start);
    - while (vma) {
    + for_each_vma(vmi, vma) {
    address = page_address_in_vma(page, vma);
    if (address != -EFAULT)
    break;
    - vma = vma->vm_next;
    }

    if (PageHuge(page)) {
    @@ -1482,6 +1490,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le
    unsigned long vmend;
    unsigned long end;
    int err = -ENOENT;
    + VMA_ITERATOR(vmi, mm, start);

    start = untagged_addr(start);
    if (start & ~PAGE_MASK)
    @@ -1507,9 +1516,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le
    if (end == start)
    return 0;
    mmap_write_lock(mm);
    - vma = find_vma(mm, start);
    - for (; vma && vma->vm_start < end; vma = vma->vm_next) {
    -
    + for_each_vma_range(vmi, vma, end) {
    vmstart = max(start, vma->vm_start);
    vmend = min(end, vma->vm_end);
    new = mpol_dup(vma_policy(vma));
    --
    2.34.1
    \
     
     \ /
      Last update: 2022-04-04 16:41    [W:4.531 / U:0.100 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site