lkml.org 
[lkml]   [2021]   [Jan]   [19]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH v6 07/14] mm: honor PF_MEMALLOC_PIN for all movable pages
    Date
    PF_MEMALLOC_PIN is only honored for CMA pages, extend
    this flag to work for any allocations from ZONE_MOVABLE by removing
    __GFP_MOVABLE from gfp_mask when this flag is passed in the current
    context.

    Add is_pinnable_page() to return true if page is in a pinnable page.
    A pinnable page is not in ZONE_MOVABLE and not of MIGRATE_CMA type.

    Signed-off-by: Pavel Tatashin <pasha.tatashin@soleen.com>
    Acked-by: Michal Hocko <mhocko@suse.com>
    ---
    include/linux/mm.h | 11 +++++++++++
    include/linux/sched/mm.h | 6 +++++-
    mm/hugetlb.c | 2 +-
    mm/page_alloc.c | 20 +++++++++-----------
    4 files changed, 26 insertions(+), 13 deletions(-)

    diff --git a/include/linux/mm.h b/include/linux/mm.h
    index a5d618d08506..0990a76d5e6f 100644
    --- a/include/linux/mm.h
    +++ b/include/linux/mm.h
    @@ -1117,6 +1117,17 @@ static inline bool is_zone_device_page(const struct page *page)
    }
    #endif

    +static inline bool is_zone_movable_page(const struct page *page)
    +{
    + return page_zonenum(page) == ZONE_MOVABLE;
    +}
    +
    +/* MIGRATE_CMA and ZONE_MOVABLE do not allow pin pages */
    +static inline bool is_pinnable_page(struct page *page)
    +{
    + return !is_zone_movable_page(page) && !is_migrate_cma_page(page);
    +}
    +
    #ifdef CONFIG_DEV_PAGEMAP_OPS
    void free_devmap_managed_page(struct page *page);
    DECLARE_STATIC_KEY_FALSE(devmap_managed_key);
    diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h
    index 5f4dd3274734..a55277b0d475 100644
    --- a/include/linux/sched/mm.h
    +++ b/include/linux/sched/mm.h
    @@ -150,12 +150,13 @@ static inline bool in_vfork(struct task_struct *tsk)
    * Applies per-task gfp context to the given allocation flags.
    * PF_MEMALLOC_NOIO implies GFP_NOIO
    * PF_MEMALLOC_NOFS implies GFP_NOFS
    + * PF_MEMALLOC_PIN implies !GFP_MOVABLE
    */
    static inline gfp_t current_gfp_context(gfp_t flags)
    {
    unsigned int pflags = READ_ONCE(current->flags);

    - if (unlikely(pflags & (PF_MEMALLOC_NOIO | PF_MEMALLOC_NOFS))) {
    + if (unlikely(pflags & (PF_MEMALLOC_NOIO | PF_MEMALLOC_NOFS | PF_MEMALLOC_PIN))) {
    /*
    * NOIO implies both NOIO and NOFS and it is a weaker context
    * so always make sure it makes precedence
    @@ -164,6 +165,9 @@ static inline gfp_t current_gfp_context(gfp_t flags)
    flags &= ~(__GFP_IO | __GFP_FS);
    else if (pflags & PF_MEMALLOC_NOFS)
    flags &= ~__GFP_FS;
    +
    + if (pflags & PF_MEMALLOC_PIN)
    + flags &= ~__GFP_MOVABLE;
    }
    return flags;
    }
    diff --git a/mm/hugetlb.c b/mm/hugetlb.c
    index 006eccfa23d7..7613c4718d24 100644
    --- a/mm/hugetlb.c
    +++ b/mm/hugetlb.c
    @@ -1052,7 +1052,7 @@ static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid)
    bool pin = !!(current->flags & PF_MEMALLOC_PIN);

    list_for_each_entry(page, &h->hugepage_freelists[nid], lru) {
    - if (pin && is_migrate_cma_page(page))
    + if (pin && !is_pinnable_page(page))
    continue;

    if (PageHWPoison(page))
    diff --git a/mm/page_alloc.c b/mm/page_alloc.c
    index de9bcd08d002..4dcee3bfd2fe 100644
    --- a/mm/page_alloc.c
    +++ b/mm/page_alloc.c
    @@ -3805,16 +3805,13 @@ alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
    return alloc_flags;
    }

    -static inline unsigned int current_alloc_flags(gfp_t gfp_mask,
    - unsigned int alloc_flags)
    +/* Must be called after current_gfp_context() which can change gfp_mask */
    +static inline unsigned int gpf_to_alloc_flags(gfp_t gfp_mask,
    + unsigned int alloc_flags)
    {
    #ifdef CONFIG_CMA
    - unsigned int pflags = current->flags;
    -
    - if (!(pflags & PF_MEMALLOC_PIN) &&
    - gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE)
    + if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE)
    alloc_flags |= ALLOC_CMA;
    -
    #endif
    return alloc_flags;
    }
    @@ -4470,7 +4467,7 @@ gfp_to_alloc_flags(gfp_t gfp_mask)
    } else if (unlikely(rt_task(current)) && !in_interrupt())
    alloc_flags |= ALLOC_HARDER;

    - alloc_flags = current_alloc_flags(gfp_mask, alloc_flags);
    + alloc_flags = gpf_to_alloc_flags(gfp_mask, alloc_flags);

    return alloc_flags;
    }
    @@ -4772,7 +4769,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,

    reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
    if (reserve_flags)
    - alloc_flags = current_alloc_flags(gfp_mask, reserve_flags);
    + alloc_flags = gpf_to_alloc_flags(gfp_mask, reserve_flags);

    /*
    * Reset the nodemask and zonelist iterators if memory policies can be
    @@ -4941,7 +4938,7 @@ static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
    if (should_fail_alloc_page(gfp_mask, order))
    return false;

    - *alloc_flags = current_alloc_flags(gfp_mask, *alloc_flags);
    + *alloc_flags = gpf_to_alloc_flags(gfp_mask, *alloc_flags);

    /* Dirty zone balancing only done in the fast path */
    ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
    @@ -4983,7 +4980,8 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
    * Apply scoped allocation constraints. This is mainly about GFP_NOFS
    * resp. GFP_NOIO which has to be inherited for all allocation requests
    * from a particular context which has been marked by
    - * memalloc_no{fs,io}_{save,restore}.
    + * memalloc_no{fs,io}_{save,restore}. And PF_MEMALLOC_PIN which ensures
    + * movable zones are not used during allocation.
    */
    gfp_mask = current_gfp_context(gfp_mask);
    alloc_mask = gfp_mask;
    --
    2.25.1
    \
     
     \ /
      Last update: 2021-01-20 03:04    [W:5.671 / U:0.284 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site