lkml.org 
[lkml]   [2020]   [Oct]   [2]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH v5 1/4] mm: memcontrol: use helpers to read page's memcg data
    Date
    Currently there are many open-coded reads of the page->mem_cgroup
    pointer, as well as a couple of read helpers, which are barely used.

    It creates an obstacle on a way to reuse some bits of the pointer
    for storing additional bits of information. In fact, we already do
    this for slab pages, where the last bit indicates that a pointer has
    an attached vector of objcg pointers instead of a regular memcg
    pointer.

    This commits uses 2 existing helpers and introduces a new helper to
    converts all read sides to calls of these helpers:
    struct mem_cgroup *page_memcg(struct page *page);
    struct mem_cgroup *page_memcg_rcu(struct page *page);
    struct mem_cgroup *page_memcg_check(struct page *page);

    page_memcg_check() is intended to be used in cases when the page
    can be a slab page and have a memcg pointer pointing at objcg vector.
    It does check the lowest bit, and if set, returns NULL.
    page_memcg() contains a VM_BUG_ON_PAGE() check for the page not
    being a slab page.

    To make sure nobody uses a direct access, struct page's
    mem_cgroup/obj_cgroups is converted to unsigned long memcg_data.

    Signed-off-by: Roman Gushchin <guro@fb.com>
    Acked-by: Johannes Weiner <hannes@cmpxchg.org>
    ---
    fs/buffer.c | 2 +-
    fs/iomap/buffered-io.c | 2 +-
    include/linux/memcontrol.h | 114 ++++++++++++++++++++++++++---
    include/linux/mm.h | 22 ------
    include/linux/mm_types.h | 5 +-
    include/trace/events/writeback.h | 2 +-
    kernel/fork.c | 7 +-
    mm/debug.c | 4 +-
    mm/huge_memory.c | 4 +-
    mm/memcontrol.c | 121 ++++++++++++++-----------------
    mm/page_alloc.c | 4 +-
    mm/page_io.c | 6 +-
    mm/slab.h | 9 +--
    mm/workingset.c | 2 +-
    14 files changed, 184 insertions(+), 120 deletions(-)

    diff --git a/fs/buffer.c b/fs/buffer.c
    index bf4d8037f91b..64564ac7dcc5 100644
    --- a/fs/buffer.c
    +++ b/fs/buffer.c
    @@ -657,7 +657,7 @@ int __set_page_dirty_buffers(struct page *page)
    } while (bh != head);
    }
    /*
    - * Lock out page->mem_cgroup migration to keep PageDirty
    + * Lock out page's memcg migration to keep PageDirty
    * synchronized with per-memcg dirty page counters.
    */
    lock_page_memcg(page);
    diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c
    index 8180061b9e16..f96094a32851 100644
    --- a/fs/iomap/buffered-io.c
    +++ b/fs/iomap/buffered-io.c
    @@ -650,7 +650,7 @@ iomap_set_page_dirty(struct page *page)
    return !TestSetPageDirty(page);

    /*
    - * Lock out page->mem_cgroup migration to keep PageDirty
    + * Lock out page's memcg migration to keep PageDirty
    * synchronized with per-memcg dirty page counters.
    */
    lock_page_memcg(page);
    diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
    index e391e3c56de5..f95c1433461c 100644
    --- a/include/linux/memcontrol.h
    +++ b/include/linux/memcontrol.h
    @@ -343,6 +343,79 @@ struct mem_cgroup {

    extern struct mem_cgroup *root_mem_cgroup;

    +/*
    + * page_memcg - get the memory cgroup associated with a page
    + * @page: a pointer to the page struct
    + *
    + * Returns a pointer to the memory cgroup associated with the page,
    + * or NULL. This function assumes that the page is known to have a
    + * proper memory cgroup pointer. It's not safe to call this function
    + * against some type of pages, e.g. slab pages or ex-slab pages.
    + *
    + * Any of the following ensures page and memcg binding stability:
    + * - the page lock
    + * - LRU isolation
    + * - lock_page_memcg()
    + * - exclusive reference
    + */
    +static inline struct mem_cgroup *page_memcg(struct page *page)
    +{
    + VM_BUG_ON_PAGE(PageSlab(page), page);
    + return (struct mem_cgroup *)page->memcg_data;
    +}
    +
    +/*
    + * page_memcg_rcu - locklessly get the memory cgroup associated with a page
    + * @page: a pointer to the page struct
    + *
    + * Returns a pointer to the memory cgroup associated with the page,
    + * or NULL. This function assumes that the page is known to have a
    + * proper memory cgroup pointer. It's not safe to call this function
    + * against some type of pages, e.g. slab pages or ex-slab pages.
    + */
    +static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
    +{
    + VM_BUG_ON_PAGE(PageSlab(page), page);
    + WARN_ON_ONCE(!rcu_read_lock_held());
    +
    + return (struct mem_cgroup *)READ_ONCE(page->memcg_data);
    +}
    +
    +/*
    + * page_memcg_check - get the memory cgroup associated with a page
    + * @page: a pointer to the page struct
    + *
    + * Returns a pointer to the memory cgroup associated with the page,
    + * or NULL. This function unlike page_memcg() can take any page
    + * as an argument. It has to be used in cases when it's not known if a page
    + * has an associated memory cgroup pointer or an object cgroups vector.
    + *
    + * Any of the following ensures page and memcg binding stability:
    + * - the page lock
    + * - LRU isolation
    + * - lock_page_memcg()
    + * - exclusive reference
    + */
    +static inline struct mem_cgroup *page_memcg_check(struct page *page)
    +{
    + /*
    + * Because page->memcg_data might be changed asynchronously
    + * for slab pages, READ_ONCE() should be used here.
    + */
    + unsigned long memcg_data = READ_ONCE(page->memcg_data);
    +
    + /*
    + * The lowest bit set means that memcg isn't a valid
    + * memcg pointer, but a obj_cgroups pointer.
    + * In this case the page is shared and doesn't belong
    + * to any specific memory cgroup.
    + */
    + if (memcg_data & 0x1UL)
    + return NULL;
    +
    + return (struct mem_cgroup *)memcg_data;
    +}
    +
    static __always_inline bool memcg_stat_item_in_bytes(int idx)
    {
    if (idx == MEMCG_PERCPU_B)
    @@ -743,15 +816,19 @@ static inline void mod_memcg_state(struct mem_cgroup *memcg,
    static inline void __mod_memcg_page_state(struct page *page,
    int idx, int val)
    {
    - if (page->mem_cgroup)
    - __mod_memcg_state(page->mem_cgroup, idx, val);
    + struct mem_cgroup *memcg = page_memcg(page);
    +
    + if (memcg)
    + __mod_memcg_state(memcg, idx, val);
    }

    static inline void mod_memcg_page_state(struct page *page,
    int idx, int val)
    {
    - if (page->mem_cgroup)
    - mod_memcg_state(page->mem_cgroup, idx, val);
    + struct mem_cgroup *memcg = page_memcg(page);
    +
    + if (memcg)
    + mod_memcg_state(memcg, idx, val);
    }

    static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
    @@ -834,16 +911,17 @@ static inline void __mod_lruvec_page_state(struct page *page,
    enum node_stat_item idx, int val)
    {
    struct page *head = compound_head(page); /* rmap on tail pages */
    + struct mem_cgroup *memcg = page_memcg(head);
    pg_data_t *pgdat = page_pgdat(page);
    struct lruvec *lruvec;

    /* Untracked pages have no memcg, no lruvec. Update only the node */
    - if (!head->mem_cgroup) {
    + if (!memcg) {
    __mod_node_page_state(pgdat, idx, val);
    return;
    }

    - lruvec = mem_cgroup_lruvec(head->mem_cgroup, pgdat);
    + lruvec = mem_cgroup_lruvec(memcg, pgdat);
    __mod_lruvec_state(lruvec, idx, val);
    }

    @@ -878,8 +956,10 @@ static inline void count_memcg_events(struct mem_cgroup *memcg,
    static inline void count_memcg_page_event(struct page *page,
    enum vm_event_item idx)
    {
    - if (page->mem_cgroup)
    - count_memcg_events(page->mem_cgroup, idx, 1);
    + struct mem_cgroup *memcg = page_memcg(page);
    +
    + if (memcg)
    + count_memcg_events(memcg, idx, 1);
    }

    static inline void count_memcg_event_mm(struct mm_struct *mm,
    @@ -941,6 +1021,22 @@ void mem_cgroup_split_huge_fixup(struct page *head);

    struct mem_cgroup;

    +static inline struct mem_cgroup *page_memcg(struct page *page)
    +{
    + return NULL;
    +}
    +
    +static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
    +{
    + WARN_ON_ONCE(!rcu_read_lock_held());
    + return NULL;
    +}
    +
    +static inline struct mem_cgroup *page_memcg_check(struct page *page)
    +{
    + return NULL;
    +}
    +
    static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
    {
    return true;
    @@ -1430,7 +1526,7 @@ static inline void mem_cgroup_track_foreign_dirty(struct page *page,
    if (mem_cgroup_disabled())
    return;

    - if (unlikely(&page->mem_cgroup->css != wb->memcg_css))
    + if (unlikely(&page_memcg(page)->css != wb->memcg_css))
    mem_cgroup_track_foreign_dirty_slowpath(page, wb);
    }

    diff --git a/include/linux/mm.h b/include/linux/mm.h
    index 27c64d0d7520..6707801b014d 100644
    --- a/include/linux/mm.h
    +++ b/include/linux/mm.h
    @@ -1484,28 +1484,6 @@ static inline void set_page_links(struct page *page, enum zone_type zone,
    #endif
    }

    -#ifdef CONFIG_MEMCG
    -static inline struct mem_cgroup *page_memcg(struct page *page)
    -{
    - return page->mem_cgroup;
    -}
    -static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
    -{
    - WARN_ON_ONCE(!rcu_read_lock_held());
    - return READ_ONCE(page->mem_cgroup);
    -}
    -#else
    -static inline struct mem_cgroup *page_memcg(struct page *page)
    -{
    - return NULL;
    -}
    -static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
    -{
    - WARN_ON_ONCE(!rcu_read_lock_held());
    - return NULL;
    -}
    -#endif
    -
    /*
    * Some inline functions in vmstat.h depend on page_zone()
    */
    diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
    index 5a9238f6caad..80f5d755c037 100644
    --- a/include/linux/mm_types.h
    +++ b/include/linux/mm_types.h
    @@ -199,10 +199,7 @@ struct page {
    atomic_t _refcount;

    #ifdef CONFIG_MEMCG
    - union {
    - struct mem_cgroup *mem_cgroup;
    - struct obj_cgroup **obj_cgroups;
    - };
    + unsigned long memcg_data;
    #endif

    /*
    diff --git a/include/trace/events/writeback.h b/include/trace/events/writeback.h
    index e7cbccc7c14c..39a40dfb578a 100644
    --- a/include/trace/events/writeback.h
    +++ b/include/trace/events/writeback.h
    @@ -257,7 +257,7 @@ TRACE_EVENT(track_foreign_dirty,
    __entry->ino = inode ? inode->i_ino : 0;
    __entry->memcg_id = wb->memcg_css->id;
    __entry->cgroup_ino = __trace_wb_assign_cgroup(wb);
    - __entry->page_cgroup_ino = cgroup_ino(page->mem_cgroup->css.cgroup);
    + __entry->page_cgroup_ino = cgroup_ino(page_memcg(page)->css.cgroup);
    ),

    TP_printk("bdi %s[%llu]: ino=%lu memcg_id=%u cgroup_ino=%lu page_cgroup_ino=%lu",
    diff --git a/kernel/fork.c b/kernel/fork.c
    index 9be04e54de7f..74e5776f2cea 100644
    --- a/kernel/fork.c
    +++ b/kernel/fork.c
    @@ -403,9 +403,10 @@ static int memcg_charge_kernel_stack(struct task_struct *tsk)

    for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
    /*
    - * If memcg_kmem_charge_page() fails, page->mem_cgroup
    - * pointer is NULL, and memcg_kmem_uncharge_page() in
    - * free_thread_stack() will ignore this page.
    + * If memcg_kmem_charge_page() fails, page's
    + * memory cgroup pointer is NULL, and
    + * memcg_kmem_uncharge_page() in free_thread_stack()
    + * will ignore this page.
    */
    ret = memcg_kmem_charge_page(vm->pages[i], GFP_KERNEL,
    0);
    diff --git a/mm/debug.c b/mm/debug.c
    index ccca576b2899..8a40b3fefbeb 100644
    --- a/mm/debug.c
    +++ b/mm/debug.c
    @@ -182,8 +182,8 @@ void __dump_page(struct page *page, const char *reason)
    pr_warn("page dumped because: %s\n", reason);

    #ifdef CONFIG_MEMCG
    - if (!page_poisoned && page->mem_cgroup)
    - pr_warn("page->mem_cgroup:%px\n", page->mem_cgroup);
    + if (!page_poisoned && page->memcg_data)
    + pr_warn("pages's memcg:%lx\n", page->memcg_data);
    #endif
    }

    diff --git a/mm/huge_memory.c b/mm/huge_memory.c
    index cba3812a5c3e..999f8a4dcb56 100644
    --- a/mm/huge_memory.c
    +++ b/mm/huge_memory.c
    @@ -470,7 +470,7 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
    #ifdef CONFIG_MEMCG
    static inline struct deferred_split *get_deferred_split_queue(struct page *page)
    {
    - struct mem_cgroup *memcg = compound_head(page)->mem_cgroup;
    + struct mem_cgroup *memcg = page_memcg(compound_head(page));
    struct pglist_data *pgdat = NODE_DATA(page_to_nid(page));

    if (memcg)
    @@ -2766,7 +2766,7 @@ void deferred_split_huge_page(struct page *page)
    {
    struct deferred_split *ds_queue = get_deferred_split_queue(page);
    #ifdef CONFIG_MEMCG
    - struct mem_cgroup *memcg = compound_head(page)->mem_cgroup;
    + struct mem_cgroup *memcg = page_memcg(compound_head(page));
    #endif
    unsigned long flags;

    diff --git a/mm/memcontrol.c b/mm/memcontrol.c
    index 888cbc48311f..c2e6c75ed988 100644
    --- a/mm/memcontrol.c
    +++ b/mm/memcontrol.c
    @@ -533,7 +533,7 @@ struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
    {
    struct mem_cgroup *memcg;

    - memcg = page->mem_cgroup;
    + memcg = page_memcg(page);

    if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
    memcg = root_mem_cgroup;
    @@ -560,16 +560,7 @@ ino_t page_cgroup_ino(struct page *page)
    unsigned long ino = 0;

    rcu_read_lock();
    - memcg = page->mem_cgroup;
    -
    - /*
    - * The lowest bit set means that memcg isn't a valid
    - * memcg pointer, but a obj_cgroups pointer.
    - * In this case the page is shared and doesn't belong
    - * to any specific memory cgroup.
    - */
    - if ((unsigned long) memcg & 0x1UL)
    - memcg = NULL;
    + memcg = page_memcg_check(page);

    while (memcg && !(memcg->css.flags & CSS_ONLINE))
    memcg = parent_mem_cgroup(memcg);
    @@ -1050,7 +1041,7 @@ EXPORT_SYMBOL(get_mem_cgroup_from_mm);
    */
    struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
    {
    - struct mem_cgroup *memcg = page->mem_cgroup;
    + struct mem_cgroup *memcg = page_memcg(page);

    if (mem_cgroup_disabled())
    return NULL;
    @@ -1349,7 +1340,7 @@ struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgd
    goto out;
    }

    - memcg = page->mem_cgroup;
    + memcg = page_memcg(page);
    /*
    * Swapcache readahead pages are added to the LRU - and
    * possibly migrated - before they are charged.
    @@ -2109,7 +2100,7 @@ void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
    }

    /**
    - * lock_page_memcg - lock a page->mem_cgroup binding
    + * lock_page_memcg - lock a page and memcg binding
    * @page: the page
    *
    * This function protects unlocked LRU pages from being moved to
    @@ -2141,7 +2132,7 @@ struct mem_cgroup *lock_page_memcg(struct page *page)
    if (mem_cgroup_disabled())
    return NULL;
    again:
    - memcg = head->mem_cgroup;
    + memcg = page_memcg(head);
    if (unlikely(!memcg))
    return NULL;

    @@ -2149,7 +2140,7 @@ struct mem_cgroup *lock_page_memcg(struct page *page)
    return memcg;

    spin_lock_irqsave(&memcg->move_lock, flags);
    - if (memcg != head->mem_cgroup) {
    + if (memcg != page_memcg(head)) {
    spin_unlock_irqrestore(&memcg->move_lock, flags);
    goto again;
    }
    @@ -2187,14 +2178,14 @@ void __unlock_page_memcg(struct mem_cgroup *memcg)
    }

    /**
    - * unlock_page_memcg - unlock a page->mem_cgroup binding
    + * unlock_page_memcg - unlock a page and memcg binding
    * @page: the page
    */
    void unlock_page_memcg(struct page *page)
    {
    struct page *head = compound_head(page);

    - __unlock_page_memcg(head->mem_cgroup);
    + __unlock_page_memcg(page_memcg(head));
    }
    EXPORT_SYMBOL(unlock_page_memcg);

    @@ -2884,7 +2875,7 @@ static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)

    static void commit_charge(struct page *page, struct mem_cgroup *memcg)
    {
    - VM_BUG_ON_PAGE(page->mem_cgroup, page);
    + VM_BUG_ON_PAGE(page_memcg(page), page);
    /*
    * Any of the following ensures page->mem_cgroup stability:
    *
    @@ -2893,7 +2884,7 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg)
    * - lock_page_memcg()
    * - exclusive reference
    */
    - page->mem_cgroup = memcg;
    + page->memcg_data = (unsigned long)memcg;
    }

    #ifdef CONFIG_MEMCG_KMEM
    @@ -2908,8 +2899,7 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
    if (!vec)
    return -ENOMEM;

    - if (cmpxchg(&page->obj_cgroups, NULL,
    - (struct obj_cgroup **) ((unsigned long)vec | 0x1UL)))
    + if (cmpxchg(&page->memcg_data, 0, (unsigned long)vec | 0x1UL))
    kfree(vec);
    else
    kmemleak_not_leak(vec);
    @@ -2920,6 +2910,12 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
    /*
    * Returns a pointer to the memory cgroup to which the kernel object is charged.
    *
    + * A passed kernel object can be a slab object or a generic kernel page, so
    + * different mechanisms for getting the memory cgroup pointer should be used.
    + * In certain cases (e.g. kernel stacks or large kmallocs with SLUB) the caller
    + * can not know for sure how the kernel object is implemented.
    + * mem_cgroup_from_obj() can be safely used in such cases.
    + *
    * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
    * cgroup_mutex, etc.
    */
    @@ -2932,17 +2928,6 @@ struct mem_cgroup *mem_cgroup_from_obj(void *p)

    page = virt_to_head_page(p);

    - /*
    - * If page->mem_cgroup is set, it's either a simple mem_cgroup pointer
    - * or a pointer to obj_cgroup vector. In the latter case the lowest
    - * bit of the pointer is set.
    - * The page->mem_cgroup pointer can be asynchronously changed
    - * from NULL to (obj_cgroup_vec | 0x1UL), but can't be changed
    - * from a valid memcg pointer to objcg vector or back.
    - */
    - if (!page->mem_cgroup)
    - return NULL;
    -
    /*
    * Slab objects are accounted individually, not per-page.
    * Memcg membership data for each individual object is saved in
    @@ -2960,8 +2945,14 @@ struct mem_cgroup *mem_cgroup_from_obj(void *p)
    return NULL;
    }

    - /* All other pages use page->mem_cgroup */
    - return page->mem_cgroup;
    + /*
    + * page_memcg_check() is used here, because page_has_obj_cgroups()
    + * check above could fail because the object cgroups vector wasn't set
    + * at that moment, but it can be set concurrently.
    + * page_memcg_check(page) will guarantee that a proper memory
    + * cgroup pointer or NULL will be returned.
    + */
    + return page_memcg_check(page);
    }

    __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void)
    @@ -3099,7 +3090,7 @@ int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
    if (memcg && !mem_cgroup_is_root(memcg)) {
    ret = __memcg_kmem_charge(memcg, gfp, 1 << order);
    if (!ret) {
    - page->mem_cgroup = memcg;
    + page->memcg_data = (unsigned long)memcg;
    __SetPageKmemcg(page);
    return 0;
    }
    @@ -3115,7 +3106,7 @@ int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
    */
    void __memcg_kmem_uncharge_page(struct page *page, int order)
    {
    - struct mem_cgroup *memcg = page->mem_cgroup;
    + struct mem_cgroup *memcg = page_memcg(page);
    unsigned int nr_pages = 1 << order;

    if (!memcg)
    @@ -3123,7 +3114,7 @@ void __memcg_kmem_uncharge_page(struct page *page, int order)

    VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
    __memcg_kmem_uncharge(memcg, nr_pages);
    - page->mem_cgroup = NULL;
    + page->memcg_data = 0;
    css_put(&memcg->css);

    /* slab pages do not have PageKmemcg flag set */
    @@ -3274,7 +3265,7 @@ void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)
    */
    void mem_cgroup_split_huge_fixup(struct page *head)
    {
    - struct mem_cgroup *memcg = head->mem_cgroup;
    + struct mem_cgroup *memcg = page_memcg(head);
    int i;

    if (mem_cgroup_disabled())
    @@ -3282,7 +3273,7 @@ void mem_cgroup_split_huge_fixup(struct page *head)

    for (i = 1; i < HPAGE_PMD_NR; i++) {
    css_get(&memcg->css);
    - head[i].mem_cgroup = memcg;
    + head[i].memcg_data = (unsigned long)memcg;
    }
    }
    #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
    @@ -4658,7 +4649,7 @@ void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
    void mem_cgroup_track_foreign_dirty_slowpath(struct page *page,
    struct bdi_writeback *wb)
    {
    - struct mem_cgroup *memcg = page->mem_cgroup;
    + struct mem_cgroup *memcg = page_memcg(page);
    struct memcg_cgwb_frn *frn;
    u64 now = get_jiffies_64();
    u64 oldest_at = now;
    @@ -5627,14 +5618,14 @@ static int mem_cgroup_move_account(struct page *page,

    /*
    * Prevent mem_cgroup_migrate() from looking at
    - * page->mem_cgroup of its source page while we change it.
    + * page's memory cgroup of its source page while we change it.
    */
    ret = -EBUSY;
    if (!trylock_page(page))
    goto out;

    ret = -EINVAL;
    - if (page->mem_cgroup != from)
    + if (page_memcg(page) != from)
    goto out_unlock;

    pgdat = page_pgdat(page);
    @@ -5689,13 +5680,13 @@ static int mem_cgroup_move_account(struct page *page,
    /*
    * All state has been migrated, let's switch to the new memcg.
    *
    - * It is safe to change page->mem_cgroup here because the page
    + * It is safe to change page's memcg here because the page
    * is referenced, charged, isolated, and locked: we can't race
    * with (un)charging, migration, LRU putback, or anything else
    - * that would rely on a stable page->mem_cgroup.
    + * that would rely on a stable page's memory cgroup.
    *
    * Note that lock_page_memcg is a memcg lock, not a page lock,
    - * to save space. As soon as we switch page->mem_cgroup to a
    + * to save space. As soon as we switch page's memory cgroup to a
    * new memcg that isn't locked, the above state can change
    * concurrently again. Make sure we're truly done with it.
    */
    @@ -5704,7 +5695,7 @@ static int mem_cgroup_move_account(struct page *page,
    css_get(&to->css);
    css_put(&from->css);

    - page->mem_cgroup = to;
    + page->memcg_data = (unsigned long)to;

    __unlock_page_memcg(from);

    @@ -5770,7 +5761,7 @@ static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
    * mem_cgroup_move_account() checks the page is valid or
    * not under LRU exclusion.
    */
    - if (page->mem_cgroup == mc.from) {
    + if (page_memcg(page) == mc.from) {
    ret = MC_TARGET_PAGE;
    if (is_device_private_page(page))
    ret = MC_TARGET_DEVICE;
    @@ -5814,7 +5805,7 @@ static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
    VM_BUG_ON_PAGE(!page || !PageHead(page), page);
    if (!(mc.flags & MOVE_ANON))
    return ret;
    - if (page->mem_cgroup == mc.from) {
    + if (page_memcg(page) == mc.from) {
    ret = MC_TARGET_PAGE;
    if (target) {
    get_page(page);
    @@ -6760,12 +6751,12 @@ int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
    /*
    * Every swap fault against a single page tries to charge the
    * page, bail as early as possible. shmem_unuse() encounters
    - * already charged pages, too. page->mem_cgroup is protected
    - * by the page lock, which serializes swap cache removal, which
    - * in turn serializes uncharging.
    + * already charged pages, too. page and memcg binding is
    + * protected by the page lock, which serializes swap cache
    + * removal, which in turn serializes uncharging.
    */
    VM_BUG_ON_PAGE(!PageLocked(page), page);
    - if (compound_head(page)->mem_cgroup)
    + if (page_memcg(compound_head(page)))
    goto out;

    id = lookup_swap_cgroup_id(ent);
    @@ -6849,21 +6840,21 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug)

    VM_BUG_ON_PAGE(PageLRU(page), page);

    - if (!page->mem_cgroup)
    + if (!page_memcg(page))
    return;

    /*
    * Nobody should be changing or seriously looking at
    - * page->mem_cgroup at this point, we have fully
    + * page_memcg(page) at this point, we have fully
    * exclusive access to the page.
    */

    - if (ug->memcg != page->mem_cgroup) {
    + if (ug->memcg != page_memcg(page)) {
    if (ug->memcg) {
    uncharge_batch(ug);
    uncharge_gather_clear(ug);
    }
    - ug->memcg = page->mem_cgroup;
    + ug->memcg = page_memcg(page);

    /* pairs with css_put in uncharge_batch */
    css_get(&ug->memcg->css);
    @@ -6880,7 +6871,7 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug)
    }

    ug->dummy_page = page;
    - page->mem_cgroup = NULL;
    + page->memcg_data = 0;
    css_put(&ug->memcg->css);
    }

    @@ -6923,7 +6914,7 @@ void mem_cgroup_uncharge(struct page *page)
    return;

    /* Don't touch page->lru of any random page, pre-check: */
    - if (!page->mem_cgroup)
    + if (!page_memcg(page))
    return;

    uncharge_gather_clear(&ug);
    @@ -6973,11 +6964,11 @@ void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
    return;

    /* Page cache replacement: new page already charged? */
    - if (newpage->mem_cgroup)
    + if (page_memcg(newpage))
    return;

    /* Swapcache readahead pages can get replaced before being charged */
    - memcg = oldpage->mem_cgroup;
    + memcg = page_memcg(oldpage);
    if (!memcg)
    return;

    @@ -7172,7 +7163,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
    if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
    return;

    - memcg = page->mem_cgroup;
    + memcg = page_memcg(page);

    /* Readahead page, never charged */
    if (!memcg)
    @@ -7193,7 +7184,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
    VM_BUG_ON_PAGE(oldid, page);
    mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);

    - page->mem_cgroup = NULL;
    + page->memcg_data = 0;

    if (!mem_cgroup_is_root(memcg))
    page_counter_uncharge(&memcg->memory, nr_entries);
    @@ -7236,7 +7227,7 @@ int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
    if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
    return 0;

    - memcg = page->mem_cgroup;
    + memcg = page_memcg(page);

    /* Readahead page, never charged */
    if (!memcg)
    @@ -7317,7 +7308,7 @@ bool mem_cgroup_swap_full(struct page *page)
    if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
    return false;

    - memcg = page->mem_cgroup;
    + memcg = page_memcg(page);
    if (!memcg)
    return false;

    diff --git a/mm/page_alloc.c b/mm/page_alloc.c
    index eb0962976a0e..e9f0fe4a143e 100644
    --- a/mm/page_alloc.c
    +++ b/mm/page_alloc.c
    @@ -1057,7 +1057,7 @@ static inline bool page_expected_state(struct page *page,
    if (unlikely((unsigned long)page->mapping |
    page_ref_count(page) |
    #ifdef CONFIG_MEMCG
    - (unsigned long)page->mem_cgroup |
    + (unsigned long)page_memcg(page) |
    #endif
    (page->flags & check_flags)))
    return false;
    @@ -1082,7 +1082,7 @@ static const char *page_bad_reason(struct page *page, unsigned long flags)
    bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
    }
    #ifdef CONFIG_MEMCG
    - if (unlikely(page->mem_cgroup))
    + if (unlikely(page_memcg(page)))
    bad_reason = "page still charged to cgroup";
    #endif
    return bad_reason;
    diff --git a/mm/page_io.c b/mm/page_io.c
    index 433df1263349..9bca17ecc4df 100644
    --- a/mm/page_io.c
    +++ b/mm/page_io.c
    @@ -291,12 +291,14 @@ static inline void count_swpout_vm_event(struct page *page)
    static void bio_associate_blkg_from_page(struct bio *bio, struct page *page)
    {
    struct cgroup_subsys_state *css;
    + struct mem_cgroup *memcg;

    - if (!page->mem_cgroup)
    + memcg = page_memcg(page);
    + if (!memcg)
    return;

    rcu_read_lock();
    - css = cgroup_e_css(page->mem_cgroup->css.cgroup, &io_cgrp_subsys);
    + css = cgroup_e_css(memcg->css.cgroup, &io_cgrp_subsys);
    bio_associate_blkg_from_css(bio, css);
    rcu_read_unlock();
    }
    diff --git a/mm/slab.h b/mm/slab.h
    index 4a24e1702923..5ac89260f329 100644
    --- a/mm/slab.h
    +++ b/mm/slab.h
    @@ -242,18 +242,17 @@ static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t fla
    static inline struct obj_cgroup **page_obj_cgroups(struct page *page)
    {
    /*
    - * page->mem_cgroup and page->obj_cgroups are sharing the same
    + * Page's memory cgroup and obj_cgroups vector are sharing the same
    * space. To distinguish between them in case we don't know for sure
    * that the page is a slab page (e.g. page_cgroup_ino()), let's
    * always set the lowest bit of obj_cgroups.
    */
    - return (struct obj_cgroup **)
    - ((unsigned long)page->obj_cgroups & ~0x1UL);
    + return (struct obj_cgroup **)(page->memcg_data & ~0x1UL);
    }

    static inline bool page_has_obj_cgroups(struct page *page)
    {
    - return ((unsigned long)page->obj_cgroups & 0x1UL);
    + return page->memcg_data & 0x1UL;
    }

    int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
    @@ -262,7 +261,7 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
    static inline void memcg_free_page_obj_cgroups(struct page *page)
    {
    kfree(page_obj_cgroups(page));
    - page->obj_cgroups = NULL;
    + page->memcg_data = 0;
    }

    static inline size_t obj_full_size(struct kmem_cache *s)
    diff --git a/mm/workingset.c b/mm/workingset.c
    index 8ed8e6296d8c..ef563f94a0e3 100644
    --- a/mm/workingset.c
    +++ b/mm/workingset.c
    @@ -257,7 +257,7 @@ void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg)
    struct lruvec *lruvec;
    int memcgid;

    - /* Page is fully exclusive and pins page->mem_cgroup */
    + /* Page is fully exclusive and pins page's memory cgroup pointer */
    VM_BUG_ON_PAGE(PageLRU(page), page);
    VM_BUG_ON_PAGE(page_count(page), page);
    VM_BUG_ON_PAGE(!PageLocked(page), page);
    --
    2.26.2
    \
     
     \ /
      Last update: 2020-10-02 19:26    [W:2.506 / U:0.276 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site