lkml.org 
[lkml]   [2021]   [Jul]   [14]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH v14 011/138] mm/lru: Add folio LRU functions
    Date
    Handle arbitrary-order folios being added to the LRU.  By definition,
    all pages being added to the LRU were already head or base pages, but
    call page_folio() on them anyway to get the type right and avoid the
    buried calls to compound_head().

    Saves 783 bytes of kernel text; no functions grow.

    Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
    Reviewed-by: Yu Zhao <yuzhao@google.com>
    Reviewed-by: Christoph Hellwig <hch@lst.de>
    Reviewed-by: David Howells <dhowells@redhat.com>
    Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
    ---
    include/linux/mm_inline.h | 98 ++++++++++++++++++++++------------
    include/trace/events/pagemap.h | 2 +-
    2 files changed, 65 insertions(+), 35 deletions(-)

    diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
    index 355ea1ee32bd..ee155d19885e 100644
    --- a/include/linux/mm_inline.h
    +++ b/include/linux/mm_inline.h
    @@ -6,22 +6,27 @@
    #include <linux/swap.h>

    /**
    - * page_is_file_lru - should the page be on a file LRU or anon LRU?
    - * @page: the page to test
    + * folio_is_file_lru - should the folio be on a file LRU or anon LRU?
    + * @folio: the folio to test
    *
    - * Returns 1 if @page is a regular filesystem backed page cache page or a lazily
    - * freed anonymous page (e.g. via MADV_FREE). Returns 0 if @page is a normal
    - * anonymous page, a tmpfs page or otherwise ram or swap backed page. Used by
    - * functions that manipulate the LRU lists, to sort a page onto the right LRU
    - * list.
    + * Returns 1 if @folio is a regular filesystem backed page cache folio
    + * or a lazily freed anonymous folio (e.g. via MADV_FREE). Returns 0 if
    + * @folio is a normal anonymous folio, a tmpfs folio or otherwise ram or
    + * swap backed folio. Used by functions that manipulate the LRU lists,
    + * to sort a folio onto the right LRU list.
    *
    * We would like to get this info without a page flag, but the state
    - * needs to survive until the page is last deleted from the LRU, which
    + * needs to survive until the folio is last deleted from the LRU, which
    * could be as far down as __page_cache_release.
    */
    +static inline int folio_is_file_lru(struct folio *folio)
    +{
    + return !folio_test_swapbacked(folio);
    +}
    +
    static inline int page_is_file_lru(struct page *page)
    {
    - return !PageSwapBacked(page);
    + return folio_is_file_lru(page_folio(page));
    }

    static __always_inline void update_lru_size(struct lruvec *lruvec,
    @@ -39,69 +44,94 @@ static __always_inline void update_lru_size(struct lruvec *lruvec,
    }

    /**
    - * __clear_page_lru_flags - clear page lru flags before releasing a page
    - * @page: the page that was on lru and now has a zero reference
    + * __folio_clear_lru_flags - clear page lru flags before releasing a page
    + * @folio: The folio that was on lru and now has a zero reference
    */
    -static __always_inline void __clear_page_lru_flags(struct page *page)
    +static __always_inline void __folio_clear_lru_flags(struct folio *folio)
    {
    - VM_BUG_ON_PAGE(!PageLRU(page), page);
    + VM_BUG_ON_FOLIO(!folio_test_lru(folio), folio);

    - __ClearPageLRU(page);
    + __folio_clear_lru(folio);

    /* this shouldn't happen, so leave the flags to bad_page() */
    - if (PageActive(page) && PageUnevictable(page))
    + if (folio_test_active(folio) && folio_test_unevictable(folio))
    return;

    - __ClearPageActive(page);
    - __ClearPageUnevictable(page);
    + __folio_clear_active(folio);
    + __folio_clear_unevictable(folio);
    +}
    +
    +static __always_inline void __clear_page_lru_flags(struct page *page)
    +{
    + __folio_clear_lru_flags(page_folio(page));
    }

    /**
    - * page_lru - which LRU list should a page be on?
    - * @page: the page to test
    + * folio_lru_list - which LRU list should a folio be on?
    + * @folio: the folio to test
    *
    - * Returns the LRU list a page should be on, as an index
    + * Returns the LRU list a folio should be on, as an index
    * into the array of LRU lists.
    */
    -static __always_inline enum lru_list page_lru(struct page *page)
    +static __always_inline enum lru_list folio_lru_list(struct folio *folio)
    {
    enum lru_list lru;

    - VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page);
    + VM_BUG_ON_FOLIO(folio_test_active(folio) && folio_test_unevictable(folio), folio);

    - if (PageUnevictable(page))
    + if (folio_test_unevictable(folio))
    return LRU_UNEVICTABLE;

    - lru = page_is_file_lru(page) ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON;
    - if (PageActive(page))
    + lru = folio_is_file_lru(folio) ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON;
    + if (folio_test_active(folio))
    lru += LRU_ACTIVE;

    return lru;
    }

    +static __always_inline
    +void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)
    +{
    + enum lru_list lru = folio_lru_list(folio);
    +
    + update_lru_size(lruvec, lru, folio_zonenum(folio),
    + folio_nr_pages(folio));
    + list_add(&folio->lru, &lruvec->lists[lru]);
    +}
    +
    static __always_inline void add_page_to_lru_list(struct page *page,
    struct lruvec *lruvec)
    {
    - enum lru_list lru = page_lru(page);
    + lruvec_add_folio(lruvec, page_folio(page));
    +}
    +
    +static __always_inline
    +void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio)
    +{
    + enum lru_list lru = folio_lru_list(folio);

    - update_lru_size(lruvec, lru, page_zonenum(page), thp_nr_pages(page));
    - list_add(&page->lru, &lruvec->lists[lru]);
    + update_lru_size(lruvec, lru, folio_zonenum(folio),
    + folio_nr_pages(folio));
    + list_add_tail(&folio->lru, &lruvec->lists[lru]);
    }

    static __always_inline void add_page_to_lru_list_tail(struct page *page,
    struct lruvec *lruvec)
    {
    - enum lru_list lru = page_lru(page);
    + lruvec_add_folio_tail(lruvec, page_folio(page));
    +}

    - update_lru_size(lruvec, lru, page_zonenum(page), thp_nr_pages(page));
    - list_add_tail(&page->lru, &lruvec->lists[lru]);
    +static __always_inline
    +void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)
    +{
    + list_del(&folio->lru);
    + update_lru_size(lruvec, folio_lru_list(folio), folio_zonenum(folio),
    + -folio_nr_pages(folio));
    }

    static __always_inline void del_page_from_lru_list(struct page *page,
    struct lruvec *lruvec)
    {
    - list_del(&page->lru);
    - update_lru_size(lruvec, page_lru(page), page_zonenum(page),
    - -thp_nr_pages(page));
    + lruvec_del_folio(lruvec, page_folio(page));
    }
    #endif
    diff --git a/include/trace/events/pagemap.h b/include/trace/events/pagemap.h
    index 1d28431e85bd..92ad176210ff 100644
    --- a/include/trace/events/pagemap.h
    +++ b/include/trace/events/pagemap.h
    @@ -41,7 +41,7 @@ TRACE_EVENT(mm_lru_insertion,
    TP_fast_assign(
    __entry->page = page;
    __entry->pfn = page_to_pfn(page);
    - __entry->lru = page_lru(page);
    + __entry->lru = folio_lru_list(page_folio(page));
    __entry->flags = trace_pagemap_flags(page);
    ),

    --
    2.30.2
    \
     
     \ /
      Last update: 2021-07-15 05:46    [W:4.046 / U:0.148 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site