mm: introduce a pageflag for partially mapped folios

Currently folio->_deferred_list is used to keep track of partially_mapped
folios that are going to be split under memory pressure.  In the next
patch, all THPs that are faulted in and collapsed by khugepaged are also
going to be tracked using _deferred_list.

This patch introduces a pageflag to be able to distinguish between
partially mapped folios and others in the deferred_list at split time in
deferred_split_scan.  Its needed as __folio_remove_rmap decrements
_mapcount, _large_mapcount and _entire_mapcount, hence it won't be
possible to distinguish between partially mapped folios and others in
deferred_split_scan.

Eventhough it introduces an extra flag to track if the folio is partially
mapped, there is no functional change intended with this patch and the
flag is not useful in this patch itself, it will become useful in the next
patch when _deferred_list has non partially mapped folios.

Link: https://lkml.kernel.org/r/20240830100438.3623486-5-usamaarif642@gmail.com
Signed-off-by: Usama Arif <usamaarif642@gmail.com>
Cc: Alexander Zhu <alexlzhu@fb.com>
Cc: Barry Song <baohua@kernel.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: Domenico Cerasuolo <cerasuolodomenico@gmail.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Kairui Song <ryncsn@gmail.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Nico Pache <npache@redhat.com>
Cc: Rik van Riel <riel@surriel.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Shuang Zhai <zhais@google.com>
Cc: Yu Zhao <yuzhao@google.com>
Cc: Shuang Zhai <szhai2@cs.rochester.edu>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 4902e2f7e896f9d3c8edcf67f04c0d882ae45043..e3896ebf0f94a52a17da033b11d1e982eaba0fce 100644 (file)
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -333,7 +333,7 @@ static inline int split_huge_page(struct page *page)
 {
        return split_huge_page_to_list_to_order(page, NULL, 0);
 }
-void deferred_split_folio(struct folio *folio);
+void deferred_split_folio(struct folio *folio, bool partially_mapped);
 
 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
                unsigned long address, bool freeze, struct folio *folio);
@@ -507,7 +507,7 @@ static inline int split_huge_page(struct page *page)
 {
        return 0;
 }
-static inline void deferred_split_folio(struct folio *folio) {}
+static inline void deferred_split_folio(struct folio *folio, bool partially_mapped) {}
 #define split_huge_pmd(__vma, __pmd, __address)        \
        do { } while (0)
 

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 2175ebceb41cb9823f53705e60e5c6ff7e5555d9..1b3a7671048788d8adbe29bc863209d66317cb06 100644 (file)
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -186,6 +186,7 @@ enum pageflags {
        /* At least one page in this folio has the hwpoison flag set */
        PG_has_hwpoisoned = PG_active,
        PG_large_rmappable = PG_workingset, /* anon or file-backed */
+       PG_partially_mapped = PG_reclaim, /* was identified to be partially mapped */
 };
 
 #define PAGEFLAGS_MASK         ((1UL << NR_PAGEFLAGS) - 1)
@@ -859,8 +860,18 @@ static inline void ClearPageCompound(struct page *page)
        ClearPageHead(page);
 }
 FOLIO_FLAG(large_rmappable, FOLIO_SECOND_PAGE)
+FOLIO_TEST_FLAG(partially_mapped, FOLIO_SECOND_PAGE)
+/*
+ * PG_partially_mapped is protected by deferred_split split_queue_lock,
+ * so its safe to use non-atomic set/clear.
+ */
+__FOLIO_SET_FLAG(partially_mapped, FOLIO_SECOND_PAGE)
+__FOLIO_CLEAR_FLAG(partially_mapped, FOLIO_SECOND_PAGE)
 #else
 FOLIO_FLAG_FALSE(large_rmappable)
+FOLIO_TEST_FLAG_FALSE(partially_mapped)
+__FOLIO_SET_FLAG_NOOP(partially_mapped)
+__FOLIO_CLEAR_FLAG_NOOP(partially_mapped)
 #endif
 
 #define PG_head_mask ((1UL << PG_head))
@@ -1171,7 +1182,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page)
  */
 #define PAGE_FLAGS_SECOND                                              \
        (0xffUL /* order */             | 1UL << PG_has_hwpoisoned |    \
-        1UL << PG_large_rmappable)
+        1UL << PG_large_rmappable      | 1UL << PG_partially_mapped)
 
 #define PAGE_FLAGS_PRIVATE                             \
        (1UL << PG_private | 1UL << PG_private_2)

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 6b69309e002fb1b44c9c32dd41884f6d498cc5e1..9bc435bef2e62f6ea22a55109f91fe95d722bf63 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -3459,7 +3459,11 @@ int split_huge_page_to_list_to_order(struct page *page, struct list_head *list,
                if (folio_order(folio) > 1 &&
                    !list_empty(&folio->_deferred_list)) {
                        ds_queue->split_queue_len--;
-                       mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
+                       if (folio_test_partially_mapped(folio)) {
+                               __folio_clear_partially_mapped(folio);
+                               mod_mthp_stat(folio_order(folio),
+                                             MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
+                       }
                        /*
                         * Reinitialize page_deferred_list after removing the
                         * page from the split_queue, otherwise a subsequent
@@ -3526,13 +3530,18 @@ void __folio_undo_large_rmappable(struct folio *folio)
        spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
        if (!list_empty(&folio->_deferred_list)) {
                ds_queue->split_queue_len--;
-               mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
+               if (folio_test_partially_mapped(folio)) {
+                       __folio_clear_partially_mapped(folio);
+                       mod_mthp_stat(folio_order(folio),
+                                     MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
+               }
                list_del_init(&folio->_deferred_list);
        }
        spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
 }
 
-void deferred_split_folio(struct folio *folio)
+/* partially_mapped=false won't clear PG_partially_mapped folio flag */
+void deferred_split_folio(struct folio *folio, bool partially_mapped)
 {
        struct deferred_split *ds_queue = get_deferred_split_queue(folio);
 #ifdef CONFIG_MEMCG
@@ -3560,15 +3569,21 @@ void deferred_split_folio(struct folio *folio)
        if (folio_test_swapcache(folio))
                return;
 
-       if (!list_empty(&folio->_deferred_list))
-               return;
-
        spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
+       if (partially_mapped) {
+               if (!folio_test_partially_mapped(folio)) {
+                       __folio_set_partially_mapped(folio);
+                       if (folio_test_pmd_mappable(folio))
+                               count_vm_event(THP_DEFERRED_SPLIT_PAGE);
+                       count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED);
+                       mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1);
+
+               }
+       } else {
+               /* partially mapped folios cannot become non-partially mapped */
+               VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio);
+       }
        if (list_empty(&folio->_deferred_list)) {
-               if (folio_test_pmd_mappable(folio))
-                       count_vm_event(THP_DEFERRED_SPLIT_PAGE);
-               count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED);
-               mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1);
                list_add_tail(&folio->_deferred_list, &ds_queue->split_queue);
                ds_queue->split_queue_len++;
 #ifdef CONFIG_MEMCG
@@ -3616,7 +3631,11 @@ static unsigned long deferred_split_scan(struct shrinker *shrink,
                        list_move(&folio->_deferred_list, &list);
                } else {
                        /* We lost race with folio_put() */
-                       mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
+                       if (folio_test_partially_mapped(folio)) {
+                               __folio_clear_partially_mapped(folio);
+                               mod_mthp_stat(folio_order(folio),
+                                             MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
+                       }
                        list_del_init(&folio->_deferred_list);
                        ds_queue->split_queue_len--;
                }

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 0370ce03ce4cc50657cced16aea8cfb80bc72f33..5abff3b0cae5405c7995f87181d956c5b6416cc7 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -4645,7 +4645,8 @@ static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug)
        VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
        VM_BUG_ON_FOLIO(folio_order(folio) > 1 &&
                        !folio_test_hugetlb(folio) &&
-                       !list_empty(&folio->_deferred_list), folio);
+                       !list_empty(&folio->_deferred_list) &&
+                       folio_test_partially_mapped(folio), folio);
 
        /*
         * Nobody should be changing or seriously looking at

diff --git a/mm/migrate.c b/mm/migrate.c
index d039863e014bece30a45dcb168d6d8623b360ae1..35cc9d35064b0a78b7456c34dfc1a6972bc5950c 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1766,7 +1766,8 @@ static int migrate_pages_batch(struct list_head *from,
                         * use _deferred_list.
                         */
                        if (nr_pages > 2 &&
-                          !list_empty(&folio->_deferred_list)) {
+                          !list_empty(&folio->_deferred_list) &&
+                          folio_test_partially_mapped(folio)) {
                                if (!try_split_folio(folio, split_folios, mode)) {
                                        nr_failed++;
                                        stats->nr_thp_failed += is_thp;

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 23509b823e2ba6a58909b8b43ad4c6138ab125a1..ea4c44c3cfc0ab079571cac54981ae15747fad24 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -962,8 +962,9 @@ static int free_tail_page_prepare(struct page *head_page, struct page *page)
                break;
        case 2:
                /* the second tail page: deferred_list overlaps ->mapping */
-               if (unlikely(!list_empty(&folio->_deferred_list))) {
-                       bad_page(page, "on deferred list");
+               if (unlikely(!list_empty(&folio->_deferred_list) &&
+                   folio_test_partially_mapped(folio))) {
+                       bad_page(page, "partially mapped folio on deferred list");
                        goto out;
                }
                break;

diff --git a/mm/rmap.c b/mm/rmap.c
index 78529cf0fd6685b9ca48c451efdaf411db393156..a8797d1b3d4963208f8b90187368d00eff0610f7 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1579,8 +1579,9 @@ static __always_inline void __folio_remove_rmap(struct folio *folio,
         * Check partially_mapped first to ensure it is a large folio.
         */
        if (partially_mapped && folio_test_anon(folio) &&
-           list_empty(&folio->_deferred_list))
-               deferred_split_folio(folio);
+           !folio_test_partially_mapped(folio))
+               deferred_split_folio(folio, true);
+
        __folio_mod_stat(folio, -nr, -nr_pmdmapped);
 
        /*

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 1b1fad0c1e113a90da9e23cc3e156bf3dc564f62..8163b24bec759fab9f351640f67d4929bb9d5599 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1238,7 +1238,8 @@ retry:
                                         * Split partially mapped folios right away.
                                         * We can free the unmapped pages without IO.
                                         */
-                                       if (data_race(!list_empty(&folio->_deferred_list)) &&
+                                       if (data_race(!list_empty(&folio->_deferred_list) &&
+                                           folio_test_partially_mapped(folio)) &&
                                            split_folio_to_list(folio, folio_list))
                                                goto activate_locked;
                                }