mm-memory_hotplug-make-hugetlb_optimize_vmemmap-compatible-with-memmap_on_memory-v5

walk vmemmap page tables to avoid false-positive

Link: https://lkml.kernel.org/r/20220620110616.12056-3-songmuchun@bytedance.com
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Co-developed-by: Oscar Salvador <osalvador@suse.de>
Signed-off-by: Oscar Salvador <osalvador@suse.de>
Cc: David Hildenbrand <david@redhat.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Paul E. McKenney <paulmck@kernel.org>
Cc: Xiongchun Duan <duanxiongchun@bytedance.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 982f27c24a951576c5ff42d610c8adf6b80a3bf8..6d9801bb3fec4176ca172fbab2378608a0e19e18 100644 (file)
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -10,6 +10,7 @@
  */
 #define pr_fmt(fmt)    "HugeTLB: " fmt
 
+#include <linux/memory.h>
 #include "hugetlb_vmemmap.h"
 
 /*
@@ -99,34 +100,52 @@ int hugetlb_vmemmap_alloc(struct hstate *h, struct page *head)
 static unsigned int vmemmap_optimizable_pages(struct hstate *h,
                                              struct page *head)
 {
-       struct mem_section *ms;
-       struct page *vmemmap_page;
-       unsigned long pfn = page_to_pfn(head);
-
        if (READ_ONCE(vmemmap_optimize_mode) == VMEMMAP_OPTIMIZE_OFF)
                return 0;
 
-       ms = __pfn_to_section(pfn);
-       vmemmap_page = sparse_decode_mem_map(ms->section_mem_map,
-                                            pfn_to_section_nr(pfn));
-       /*
-        * Only the vmemmap pages' vmemmap may be marked as VmemmapSelfHosted.
-        *
-        * Due to HugeTLB alignment requirements, and the vmemmap pages being
-        * at the start of the hotplugged memory region. Checking any vmemmap
-        * page's vmemmap is fine.
-        *
-        * [      hotplugged memory     ]
-        * [ vmemmap ][  usable memory  ]
-        *   ^   |      |            |
-        *   +---+      |            |
-        *     ^        |            |
-        *     +--------+            |
-        *         ^                 |
-        *         +-----------------+
-        */
-       if (PageVmemmapSelfHosted(vmemmap_page))
-               return 0;
+       if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG)) {
+               pmd_t *pmdp, pmd;
+               struct page *vmemmap_page;
+               unsigned long vaddr = (unsigned long)head;
+
+               /*
+                * Only the vmemmap page's vmemmap page can be self-hosted.
+                * Walking the page tables to find the backing page of the
+                * vmemmap page.
+                */
+               pmdp = pmd_off_k(vaddr);
+               /*
+                * The READ_ONCE() is used to stabilize *pmdp in a register or
+                * on the stack so that it will stop changing under the code.
+                * The only concurrent operation where it can be changed is
+                * split_vmemmap_huge_pmd() (*pmdp will be stable after this
+                * operation).
+                */
+               pmd = READ_ONCE(*pmdp);
+               if (pmd_leaf(pmd))
+                       vmemmap_page = pmd_page(pmd) + pte_index(vaddr);
+               else
+                       vmemmap_page = pte_page(*pte_offset_kernel(pmdp, vaddr));
+               /*
+                * Due to HugeTLB alignment requirements and the vmemmap pages
+                * being at the start of the hotplugged memory region in
+                * memory_hotplug.memmap_on_memory case. Checking any vmemmap
+                * page's vmemmap page if it is marked as VmemmapSelfHosted is
+                * sufficient.
+                *
+                * [                  hotplugged memory                  ]
+                * [        section        ][...][        section        ]
+                * [ vmemmap ][              usable memory               ]
+                *   ^   |     |                                        |
+                *   +---+     |                                        |
+                *     ^       |                                        |
+                *     +-------+                                        |
+                *          ^                                           |
+                *          +-------------------------------------------+
+                */
+               if (PageVmemmapSelfHosted(vmemmap_page))
+                       return 0;
+       }
 
        return hugetlb_optimize_vmemmap_pages(h);
 }