mm: hugetlb: replace hugetlb_free_vmemmap_enabled with a static_key

The page_fixed_fake_head() is used throughout memory management and the
conditional check requires checking a global variable, although the
overhead of this check may be small, it increases when the memory cache
comes under pressure.  Also, the global variable will not be modified
after system boot, so it is very appropriate to use static key machanism.

Link: https://lkml.kernel.org/r/20211101031651.75851-3-songmuchun@bytedance.com
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Reviewed-by: Barry Song <song.bao.hua@hisilicon.com>
Cc: Bodeddula Balasubramaniam <bodeddub@amazon.com>
Cc: Chen Huang <chenhuang5@huawei.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Fam Zheng <fam.zheng@bytedance.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Qi Zheng <zhengqi.arch@bytedance.com>
Cc: Xiongchun Duan <duanxiongchun@bytedance.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 52c462390aee374497a8f7feb635366cba992fc3..08357b4c7be73c15221cd07e9b4e7fb0631f2acd 100644 (file)
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -1075,12 +1075,6 @@ static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr
 }
 #endif /* CONFIG_HUGETLB_PAGE */
 
-#ifdef CONFIG_HUGETLB_PAGE_FREE_VMEMMAP
-extern bool hugetlb_free_vmemmap_enabled;
-#else
-#define hugetlb_free_vmemmap_enabled   false
-#endif
-
 static inline spinlock_t *huge_pte_lock(struct hstate *h,
                                        struct mm_struct *mm, pte_t *pte)
 {

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 111e453f23d22793a0fd2c06a3a0cb2e367360da..340cb81565683f45134468efce45a71d6df6b9eb 100644 (file)
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -191,7 +191,14 @@ enum pageflags {
 #ifndef __GENERATING_BOUNDS_H
 
 #ifdef CONFIG_HUGETLB_PAGE_FREE_VMEMMAP
-extern bool hugetlb_free_vmemmap_enabled;
+DECLARE_STATIC_KEY_MAYBE(CONFIG_HUGETLB_PAGE_FREE_VMEMMAP_DEFAULT_ON,
+                        hugetlb_free_vmemmap_enabled_key);
+
+static __always_inline bool hugetlb_free_vmemmap_enabled(void)
+{
+       return static_branch_maybe(CONFIG_HUGETLB_PAGE_FREE_VMEMMAP_DEFAULT_ON,
+                                  &hugetlb_free_vmemmap_enabled_key);
+}
 
 /*
  * If the feature of freeing some vmemmap pages associated with each HugeTLB
@@ -211,7 +218,7 @@ extern bool hugetlb_free_vmemmap_enabled;
  */
 static __always_inline const struct page *page_fixed_fake_head(const struct page *page)
 {
-       if (!hugetlb_free_vmemmap_enabled)
+       if (!hugetlb_free_vmemmap_enabled())
                return page;
 
        /*
@@ -239,6 +246,11 @@ static inline const struct page *page_fixed_fake_head(const struct page *page)
 {
        return page;
 }
+
+static inline bool hugetlb_free_vmemmap_enabled(void)
+{
+       return false;
+}
 #endif
 
 static __always_inline int page_is_fake_head(struct page *page)

diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 4977f5a520c22fc4fc626987ffbc05aab2300449..791626983c2e1c65e040d52b878877b85322952b 100644 (file)
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -188,9 +188,9 @@
 #define RESERVE_VMEMMAP_NR             1U
 #define RESERVE_VMEMMAP_SIZE           (RESERVE_VMEMMAP_NR << PAGE_SHIFT)
 
-bool hugetlb_free_vmemmap_enabled __read_mostly =
-       IS_ENABLED(CONFIG_HUGETLB_PAGE_FREE_VMEMMAP_DEFAULT_ON);
-EXPORT_SYMBOL(hugetlb_free_vmemmap_enabled);
+DEFINE_STATIC_KEY_MAYBE(CONFIG_HUGETLB_PAGE_FREE_VMEMMAP_DEFAULT_ON,
+                       hugetlb_free_vmemmap_enabled_key);
+EXPORT_SYMBOL(hugetlb_free_vmemmap_enabled_key);
 
 static int __init early_hugetlb_free_vmemmap_param(char *buf)
 {
@@ -204,9 +204,9 @@ static int __init early_hugetlb_free_vmemmap_param(char *buf)
                return -EINVAL;
 
        if (!strcmp(buf, "on"))
-               hugetlb_free_vmemmap_enabled = true;
+               static_branch_enable(&hugetlb_free_vmemmap_enabled_key);
        else if (!strcmp(buf, "off"))
-               hugetlb_free_vmemmap_enabled = false;
+               static_branch_disable(&hugetlb_free_vmemmap_enabled_key);
        else
                return -EINVAL;
 
@@ -284,7 +284,7 @@ void __init hugetlb_vmemmap_init(struct hstate *h)
        BUILD_BUG_ON(__NR_USED_SUBPAGE >=
                     RESERVE_VMEMMAP_SIZE / sizeof(struct page));
 
-       if (!hugetlb_free_vmemmap_enabled)
+       if (!hugetlb_free_vmemmap_enabled())
                return;
 
        vmemmap_pages = (nr_pages * sizeof(struct page)) >> PAGE_SHIFT;

diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 2a9627dc784c31072f39527f0a52953f72257aff..0139b77c51d5d238031fdea07df54d95089e4d7a 100644 (file)
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -1327,7 +1327,7 @@ bool mhp_supports_memmap_on_memory(unsigned long size)
         *       populate a single PMD.
         */
        return memmap_on_memory &&
-              !hugetlb_free_vmemmap_enabled &&
+              !hugetlb_free_vmemmap_enabled() &&
               IS_ENABLED(CONFIG_MHP_MEMMAP_ON_MEMORY) &&
               size == memory_block_size_bytes() &&
               IS_ALIGNED(vmemmap_size, PMD_SIZE) &&