alloc_tag: uninline code gated by mem_alloc_profiling_key in page allocator

When a sizable code section is protected by a disabled static key, that
code gets into the instruction cache even though it's not executed and
consumes the cache, increasing cache misses. This can be remedied by
moving such code into a separate uninlined function.
On a Pixel6 phone, page allocation profiling overhead measured with
CONFIG_MEM_ALLOC_PROFILING=y and profiling disabled is:

             baseline             modified
Big core     4.93%                1.53%
Medium core  4.39%                1.41%
Little core  1.02%                0.36%

This improvement comes at the expense of the configuration when profiling
gets enabled, since there is now an additional function call. The overhead
from this additional call on Pixel6 is:

Big core     0.24%
Middle core  0.63%
Little core  1.1%

However this is negligible when compared with the overall overhead of the
memory allocation profiling when it is enabled.
On x86 this patch does not make noticeable difference because the overhead
with mem_alloc_profiling_key disabled is much lower (under 1%) to start
with, so any improvement is less visible and hard to distinguish from the
noise. The overhead from additional call when profiling is enabled is also
within noise levels.

Link: https://lkml.kernel.org/r/20250201231803.2661189-3-surenb@google.com
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Reviewed-by: Shakeel Butt <shakeel.butt@linux.dev>
Cc: David Wang <00107082@163.com>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: Minchan Kim <minchan@google.com>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Sourav Panda <souravpanda@google.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Yu Zhao <yuzhao@google.com>
Cc: Zhenhua Huang <quic_zhenhuah@quicinc.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/pgalloc_tag.h b/include/linux/pgalloc_tag.h
index 4a82b6b4820e952751f370ced9193dec0c27cb90..c740779778304a98b12088bc8082d88ffbc7c5d5 100644 (file)
--- a/include/linux/pgalloc_tag.h
+++ b/include/linux/pgalloc_tag.h
@@ -162,47 +162,13 @@ static inline void update_page_tag_ref(union pgtag_ref_handle handle, union code
        }
 }
 
-static inline void clear_page_tag_ref(struct page *page)
-{
-       if (mem_alloc_profiling_enabled()) {
-               union pgtag_ref_handle handle;
-               union codetag_ref ref;
-
-               if (get_page_tag_ref(page, &ref, &handle)) {
-                       set_codetag_empty(&ref);
-                       update_page_tag_ref(handle, &ref);
-                       put_page_tag_ref(handle);
-               }
-       }
-}
-
-static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
-                                  unsigned int nr)
-{
-       if (mem_alloc_profiling_enabled()) {
-               union pgtag_ref_handle handle;
-               union codetag_ref ref;
-
-               if (get_page_tag_ref(page, &ref, &handle)) {
-                       alloc_tag_add(&ref, task->alloc_tag, PAGE_SIZE * nr);
-                       update_page_tag_ref(handle, &ref);
-                       put_page_tag_ref(handle);
-               }
-       }
-}
+/* Should be called only if mem_alloc_profiling_enabled() */
+void __clear_page_tag_ref(struct page *page);
 
-static inline void pgalloc_tag_sub(struct page *page, unsigned int nr)
+static inline void clear_page_tag_ref(struct page *page)
 {
-       if (mem_alloc_profiling_enabled()) {
-               union pgtag_ref_handle handle;
-               union codetag_ref ref;
-
-               if (get_page_tag_ref(page, &ref, &handle)) {
-                       alloc_tag_sub(&ref, PAGE_SIZE * nr);
-                       update_page_tag_ref(handle, &ref);
-                       put_page_tag_ref(handle);
-               }
-       }
+       if (mem_alloc_profiling_enabled())
+               __clear_page_tag_ref(page);
 }
 
 /* Should be called only if mem_alloc_profiling_enabled() */
@@ -222,18 +188,6 @@ static inline struct alloc_tag *__pgalloc_tag_get(struct page *page)
        return tag;
 }
 
-static inline void pgalloc_tag_sub_pages(struct page *page, unsigned int nr)
-{
-       struct alloc_tag *tag;
-
-       if (!mem_alloc_profiling_enabled())
-               return;
-
-       tag = __pgalloc_tag_get(page);
-       if (tag)
-               this_cpu_sub(tag->counters->bytes, PAGE_SIZE * nr);
-}
-
 void pgalloc_tag_split(struct folio *folio, int old_order, int new_order);
 void pgalloc_tag_swap(struct folio *new, struct folio *old);
 
@@ -242,10 +196,6 @@ void __init alloc_tag_sec_init(void);
 #else /* CONFIG_MEM_ALLOC_PROFILING */
 
 static inline void clear_page_tag_ref(struct page *page) {}
-static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
-                                  unsigned int nr) {}
-static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {}
-static inline void pgalloc_tag_sub_pages(struct page *page, unsigned int nr) {}
 static inline void alloc_tag_sec_init(void) {}
 static inline void pgalloc_tag_split(struct folio *folio, int old_order, int new_order) {}
 static inline void pgalloc_tag_swap(struct folio *new, struct folio *old) {}

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 38c2b2d20b1df07815eca480f6761c606feaf829..d875f055aa53c5b8061a62f6505414f0a708bfab 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1041,6 +1041,84 @@ static void kernel_init_pages(struct page *page, int numpages)
        kasan_enable_current();
 }
 
+#ifdef CONFIG_MEM_ALLOC_PROFILING
+
+/* Should be called only if mem_alloc_profiling_enabled() */
+void __clear_page_tag_ref(struct page *page)
+{
+       union pgtag_ref_handle handle;
+       union codetag_ref ref;
+
+       if (get_page_tag_ref(page, &ref, &handle)) {
+               set_codetag_empty(&ref);
+               update_page_tag_ref(handle, &ref);
+               put_page_tag_ref(handle);
+       }
+}
+
+/* Should be called only if mem_alloc_profiling_enabled() */
+static noinline
+void __pgalloc_tag_add(struct page *page, struct task_struct *task,
+                      unsigned int nr)
+{
+       union pgtag_ref_handle handle;
+       union codetag_ref ref;
+
+       if (get_page_tag_ref(page, &ref, &handle)) {
+               alloc_tag_add(&ref, task->alloc_tag, PAGE_SIZE * nr);
+               update_page_tag_ref(handle, &ref);
+               put_page_tag_ref(handle);
+       }
+}
+
+static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
+                                  unsigned int nr)
+{
+       if (mem_alloc_profiling_enabled())
+               __pgalloc_tag_add(page, task, nr);
+}
+
+/* Should be called only if mem_alloc_profiling_enabled() */
+static noinline
+void __pgalloc_tag_sub(struct page *page, unsigned int nr)
+{
+       union pgtag_ref_handle handle;
+       union codetag_ref ref;
+
+       if (get_page_tag_ref(page, &ref, &handle)) {
+               alloc_tag_sub(&ref, PAGE_SIZE * nr);
+               update_page_tag_ref(handle, &ref);
+               put_page_tag_ref(handle);
+       }
+}
+
+static inline void pgalloc_tag_sub(struct page *page, unsigned int nr)
+{
+       if (mem_alloc_profiling_enabled())
+               __pgalloc_tag_sub(page, nr);
+}
+
+static inline void pgalloc_tag_sub_pages(struct page *page, unsigned int nr)
+{
+       struct alloc_tag *tag;
+
+       if (!mem_alloc_profiling_enabled())
+               return;
+
+       tag = __pgalloc_tag_get(page);
+       if (tag)
+               this_cpu_sub(tag->counters->bytes, PAGE_SIZE * nr);
+}
+
+#else /* CONFIG_MEM_ALLOC_PROFILING */
+
+static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
+                                  unsigned int nr) {}
+static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {}
+static inline void pgalloc_tag_sub_pages(struct page *page, unsigned int nr) {}
+
+#endif /* CONFIG_MEM_ALLOC_PROFILING */
+
 __always_inline bool free_pages_prepare(struct page *page,
                        unsigned int order)
 {