mm: avoid extra mem_alloc_profiling_enabled() checks

Refactor code to avoid extra mem_alloc_profiling_enabled() checks inside
pgalloc_tag_get() function which is often called after that check was
already done.

Link: https://lkml.kernel.org/r/20250201231803.2661189-1-surenb@google.com
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Reviewed-by: Shakeel Butt <shakeel.butt@linux.dev>
Cc: David Wang <00107082@163.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: Minchan Kim <minchan@google.com>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Sourav Panda <souravpanda@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Yu Zhao <yuzhao@google.com>
Cc: Zhenhua Huang <quic_zhenhuah@quicinc.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/pgalloc_tag.h b/include/linux/pgalloc_tag.h
index 3469c4b2010536f2c63ae8a60fbbfb39ec86b6b9..4a82b6b4820e952751f370ced9193dec0c27cb90 100644 (file)
--- a/include/linux/pgalloc_tag.h
+++ b/include/linux/pgalloc_tag.h
@@ -205,28 +205,32 @@ static inline void pgalloc_tag_sub(struct page *page, unsigned int nr)
        }
 }
 
-static inline struct alloc_tag *pgalloc_tag_get(struct page *page)
+/* Should be called only if mem_alloc_profiling_enabled() */
+static inline struct alloc_tag *__pgalloc_tag_get(struct page *page)
 {
        struct alloc_tag *tag = NULL;
-
-       if (mem_alloc_profiling_enabled()) {
-               union pgtag_ref_handle handle;
-               union codetag_ref ref;
-
-               if (get_page_tag_ref(page, &ref, &handle)) {
-                       alloc_tag_sub_check(&ref);
-                       if (ref.ct)
-                               tag = ct_to_alloc_tag(ref.ct);
-                       put_page_tag_ref(handle);
-               }
+       union pgtag_ref_handle handle;
+       union codetag_ref ref;
+
+       if (get_page_tag_ref(page, &ref, &handle)) {
+               alloc_tag_sub_check(&ref);
+               if (ref.ct)
+                       tag = ct_to_alloc_tag(ref.ct);
+               put_page_tag_ref(handle);
        }
 
        return tag;
 }
 
-static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)
+static inline void pgalloc_tag_sub_pages(struct page *page, unsigned int nr)
 {
-       if (mem_alloc_profiling_enabled() && tag)
+       struct alloc_tag *tag;
+
+       if (!mem_alloc_profiling_enabled())
+               return;
+
+       tag = __pgalloc_tag_get(page);
+       if (tag)
                this_cpu_sub(tag->counters->bytes, PAGE_SIZE * nr);
 }
 
@@ -241,8 +245,7 @@ static inline void clear_page_tag_ref(struct page *page) {}
 static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
                                   unsigned int nr) {}
 static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {}
-static inline struct alloc_tag *pgalloc_tag_get(struct page *page) { return NULL; }
-static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {}
+static inline void pgalloc_tag_sub_pages(struct page *page, unsigned int nr) {}
 static inline void alloc_tag_sec_init(void) {}
 static inline void pgalloc_tag_split(struct folio *folio, int old_order, int new_order) {}
 static inline void pgalloc_tag_swap(struct folio *new, struct folio *old) {}

diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c
index 19b45617bdcf6908d9267d9ef2b667f662956a5a..1d893e31361493326bd65996beef0fcfd9d20fa6 100644 (file)
--- a/lib/alloc_tag.c
+++ b/lib/alloc_tag.c
@@ -174,7 +174,7 @@ void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
        if (!mem_alloc_profiling_enabled())
                return;
 
-       tag = pgalloc_tag_get(&folio->page);
+       tag = __pgalloc_tag_get(&folio->page);
        if (!tag)
                return;
 
@@ -200,10 +200,10 @@ void pgalloc_tag_swap(struct folio *new, struct folio *old)
        if (!mem_alloc_profiling_enabled())
                return;
 
-       tag_old = pgalloc_tag_get(&old->page);
+       tag_old = __pgalloc_tag_get(&old->page);
        if (!tag_old)
                return;
-       tag_new = pgalloc_tag_get(&new->page);
+       tag_new = __pgalloc_tag_get(&new->page);
        if (!tag_new)
                return;
 

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 542d25f77be80304b731411ffd29b276ee13be0c..38c2b2d20b1df07815eca480f6761c606feaf829 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -4833,12 +4833,11 @@ void __free_pages(struct page *page, unsigned int order)
 {
        /* get PageHead before we drop reference */
        int head = PageHead(page);
-       struct alloc_tag *tag = pgalloc_tag_get(page);
 
        if (put_page_testzero(page))
                free_frozen_pages(page, order);
        else if (!head) {
-               pgalloc_tag_sub_pages(tag, (1 << order) - 1);
+               pgalloc_tag_sub_pages(page, (1 << order) - 1);
                while (order-- > 0)
                        free_frozen_pages(page + (1 << order), order);
        }