alloc_tag: avoid current->alloc_tag manipulations when profiling is disabled

When memory allocation profiling is disabled there is no need to update
current->alloc_tag and these manipulations add unnecessary overhead.  Fix
the overhead by skipping these extra updates.

I ran comprehensive testing on Pixel 6 on Big, Medium and Little cores:

                 Overhead before fixes            Overhead after fixes
                 slab alloc      page alloc          slab alloc      page alloc
Big               6.21%           5.32%                3.31%          4.93%
Medium            4.51%           5.05%                3.79%          4.39%
Little            7.62%           1.82%                6.68%          1.02%

This is an allocation microbenchmark doing allocations in a tight loop.
Not a really realistic scenario and useful only to make performance
comparisons.

Link: https://lkml.kernel.org/r/20241226211639.1357704-1-surenb@google.com
Fixes: b951aaff5035 ("mm: enable page allocation tagging")
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Cc: David Wang <00107082@163.com>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: Yu Zhao <yuzhao@google.com>
Cc: Zhenhua Huang <quic_zhenhuah@quicinc.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h
index 0bbbe537c5f9fbb76f0414ce2479c2935fbdf696..a946e0203e6d600d451542da0e6cd90d576fb9b6 100644 (file)
--- a/include/linux/alloc_tag.h
+++ b/include/linux/alloc_tag.h
@@ -224,9 +224,14 @@ static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {}
 
 #define alloc_hooks_tag(_tag, _do_alloc)                               \
 ({                                                                     \
-       struct alloc_tag * __maybe_unused _old = alloc_tag_save(_tag);  \
-       typeof(_do_alloc) _res = _do_alloc;                             \
-       alloc_tag_restore(_tag, _old);                                  \
+       typeof(_do_alloc) _res;                                         \
+       if (mem_alloc_profiling_enabled()) {                            \
+               struct alloc_tag * __maybe_unused _old;                 \
+               _old = alloc_tag_save(_tag);                            \
+               _res = _do_alloc;                                       \
+               alloc_tag_restore(_tag, _old);                          \
+       } else                                                          \
+               _res = _do_alloc;                                       \
        _res;                                                           \
 })
 

diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c
index 4bb778be4476434e6316653391c03e0fb809329f..894f19694010da1f65213a7537ce9c7f93aeca49 100644 (file)
--- a/lib/alloc_tag.c
+++ b/lib/alloc_tag.c
@@ -29,6 +29,8 @@ EXPORT_SYMBOL(_shared_alloc_tag);
 
 DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
                        mem_alloc_profiling_key);
+EXPORT_SYMBOL(mem_alloc_profiling_key);
+
 DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed);
 
 struct alloc_tag_kernel_section kernel_tags = { NULL, 0 };