net: move sysctl_mem_pcpu_rsv to net_hotdata

sysctl_mem_pcpu_rsv is used in TCP fast path,
move it to net_hodata for better cache locality.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: David Ahern <dsahern@kernel.org>
Link: https://lore.kernel.org/r/20240429134025.1233626-6-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/net/hotdata.h b/include/net/hotdata.h
index 290499f72e18d45a8ed7bd9e8880a51f7ef2b94c..30e9570beb2afbcb4ece641d6042bdc0de80bd38 100644 (file)
--- a/include/net/hotdata.h
+++ b/include/net/hotdata.h
@@ -40,6 +40,7 @@ struct net_hotdata {
        int                     dev_rx_weight;
        int                     sysctl_max_skb_frags;
        int                     sysctl_skb_defer_max;
+       int                     sysctl_mem_pcpu_rsv;
 };
 
 #define inet_ehash_secret      net_hotdata.tcp_protocol.secret

diff --git a/include/net/proto_memory.h b/include/net/proto_memory.h
index 41404d4bb6f08e84030838f0e63ef240587f65dd..a6ab2f4f5e28a21ab63acb38caf157f6a7415a0b 100644 (file)
--- a/include/net/proto_memory.h
+++ b/include/net/proto_memory.h
@@ -3,10 +3,10 @@
 #define _PROTO_MEMORY_H
 
 #include <net/sock.h>
+#include <net/hotdata.h>
 
 /* 1 MB per cpu, in page units */
 #define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT))
-extern int sysctl_mem_pcpu_rsv;
 
 static inline bool sk_has_memory_pressure(const struct sock *sk)
 {
@@ -65,7 +65,7 @@ sk_memory_allocated_add(const struct sock *sk, int val)
 
        val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val);
 
-       if (unlikely(val >= READ_ONCE(sysctl_mem_pcpu_rsv)))
+       if (unlikely(val >= READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))
                proto_memory_pcpu_drain(proto);
 }
 
@@ -76,7 +76,7 @@ sk_memory_allocated_sub(const struct sock *sk, int val)
 
        val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val);
 
-       if (unlikely(val <= -READ_ONCE(sysctl_mem_pcpu_rsv)))
+       if (unlikely(val <= -READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))
                proto_memory_pcpu_drain(proto);
 }
 

diff --git a/net/core/hotdata.c b/net/core/hotdata.c
index a359ff160d54ad379eac7e56e1810a7e840f9275..d0aaaaa556f229ded4e1997bf814a2b690b46920 100644 (file)
--- a/net/core/hotdata.c
+++ b/net/core/hotdata.c
@@ -1,9 +1,9 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
-#include <net/hotdata.h>
 #include <linux/cache.h>
 #include <linux/jiffies.h>
 #include <linux/list.h>
-
+#include <net/hotdata.h>
+#include <net/proto_memory.h>
 
 struct net_hotdata net_hotdata __cacheline_aligned = {
        .offload_base = LIST_HEAD_INIT(net_hotdata.offload_base),
@@ -20,5 +20,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = {
        .dev_rx_weight = 64,
        .sysctl_max_skb_frags = MAX_SKB_FRAGS,
        .sysctl_skb_defer_max = 64,
+       .sysctl_mem_pcpu_rsv = SK_MEMORY_PCPU_RESERVE
 };
 EXPORT_SYMBOL(net_hotdata);

diff --git a/net/core/sock.c b/net/core/sock.c
index e0692b752369cc1cbf34253a40c06d18dcf350fe..8d6e638b5426daf84ff71d22101252943f9c5466 100644 (file)
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -284,7 +284,6 @@ __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
 EXPORT_SYMBOL(sysctl_rmem_max);
 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
-int sysctl_mem_pcpu_rsv __read_mostly = SK_MEMORY_PCPU_RESERVE;
 
 int sysctl_tstamp_allow_data __read_mostly = 1;
 

diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index a452a330d0ed649a2d1b8e65c81aaa3ff3d826f8..6da5995ac86a08a58e63af9a1e62c7a136c08dc9 100644 (file)
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -416,7 +416,7 @@ static struct ctl_table net_core_table[] = {
        },
        {
                .procname       = "mem_pcpu_rsv",
-               .data           = &sysctl_mem_pcpu_rsv,
+               .data           = &net_hotdata.sysctl_mem_pcpu_rsv,
                .maxlen         = sizeof(int),
                .mode           = 0644,
                .proc_handler   = proc_dointvec_minmax,