tcp: move tcp_memory_allocated into net_aligned_data

____cacheline_aligned_in_smp attribute only makes sure to align
a field to a cache line. It does not prevent the linker to use
the remaining of the cache line for other variables, causing
potential false sharing.

Move tcp_memory_allocated into a dedicated cache line.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Link: https://patch.msgid.link/20250630093540.3052835-4-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/net/aligned_data.h b/include/net/aligned_data.h
index 5c7badf71f043a2bbc871b0063c9a2d2a4ffbfcb..bedb4f86b0fea1ce13a10895182681c37ef7e904 100644 (file)
--- a/include/net/aligned_data.h
+++ b/include/net/aligned_data.h
@@ -11,6 +11,9 @@
  */
 struct net_aligned_data {
        atomic64_t      net_cookie ____cacheline_aligned_in_smp;
+#if defined(CONFIG_INET)
+       atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp;
+#endif
 };
 
 extern struct net_aligned_data net_aligned_data;

diff --git a/include/net/tcp.h b/include/net/tcp.h
index 761c4a0ad386f95f73d72dc013a0952187342b51..bc08de49805cf6fc2ffbec96e42bf12378fd10cf 100644 (file)
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -267,7 +267,6 @@ extern long sysctl_tcp_mem[3];
 #define TCP_RACK_STATIC_REO_WND  0x2 /* Use static RACK reo wnd */
 #define TCP_RACK_NO_DUPTHRESH    0x4 /* Do not use DUPACK threshold in RACK */
 
-extern atomic_long_t tcp_memory_allocated;
 DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
 
 extern struct percpu_counter tcp_sockets_allocated;

diff --git a/net/core/hotdata.c b/net/core/hotdata.c
index e9c03491ab001cc85fd60ad28533649b32d8a003..95d0a4df10069e4529fb9e5b58e8391574085cf1 100644 (file)
--- a/net/core/hotdata.c
+++ b/net/core/hotdata.c
@@ -4,6 +4,7 @@
 #include <linux/list.h>
 #include <net/aligned_data.h>
 #include <net/hotdata.h>
+#include <net/ip.h>
 #include <net/proto_memory.h>
 
 struct net_hotdata net_hotdata __cacheline_aligned = {
@@ -25,3 +26,4 @@ struct net_hotdata net_hotdata __cacheline_aligned = {
 EXPORT_SYMBOL(net_hotdata);
 
 struct net_aligned_data net_aligned_data;
+EXPORT_IPV6_MOD(net_aligned_data);

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 8a3c99246d2ed32ba45849b56830bf128e265763..925b2c572ca23b3f2eba48a38820f6553a2724f4 100644 (file)
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -302,8 +302,6 @@ EXPORT_PER_CPU_SYMBOL_GPL(tcp_tw_isn);
 long sysctl_tcp_mem[3] __read_mostly;
 EXPORT_IPV6_MOD(sysctl_tcp_mem);
 
-atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp;       /* Current allocated memory. */
-EXPORT_IPV6_MOD(tcp_memory_allocated);
 DEFINE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
 EXPORT_PER_CPU_SYMBOL_GPL(tcp_memory_per_cpu_fw_alloc);
 

diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 56223338bc0f070179efb2ce9996fa7146782adc..b406fd012b2e6a4f5b0bca9d26298bdf26b7989a 100644 (file)
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -59,6 +59,7 @@
 #include <linux/slab.h>
 #include <linux/sched.h>
 
+#include <net/aligned_data.h>
 #include <net/net_namespace.h>
 #include <net/icmp.h>
 #include <net/inet_hashtables.h>
@@ -3390,7 +3391,7 @@ struct proto tcp_prot = {
        .sockets_allocated      = &tcp_sockets_allocated,
        .orphan_count           = &tcp_orphan_count,
 
-       .memory_allocated       = &tcp_memory_allocated,
+       .memory_allocated       = &net_aligned_data.tcp_memory_allocated,
        .per_cpu_fw_alloc       = &tcp_memory_per_cpu_fw_alloc,
 
        .memory_pressure        = &tcp_memory_pressure,

diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index 9fb614e17bde99e5806cd56fdbc4d0b0b74a3f57..ed0b891885d848b25667edbaf4e925b3ece46033 100644 (file)
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -41,6 +41,7 @@
 #include <linux/random.h>
 #include <linux/indirect_call_wrapper.h>
 
+#include <net/aligned_data.h>
 #include <net/tcp.h>
 #include <net/ndisc.h>
 #include <net/inet6_hashtables.h>
@@ -2356,7 +2357,7 @@ struct proto tcpv6_prot = {
        .stream_memory_free     = tcp_stream_memory_free,
        .sockets_allocated      = &tcp_sockets_allocated,
 
-       .memory_allocated       = &tcp_memory_allocated,
+       .memory_allocated       = &net_aligned_data.tcp_memory_allocated,
        .per_cpu_fw_alloc       = &tcp_memory_per_cpu_fw_alloc,
 
        .memory_pressure        = &tcp_memory_pressure,

diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c
index e7972e633236e0451f0321ff4b0a8d1b37282d5f..5f904fc5ac4c63e8b6c7c9aa79f17e8dcdf1a007 100644 (file)
--- a/net/mptcp/protocol.c
+++ b/net/mptcp/protocol.c
@@ -11,6 +11,7 @@
 #include <linux/netdevice.h>
 #include <linux/sched/signal.h>
 #include <linux/atomic.h>
+#include <net/aligned_data.h>
 #include <net/sock.h>
 #include <net/inet_common.h>
 #include <net/inet_hashtables.h>
@@ -3729,7 +3730,7 @@ static struct proto mptcp_prot = {
        .stream_memory_free     = mptcp_stream_memory_free,
        .sockets_allocated      = &mptcp_sockets_allocated,
 
-       .memory_allocated       = &tcp_memory_allocated,
+       .memory_allocated       = &net_aligned_data.tcp_memory_allocated,
        .per_cpu_fw_alloc       = &tcp_memory_per_cpu_fw_alloc,
 
        .memory_pressure        = &tcp_memory_pressure,