ipv6: move tcpv6_protocol and udpv6_protocol to net_hotdata

These structures are read in rx path, move them to net_hotdata
for better cache locality.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Soheil Hassas Yeganeh <soheil@google.com>
Reviewed-by: David Ahern <dsahern@kernel.org>
Link: https://lore.kernel.org/r/20240306160031.874438-13-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/net/hotdata.h b/include/net/hotdata.h
index daeee8ce8084e885984716113779750ca7b72147..03d758d25c02864b00e0a557603b64dc9d749b9c 100644 (file)
--- a/include/net/hotdata.h
+++ b/include/net/hotdata.h
@@ -14,6 +14,10 @@ struct net_hotdata {
        struct net_offload      udpv4_offload;
        struct packet_offload   ipv6_packet_offload;
        struct net_offload      tcpv6_offload;
+#if IS_ENABLED(CONFIG_IPV6)
+       struct inet6_protocol   tcpv6_protocol;
+       struct inet6_protocol   udpv6_protocol;
+#endif
        struct net_offload      udpv6_offload;
 #endif
        struct list_head        offload_base;

diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index f677f0fa51968d00c3571d55ae7850742387f2d1..3f4cba49e9ee6520987993dcea082e6065b4688b 100644 (file)
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -58,6 +58,7 @@
 #include <net/timewait_sock.h>
 #include <net/inet_common.h>
 #include <net/secure_seq.h>
+#include <net/hotdata.h>
 #include <net/busy_poll.h>
 
 #include <linux/proc_fs.h>
@@ -2367,11 +2368,6 @@ struct proto tcpv6_prot = {
 };
 EXPORT_SYMBOL_GPL(tcpv6_prot);
 
-static const struct inet6_protocol tcpv6_protocol = {
-       .handler        =       tcp_v6_rcv,
-       .err_handler    =       tcp_v6_err,
-       .flags          =       INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
-};
 
 static struct inet_protosw tcpv6_protosw = {
        .type           =       SOCK_STREAM,
@@ -2408,7 +2404,12 @@ int __init tcpv6_init(void)
 {
        int ret;
 
-       ret = inet6_add_protocol(&tcpv6_protocol, IPPROTO_TCP);
+       net_hotdata.tcpv6_protocol = (struct inet6_protocol) {
+               .handler     = tcp_v6_rcv,
+               .err_handler = tcp_v6_err,
+               .flags       = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL,
+       };
+       ret = inet6_add_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
        if (ret)
                goto out;
 
@@ -2433,7 +2434,7 @@ out_tcpv6_pernet_subsys:
 out_tcpv6_protosw:
        inet6_unregister_protosw(&tcpv6_protosw);
 out_tcpv6_protocol:
-       inet6_del_protocol(&tcpv6_protocol, IPPROTO_TCP);
+       inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
        goto out;
 }
 
@@ -2441,5 +2442,5 @@ void tcpv6_exit(void)
 {
        unregister_pernet_subsys(&tcpv6_net_ops);
        inet6_unregister_protosw(&tcpv6_protosw);
-       inet6_del_protocol(&tcpv6_protocol, IPPROTO_TCP);
+       inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
 }

diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 3f2249b4cd5f6a594dd9768e29f20f0d9a57faed..97d86909aabb6588d0bba901f6df1f23a4f2e561 100644 (file)
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -1702,11 +1702,6 @@ int udpv6_getsockopt(struct sock *sk, int level, int optname,
        return ipv6_getsockopt(sk, level, optname, optval, optlen);
 }
 
-static const struct inet6_protocol udpv6_protocol = {
-       .handler        =       udpv6_rcv,
-       .err_handler    =       udpv6_err,
-       .flags          =       INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
-};
 
 /* ------------------------------------------------------------------------ */
 #ifdef CONFIG_PROC_FS
@@ -1803,7 +1798,12 @@ int __init udpv6_init(void)
 {
        int ret;
 
-       ret = inet6_add_protocol(&udpv6_protocol, IPPROTO_UDP);
+       net_hotdata.udpv6_protocol = (struct inet6_protocol) {
+               .handler     = udpv6_rcv,
+               .err_handler = udpv6_err,
+               .flags       = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL,
+       };
+       ret = inet6_add_protocol(&net_hotdata.udpv6_protocol, IPPROTO_UDP);
        if (ret)
                goto out;
 
@@ -1814,12 +1814,12 @@ out:
        return ret;
 
 out_udpv6_protocol:
-       inet6_del_protocol(&udpv6_protocol, IPPROTO_UDP);
+       inet6_del_protocol(&net_hotdata.udpv6_protocol, IPPROTO_UDP);
        goto out;
 }
 
 void udpv6_exit(void)
 {
        inet6_unregister_protosw(&udpv6_protosw);
-       inet6_del_protocol(&udpv6_protocol, IPPROTO_UDP);
+       inet6_del_protocol(&net_hotdata.udpv6_protocol, IPPROTO_UDP);
 }