ethernet: avoid retpoline overhead on TEB (GENEVE, NvGRE, VxLAN) GRO

The two most popular headers going after Ethernet are IPv4 and IPv6.
Retpoline overhead for them is addressed only in dev_gro_receive(),
when they lie right after the outermost Ethernet header.
Use the indirect call wrappers in TEB (Transparent Ethernet Bridging,
such as GENEVE, NvGRE, VxLAN etc.) GRO receive code to reduce the
penalty when processing the inner headers.

Signed-off-by: Alexander Lobakin <alobakin@pm.me>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/ethernet/eth.c b/net/ethernet/eth.c
index e01cf766d2c50a4ae1b4e6dffedb07acfd16ff87..933b427122be5fb8b9181a9a51cb1594743f8343 100644 (file)
--- a/net/ethernet/eth.c
+++ b/net/ethernet/eth.c
@@ -58,6 +58,7 @@
 #include <net/ip.h>
 #include <net/dsa.h>
 #include <net/flow_dissector.h>
+#include <net/gro.h>
 #include <linux/uaccess.h>
 #include <net/pkt_sched.h>
 
@@ -449,7 +450,10 @@ struct sk_buff *eth_gro_receive(struct list_head *head, struct sk_buff *skb)
 
        skb_gro_pull(skb, sizeof(*eh));
        skb_gro_postpull_rcsum(skb, eh, sizeof(*eh));
-       pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb);
+
+       pp = indirect_call_gro_receive_inet(ptype->callbacks.gro_receive,
+                                           ipv6_gro_receive, inet_gro_receive,
+                                           head, skb);
 
 out_unlock:
        rcu_read_unlock();
@@ -473,8 +477,9 @@ int eth_gro_complete(struct sk_buff *skb, int nhoff)
        rcu_read_lock();
        ptype = gro_find_complete_by_type(type);
        if (ptype != NULL)
-               err = ptype->callbacks.gro_complete(skb, nhoff +
-                                                   sizeof(struct ethhdr));
+               err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
+                                        ipv6_gro_complete, inet_gro_complete,
+                                        skb, nhoff + sizeof(*eh));
 
        rcu_read_unlock();
        return err;