net: optimize GRO for the common case.

After the previous patches, at GRO time, skb->slow_gro is
usually 0, unless the packets comes from some H/W offload
slowpath or tunnel.

We can optimize the GRO code assuming !skb->slow_gro is likely.
This remove multiple conditionals in the most common path, at the
price of an additional one when we hit the above "slow-paths".

Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/core/dev.c b/net/core/dev.c
index fb5d12a3d52d62b45234dd298746e626ff3f52b1..19565f7497ee4516fbdf58b534d6488ba1e72688 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -6014,7 +6014,6 @@ static void gro_list_prepare(const struct list_head *head,
                diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb);
                if (skb_vlan_tag_present(p))
                        diffs |= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb);
-               diffs |= skb_metadata_dst_cmp(p, skb);
                diffs |= skb_metadata_differs(p, skb);
                if (maclen == ETH_HLEN)
                        diffs |= compare_ether_header(skb_mac_header(p),
@@ -6024,17 +6023,29 @@ static void gro_list_prepare(const struct list_head *head,
                                       skb_mac_header(skb),
                                       maclen);
 
-               diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb);
+               /* in most common scenarions _state is 0
+                * otherwise we are already on some slower paths
+                * either skip all the infrequent tests altogether or
+                * avoid trying too hard to skip each of them individually
+                */
+               if (!diffs && unlikely(skb->slow_gro | p->slow_gro)) {
+#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
+                       struct tc_skb_ext *skb_ext;
+                       struct tc_skb_ext *p_ext;
+#endif
+
+                       diffs |= skb_metadata_dst_cmp(p, skb);
+                       diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb);
+
 #if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
-               if (!diffs) {
-                       struct tc_skb_ext *skb_ext = skb_ext_find(skb, TC_SKB_EXT);
-                       struct tc_skb_ext *p_ext = skb_ext_find(p, TC_SKB_EXT);
+                       skb_ext = skb_ext_find(skb, TC_SKB_EXT);
+                       p_ext = skb_ext_find(p, TC_SKB_EXT);
 
                        diffs |= (!!p_ext) ^ (!!skb_ext);
                        if (!diffs && unlikely(skb_ext))
                                diffs |= p_ext->chain ^ skb_ext->chain;
-               }
 #endif
+               }
 
                NAPI_GRO_CB(p)->same_flow = !diffs;
        }
@@ -6299,8 +6310,11 @@ static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
        skb->encapsulation = 0;
        skb_shinfo(skb)->gso_type = 0;
        skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
-       skb_ext_reset(skb);
-       nf_reset_ct(skb);
+       if (unlikely(skb->slow_gro)) {
+               skb_ext_reset(skb);
+               nf_reset_ct(skb);
+               skb->slow_gro = 0;
+       }
 
        napi->skb = skb;
 }

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index c5b02edd88815fb3015f97d37a02a71b083a3677..d04e286149cc98169e5ce4ee70e4a4287a327d0d 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -954,9 +954,12 @@ void __kfree_skb_defer(struct sk_buff *skb)
 
 void napi_skb_free_stolen_head(struct sk_buff *skb)
 {
-       nf_reset_ct(skb);
-       skb_dst_drop(skb);
-       skb_ext_put(skb);
+       if (unlikely(skb->slow_gro)) {
+               nf_reset_ct(skb);
+               skb_dst_drop(skb);
+               skb_ext_put(skb);
+               skb->slow_gro = 0;
+       }
        napi_skb_cache_put(skb);
 }