net: gro: enable fast path for more cases

Currently the so-called GRO fast path is only enabled for
napi_frags_skb() callers.

After the prior patch, we no longer have to clear frag0 whenever
we pulled bytes to skb->head.

We therefore can initialize frag0 to skb->data so that GRO
fast path can be used in the following additional cases:

- Drivers using header split (populating skb->data with headers,
  and having payload in one or more page fragments).

- Drivers not using any page frag (entire packet is in skb->data)

Add a likely() in skb_gro_may_pull() to help the compiler
to generate better code if possible.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>

diff --git a/include/net/gro.h b/include/net/gro.h
index 3c3666e46b3055caa619f2da0b6b8b20192a03b4..2b58671a65492bf3f9dabf1e7a2d985cee007e11 100644 (file)
--- a/include/net/gro.h
+++ b/include/net/gro.h
@@ -148,7 +148,7 @@ static inline void *skb_gro_header_fast(const struct sk_buff *skb,
 static inline bool skb_gro_may_pull(const struct sk_buff *skb,
                                    unsigned int hlen)
 {
-       return hlen <= NAPI_GRO_CB(skb)->frag0_len;
+       return likely(hlen <= NAPI_GRO_CB(skb)->frag0_len);
 }
 
 static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,

diff --git a/net/core/gro.c b/net/core/gro.c
index 927ccf68149093d6dfd66a622a7db5215a483876..6a0edbd826a17573b51c5f71e20ff0c09364fc21 100644 (file)
--- a/net/core/gro.c
+++ b/net/core/gro.c
@@ -369,15 +369,21 @@ static void gro_list_prepare(const struct list_head *head,
 
 static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff)
 {
-       const struct skb_shared_info *pinfo = skb_shinfo(skb);
-       const skb_frag_t *frag0 = &pinfo->frags[0];
+       const struct skb_shared_info *pinfo;
+       const skb_frag_t *frag0;
+       unsigned int headlen;
 
        NAPI_GRO_CB(skb)->data_offset = 0;
-       NAPI_GRO_CB(skb)->frag0 = NULL;
-       NAPI_GRO_CB(skb)->frag0_len = 0;
+       headlen = skb_headlen(skb);
+       NAPI_GRO_CB(skb)->frag0 = skb->data;
+       NAPI_GRO_CB(skb)->frag0_len = headlen;
+       if (headlen)
+               return;
+
+       pinfo = skb_shinfo(skb);
+       frag0 = &pinfo->frags[0];
 
-       if (!skb_headlen(skb) && pinfo->nr_frags &&
-           !PageHighMem(skb_frag_page(frag0)) &&
+       if (pinfo->nr_frags && !PageHighMem(skb_frag_page(frag0)) &&
            (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) {
                NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
                NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
@@ -710,7 +716,10 @@ static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
                }
        } else {
                eth = (const struct ethhdr *)skb->data;
-               gro_pull_from_frag0(skb, hlen);
+
+               if (NAPI_GRO_CB(skb)->frag0 != skb->data)
+                       gro_pull_from_frag0(skb, hlen);
+
                NAPI_GRO_CB(skb)->frag0 += hlen;
                NAPI_GRO_CB(skb)->frag0_len -= hlen;
        }