net: Take common prefetch code structure into a function

Many device drivers use the same prefetch code structure to
deal with small L1 cacheline size.
Take this code into a function and call it from the drivers.

Suggested-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Signed-off-by: Tariq Toukan <tariqt@mellanox.com>
Reviewed-by: Saeed Mahameed <saeedm@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/drivers/net/ethernet/chelsio/cxgb3/sge.c b/drivers/net/ethernet/chelsio/cxgb3/sge.c
index 6dabbf1502c713fd197666dfd4d5975634c0044f..ee6188dea7056109a0c1cf11227fe2a4e6228a18 100644 (file)
--- a/drivers/net/ethernet/chelsio/cxgb3/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb3/sge.c
@@ -2372,10 +2372,7 @@ no_mem:
                        if (fl->use_pages) {
                                void *addr = fl->sdesc[fl->cidx].pg_chunk.va;
 
-                               prefetch(addr);
-#if L1_CACHE_BYTES < 128
-                               prefetch(addr + L1_CACHE_BYTES);
-#endif
+                               net_prefetch(addr);
                                __refill_fl(adap, fl);
                                if (lro > 0) {
                                        lro_add_page(adap, qs, fl,

diff --git a/drivers/net/ethernet/hisilicon/hns/hns_enet.c b/drivers/net/ethernet/hisilicon/hns/hns_enet.c
index 23f278e46975bf4b58668af1c35567584507422f..3af33ade7b60bd67ae2a34fae5a797fc45911c7f 100644 (file)
--- a/drivers/net/ethernet/hisilicon/hns/hns_enet.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_enet.c
@@ -557,10 +557,7 @@ static int hns_nic_poll_rx_skb(struct hns_nic_ring_data *ring_data,
        va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
 
        /* prefetch first cache line of first page */
-       prefetch(va);
-#if L1_CACHE_BYTES < 128
-       prefetch(va + L1_CACHE_BYTES);
-#endif
+       net_prefetch(va);
 
        skb = *out_skb = napi_alloc_skb(&ring_data->napi,
                                        HNS_RX_HEAD_SIZE);

diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
index 87776ce3539b487523965c3fe549ff57f1aa2aaf..1a1ba6a41bfec5f1eebb7e2ac8a3ff94f45c8ff2 100644 (file)
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
@@ -3091,10 +3091,7 @@ static int hns3_handle_rx_bd(struct hns3_enet_ring *ring)
         * lines. In such a case, single fetch would suffice to cache in the
         * relevant part of the header.
         */
-       prefetch(ring->va);
-#if L1_CACHE_BYTES < 128
-       prefetch(ring->va + L1_CACHE_BYTES);
-#endif
+       net_prefetch(ring->va);
 
        if (!skb) {
                ret = hns3_alloc_skb(ring, length, ring->va);

diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_main.c b/drivers/net/ethernet/intel/fm10k/fm10k_main.c
index d88dd41a94422b38a6914be96ca264f5ffc27b3b..99b8252eb969ebea991ca374427822504c8cddc9 100644 (file)
--- a/drivers/net/ethernet/intel/fm10k/fm10k_main.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_main.c
@@ -310,10 +310,7 @@ static struct sk_buff *fm10k_fetch_rx_buffer(struct fm10k_ring *rx_ring,
                                  rx_buffer->page_offset;
 
                /* prefetch first cache line of first page */
-               prefetch(page_addr);
-#if L1_CACHE_BYTES < 128
-               prefetch((void *)((u8 *)page_addr + L1_CACHE_BYTES));
-#endif
+               net_prefetch(page_addr);
 
                /* allocate a skb to store the frags */
                skb = napi_alloc_skb(&rx_ring->q_vector->napi,

diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
index 3e5c566ceb012efc82cec3e30d14c16dd48de522..432a984ac335db15a42f713387642cbb7da01520 100644 (file)
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
@@ -1992,10 +1992,8 @@ static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring,
        struct sk_buff *skb;
 
        /* prefetch first cache line of first page */
-       prefetch(xdp->data);
-#if L1_CACHE_BYTES < 128
-       prefetch(xdp->data + L1_CACHE_BYTES);
-#endif
+       net_prefetch(xdp->data);
+
        /* Note, we get here by enabling legacy-rx via:
         *
         *    ethtool --set-priv-flags <dev> legacy-rx on
@@ -2078,10 +2076,8 @@ static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring,
         * likely have a consumer accessing first few bytes of meta
         * data, and then actual data.
         */
-       prefetch(xdp->data_meta);
-#if L1_CACHE_BYTES < 128
-       prefetch(xdp->data_meta + L1_CACHE_BYTES);
-#endif
+       net_prefetch(xdp->data_meta);
+
        /* build an skb around the page buffer */
        skb = build_skb(xdp->data_hard_start, truesize);
        if (unlikely(!skb))

diff --git a/drivers/net/ethernet/intel/iavf/iavf_txrx.c b/drivers/net/ethernet/intel/iavf/iavf_txrx.c
index ca041b39ffda9ade5d10882e9531dcde44f60304..256fa07d54d5d94fd95fb1a05006c5d5afa11f33 100644 (file)
--- a/drivers/net/ethernet/intel/iavf/iavf_txrx.c
+++ b/drivers/net/ethernet/intel/iavf/iavf_txrx.c
@@ -1309,10 +1309,7 @@ static struct sk_buff *iavf_construct_skb(struct iavf_ring *rx_ring,
                return NULL;
        /* prefetch first cache line of first page */
        va = page_address(rx_buffer->page) + rx_buffer->page_offset;
-       prefetch(va);
-#if L1_CACHE_BYTES < 128
-       prefetch(va + L1_CACHE_BYTES);
-#endif
+       net_prefetch(va);
 
        /* allocate a skb to store the frags */
        skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
@@ -1376,10 +1373,8 @@ static struct sk_buff *iavf_build_skb(struct iavf_ring *rx_ring,
                return NULL;
        /* prefetch first cache line of first page */
        va = page_address(rx_buffer->page) + rx_buffer->page_offset;
-       prefetch(va);
-#if L1_CACHE_BYTES < 128
-       prefetch(va + L1_CACHE_BYTES);
-#endif
+       net_prefetch(va);
+
        /* build an skb around the page buffer */
        skb = build_skb(va - IAVF_SKB_PAD, truesize);
        if (unlikely(!skb))

diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c
index 9d0d6b0025cf7bf2d132b4b3a0ff0bab01153bd3..d2fca4a52f517de0a83ca8e91cd6107025a1dcc3 100644 (file)
--- a/drivers/net/ethernet/intel/ice/ice_txrx.c
+++ b/drivers/net/ethernet/intel/ice/ice_txrx.c
@@ -919,10 +919,7 @@ ice_build_skb(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,
         * likely have a consumer accessing first few bytes of meta
         * data, and then actual data.
         */
-       prefetch(xdp->data_meta);
-#if L1_CACHE_BYTES < 128
-       prefetch((void *)(xdp->data + L1_CACHE_BYTES));
-#endif
+       net_prefetch(xdp->data_meta);
        /* build an skb around the page buffer */
        skb = build_skb(xdp->data_hard_start, truesize);
        if (unlikely(!skb))
@@ -964,10 +961,7 @@ ice_construct_skb(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,
        struct sk_buff *skb;
 
        /* prefetch first cache line of first page */
-       prefetch(xdp->data);
-#if L1_CACHE_BYTES < 128
-       prefetch((void *)(xdp->data + L1_CACHE_BYTES));
-#endif /* L1_CACHE_BYTES */
+       net_prefetch(xdp->data);
 
        /* allocate a skb to store the frags */
        skb = __napi_alloc_skb(&rx_ring->q_vector->napi, ICE_RX_HDR_SIZE,

diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index 4f05f6efe6af168f53fbe40db6afef6f96f19725..698bb6a4b088c1834fe8382f6bb6938a196c2749 100644 (file)
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -8047,10 +8047,7 @@ static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring,
        struct sk_buff *skb;
 
        /* prefetch first cache line of first page */
-       prefetch(va);
-#if L1_CACHE_BYTES < 128
-       prefetch(va + L1_CACHE_BYTES);
-#endif
+       net_prefetch(va);
 
        /* allocate a skb to store the frags */
        skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN);
@@ -8104,10 +8101,7 @@ static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring,
        struct sk_buff *skb;
 
        /* prefetch first cache line of first page */
-       prefetch(va);
-#if L1_CACHE_BYTES < 128
-       prefetch(va + L1_CACHE_BYTES);
-#endif
+       net_prefetch(va);
 
        /* build an skb around the page buffer */
        skb = build_skb(va - IGB_SKB_PAD, truesize);

diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c
index 9593aa4eea369e78d37d137ed5db7c3a13c7f9ed..c6968fdb6caa2c5a488aca343217a3599f7cbe8b 100644 (file)
--- a/drivers/net/ethernet/intel/igc/igc_main.c
+++ b/drivers/net/ethernet/intel/igc/igc_main.c
@@ -1550,10 +1550,7 @@ static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring,
        struct sk_buff *skb;
 
        /* prefetch first cache line of first page */
-       prefetch(va);
-#if L1_CACHE_BYTES < 128
-       prefetch(va + L1_CACHE_BYTES);
-#endif
+       net_prefetch(va);
 
        /* build an skb around the page buffer */
        skb = build_skb(va - IGC_SKB_PAD, truesize);
@@ -1589,10 +1586,7 @@ static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring,
        struct sk_buff *skb;
 
        /* prefetch first cache line of first page */
-       prefetch(va);
-#if L1_CACHE_BYTES < 128
-       prefetch(va + L1_CACHE_BYTES);
-#endif
+       net_prefetch(va);
 
        /* allocate a skb to store the frags */
        skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN);

diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index 2f8a4cfc5fa1ffe6a8fa77c3add02010b0b5c438..f4f2198f388b2177935f0b526fd539028cff1786 100644 (file)
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -2095,10 +2095,8 @@ static struct sk_buff *ixgbe_construct_skb(struct ixgbe_ring *rx_ring,
        struct sk_buff *skb;
 
        /* prefetch first cache line of first page */
-       prefetch(xdp->data);
-#if L1_CACHE_BYTES < 128
-       prefetch(xdp->data + L1_CACHE_BYTES);
-#endif
+       net_prefetch(xdp->data);
+
        /* Note, we get here by enabling legacy-rx via:
         *
         *    ethtool --set-priv-flags <dev> legacy-rx on
@@ -2161,10 +2159,7 @@ static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring,
         * likely have a consumer accessing first few bytes of meta
         * data, and then actual data.
         */
-       prefetch(xdp->data_meta);
-#if L1_CACHE_BYTES < 128
-       prefetch(xdp->data_meta + L1_CACHE_BYTES);
-#endif
+       net_prefetch(xdp->data_meta);
 
        /* build an skb to around the page buffer */
        skb = build_skb(xdp->data_hard_start, truesize);

diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
index a428113e6d54088d3385a54633389930ea524274..50afec43e001ea32045ad235f8ecbe2161edf8c2 100644 (file)
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
@@ -866,10 +866,8 @@ struct sk_buff *ixgbevf_construct_skb(struct ixgbevf_ring *rx_ring,
        struct sk_buff *skb;
 
        /* prefetch first cache line of first page */
-       prefetch(xdp->data);
-#if L1_CACHE_BYTES < 128
-       prefetch(xdp->data + L1_CACHE_BYTES);
-#endif
+       net_prefetch(xdp->data);
+
        /* Note, we get here by enabling legacy-rx via:
         *
         *    ethtool --set-priv-flags <dev> legacy-rx on
@@ -947,10 +945,7 @@ static struct sk_buff *ixgbevf_build_skb(struct ixgbevf_ring *rx_ring,
         * have a consumer accessing first few bytes of meta data,
         * and then actual data.
         */
-       prefetch(xdp->data_meta);
-#if L1_CACHE_BYTES < 128
-       prefetch(xdp->data_meta + L1_CACHE_BYTES);
-#endif
+       net_prefetch(xdp->data_meta);
 
        /* build an skb around the page buffer */
        skb = build_skb(xdp->data_hard_start, truesize);

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index b0e303f6603f0bb7ef2a45b1d485e13822969382..b8abe1d7aa0bf8316b292fb2fe00f955e52c7001 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -2193,6 +2193,22 @@ int netdev_get_num_tc(struct net_device *dev)
        return dev->num_tc;
 }
 
+static inline void net_prefetch(void *p)
+{
+       prefetch(p);
+#if L1_CACHE_BYTES < 128
+       prefetch((u8 *)p + L1_CACHE_BYTES);
+#endif
+}
+
+static inline void net_prefetchw(void *p)
+{
+       prefetchw(p);
+#if L1_CACHE_BYTES < 128
+       prefetchw((u8 *)p + L1_CACHE_BYTES);
+#endif
+}
+
 void netdev_unbind_sb_channel(struct net_device *dev,
                              struct net_device *sb_dev);
 int netdev_bind_sb_channel_queue(struct net_device *dev,