bnxt_en: Add support for XDP_TX action.

Add dedicated transmit function and transmit completion handler for
XDP.  The XDP transmit logic and completion logic are different than
regular TX ring.  The TX buffer is recycled back to the RX ring when
it completes.

v3: Improved the buffer recyling scheme for XDP_TX.

v2: Add trace_xdp_exception().
    Add dma_sync.

Signed-off-by: Michael Chan <michael.chan@broadcom.com>
Tested-by: Andy Gospodarek <gospo@broadcom.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
[Upstream commit 38413406277fd060f46855ad527f6f8d4cf2652d]
Orabug: 27988326

Signed-off-by: Somasundaram Krishnasamy <somasundaram.krishnasamy@oracle.com>
Reviewed-by: Brian Maly <brian.maly@oracle.com>
Signed-off-by: Brian Maly <brian.maly@oracle.com>

diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index 1ab36080248164567a674dffd8887f3210d3d53a..a9c9f99d711ed561274b6f31053d8025f6a892ff 100644 (file)
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -256,16 +256,7 @@ static bool bnxt_vf_pciid(enum board_idx idx)
 #define BNXT_DMA_ATTRS(a)  do { } while (0)
 #endif
 
-static inline u32 bnxt_tx_avail(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
-{
-       /* Tell compiler to fetch tx indices from memory. */
-       barrier();
-
-       return bp->tx_ring_size -
-               ((txr->tx_prod - txr->tx_cons) & bp->tx_ring_mask);
-}
-
-static const u16 bnxt_lhint_arr[] = {
+const u16 bnxt_lhint_arr[] = {
        TX_BD_FLAGS_LHINT_512_AND_SMALLER,
        TX_BD_FLAGS_LHINT_512_TO_1023,
        TX_BD_FLAGS_LHINT_1024_TO_2047,
@@ -667,9 +658,8 @@ static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
        return data;
 }
 
-static inline int bnxt_alloc_rx_data(struct bnxt *bp,
-                                    struct bnxt_rx_ring_info *rxr,
-                                    u16 prod, gfp_t gfp)
+int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
+                      u16 prod, gfp_t gfp)
 {
        struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
        struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
@@ -1908,6 +1898,18 @@ static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
                        break;
        }
 
+       if (event & BNXT_TX_EVENT) {
+               struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
+               void __iomem *db = txr->tx_doorbell;
+               u16 prod = txr->tx_prod;
+
+               /* Sync BD data before updating doorbell */
+               wmb();
+
+               writel(DB_KEY_TX | prod, db);
+               writel(DB_KEY_TX | prod, db);
+       }
+
        cpr->cp_raw_cons = raw_cons;
        /* ACK completion ring before freeing tx ring and producing new
         * buffers in rx/agg rings to prevent overflowing the completion
@@ -3255,12 +3257,14 @@ static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
                        bp->tx_ring[i].bnapi = bp->bnapi[j];
                        bp->bnapi[j]->tx_ring = &bp->tx_ring[i];
                        bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
-                       if (i >= bp->tx_nr_rings_xdp)
+                       if (i >= bp->tx_nr_rings_xdp) {
                                bp->tx_ring[i].txq_index = i -
                                        bp->tx_nr_rings_xdp;
-                       else
+                               bp->bnapi[j]->tx_int = bnxt_tx_int;
+                       } else {
                                bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
-                       bp->bnapi[j]->tx_int = bnxt_tx_int;
+                               bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
+                       }
                }
 
                rc = bnxt_alloc_stats(bp);

diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
index 5be2446d98da2c1da3bf37f20052bfb950223ac2..826e02ec797005aa24230fbb0cf2c8866b3d2942 100644 (file)
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
@@ -518,13 +518,17 @@ struct rx_tpa_end_cmp_ext {
 
 #define BNXT_RX_EVENT  1
 #define BNXT_AGG_EVENT 2
+#define BNXT_TX_EVENT  4
 
 struct bnxt_sw_tx_bd {
        struct sk_buff          *skb;
        DEFINE_DMA_UNMAP_ADDR(mapping);
        u8                      is_gso;
        u8                      is_push;
-       unsigned short          nr_frags;
+       union {
+               unsigned short          nr_frags;
+               u16                     rx_prod;
+       };
 };
 
 struct bnxt_sw_rx_bd {
@@ -1294,6 +1298,19 @@ static inline void bnxt_db_write(struct bnxt *bp, void __iomem *db, u32 val)
                writel(val, db);
 }
 
+static inline u32 bnxt_tx_avail(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
+{
+       /* Tell compiler to fetch tx indices from memory. */
+       barrier();
+
+       return bp->tx_ring_size -
+               ((txr->tx_prod - txr->tx_cons) & bp->tx_ring_mask);
+}
+
+extern const u16 bnxt_lhint_arr[];
+
+int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
+                      u16 prod, gfp_t gfp);
 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data);
 void bnxt_set_tpa_flags(struct bnxt *bp);
 void bnxt_set_ring_params(struct bnxt *);

diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
index 46c2e860394d00cdf56dc48a604f74c53be2aab5..d1e0d5ab3ddcc54056e8a7c707ced60f4f325ccc 100644 (file)
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
@@ -23,7 +23,66 @@
 #include "bnxt.h"
 #include "bnxt_xdp.h"
 
+static void bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
+                         dma_addr_t mapping, u32 len, u16 rx_prod)
+{
+       struct bnxt_sw_tx_bd *tx_buf;
+       struct tx_bd_ext *txbd1;
+       struct tx_bd *txbd;
+       u32 flags;
+       u16 prod;
+
+       prod = txr->tx_prod;
+       tx_buf = &txr->tx_buf_ring[prod];
+       tx_buf->rx_prod = rx_prod;
+
+       txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
+       flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
+               (2 << TX_BD_FLAGS_BD_CNT_SHIFT) | TX_BD_FLAGS_COAL_NOW |
+               TX_BD_FLAGS_PACKET_END | bnxt_lhint_arr[len >> 9];
+       txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
+       txbd->tx_bd_opaque = prod;
+       txbd->tx_bd_haddr = cpu_to_le64(mapping);
+
+       prod = NEXT_TX(prod);
+       txbd1 = (struct tx_bd_ext *)
+               &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
+
+       txbd1->tx_bd_hsize_lflags = cpu_to_le32(0);
+       txbd1->tx_bd_mss = cpu_to_le32(0);
+       txbd1->tx_bd_cfa_action = cpu_to_le32(0);
+       txbd1->tx_bd_cfa_meta = cpu_to_le32(0);
+
+       prod = NEXT_TX(prod);
+       txr->tx_prod = prod;
+}
+
 #ifdef HAVE_NDO_XDP
+void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
+{
+       struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
+       struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
+       struct bnxt_sw_tx_bd *tx_buf;
+       u16 tx_cons = txr->tx_cons;
+       u16 last_tx_cons = tx_cons;
+       u16 rx_prod;
+       int i;
+
+       for (i = 0; i < nr_pkts; i++) {
+               last_tx_cons = tx_cons;
+               tx_cons = NEXT_TX(tx_cons);
+               tx_cons = NEXT_TX(tx_cons);
+       }
+       txr->tx_cons = tx_cons;
+       if (bnxt_tx_avail(bp, txr) == bp->tx_ring_size) {
+               rx_prod = rxr->rx_prod;
+       } else {
+               tx_buf = &txr->tx_buf_ring[last_tx_cons];
+               rx_prod = tx_buf->rx_prod;
+       }
+       writel(DB_KEY_RX | rx_prod, rxr->rx_doorbell);
+}
+
 /* returns the following:
  * true    - packet consumed by XDP and new buffer is allocated.
  * false   - packet should be passed to the stack.
@@ -32,11 +91,13 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
                 struct page *page, u8 **data_ptr, unsigned int *len, u8 *event)
 {
        struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog);
+       struct bnxt_tx_ring_info *txr;
        struct bnxt_sw_rx_bd *rx_buf;
        struct pci_dev *pdev;
        struct xdp_buff xdp;
        dma_addr_t mapping;
        void *orig_data;
+       u32 tx_avail;
        u32 offset;
        u32 act;
 
@@ -44,6 +105,7 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
                return false;
 
        pdev = bp->pdev;
+       txr = rxr->bnapi->tx_ring;
        rx_buf = &rxr->rx_buf_ring[cons];
        offset = bp->rx_offset;
 
@@ -59,6 +121,13 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
        act = bpf_prog_run_xdp(xdp_prog, &xdp);
        rcu_read_unlock();
 
+       tx_avail = bnxt_tx_avail(bp, txr);
+       /* If the tx ring is not full, we must not update the rx producer yet
+        * because we may still be transmitting on some BDs.
+        */
+       if (tx_avail != bp->tx_ring_size)
+               *event &= ~BNXT_RX_EVENT;
+
        if (orig_data != xdp.data) {
                offset = xdp.data - xdp.data_hard_start;
                *data_ptr = xdp.data_hard_start + offset;
@@ -68,6 +137,20 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
        case XDP_PASS:
                return false;
 
+       case XDP_TX:
+               if (tx_avail < 2) {
+                       trace_xdp_exception(bp->dev, xdp_prog, act);
+                       bnxt_reuse_rx_data(rxr, cons, page);
+                       return true;
+               }
+
+               *event = BNXT_TX_EVENT;
+               dma_sync_single_for_device(&pdev->dev, mapping + offset, *len,
+                                          bp->rx_dir);
+               bnxt_xmit_xdp(bp, txr, mapping + offset, *len,
+                             NEXT_RX(rxr->rx_prod));
+               bnxt_reuse_rx_data(rxr, cons, page);
+               return true;
        default:
                bpf_warn_invalid_xdp_action(act);
                /* Fall thru */
@@ -160,4 +243,14 @@ int bnxt_xdp(struct net_device *dev, struct netdev_xdp *xdp)
        }
        return rc;
 }
+#else
+void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
+{
+}
+
+bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
+                void *page, u8 **data_ptr, unsigned int *len, u8 *event)
+{
+       return false;
+}
 #endif

diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h
index 3483ac42d8837dc321debcf1b6823796cd7a6ac6..f10bdbbc4ce6c2bb84ddf3586603ec70c20328ef 100644 (file)
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h
@@ -12,9 +12,16 @@
 
 struct netdev_xdp;
 
+void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts);
+#ifdef HAVE_NDO_XDP
 bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
                 struct page *page, u8 **data_ptr, unsigned int *len,
                 u8 *event);
+#else
+bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
+                void *page, u8 **data_ptr, unsigned int *len,
+                u8 *event);
+#endif
 int bnxt_xdp(struct net_device *dev, struct netdev_xdp *xdp);
 
 #endif