From efabce29015189cb5cd8066cf29eb1d754de6c3c Mon Sep 17 00:00:00 2001 From: Suman Ghosh Date: Thu, 13 Feb 2025 11:01:38 +0530 Subject: [PATCH 01/16] octeontx2-pf: AF_XDP zero copy receive support This patch adds support to AF_XDP zero copy for CN10K. This patch specifically adds receive side support. In this approach once a xdp program with zero copy support on a specific rx queue is enabled, then that receive quse is disabled/detached from the existing kernel queue and re-assigned to the umem memory. Signed-off-by: Suman Ghosh Signed-off-by: Paolo Abeni --- .../ethernet/marvell/octeontx2/nic/Makefile | 2 +- .../ethernet/marvell/octeontx2/nic/cn10k.c | 7 +- .../marvell/octeontx2/nic/otx2_common.c | 114 ++++++++--- .../marvell/octeontx2/nic/otx2_common.h | 6 +- .../ethernet/marvell/octeontx2/nic/otx2_pf.c | 25 ++- .../marvell/octeontx2/nic/otx2_txrx.c | 73 +++++-- .../marvell/octeontx2/nic/otx2_txrx.h | 6 + .../ethernet/marvell/octeontx2/nic/otx2_vf.c | 12 +- .../ethernet/marvell/octeontx2/nic/otx2_xsk.c | 182 ++++++++++++++++++ .../ethernet/marvell/octeontx2/nic/otx2_xsk.h | 21 ++ .../ethernet/marvell/octeontx2/nic/qos_sq.c | 2 +- 11 files changed, 389 insertions(+), 61 deletions(-) create mode 100644 drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c create mode 100644 drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/Makefile b/drivers/net/ethernet/marvell/octeontx2/nic/Makefile index cb6513ab35e7..69e0778f9ac1 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/Makefile +++ b/drivers/net/ethernet/marvell/octeontx2/nic/Makefile @@ -9,7 +9,7 @@ obj-$(CONFIG_RVU_ESWITCH) += rvu_rep.o rvu_nicpf-y := otx2_pf.o otx2_common.o otx2_txrx.o otx2_ethtool.o \ otx2_flows.o otx2_tc.o cn10k.o otx2_dmac_flt.o \ - otx2_devlink.o qos_sq.o qos.o + otx2_devlink.o qos_sq.o qos.o otx2_xsk.o rvu_nicvf-y := otx2_vf.o rvu_rep-y := rep.o diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c index a15cc86635d6..c3b6e0f60a79 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c @@ -112,9 +112,12 @@ int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq) struct otx2_nic *pfvf = dev; int cnt = cq->pool_ptrs; u64 ptrs[NPA_MAX_BURST]; + struct otx2_pool *pool; dma_addr_t bufptr; int num_ptrs = 1; + pool = &pfvf->qset.pool[cq->cq_idx]; + /* Refill pool with new buffers */ while (cq->pool_ptrs) { if (otx2_alloc_buffer(pfvf, cq, &bufptr)) { @@ -124,7 +127,9 @@ int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq) break; } cq->pool_ptrs--; - ptrs[num_ptrs] = (u64)bufptr + OTX2_HEAD_ROOM; + ptrs[num_ptrs] = pool->xsk_pool ? + (u64)bufptr : (u64)bufptr + OTX2_HEAD_ROOM; + num_ptrs++; if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) { __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs, diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c index 161cf33ef89e..92b0dba07853 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c @@ -17,6 +17,7 @@ #include "otx2_common.h" #include "otx2_struct.h" #include "cn10k.h" +#include "otx2_xsk.h" static bool otx2_is_pfc_enabled(struct otx2_nic *pfvf) { @@ -549,10 +550,13 @@ static int otx2_alloc_pool_buf(struct otx2_nic *pfvf, struct otx2_pool *pool, } static int __otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, - dma_addr_t *dma) + dma_addr_t *dma, int qidx, int idx) { u8 *buf; + if (pool->xsk_pool) + return otx2_xsk_pool_alloc_buf(pfvf, pool, dma, idx); + if (pool->page_pool) return otx2_alloc_pool_buf(pfvf, pool, dma); @@ -571,12 +575,12 @@ static int __otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, } int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, - dma_addr_t *dma) + dma_addr_t *dma, int qidx, int idx) { int ret; local_bh_disable(); - ret = __otx2_alloc_rbuf(pfvf, pool, dma); + ret = __otx2_alloc_rbuf(pfvf, pool, dma, qidx, idx); local_bh_enable(); return ret; } @@ -584,7 +588,8 @@ int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, int otx2_alloc_buffer(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, dma_addr_t *dma) { - if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, dma))) + if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, dma, + cq->cq_idx, cq->pool_ptrs - 1))) return -ENOMEM; return 0; } @@ -884,7 +889,7 @@ void otx2_sqb_flush(struct otx2_nic *pfvf) #define RQ_PASS_LVL_AURA (255 - ((95 * 256) / 100)) /* RED when 95% is full */ #define RQ_DROP_LVL_AURA (255 - ((99 * 256) / 100)) /* Drop when 99% is full */ -static int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura) +int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura) { struct otx2_qset *qset = &pfvf->qset; struct nix_aq_enq_req *aq; @@ -1041,7 +1046,7 @@ int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura) } -static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx) +int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx) { struct otx2_qset *qset = &pfvf->qset; int err, pool_id, non_xdp_queues; @@ -1057,11 +1062,18 @@ static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx) cq->cint_idx = qidx; cq->cqe_cnt = qset->rqe_cnt; if (pfvf->xdp_prog) { - pool = &qset->pool[qidx]; xdp_rxq_info_reg(&cq->xdp_rxq, pfvf->netdev, qidx, 0); - xdp_rxq_info_reg_mem_model(&cq->xdp_rxq, - MEM_TYPE_PAGE_POOL, - pool->page_pool); + pool = &qset->pool[qidx]; + if (pool->xsk_pool) { + xdp_rxq_info_reg_mem_model(&cq->xdp_rxq, + MEM_TYPE_XSK_BUFF_POOL, + NULL); + xsk_pool_set_rxq_info(pool->xsk_pool, &cq->xdp_rxq); + } else if (pool->page_pool) { + xdp_rxq_info_reg_mem_model(&cq->xdp_rxq, + MEM_TYPE_PAGE_POOL, + pool->page_pool); + } } } else if (qidx < non_xdp_queues) { cq->cq_type = CQ_TX; @@ -1281,9 +1293,10 @@ void otx2_free_bufs(struct otx2_nic *pfvf, struct otx2_pool *pool, pa = otx2_iova_to_phys(pfvf->iommu_domain, iova); page = virt_to_head_page(phys_to_virt(pa)); - if (pool->page_pool) { page_pool_put_full_page(pool->page_pool, page, true); + } else if (pool->xsk_pool) { + /* Note: No way of identifying xdp_buff */ } else { dma_unmap_page_attrs(pfvf->dev, iova, size, DMA_FROM_DEVICE, @@ -1298,6 +1311,7 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type) int pool_id, pool_start = 0, pool_end = 0, size = 0; struct otx2_pool *pool; u64 iova; + int idx; if (type == AURA_NIX_SQ) { pool_start = otx2_get_pool_idx(pfvf, type, 0); @@ -1312,16 +1326,21 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type) /* Free SQB and RQB pointers from the aura pool */ for (pool_id = pool_start; pool_id < pool_end; pool_id++) { - iova = otx2_aura_allocptr(pfvf, pool_id); pool = &pfvf->qset.pool[pool_id]; + iova = otx2_aura_allocptr(pfvf, pool_id); while (iova) { if (type == AURA_NIX_RQ) iova -= OTX2_HEAD_ROOM; - otx2_free_bufs(pfvf, pool, iova, size); - iova = otx2_aura_allocptr(pfvf, pool_id); } + + for (idx = 0 ; idx < pool->xdp_cnt; idx++) { + if (!pool->xdp[idx]) + continue; + + xsk_buff_free(pool->xdp[idx]); + } } } @@ -1338,7 +1357,8 @@ void otx2_aura_pool_free(struct otx2_nic *pfvf) qmem_free(pfvf->dev, pool->stack); qmem_free(pfvf->dev, pool->fc_addr); page_pool_destroy(pool->page_pool); - pool->page_pool = NULL; + devm_kfree(pfvf->dev, pool->xdp); + pool->xsk_pool = NULL; } devm_kfree(pfvf->dev, pfvf->qset.pool); pfvf->qset.pool = NULL; @@ -1425,6 +1445,7 @@ int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id, int stack_pages, int numptrs, int buf_size, int type) { struct page_pool_params pp_params = { 0 }; + struct xsk_buff_pool *xsk_pool; struct npa_aq_enq_req *aq; struct otx2_pool *pool; int err; @@ -1468,21 +1489,35 @@ int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id, aq->ctype = NPA_AQ_CTYPE_POOL; aq->op = NPA_AQ_INSTOP_INIT; - if (type != AURA_NIX_RQ) { - pool->page_pool = NULL; + if (type != AURA_NIX_RQ) + return 0; + + if (!test_bit(pool_id, pfvf->af_xdp_zc_qidx)) { + pp_params.order = get_order(buf_size); + pp_params.flags = PP_FLAG_DMA_MAP; + pp_params.pool_size = min(OTX2_PAGE_POOL_SZ, numptrs); + pp_params.nid = NUMA_NO_NODE; + pp_params.dev = pfvf->dev; + pp_params.dma_dir = DMA_FROM_DEVICE; + pool->page_pool = page_pool_create(&pp_params); + if (IS_ERR(pool->page_pool)) { + netdev_err(pfvf->netdev, "Creation of page pool failed\n"); + return PTR_ERR(pool->page_pool); + } return 0; } - pp_params.order = get_order(buf_size); - pp_params.flags = PP_FLAG_DMA_MAP; - pp_params.pool_size = min(OTX2_PAGE_POOL_SZ, numptrs); - pp_params.nid = NUMA_NO_NODE; - pp_params.dev = pfvf->dev; - pp_params.dma_dir = DMA_FROM_DEVICE; - pool->page_pool = page_pool_create(&pp_params); - if (IS_ERR(pool->page_pool)) { - netdev_err(pfvf->netdev, "Creation of page pool failed\n"); - return PTR_ERR(pool->page_pool); + /* Set XSK pool to support AF_XDP zero-copy */ + xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, pool_id); + if (xsk_pool) { + pool->xsk_pool = xsk_pool; + pool->xdp_cnt = numptrs; + pool->xdp = devm_kcalloc(pfvf->dev, + numptrs, sizeof(struct xdp_buff *), GFP_KERNEL); + if (IS_ERR(pool->xdp)) { + netdev_err(pfvf->netdev, "Creation of xsk pool failed\n"); + return PTR_ERR(pool->xdp); + } } return 0; @@ -1543,9 +1578,18 @@ int otx2_sq_aura_pool_init(struct otx2_nic *pfvf) } for (ptr = 0; ptr < num_sqbs; ptr++) { - err = otx2_alloc_rbuf(pfvf, pool, &bufptr); - if (err) + err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr); + if (err) { + if (pool->xsk_pool) { + ptr--; + while (ptr >= 0) { + xsk_buff_free(pool->xdp[ptr]); + ptr--; + } + } goto err_mem; + } + pfvf->hw_ops->aura_freeptr(pfvf, pool_id, bufptr); sq->sqb_ptrs[sq->sqb_count++] = (u64)bufptr; } @@ -1595,11 +1639,19 @@ int otx2_rq_aura_pool_init(struct otx2_nic *pfvf) /* Allocate pointers and free them to aura/pool */ for (pool_id = 0; pool_id < hw->rqpool_cnt; pool_id++) { pool = &pfvf->qset.pool[pool_id]; + for (ptr = 0; ptr < num_ptrs; ptr++) { - err = otx2_alloc_rbuf(pfvf, pool, &bufptr); - if (err) + err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr); + if (err) { + if (pool->xsk_pool) { + while (ptr) + xsk_buff_free(pool->xdp[--ptr]); + } return -ENOMEM; + } + pfvf->hw_ops->aura_freeptr(pfvf, pool_id, + pool->xsk_pool ? bufptr : bufptr + OTX2_HEAD_ROOM); } } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h index d5fbccb289df..60508971b62f 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h @@ -532,6 +532,8 @@ struct otx2_nic { /* Inline ipsec */ struct cn10k_ipsec ipsec; + /* af_xdp zero-copy */ + unsigned long *af_xdp_zc_qidx; }; static inline bool is_otx2_lbkvf(struct pci_dev *pdev) @@ -1003,7 +1005,7 @@ void otx2_txschq_free_one(struct otx2_nic *pfvf, u16 lvl, u16 schq); void otx2_free_pending_sqe(struct otx2_nic *pfvf); void otx2_sqb_flush(struct otx2_nic *pfvf); int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, - dma_addr_t *dma); + dma_addr_t *dma, int qidx, int idx); int otx2_rxtx_enable(struct otx2_nic *pfvf, bool enable); void otx2_ctx_disable(struct mbox *mbox, int type, bool npa); int otx2_nix_config_bp(struct otx2_nic *pfvf, bool enable); @@ -1033,6 +1035,8 @@ void otx2_pfaf_mbox_destroy(struct otx2_nic *pf); void otx2_disable_mbox_intr(struct otx2_nic *pf); void otx2_disable_napi(struct otx2_nic *pf); irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq); +int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura); +int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx); /* RSS configuration APIs*/ int otx2_rss_init(struct otx2_nic *pfvf); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index 4347a3c95350..50a42cd5d50a 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -27,6 +27,7 @@ #include "qos.h" #include #include "cn10k_ipsec.h" +#include "otx2_xsk.h" #define DRV_NAME "rvu_nicpf" #define DRV_STRING "Marvell RVU NIC Physical Function Driver" @@ -1662,9 +1663,7 @@ void otx2_free_hw_resources(struct otx2_nic *pf) struct nix_lf_free_req *free_req; struct mbox *mbox = &pf->mbox; struct otx2_cq_queue *cq; - struct otx2_pool *pool; struct msg_req *req; - int pool_id; int qidx; /* Ensure all SQE are processed */ @@ -1705,13 +1704,6 @@ void otx2_free_hw_resources(struct otx2_nic *pf) /* Free RQ buffer pointers*/ otx2_free_aura_ptr(pf, AURA_NIX_RQ); - for (qidx = 0; qidx < pf->hw.rx_queues; qidx++) { - pool_id = otx2_get_pool_idx(pf, AURA_NIX_RQ, qidx); - pool = &pf->qset.pool[pool_id]; - page_pool_destroy(pool->page_pool); - pool->page_pool = NULL; - } - otx2_free_cq_res(pf); /* Free all ingress bandwidth profiles allocated */ @@ -2788,6 +2780,8 @@ static int otx2_xdp(struct net_device *netdev, struct netdev_bpf *xdp) switch (xdp->command) { case XDP_SETUP_PROG: return otx2_xdp_setup(pf, xdp->prog); + case XDP_SETUP_XSK_POOL: + return otx2_xsk_pool_setup(pf, xdp->xsk.pool, xdp->xsk.queue_id); default: return -EINVAL; } @@ -2865,6 +2859,7 @@ static const struct net_device_ops otx2_netdev_ops = { .ndo_set_vf_vlan = otx2_set_vf_vlan, .ndo_get_vf_config = otx2_get_vf_config, .ndo_bpf = otx2_xdp, + .ndo_xsk_wakeup = otx2_xsk_wakeup, .ndo_xdp_xmit = otx2_xdp_xmit, .ndo_setup_tc = otx2_setup_tc, .ndo_set_vf_trust = otx2_ndo_set_vf_trust, @@ -3203,16 +3198,26 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) /* Enable link notifications */ otx2_cgx_config_linkevents(pf, true); + pf->af_xdp_zc_qidx = bitmap_zalloc(qcount, GFP_KERNEL); + if (!pf->af_xdp_zc_qidx) { + err = -ENOMEM; + goto err_sriov_cleannup; + } + #ifdef CONFIG_DCB err = otx2_dcbnl_set_ops(netdev); if (err) - goto err_pf_sriov_init; + goto err_free_zc_bmap; #endif otx2_qos_init(pf, qos_txqs); return 0; +err_free_zc_bmap: + bitmap_free(pf->af_xdp_zc_qidx); +err_sriov_cleannup: + otx2_sriov_vfcfg_cleanup(pf); err_pf_sriov_init: otx2_shutdown_tc(pf); err_mcam_flow_del: diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c index 4a7275043103..00b6903ba250 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "otx2_reg.h" #include "otx2_common.h" @@ -523,9 +524,10 @@ static void otx2_adjust_adaptive_coalese(struct otx2_nic *pfvf, struct otx2_cq_p int otx2_napi_handler(struct napi_struct *napi, int budget) { struct otx2_cq_queue *rx_cq = NULL; + struct otx2_cq_queue *cq = NULL; + struct otx2_pool *pool = NULL; struct otx2_cq_poll *cq_poll; int workdone = 0, cq_idx, i; - struct otx2_cq_queue *cq; struct otx2_qset *qset; struct otx2_nic *pfvf; int filled_cnt = -1; @@ -550,6 +552,7 @@ int otx2_napi_handler(struct napi_struct *napi, int budget) if (rx_cq && rx_cq->pool_ptrs) filled_cnt = pfvf->hw_ops->refill_pool_ptrs(pfvf, rx_cq); + /* Clear the IRQ */ otx2_write64(pfvf, NIX_LF_CINTX_INT(cq_poll->cint_idx), BIT_ULL(0)); @@ -562,20 +565,31 @@ int otx2_napi_handler(struct napi_struct *napi, int budget) if (pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED) otx2_adjust_adaptive_coalese(pfvf, cq_poll); + if (likely(cq)) + pool = &pfvf->qset.pool[cq->cq_idx]; + if (unlikely(!filled_cnt)) { struct refill_work *work; struct delayed_work *dwork; - work = &pfvf->refill_wrk[cq->cq_idx]; - dwork = &work->pool_refill_work; - /* Schedule a task if no other task is running */ - if (!cq->refill_task_sched) { - work->napi = napi; - cq->refill_task_sched = true; - schedule_delayed_work(dwork, - msecs_to_jiffies(100)); + if (likely(cq)) { + work = &pfvf->refill_wrk[cq->cq_idx]; + dwork = &work->pool_refill_work; + /* Schedule a task if no other task is running */ + if (!cq->refill_task_sched) { + work->napi = napi; + cq->refill_task_sched = true; + schedule_delayed_work(dwork, + msecs_to_jiffies(100)); + } + /* Call wake-up for not able to fill buffers */ + if (pool->xsk_pool) + xsk_set_rx_need_wakeup(pool->xsk_pool); } } else { + /* Clear wake-up, since buffers are filled successfully */ + if (pool && pool->xsk_pool) + xsk_clear_rx_need_wakeup(pool->xsk_pool); /* Re-enable interrupts */ otx2_write64(pfvf, NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx), @@ -1226,15 +1240,19 @@ void otx2_cleanup_rx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, int q u16 pool_id; u64 iova; - if (pfvf->xdp_prog) + pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_RQ, qidx); + pool = &pfvf->qset.pool[pool_id]; + + if (pfvf->xdp_prog) { + if (pool->page_pool) + xdp_rxq_info_unreg_mem_model(&cq->xdp_rxq); + xdp_rxq_info_unreg(&cq->xdp_rxq); + } if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe) return; - pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_RQ, qidx); - pool = &pfvf->qset.pool[pool_id]; - while (cq->pend_cqe) { cqe = (struct nix_cqe_rx_s *)otx2_get_next_cqe(cq); processed_cqe++; @@ -1418,17 +1436,28 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, bool *need_xdp_flush) { + struct xdp_buff xdp, *xsk_buff = NULL; unsigned char *hard_start; struct otx2_pool *pool; struct xdp_frame *xdpf; int qidx = cq->cq_idx; - struct xdp_buff xdp; struct page *page; u64 iova, pa; u32 act; int err; pool = &pfvf->qset.pool[qidx]; + + if (pool->xsk_pool) { + xsk_buff = pool->xdp[--cq->rbpool->xdp_top]; + if (!xsk_buff) + return false; + + xsk_buff->data_end = xsk_buff->data + cqe->sg.seg_size; + act = bpf_prog_run_xdp(prog, xsk_buff); + goto handle_xdp_verdict; + } + iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM; pa = otx2_iova_to_phys(pfvf->iommu_domain, iova); page = virt_to_page(phys_to_virt(pa)); @@ -1441,6 +1470,7 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf, act = bpf_prog_run_xdp(prog, &xdp); +handle_xdp_verdict: switch (act) { case XDP_PASS: break; @@ -1452,6 +1482,15 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf, cqe->sg.seg_size, qidx, XDP_TX); case XDP_REDIRECT: cq->pool_ptrs++; + if (xsk_buff) { + err = xdp_do_redirect(pfvf->netdev, xsk_buff, prog); + if (!err) { + *need_xdp_flush = true; + return true; + } + return false; + } + err = xdp_do_redirect(pfvf->netdev, &xdp, prog); if (!err) { *need_xdp_flush = true; @@ -1467,11 +1506,15 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf, bpf_warn_invalid_xdp_action(pfvf->netdev, prog, act); break; case XDP_ABORTED: + if (xsk_buff) + xsk_buff_free(xsk_buff); trace_xdp_exception(pfvf->netdev, prog, act); break; case XDP_DROP: cq->pool_ptrs++; - if (page->pp) { + if (xsk_buff) { + xsk_buff_free(xsk_buff); + } else if (page->pp) { page_pool_recycle_direct(pool->page_pool, page); } else { otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize, diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h index 92e1e84cad75..8f346fbc8221 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h @@ -12,6 +12,7 @@ #include #include #include +#include #define LBK_CHAN_BASE 0x000 #define SDP_CHAN_BASE 0x700 @@ -128,7 +129,11 @@ struct otx2_pool { struct qmem *stack; struct qmem *fc_addr; struct page_pool *page_pool; + struct xsk_buff_pool *xsk_pool; + struct xdp_buff **xdp; + u16 xdp_cnt; u16 rbsize; + u16 xdp_top; }; struct otx2_cq_queue { @@ -145,6 +150,7 @@ struct otx2_cq_queue { void *cqe_base; struct qmem *cqe; struct otx2_pool *rbpool; + bool xsk_zc_en; struct xdp_rxq_info xdp_rxq; } ____cacheline_aligned_in_smp; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c index e926c6ce96cf..63ddd262d122 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c @@ -722,15 +722,25 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) goto err_shutdown_tc; + vf->af_xdp_zc_qidx = bitmap_zalloc(qcount, GFP_KERNEL); + if (!vf->af_xdp_zc_qidx) { + err = -ENOMEM; + goto err_unreg_devlink; + } + #ifdef CONFIG_DCB err = otx2_dcbnl_set_ops(netdev); if (err) - goto err_shutdown_tc; + goto err_free_zc_bmap; #endif otx2_qos_init(vf, qos_txqs); return 0; +err_free_zc_bmap: + bitmap_free(vf->af_xdp_zc_qidx); +err_unreg_devlink: + otx2_unregister_dl(vf); err_shutdown_tc: otx2_shutdown_tc(vf); err_unreg_netdev: diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c new file mode 100644 index 000000000000..894c1e0aea6f --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell RVU Ethernet driver + * + * Copyright (C) 2024 Marvell. + * + */ + +#include +#include +#include +#include + +#include "otx2_common.h" +#include "otx2_xsk.h" + +int otx2_xsk_pool_alloc_buf(struct otx2_nic *pfvf, struct otx2_pool *pool, + dma_addr_t *dma, int idx) +{ + struct xdp_buff *xdp; + int delta; + + xdp = xsk_buff_alloc(pool->xsk_pool); + if (!xdp) + return -ENOMEM; + + pool->xdp[pool->xdp_top++] = xdp; + *dma = OTX2_DATA_ALIGN(xsk_buff_xdp_get_dma(xdp)); + /* Adjust xdp->data for unaligned addresses */ + delta = *dma - xsk_buff_xdp_get_dma(xdp); + xdp->data += delta; + + return 0; +} + +static int otx2_xsk_ctx_disable(struct otx2_nic *pfvf, u16 qidx, int aura_id) +{ + struct nix_cn10k_aq_enq_req *cn10k_rq_aq; + struct npa_aq_enq_req *aura_aq; + struct npa_aq_enq_req *pool_aq; + struct nix_aq_enq_req *rq_aq; + + if (test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) { + cn10k_rq_aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); + if (!cn10k_rq_aq) + return -ENOMEM; + cn10k_rq_aq->qidx = qidx; + cn10k_rq_aq->rq.ena = 0; + cn10k_rq_aq->rq_mask.ena = 1; + cn10k_rq_aq->ctype = NIX_AQ_CTYPE_RQ; + cn10k_rq_aq->op = NIX_AQ_INSTOP_WRITE; + } else { + rq_aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); + if (!rq_aq) + return -ENOMEM; + rq_aq->qidx = qidx; + rq_aq->sq.ena = 0; + rq_aq->sq_mask.ena = 1; + rq_aq->ctype = NIX_AQ_CTYPE_RQ; + rq_aq->op = NIX_AQ_INSTOP_WRITE; + } + + aura_aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); + if (!aura_aq) + goto fail; + + aura_aq->aura_id = aura_id; + aura_aq->aura.ena = 0; + aura_aq->aura_mask.ena = 1; + aura_aq->ctype = NPA_AQ_CTYPE_AURA; + aura_aq->op = NPA_AQ_INSTOP_WRITE; + + pool_aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); + if (!pool_aq) + goto fail; + + pool_aq->aura_id = aura_id; + pool_aq->pool.ena = 0; + pool_aq->pool_mask.ena = 1; + + pool_aq->ctype = NPA_AQ_CTYPE_POOL; + pool_aq->op = NPA_AQ_INSTOP_WRITE; + + return otx2_sync_mbox_msg(&pfvf->mbox); + +fail: + otx2_mbox_reset(&pfvf->mbox.mbox, 0); + return -ENOMEM; +} + +static void otx2_clean_up_rq(struct otx2_nic *pfvf, int qidx) +{ + struct otx2_qset *qset = &pfvf->qset; + struct otx2_cq_queue *cq; + struct otx2_pool *pool; + u64 iova; + + /* If the DOWN flag is set SQs are already freed */ + if (pfvf->flags & OTX2_FLAG_INTF_DOWN) + return; + + cq = &qset->cq[qidx]; + if (cq) + otx2_cleanup_rx_cqes(pfvf, cq, qidx); + + pool = &pfvf->qset.pool[qidx]; + iova = otx2_aura_allocptr(pfvf, qidx); + while (iova) { + iova -= OTX2_HEAD_ROOM; + otx2_free_bufs(pfvf, pool, iova, pfvf->rbsize); + iova = otx2_aura_allocptr(pfvf, qidx); + } + + mutex_lock(&pfvf->mbox.lock); + otx2_xsk_ctx_disable(pfvf, qidx, qidx); + mutex_unlock(&pfvf->mbox.lock); +} + +int otx2_xsk_pool_enable(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qidx) +{ + u16 rx_queues = pf->hw.rx_queues; + u16 tx_queues = pf->hw.tx_queues; + int err; + + if (qidx >= rx_queues || qidx >= tx_queues) + return -EINVAL; + + err = xsk_pool_dma_map(pool, pf->dev, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); + if (err) + return err; + + set_bit(qidx, pf->af_xdp_zc_qidx); + otx2_clean_up_rq(pf, qidx); + /* Kick start the NAPI context so that receiving will start */ + return otx2_xsk_wakeup(pf->netdev, qidx, XDP_WAKEUP_RX); +} + +int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qidx) +{ + struct net_device *netdev = pf->netdev; + struct xsk_buff_pool *pool; + + pool = xsk_get_pool_from_qid(netdev, qidx); + if (!pool) + return -EINVAL; + + otx2_clean_up_rq(pf, qidx); + clear_bit(qidx, pf->af_xdp_zc_qidx); + xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); + + return 0; +} + +int otx2_xsk_pool_setup(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qidx) +{ + if (pool) + return otx2_xsk_pool_enable(pf, pool, qidx); + + return otx2_xsk_pool_disable(pf, qidx); +} + +int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) +{ + struct otx2_nic *pf = netdev_priv(dev); + struct otx2_cq_poll *cq_poll = NULL; + struct otx2_qset *qset = &pf->qset; + + if (pf->flags & OTX2_FLAG_INTF_DOWN) + return -ENETDOWN; + + if (queue_id >= pf->hw.rx_queues) + return -EINVAL; + + cq_poll = &qset->napi[queue_id]; + if (!cq_poll) + return -EINVAL; + + /* Trigger interrupt */ + if (!napi_if_scheduled_mark_missed(&cq_poll->napi)) + otx2_write64(pf, NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx), BIT_ULL(0)); + + return 0; +} diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h new file mode 100644 index 000000000000..022b3433edbb --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell RVU PF/VF Netdev Devlink + * + * Copyright (C) 2024 Marvell. + * + */ + +#ifndef OTX2_XSK_H +#define OTX2_XSK_H + +struct otx2_nic; +struct xsk_buff_pool; + +int otx2_xsk_pool_setup(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qid); +int otx2_xsk_pool_enable(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qid); +int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qid); +int otx2_xsk_pool_alloc_buf(struct otx2_nic *pfvf, struct otx2_pool *pool, + dma_addr_t *dma, int idx); +int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags); + +#endif /* OTX2_XSK_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/qos_sq.c b/drivers/net/ethernet/marvell/octeontx2/nic/qos_sq.c index 9d887bfc3108..c5dbae0e513b 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/qos_sq.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/qos_sq.c @@ -82,7 +82,7 @@ static int otx2_qos_sq_aura_pool_init(struct otx2_nic *pfvf, int qidx) } for (ptr = 0; ptr < num_sqbs; ptr++) { - err = otx2_alloc_rbuf(pfvf, pool, &bufptr); + err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr); if (err) goto sqb_free; pfvf->hw_ops->aura_freeptr(pfvf, pool_id, bufptr); -- 2.50.1 From 25b07c1a869457fb1fcbbb744123d4ab05666709 Mon Sep 17 00:00:00 2001 From: Suman Ghosh Date: Thu, 13 Feb 2025 11:01:39 +0530 Subject: [PATCH 02/16] octeontx2-pf: Reconfigure RSS table after enabling AF_XDP zerocopy on rx queue RSS table needs to be reconfigured once a rx queue is enabled or disabled for AF_XDP zerocopy support. After enabling UMEM on a rx queue, that queue should not be part of RSS queue selection algorithm. Similarly the queue should be considered again after UMEM is disabled. Signed-off-by: Suman Ghosh Signed-off-by: Paolo Abeni --- drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c | 4 ++++ drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c | 6 +++++- drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c index 92b0dba07853..6f7c11d59942 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c @@ -331,6 +331,10 @@ int otx2_set_rss_table(struct otx2_nic *pfvf, int ctx_id) rss_ctx = rss->rss_ctx[ctx_id]; /* Get memory to put this msg */ for (idx = 0; idx < rss->rss_size; idx++) { + /* Ignore the queue if AF_XDP zero copy is enabled */ + if (test_bit(rss_ctx->ind_tbl[idx], pfvf->af_xdp_zc_qidx)) + continue; + aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox); if (!aq) { /* The shared memory buffer can be full. diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c index 2d53dc77ef1e..010385b29988 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c @@ -910,8 +910,12 @@ static int otx2_get_rxfh(struct net_device *dev, return -ENOENT; if (indir) { - for (idx = 0; idx < rss->rss_size; idx++) + for (idx = 0; idx < rss->rss_size; idx++) { + /* Ignore if the rx queue is AF_XDP zero copy enabled */ + if (test_bit(rss_ctx->ind_tbl[idx], pfvf->af_xdp_zc_qidx)) + continue; indir[idx] = rss_ctx->ind_tbl[idx]; + } } if (rxfh->key) memcpy(rxfh->key, rss->key, sizeof(rss->key)); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c index 894c1e0aea6f..3bdee1603fac 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c @@ -130,6 +130,8 @@ int otx2_xsk_pool_enable(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qi set_bit(qidx, pf->af_xdp_zc_qidx); otx2_clean_up_rq(pf, qidx); + /* Reconfigure RSS table as 'qidx' cannot be part of RSS now */ + otx2_set_rss_table(pf, DEFAULT_RSS_CONTEXT_GROUP); /* Kick start the NAPI context so that receiving will start */ return otx2_xsk_wakeup(pf->netdev, qidx, XDP_WAKEUP_RX); } @@ -146,6 +148,8 @@ int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qidx) otx2_clean_up_rq(pf, qidx); clear_bit(qidx, pf->af_xdp_zc_qidx); xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); + /* Reconfigure RSS table as 'qidx' now need to be part of RSS now */ + otx2_set_rss_table(pf, DEFAULT_RSS_CONTEXT_GROUP); return 0; } -- 2.50.1 From c5c2398eb88b314918a30ce4e2b0e088c3741f33 Mon Sep 17 00:00:00 2001 From: Suman Ghosh Date: Thu, 13 Feb 2025 11:01:40 +0530 Subject: [PATCH 03/16] octeontx2-pf: Prepare for AF_XDP Implement necessary APIs required for AF_XDP transmit. Signed-off-by: Hariprasad Kelam Signed-off-by: Suman Ghosh Signed-off-by: Paolo Abeni --- .../marvell/octeontx2/nic/otx2_common.h | 1 + .../marvell/octeontx2/nic/otx2_txrx.c | 25 +++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h index 60508971b62f..19e9e2e72233 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h @@ -1181,4 +1181,5 @@ static inline int mcam_entry_cmp(const void *a, const void *b) dma_addr_t otx2_dma_map_skb_frag(struct otx2_nic *pfvf, struct sk_buff *skb, int seg, int *len); void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg); +int otx2_read_free_sqe(struct otx2_nic *pfvf, u16 qidx); #endif /* OTX2_COMMON_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c index 00b6903ba250..9a6c1f1a3ee0 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c @@ -30,6 +30,12 @@ DEFINE_STATIC_KEY_FALSE(cn10k_ipsec_sa_enabled); +static int otx2_get_free_sqe(struct otx2_snd_queue *sq) +{ + return (sq->cons_head - sq->head - 1 + sq->sqe_cnt) + & (sq->sqe_cnt - 1); +} + static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf, struct bpf_prog *prog, struct nix_cqe_rx_s *cqe, @@ -1157,7 +1163,7 @@ bool otx2_sq_append_skb(void *dev, struct netdev_queue *txq, /* Check if there is enough room between producer * and consumer index. */ - free_desc = (sq->cons_head - sq->head - 1 + sq->sqe_cnt) & (sq->sqe_cnt - 1); + free_desc = otx2_get_free_sqe(sq); if (free_desc < sq->sqe_thresh) return false; @@ -1396,6 +1402,21 @@ static void otx2_xdp_sqe_add_sg(struct otx2_snd_queue *sq, sq->sg[sq->head].skb = (u64)xdpf; } +int otx2_read_free_sqe(struct otx2_nic *pfvf, u16 qidx) +{ + struct otx2_snd_queue *sq; + int free_sqe; + + sq = &pfvf->qset.sq[qidx]; + free_sqe = otx2_get_free_sqe(sq); + if (free_sqe < sq->sqe_thresh) { + netdev_warn(pfvf->netdev, "No free sqe for Send queue%d\n", qidx); + return 0; + } + + return free_sqe - sq->sqe_thresh; +} + bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, struct xdp_frame *xdpf, u64 iova, int len, u16 qidx, u16 flags) { @@ -1404,7 +1425,7 @@ bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, struct xdp_frame *xdpf, int offset, free_sqe; sq = &pfvf->qset.sq[qidx]; - free_sqe = (sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb; + free_sqe = otx2_get_free_sqe(sq); if (free_sqe < sq->sqe_thresh) return false; -- 2.50.1 From 53616af09b5a55174e470b456461fa45d8abeb35 Mon Sep 17 00:00:00 2001 From: Suman Ghosh Date: Thu, 13 Feb 2025 11:01:41 +0530 Subject: [PATCH 04/16] octeontx2-pf: AF_XDP zero copy transmit support This patch implements below changes, 1. To avoid concurrency with normal traffic uses XDP queues. 2. Since there are chances that XDP and AF_XDP can fall under same queue uses separate flags to handle dma buffers. Signed-off-by: Hariprasad Kelam Signed-off-by: Suman Ghosh Signed-off-by: Paolo Abeni --- .../marvell/octeontx2/nic/otx2_common.c | 4 ++ .../marvell/octeontx2/nic/otx2_common.h | 6 +++ .../ethernet/marvell/octeontx2/nic/otx2_pf.c | 2 +- .../marvell/octeontx2/nic/otx2_txrx.c | 49 +++++++++++++++---- .../marvell/octeontx2/nic/otx2_txrx.h | 2 + .../ethernet/marvell/octeontx2/nic/otx2_xsk.c | 43 +++++++++++++++- .../ethernet/marvell/octeontx2/nic/otx2_xsk.h | 3 ++ 7 files changed, 96 insertions(+), 13 deletions(-) diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c index 6f7c11d59942..84cd029a85aa 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c @@ -1037,6 +1037,10 @@ int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura) sq->stats.bytes = 0; sq->stats.pkts = 0; + /* Attach XSK_BUFF_POOL to XDP queue */ + if (qidx > pfvf->hw.xdp_queues) + otx2_attach_xsk_buff(pfvf, sq, (qidx - pfvf->hw.xdp_queues)); + chan_offset = qidx % pfvf->hw.tx_chan_cnt; err = pfvf->hw_ops->sq_aq_init(pfvf, qidx, chan_offset, sqb_aura); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h index 19e9e2e72233..1e88422825be 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h @@ -129,6 +129,12 @@ enum otx2_errcodes_re { ERRCODE_IL4_CSUM = 0x22, }; +enum otx2_xdp_action { + OTX2_XDP_TX = BIT(0), + OTX2_XDP_REDIRECT = BIT(1), + OTX2_AF_XDP_FRAME = BIT(2), +}; + struct otx2_dev_stats { u64 rx_bytes; u64 rx_frames; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index 50a42cd5d50a..c7c562f0f5e5 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -2693,7 +2693,7 @@ static int otx2_xdp_xmit_tx(struct otx2_nic *pf, struct xdp_frame *xdpf, return -ENOMEM; err = otx2_xdp_sq_append_pkt(pf, xdpf, dma_addr, xdpf->len, - qidx, XDP_REDIRECT); + qidx, OTX2_XDP_REDIRECT); if (!err) { otx2_dma_unmap_page(pf, dma_addr, xdpf->len, DMA_TO_DEVICE); xdp_return_frame(xdpf); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c index 9a6c1f1a3ee0..6bc5ce5a9f61 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c @@ -20,6 +20,7 @@ #include "otx2_txrx.h" #include "otx2_ptp.h" #include "cn10k.h" +#include "otx2_xsk.h" #define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx))) #define PTP_PORT 0x13F @@ -103,13 +104,19 @@ static unsigned int frag_num(unsigned int i) static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, - struct nix_cqe_tx_s *cqe) + struct nix_cqe_tx_s *cqe, + int *xsk_frames) { struct nix_send_comp_s *snd_comp = &cqe->comp; struct sg_list *sg; sg = &sq->sg[snd_comp->sqe_id]; - if (sg->flags & XDP_REDIRECT) + if (sg->flags & OTX2_AF_XDP_FRAME) { + (*xsk_frames)++; + return; + } + + if (sg->flags & OTX2_XDP_REDIRECT) otx2_dma_unmap_page(pfvf, sg->dma_addr[0], sg->size[0], DMA_TO_DEVICE); xdp_return_frame((struct xdp_frame *)sg->skb); sg->skb = (u64)NULL; @@ -434,6 +441,18 @@ int otx2_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq) return cnt - cq->pool_ptrs; } +static void otx2_zc_submit_pkts(struct otx2_nic *pfvf, struct xsk_buff_pool *xsk_pool, + int *xsk_frames, int qidx, int budget) +{ + if (*xsk_frames) + xsk_tx_completed(xsk_pool, *xsk_frames); + + if (xsk_uses_need_wakeup(xsk_pool)) + xsk_set_tx_need_wakeup(xsk_pool); + + otx2_zc_napi_handler(pfvf, xsk_pool, qidx, budget); +} + static int otx2_tx_napi_handler(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, int budget) { @@ -442,16 +461,22 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf, struct nix_cqe_tx_s *cqe; struct net_device *ndev; int processed_cqe = 0; + int xsk_frames = 0; + + qidx = cq->cq_idx - pfvf->hw.rx_queues; + sq = &pfvf->qset.sq[qidx]; if (cq->pend_cqe >= budget) goto process_cqe; - if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe) + if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe) { + if (sq->xsk_pool) + otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames, + qidx, budget); return 0; + } process_cqe: - qidx = cq->cq_idx - pfvf->hw.rx_queues; - sq = &pfvf->qset.sq[qidx]; while (likely(processed_cqe < budget) && cq->pend_cqe) { cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq); @@ -461,10 +486,8 @@ process_cqe: break; } - qidx = cq->cq_idx - pfvf->hw.rx_queues; - if (cq->cq_type == CQ_XDP) - otx2_xdp_snd_pkt_handler(pfvf, sq, cqe); + otx2_xdp_snd_pkt_handler(pfvf, sq, cqe, &xsk_frames); else otx2_snd_pkt_handler(pfvf, cq, &pfvf->qset.sq[qidx], cqe, budget, &tx_pkts, &tx_bytes); @@ -505,6 +528,10 @@ process_cqe: netif_carrier_ok(ndev)) netif_tx_wake_queue(txq); } + + if (sq->xsk_pool) + otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames, qidx, budget); + return 0; } @@ -1499,8 +1526,10 @@ handle_xdp_verdict: qidx += pfvf->hw.tx_queues; cq->pool_ptrs++; xdpf = xdp_convert_buff_to_frame(&xdp); - return otx2_xdp_sq_append_pkt(pfvf, xdpf, cqe->sg.seg_addr, - cqe->sg.seg_size, qidx, XDP_TX); + return otx2_xdp_sq_append_pkt(pfvf, xdpf, + cqe->sg.seg_addr, + cqe->sg.seg_size, + qidx, OTX2_XDP_TX); case XDP_REDIRECT: cq->pool_ptrs++; if (xsk_buff) { diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h index 8f346fbc8221..acf259d72008 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h @@ -106,6 +106,8 @@ struct otx2_snd_queue { /* SQE ring and CPT response queue for Inline IPSEC */ struct qmem *sqe_ring; struct qmem *cpt_resp; + /* Buffer pool for af_xdp zero-copy */ + struct xsk_buff_pool *xsk_pool; } ____cacheline_aligned_in_smp; enum cq_type { diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c index 3bdee1603fac..ce10caea8511 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c @@ -140,11 +140,14 @@ int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qidx) { struct net_device *netdev = pf->netdev; struct xsk_buff_pool *pool; + struct otx2_snd_queue *sq; pool = xsk_get_pool_from_qid(netdev, qidx); if (!pool) return -EINVAL; + sq = &pf->qset.sq[qidx + pf->hw.tx_queues]; + sq->xsk_pool = NULL; otx2_clean_up_rq(pf, qidx); clear_bit(qidx, pf->af_xdp_zc_qidx); xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); @@ -171,7 +174,7 @@ int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) if (pf->flags & OTX2_FLAG_INTF_DOWN) return -ENETDOWN; - if (queue_id >= pf->hw.rx_queues) + if (queue_id >= pf->hw.rx_queues || queue_id >= pf->hw.tx_queues) return -EINVAL; cq_poll = &qset->napi[queue_id]; @@ -179,8 +182,44 @@ int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) return -EINVAL; /* Trigger interrupt */ - if (!napi_if_scheduled_mark_missed(&cq_poll->napi)) + if (!napi_if_scheduled_mark_missed(&cq_poll->napi)) { otx2_write64(pf, NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx), BIT_ULL(0)); + otx2_write64(pf, NIX_LF_CINTX_INT_W1S(cq_poll->cint_idx), BIT_ULL(0)); + } return 0; } + +void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx) +{ + if (test_bit(qidx, pfvf->af_xdp_zc_qidx)) + sq->xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, qidx); +} + +void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool *pool, + int queue, int budget) +{ + struct xdp_desc *xdp_desc = pool->tx_descs; + int err, i, work_done = 0, batch; + + budget = min(budget, otx2_read_free_sqe(pfvf, queue)); + batch = xsk_tx_peek_release_desc_batch(pool, budget); + if (!batch) + return; + + for (i = 0; i < batch; i++) { + dma_addr_t dma_addr; + + dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc[i].addr); + err = otx2_xdp_sq_append_pkt(pfvf, NULL, dma_addr, xdp_desc[i].len, + queue, OTX2_AF_XDP_FRAME); + if (!err) { + netdev_err(pfvf->netdev, "AF_XDP: Unable to transfer packet err%d\n", err); + break; + } + work_done++; + } + + if (work_done) + xsk_tx_release(pool); +} diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h index 022b3433edbb..8047fafee8fe 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h @@ -17,5 +17,8 @@ int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qid); int otx2_xsk_pool_alloc_buf(struct otx2_nic *pfvf, struct otx2_pool *pool, dma_addr_t *dma, int idx); int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags); +void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool *pool, + int queue, int budget); +void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx); #endif /* OTX2_XSK_H */ -- 2.50.1 From 43130d02baa137033c25297aaae95fd0edc41654 Mon Sep 17 00:00:00 2001 From: Jason Xing Date: Fri, 14 Feb 2025 14:42:50 +0800 Subject: [PATCH 05/16] page_pool: avoid infinite loop to schedule delayed worker We noticed the kworker in page_pool_release_retry() was waken up repeatedly and infinitely in production because of the buggy driver causing the inflight less than 0 and warning us in page_pool_inflight()[1]. Since the inflight value goes negative, it means we should not expect the whole page_pool to get back to work normally. This patch mitigates the adverse effect by not rescheduling the kworker when detecting the inflight negative in page_pool_release_retry(). [1] [Mon Feb 10 20:36:11 2025] ------------[ cut here ]------------ [Mon Feb 10 20:36:11 2025] Negative(-51446) inflight packet-pages ... [Mon Feb 10 20:36:11 2025] Call Trace: [Mon Feb 10 20:36:11 2025] page_pool_release_retry+0x23/0x70 [Mon Feb 10 20:36:11 2025] process_one_work+0x1b1/0x370 [Mon Feb 10 20:36:11 2025] worker_thread+0x37/0x3a0 [Mon Feb 10 20:36:11 2025] kthread+0x11a/0x140 [Mon Feb 10 20:36:11 2025] ? process_one_work+0x370/0x370 [Mon Feb 10 20:36:11 2025] ? __kthread_cancel_work+0x40/0x40 [Mon Feb 10 20:36:11 2025] ret_from_fork+0x35/0x40 [Mon Feb 10 20:36:11 2025] ---[ end trace ebffe800f33e7e34 ]--- Note: before this patch, the above calltrace would flood the dmesg due to repeated reschedule of release_dw kworker. Signed-off-by: Jason Xing Reviewed-by: Mina Almasry Link: https://patch.msgid.link/20250214064250.85987-1-kerneljasonxing@gmail.com Signed-off-by: Paolo Abeni --- net/core/page_pool.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/net/core/page_pool.c b/net/core/page_pool.c index 1c6fec08bc43..acef1fcd8ddc 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -1112,7 +1112,13 @@ static void page_pool_release_retry(struct work_struct *wq) int inflight; inflight = page_pool_release(pool); - if (!inflight) + /* In rare cases, a driver bug may cause inflight to go negative. + * Don't reschedule release if inflight is 0 or negative. + * - If 0, the page_pool has been destroyed + * - if negative, we will never recover + * in both cases no reschedule is necessary. + */ + if (inflight <= 0) return; /* Periodic warning for page pools the user can't see */ -- 2.50.1 From 5afb1596b90c0055bad56c97a4ed3a92a0b9595a Mon Sep 17 00:00:00 2001 From: Petr Machata Date: Fri, 14 Feb 2025 17:18:20 +0100 Subject: [PATCH 06/16] vxlan: Drop 'changelink' parameter from vxlan_dev_configure() vxlan_dev_configure() only has a single caller that passes false for the changelink parameter. Drop the parameter and inline the sole value. Signed-off-by: Petr Machata Reviewed-by: Ido Schimmel Reviewed-by: Nikolay Aleksandrov Signed-off-by: Paolo Abeni --- drivers/net/vxlan/vxlan_core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index ac5bc78edfd7..9d3919265a52 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -3936,7 +3936,7 @@ static void vxlan_config_apply(struct net_device *dev, } static int vxlan_dev_configure(struct net *src_net, struct net_device *dev, - struct vxlan_config *conf, bool changelink, + struct vxlan_config *conf, struct netlink_ext_ack *extack) { struct vxlan_dev *vxlan = netdev_priv(dev); @@ -3947,7 +3947,7 @@ static int vxlan_dev_configure(struct net *src_net, struct net_device *dev, if (ret) return ret; - vxlan_config_apply(dev, conf, lowerdev, src_net, changelink); + vxlan_config_apply(dev, conf, lowerdev, src_net, false); return 0; } @@ -3965,7 +3965,7 @@ static int __vxlan_dev_create(struct net *net, struct net_device *dev, int err; dst = &vxlan->default_dst; - err = vxlan_dev_configure(net, dev, conf, false, extack); + err = vxlan_dev_configure(net, dev, conf, extack); if (err) return err; -- 2.50.1 From d42d543368343c0449a4e433b5f02e063a86209c Mon Sep 17 00:00:00 2001 From: Petr Machata Date: Fri, 14 Feb 2025 17:18:21 +0100 Subject: [PATCH 07/16] vxlan: Join / leave MC group after remote changes When a vxlan netdevice is brought up, if its default remote is a multicast address, the device joins the indicated group. Therefore when the multicast remote address changes, the device should leave the current group and subscribe to the new one. Similarly when the interface used for endpoint communication is changed in a situation when multicast remote is configured. This is currently not done. Both vxlan_igmp_join() and vxlan_igmp_leave() can however fail. So it is possible that with such fix, the netdevice will end up in an inconsistent situation where the old group is not joined anymore, but joining the new group fails. Should we join the new group first, and leave the old one second, we might end up in the opposite situation, where both groups are joined. Undoing any of this during rollback is going to be similarly problematic. One solution would be to just forbid the change when the netdevice is up. However in vnifilter mode, changing the group address is allowed, and these problems are simply ignored (see vxlan_vni_update_group()): # ip link add name br up type bridge vlan_filtering 1 # ip link add vx1 up master br type vxlan external vnifilter local 192.0.2.1 dev lo dstport 4789 # bridge vni add dev vx1 vni 200 group 224.0.0.1 # tcpdump -i lo & # bridge vni add dev vx1 vni 200 group 224.0.0.2 18:55:46.523438 IP 0.0.0.0 > 224.0.0.22: igmp v3 report, 1 group record(s) 18:55:46.943447 IP 0.0.0.0 > 224.0.0.22: igmp v3 report, 1 group record(s) # bridge vni dev vni group/remote vx1 200 224.0.0.2 Having two different modes of operation for conceptually the same interface is silly, so in this patch, just do what the vnifilter code does and deal with the errors by crossing fingers real hard. The vnifilter code leaves old before joining new, and in case of join / leave failures does not roll back the configuration changes that have already been applied, but bails out of joining if it could not leave. Do the same here: leave before join, apply changes unconditionally and do not attempt to join if we couldn't leave. Signed-off-by: Petr Machata Reviewed-by: Ido Schimmel Reviewed-by: Nikolay Aleksandrov Signed-off-by: Paolo Abeni --- drivers/net/vxlan/vxlan_core.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index 9d3919265a52..e2354c02def0 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -4419,6 +4419,7 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[], struct netlink_ext_ack *extack) { struct vxlan_dev *vxlan = netdev_priv(dev); + bool rem_ip_changed, change_igmp; struct net_device *lowerdev; struct vxlan_config conf; struct vxlan_rdst *dst; @@ -4442,8 +4443,13 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[], if (err) return err; + rem_ip_changed = !vxlan_addr_equal(&conf.remote_ip, &dst->remote_ip); + change_igmp = vxlan->dev->flags & IFF_UP && + (rem_ip_changed || + dst->remote_ifindex != conf.remote_ifindex); + /* handle default dst entry */ - if (!vxlan_addr_equal(&conf.remote_ip, &dst->remote_ip)) { + if (rem_ip_changed) { u32 hash_index = fdb_head_index(vxlan, all_zeros_mac, conf.vni); spin_lock_bh(&vxlan->hash_lock[hash_index]); @@ -4487,6 +4493,9 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[], } } + if (change_igmp && vxlan_addr_multicast(&dst->remote_ip)) + err = vxlan_multicast_leave(vxlan); + if (conf.age_interval != vxlan->cfg.age_interval) mod_timer(&vxlan->age_timer, jiffies); @@ -4494,7 +4503,12 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[], if (lowerdev && lowerdev != dst->remote_dev) dst->remote_dev = lowerdev; vxlan_config_apply(dev, &conf, lowerdev, vxlan->net, true); - return 0; + + if (!err && change_igmp && + vxlan_addr_multicast(&dst->remote_ip)) + err = vxlan_multicast_join(vxlan); + + return err; } static void vxlan_dellink(struct net_device *dev, struct list_head *head) -- 2.50.1 From f802f172d78b8b96dcd1832dc7c8da8ee9dcbc29 Mon Sep 17 00:00:00 2001 From: Petr Machata Date: Fri, 14 Feb 2025 17:18:22 +0100 Subject: [PATCH 08/16] selftests: forwarding: lib: Move require_command to net, generalize This helper could be useful to more than just forwarding tests. Move it upstairs and port over to log_test_skip(). Split the function into two parts: the bit that actually checks and reports skip, which is in a new function check_command(). And a bit that exits the test script if the check fails. This allows users consistent checking behavior while giving an option to bail out from a single test without bailing out of the whole script. Signed-off-by: Petr Machata Reviewed-by: Ido Schimmel Reviewed-by: Nikolay Aleksandrov Signed-off-by: Paolo Abeni --- tools/testing/selftests/net/forwarding/lib.sh | 10 ---------- tools/testing/selftests/net/lib.sh | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/tools/testing/selftests/net/forwarding/lib.sh b/tools/testing/selftests/net/forwarding/lib.sh index 8de80acf249e..508f3c700d71 100644 --- a/tools/testing/selftests/net/forwarding/lib.sh +++ b/tools/testing/selftests/net/forwarding/lib.sh @@ -291,16 +291,6 @@ if [[ "$CHECK_TC" = "yes" ]]; then check_tc_version fi -require_command() -{ - local cmd=$1; shift - - if [[ ! -x "$(command -v "$cmd")" ]]; then - echo "SKIP: $cmd not installed" - exit $ksft_skip - fi -} - # IPv6 support was added in v3.0 check_mtools_version() { diff --git a/tools/testing/selftests/net/lib.sh b/tools/testing/selftests/net/lib.sh index 0bd9a038a1f0..975be4fdbcdb 100644 --- a/tools/testing/selftests/net/lib.sh +++ b/tools/testing/selftests/net/lib.sh @@ -450,6 +450,25 @@ kill_process() { kill $pid && wait $pid; } 2>/dev/null } +check_command() +{ + local cmd=$1; shift + + if [[ ! -x "$(command -v "$cmd")" ]]; then + log_test_skip "$cmd not installed" + return $EXIT_STATUS + fi +} + +require_command() +{ + local cmd=$1; shift + + if ! check_command "$cmd"; then + exit $EXIT_STATUS + fi +} + ip_link_add() { local name=$1; shift -- 2.50.1 From 24adf47ea9acfb15d425e5b363c3f25dc6622a45 Mon Sep 17 00:00:00 2001 From: Petr Machata Date: Fri, 14 Feb 2025 17:18:23 +0100 Subject: [PATCH 09/16] selftests: test_vxlan_fdb_changelink: Convert to lib.sh Instead of inlining equivalents, use lib.sh-provided primitives. Use defer to manage vx lifetime. This will make it easier to extend the test in the next patch. Signed-off-by: Petr Machata Reviewed-by: Ido Schimmel Reviewed-by: Nikolay Aleksandrov Signed-off-by: Paolo Abeni --- .../net/test_vxlan_fdb_changelink.sh | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/tools/testing/selftests/net/test_vxlan_fdb_changelink.sh b/tools/testing/selftests/net/test_vxlan_fdb_changelink.sh index 2d442cdab11e..6f2bca4b346c 100755 --- a/tools/testing/selftests/net/test_vxlan_fdb_changelink.sh +++ b/tools/testing/selftests/net/test_vxlan_fdb_changelink.sh @@ -1,29 +1,38 @@ #!/bin/bash # SPDX-License-Identifier: GPL-2.0 -# Check FDB default-remote handling across "ip link set". +ALL_TESTS=" + test_set_remote +" +source lib.sh check_remotes() { local what=$1; shift local N=$(bridge fdb sh dev vx | grep 00:00:00:00:00:00 | wc -l) - echo -ne "expected two remotes after $what\t" - if [[ $N != 2 ]]; then - echo "[FAIL]" - EXIT_STATUS=1 - else - echo "[ OK ]" - fi + ((N == 2)) + check_err $? "expected 2 remotes after $what, got $N" +} + +# Check FDB default-remote handling across "ip link set". +test_set_remote() +{ + RET=0 + + ip_link_add vx up type vxlan id 2000 dstport 4789 + bridge fdb ap dev vx 00:00:00:00:00:00 dst 192.0.2.20 self permanent + bridge fdb ap dev vx 00:00:00:00:00:00 dst 192.0.2.30 self permanent + check_remotes "fdb append" + + ip link set dev vx type vxlan remote 192.0.2.30 + check_remotes "link set" + + log_test 'FDB default-remote handling across "ip link set"' } -ip link add name vx up type vxlan id 2000 dstport 4789 -bridge fdb ap dev vx 00:00:00:00:00:00 dst 192.0.2.20 self permanent -bridge fdb ap dev vx 00:00:00:00:00:00 dst 192.0.2.30 self permanent -check_remotes "fdb append" +trap defer_scopes_cleanup EXIT -ip link set dev vx type vxlan remote 192.0.2.30 -check_remotes "link set" +tests_run -ip link del dev vx exit $EXIT_STATUS -- 2.50.1 From eae1e92a1d414c4485c865c9f4e430b398cb8e8a Mon Sep 17 00:00:00 2001 From: Petr Machata Date: Fri, 14 Feb 2025 17:18:24 +0100 Subject: [PATCH 10/16] selftests: test_vxlan_fdb_changelink: Add a test for MC remote change Changes to MC remote need to be reflected in actual group memberships. Add a test to verify that it is the case. Signed-off-by: Petr Machata Reviewed-by: Ido Schimmel Reviewed-by: Nikolay Aleksandrov Signed-off-by: Paolo Abeni --- .../net/test_vxlan_fdb_changelink.sh | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/tools/testing/selftests/net/test_vxlan_fdb_changelink.sh b/tools/testing/selftests/net/test_vxlan_fdb_changelink.sh index 6f2bca4b346c..062f957950af 100755 --- a/tools/testing/selftests/net/test_vxlan_fdb_changelink.sh +++ b/tools/testing/selftests/net/test_vxlan_fdb_changelink.sh @@ -3,6 +3,7 @@ ALL_TESTS=" test_set_remote + test_change_mc_remote " source lib.sh @@ -31,6 +32,81 @@ test_set_remote() log_test 'FDB default-remote handling across "ip link set"' } +fmt_remote() +{ + local addr=$1; shift + + if [[ $addr == 224.* ]]; then + echo "group $addr" + else + echo "remote $addr" + fi +} + +change_remote() +{ + local remote=$1; shift + + ip link set dev vx type vxlan $(fmt_remote $remote) dev v1 +} + +check_membership() +{ + local check_vec=("$@") + + local memberships + memberships=$( + netstat -n --groups | + sed -n '/^v1\b/p' | + grep -o '[^ ]*$' + ) + check_err $? "Couldn't obtain group memberships" + + local item + for item in "${check_vec[@]}"; do + eval "local $item" + echo "$memberships" | grep -q "\b$group\b" + check_err_fail $fail $? "$group is_ex reported in IGMP query response" + done +} + +test_change_mc_remote() +{ + check_command netstat || return + + ip_link_add v1 up type veth peer name v2 + ip_link_set_up v2 + + RET=0 + + ip_link_add vx up type vxlan dstport 4789 \ + local 192.0.2.1 $(fmt_remote 224.1.1.1) dev v1 vni 1000 + + check_membership "group=224.1.1.1 fail=0" \ + "group=224.1.1.2 fail=1" \ + "group=224.1.1.3 fail=1" + + log_test "MC group report after VXLAN creation" + + RET=0 + + change_remote 224.1.1.2 + check_membership "group=224.1.1.1 fail=1" \ + "group=224.1.1.2 fail=0" \ + "group=224.1.1.3 fail=1" + + log_test "MC group report after changing VXLAN remote MC->MC" + + RET=0 + + change_remote 192.0.2.2 + check_membership "group=224.1.1.1 fail=1" \ + "group=224.1.1.2 fail=1" \ + "group=224.1.1.3 fail=1" + + log_test "MC group report after changing VXLAN remote MC->UC" +} + trap defer_scopes_cleanup EXIT tests_run -- 2.50.1 From 8dcaed624f6af020023a09f7d14decc7ec730355 Mon Sep 17 00:00:00 2001 From: Dimitri Fedrau Date: Fri, 14 Feb 2025 17:32:03 +0100 Subject: [PATCH 11/16] net: phy: marvell-88q2xxx: align defines MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Align some defines. Signed-off-by: Dimitri Fedrau Reviewed-by: Niklas Söderlund Signed-off-by: Paolo Abeni --- drivers/net/phy/marvell-88q2xxx.c | 62 +++++++++++++++---------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/drivers/net/phy/marvell-88q2xxx.c b/drivers/net/phy/marvell-88q2xxx.c index bad5e7b2357d..6e95de080bc6 100644 --- a/drivers/net/phy/marvell-88q2xxx.c +++ b/drivers/net/phy/marvell-88q2xxx.c @@ -12,29 +12,29 @@ #include #include -#define PHY_ID_88Q2220_REVB0 (MARVELL_PHY_ID_88Q2220 | 0x1) -#define PHY_ID_88Q2220_REVB1 (MARVELL_PHY_ID_88Q2220 | 0x2) -#define PHY_ID_88Q2220_REVB2 (MARVELL_PHY_ID_88Q2220 | 0x3) - -#define MDIO_MMD_AN_MV_STAT 32769 -#define MDIO_MMD_AN_MV_STAT_ANEG 0x0100 -#define MDIO_MMD_AN_MV_STAT_LOCAL_RX 0x1000 -#define MDIO_MMD_AN_MV_STAT_REMOTE_RX 0x2000 -#define MDIO_MMD_AN_MV_STAT_LOCAL_MASTER 0x4000 -#define MDIO_MMD_AN_MV_STAT_MS_CONF_FAULT 0x8000 - -#define MDIO_MMD_AN_MV_STAT2 32794 -#define MDIO_MMD_AN_MV_STAT2_AN_RESOLVED 0x0800 -#define MDIO_MMD_AN_MV_STAT2_100BT1 0x2000 -#define MDIO_MMD_AN_MV_STAT2_1000BT1 0x4000 - -#define MDIO_MMD_PCS_MV_RESET_CTRL 32768 -#define MDIO_MMD_PCS_MV_RESET_CTRL_TX_DISABLE 0x8 - -#define MDIO_MMD_PCS_MV_INT_EN 32784 -#define MDIO_MMD_PCS_MV_INT_EN_LINK_UP 0x0040 -#define MDIO_MMD_PCS_MV_INT_EN_LINK_DOWN 0x0080 -#define MDIO_MMD_PCS_MV_INT_EN_100BT1 0x1000 +#define PHY_ID_88Q2220_REVB0 (MARVELL_PHY_ID_88Q2220 | 0x1) +#define PHY_ID_88Q2220_REVB1 (MARVELL_PHY_ID_88Q2220 | 0x2) +#define PHY_ID_88Q2220_REVB2 (MARVELL_PHY_ID_88Q2220 | 0x3) + +#define MDIO_MMD_AN_MV_STAT 32769 +#define MDIO_MMD_AN_MV_STAT_ANEG 0x0100 +#define MDIO_MMD_AN_MV_STAT_LOCAL_RX 0x1000 +#define MDIO_MMD_AN_MV_STAT_REMOTE_RX 0x2000 +#define MDIO_MMD_AN_MV_STAT_LOCAL_MASTER 0x4000 +#define MDIO_MMD_AN_MV_STAT_MS_CONF_FAULT 0x8000 + +#define MDIO_MMD_AN_MV_STAT2 32794 +#define MDIO_MMD_AN_MV_STAT2_AN_RESOLVED 0x0800 +#define MDIO_MMD_AN_MV_STAT2_100BT1 0x2000 +#define MDIO_MMD_AN_MV_STAT2_1000BT1 0x4000 + +#define MDIO_MMD_PCS_MV_RESET_CTRL 32768 +#define MDIO_MMD_PCS_MV_RESET_CTRL_TX_DISABLE 0x8 + +#define MDIO_MMD_PCS_MV_INT_EN 32784 +#define MDIO_MMD_PCS_MV_INT_EN_LINK_UP 0x0040 +#define MDIO_MMD_PCS_MV_INT_EN_LINK_DOWN 0x0080 +#define MDIO_MMD_PCS_MV_INT_EN_100BT1 0x1000 #define MDIO_MMD_PCS_MV_GPIO_INT_STAT 32785 #define MDIO_MMD_PCS_MV_GPIO_INT_STAT_LINK_UP 0x0040 @@ -80,11 +80,11 @@ #define MDIO_MMD_PCS_MV_100BT1_STAT1_REMOTE_RX 0x2000 #define MDIO_MMD_PCS_MV_100BT1_STAT1_LOCAL_MASTER 0x4000 -#define MDIO_MMD_PCS_MV_100BT1_STAT2 33033 -#define MDIO_MMD_PCS_MV_100BT1_STAT2_JABBER 0x0001 -#define MDIO_MMD_PCS_MV_100BT1_STAT2_POL 0x0002 -#define MDIO_MMD_PCS_MV_100BT1_STAT2_LINK 0x0004 -#define MDIO_MMD_PCS_MV_100BT1_STAT2_ANGE 0x0008 +#define MDIO_MMD_PCS_MV_100BT1_STAT2 33033 +#define MDIO_MMD_PCS_MV_100BT1_STAT2_JABBER 0x0001 +#define MDIO_MMD_PCS_MV_100BT1_STAT2_POL 0x0002 +#define MDIO_MMD_PCS_MV_100BT1_STAT2_LINK 0x0004 +#define MDIO_MMD_PCS_MV_100BT1_STAT2_ANGE 0x0008 #define MDIO_MMD_PCS_MV_100BT1_INT_EN 33042 #define MDIO_MMD_PCS_MV_100BT1_INT_EN_LINKEVENT 0x0400 @@ -92,7 +92,7 @@ #define MDIO_MMD_PCS_MV_COPPER_INT_STAT 33043 #define MDIO_MMD_PCS_MV_COPPER_INT_STAT_LINKEVENT 0x0400 -#define MDIO_MMD_PCS_MV_RX_STAT 33328 +#define MDIO_MMD_PCS_MV_RX_STAT 33328 #define MDIO_MMD_PCS_MV_TDR_RESET 65226 #define MDIO_MMD_PCS_MV_TDR_RESET_TDR_RST 0x1000 @@ -115,8 +115,8 @@ #define MDIO_MMD_PCS_MV_TDR_OFF_CUTOFF 65246 -#define MV88Q2XXX_LED_INDEX_TX_ENABLE 0 -#define MV88Q2XXX_LED_INDEX_GPIO 1 +#define MV88Q2XXX_LED_INDEX_TX_ENABLE 0 +#define MV88Q2XXX_LED_INDEX_GPIO 1 struct mv88q2xxx_priv { bool enable_temp; -- 2.50.1 From cbe0449e8f9f306a022e99b051e263769ba6e599 Mon Sep 17 00:00:00 2001 From: Dimitri Fedrau Date: Fri, 14 Feb 2025 17:32:04 +0100 Subject: [PATCH 12/16] net: phy: marvell-88q2xxx: order includes alphabetically MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Order includes alphabetically. Signed-off-by: Dimitri Fedrau Reviewed-by: Niklas Söderlund Signed-off-by: Paolo Abeni --- drivers/net/phy/marvell-88q2xxx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/phy/marvell-88q2xxx.c b/drivers/net/phy/marvell-88q2xxx.c index 6e95de080bc6..7b0913968bb4 100644 --- a/drivers/net/phy/marvell-88q2xxx.c +++ b/drivers/net/phy/marvell-88q2xxx.c @@ -7,10 +7,10 @@ * Copyright (C) 2024 Liebherr-Electronics and Drives GmbH */ #include +#include #include #include #include -#include #define PHY_ID_88Q2220_REVB0 (MARVELL_PHY_ID_88Q2220 | 0x1) #define PHY_ID_88Q2220_REVB1 (MARVELL_PHY_ID_88Q2220 | 0x2) -- 2.50.1 From 6c806720bafe3ae9c43eb2dc942a1849d5453736 Mon Sep 17 00:00:00 2001 From: Dimitri Fedrau Date: Fri, 14 Feb 2025 17:32:05 +0100 Subject: [PATCH 13/16] net: phy: marvell-88q2xxx: enable temperature sensor in mv88q2xxx_config_init MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Temperature sensor gets enabled for 88Q222X devices in mv88q222x_config_init. Move enabling to mv88q2xxx_config_init because all 88Q2XXX devices support the temperature sensor. Signed-off-by: Dimitri Fedrau Tested-by: Niklas Söderlund Reviewed-by: Andrew Lunn Signed-off-by: Paolo Abeni --- drivers/net/phy/marvell-88q2xxx.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/drivers/net/phy/marvell-88q2xxx.c b/drivers/net/phy/marvell-88q2xxx.c index 7b0913968bb4..1859db10b391 100644 --- a/drivers/net/phy/marvell-88q2xxx.c +++ b/drivers/net/phy/marvell-88q2xxx.c @@ -513,6 +513,15 @@ static int mv88q2xxx_config_init(struct phy_device *phydev) return ret; } + /* Enable temperature sense */ + if (priv->enable_temp) { + ret = phy_modify_mmd(phydev, MDIO_MMD_PCS, + MDIO_MMD_PCS_MV_TEMP_SENSOR2, + MDIO_MMD_PCS_MV_TEMP_SENSOR2_DIS_MASK, 0); + if (ret < 0) + return ret; + } + return 0; } @@ -903,18 +912,6 @@ static int mv88q222x_revb1_revb2_config_init(struct phy_device *phydev) static int mv88q222x_config_init(struct phy_device *phydev) { - struct mv88q2xxx_priv *priv = phydev->priv; - int ret; - - /* Enable temperature sense */ - if (priv->enable_temp) { - ret = phy_modify_mmd(phydev, MDIO_MMD_PCS, - MDIO_MMD_PCS_MV_TEMP_SENSOR2, - MDIO_MMD_PCS_MV_TEMP_SENSOR2_DIS_MASK, 0); - if (ret < 0) - return ret; - } - if (phydev->c45_ids.device_ids[MDIO_MMD_PMAPMD] == PHY_ID_88Q2220_REVB0) return mv88q222x_revb0_config_init(phydev); else -- 2.50.1 From 8e677a466145174e6899ceaf189e7cb7fbd6ad3f Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Fri, 14 Feb 2025 09:07:11 -0800 Subject: [PATCH 14/16] trace: tcp: Add tracepoint for tcp_cwnd_reduction() Add a lightweight tracepoint to monitor TCP congestion window adjustments via tcp_cwnd_reduction(). This tracepoint enables tracking of: - TCP window size fluctuations - Active socket behavior - Congestion window reduction events Meta has been using BPF programs to monitor this function for years. Adding a proper tracepoint provides a stable API for all users who need to monitor TCP congestion window behavior. Use DECLARE_TRACE instead of TRACE_EVENT to avoid creating trace event infrastructure and exporting to tracefs, keeping the implementation minimal. (Thanks Steven Rostedt) Given that this patch creates a rawtracepoint, you could hook into it using regular tooling, like bpftrace, using regular rawtracepoint infrastructure, such as: rawtracepoint:tcp_cwnd_reduction_tp { .... } Signed-off-by: Breno Leitao Reviewed-by: Eric Dumazet Link: https://patch.msgid.link/20250214-cwnd_tracepoint-v2-1-ef8d15162d95@debian.org Signed-off-by: Paolo Abeni --- include/trace/events/tcp.h | 6 ++++++ net/ipv4/tcp_input.c | 2 ++ 2 files changed, 8 insertions(+) diff --git a/include/trace/events/tcp.h b/include/trace/events/tcp.h index a27c4b619dff..1a40c41ff8c3 100644 --- a/include/trace/events/tcp.h +++ b/include/trace/events/tcp.h @@ -259,6 +259,12 @@ TRACE_EVENT(tcp_retransmit_synack, __entry->saddr_v6, __entry->daddr_v6) ); +DECLARE_TRACE(tcp_cwnd_reduction_tp, + TP_PROTO(const struct sock *sk, int newly_acked_sacked, + int newly_lost, int flag), + TP_ARGS(sk, newly_acked_sacked, newly_lost, flag) +); + #include TRACE_EVENT(tcp_probe, diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 074406890552..fbb67a098543 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -2709,6 +2709,8 @@ void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, if (newly_acked_sacked <= 0 || WARN_ON_ONCE(!tp->prior_cwnd)) return; + trace_tcp_cwnd_reduction_tp(sk, newly_acked_sacked, newly_lost, flag); + tp->prr_delivered += newly_acked_sacked; if (delta < 0) { u64 dividend = (u64)tp->snd_ssthresh * tp->prr_delivered + -- 2.50.1 From 4991b88c2514e62bd8410e0a96999ef662765d9d Mon Sep 17 00:00:00 2001 From: =?utf8?q?Niklas=20S=C3=B6derlund?= Date: Fri, 14 Feb 2025 18:46:50 +0100 Subject: [PATCH 15/16] net: phy: marvell-88q2xxx: Init PHY private structure for mv88q211x MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit When adding LED support for mv88q222x devices the PHY private data structure was added to the mv88q211x code path, the data structure is however only allocated during mv88q222x probe. This results in a nullptr deference for mv88q2110 devices. Unable to handle kernel NULL pointer dereference at virtual address 0000000000000001 Mem abort info: ESR = 0x0000000096000004 EC = 0x25: DABT (current EL), IL = 32 bits SET = 0, FnV = 0 EA = 0, S1PTW = 0 FSC = 0x04: level 0 translation fault Data abort info: ISV = 0, ISS = 0x00000004, ISS2 = 0x00000000 CM = 0, WnR = 0, TnD = 0, TagAccess = 0 GCS = 0, Overlay = 0, DirtyBit = 0, Xs = 0 [0000000000000001] user address but active_mm is swapper Internal error: Oops: 0000000096000004 [#1] PREEMPT SMP CPU: 3 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.14.0-rc1-arm64-renesas-00342-ga3783dbf2574 #7 Hardware name: Renesas White Hawk Single board based on r8a779g2 (DT) pstate: 20400005 (nzCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : mv88q2xxx_config_init+0x28/0x84 lr : mv88q2110_config_init+0x98/0xb0 sp : ffff8000823eb9d0 x29: ffff8000823eb9d0 x28: ffff000440942000 x27: ffff80008144e400 x26: 0000000000001002 x25: 0000000000000000 x24: 0000000000000000 x23: 0000000000000009 x22: ffff8000810534f0 x21: ffff800081053550 x20: 0000000000000000 x19: ffff0004437d6800 x18: 0000000000000018 x17: 00000000000961c8 x16: ffff0006bef75ec0 x15: 0000000000000001 x14: 0000000000000001 x13: ffff000440218080 x12: 071c71c71c71c71c x11: ffff000440218080 x10: 0000000000001420 x9 : ffff8000823eb770 x8 : ffff8000823eb650 x7 : ffff8000823eb750 x6 : ffff8000823eb710 x5 : 0000000000000000 x4 : 0000000000000800 x3 : 0000000000000001 x2 : 0000000000000000 x1 : 00000000ffffffff x0 : ffff0004437d6800 Call trace: mv88q2xxx_config_init+0x28/0x84 (P) mv88q2110_config_init+0x98/0xb0 phy_init_hw+0x64/0x9c phy_attach_direct+0x118/0x320 phy_connect_direct+0x24/0x80 of_phy_connect+0x5c/0xa0 rtsn_open+0x5bc/0x78c __dev_open+0xf8/0x1fc __dev_change_flags+0x198/0x220 dev_change_flags+0x20/0x64 ip_auto_config+0x270/0xefc do_one_initcall+0xe4/0x22c kernel_init_freeable+0x2a8/0x308 kernel_init+0x20/0x130 ret_from_fork+0x10/0x20 Code: b907e404 f9432814 3100083f 540000e3 (39400680) ---[ end trace 0000000000000000 ]--- Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b SMP: stopping secondary CPUs Kernel Offset: disabled CPU features: 0x000,00000070,00801250,8200700b Memory Limit: none ---[ end Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b ]--- Fix this by using a generic probe function for both mv88q211x and mv88q222x devices that allocates the PHY private data structure, while only the mv88q222x probes for LED support. Fixes: a3783dbf2574 ("net: phy: marvell-88q2xxx: Add support for PHY LEDs on 88q2xxx") Signed-off-by: Niklas Söderlund Reviewed-by: Andrew Lunn Tested-by: Geert Uytterhoeven Link: https://patch.msgid.link/20250214174650.2056949-1-niklas.soderlund+renesas@ragnatech.se Signed-off-by: Paolo Abeni --- drivers/net/phy/marvell-88q2xxx.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/net/phy/marvell-88q2xxx.c b/drivers/net/phy/marvell-88q2xxx.c index 1859db10b391..15c0f8adc2f5 100644 --- a/drivers/net/phy/marvell-88q2xxx.c +++ b/drivers/net/phy/marvell-88q2xxx.c @@ -834,13 +834,24 @@ static int mv88q2xxx_leds_probe(struct phy_device *phydev) static int mv88q2xxx_probe(struct phy_device *phydev) { struct mv88q2xxx_priv *priv; - int ret; priv = devm_kzalloc(&phydev->mdio.dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; phydev->priv = priv; + + return 0; +} + +static int mv88q222x_probe(struct phy_device *phydev) +{ + int ret; + + ret = mv88q2xxx_probe(phydev); + if (ret) + return ret; + ret = mv88q2xxx_leds_probe(phydev); if (ret) return ret; @@ -1098,6 +1109,7 @@ static struct phy_driver mv88q2xxx_driver[] = { .phy_id = MARVELL_PHY_ID_88Q2110, .phy_id_mask = MARVELL_PHY_ID_MASK, .name = "mv88q2110", + .probe = mv88q2xxx_probe, .get_features = mv88q2xxx_get_features, .config_aneg = mv88q2xxx_config_aneg, .config_init = mv88q2110_config_init, @@ -1112,7 +1124,7 @@ static struct phy_driver mv88q2xxx_driver[] = { .phy_id_mask = MARVELL_PHY_ID_MASK, .name = "mv88q2220", .flags = PHY_POLL_CABLE_TEST, - .probe = mv88q2xxx_probe, + .probe = mv88q222x_probe, .get_features = mv88q2xxx_get_features, .config_aneg = mv88q2xxx_config_aneg, .aneg_done = genphy_c45_aneg_done, -- 2.50.1 From 8533b14b3d65ee666ba31254787c1bdaee56d95a Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 12 Feb 2025 17:06:32 -0800 Subject: [PATCH 16/16] eth: mlx4: create a page pool for Rx Create a pool per rx queue. Subsequent patches will make use of it. Move fcs_del to a hole to make space for the pointer. Per common "wisdom" base the page pool size on the ring size. Note that the page pool cache size is in full pages, so just round up the effective buffer size to pages. Reviewed-by: Tariq Toukan Link: https://patch.msgid.link/20250213010635.1354034-2-kuba@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx4/en_rx.c | 24 +++++++++++++++++++- drivers/net/ethernet/mellanox/mlx4/mlx4_en.h | 3 ++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c index 15c57e9517e9..a8c0cf5d0d08 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c @@ -48,6 +48,7 @@ #if IS_ENABLED(CONFIG_IPV6) #include #endif +#include #include "mlx4_en.h" @@ -268,6 +269,7 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, u32 size, u16 stride, int node, int queue_index) { struct mlx4_en_dev *mdev = priv->mdev; + struct page_pool_params pp = {}; struct mlx4_en_rx_ring *ring; int err = -ENOMEM; int tmp; @@ -286,9 +288,26 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, ring->log_stride = ffs(ring->stride) - 1; ring->buf_size = ring->size * ring->stride + TXBB_SIZE; - if (xdp_rxq_info_reg(&ring->xdp_rxq, priv->dev, queue_index, 0) < 0) + pp.flags = PP_FLAG_DMA_MAP; + pp.pool_size = size * DIV_ROUND_UP(priv->rx_skb_size, PAGE_SIZE); + pp.nid = node; + pp.napi = &priv->rx_cq[queue_index]->napi; + pp.netdev = priv->dev; + pp.dev = &mdev->dev->persist->pdev->dev; + pp.dma_dir = priv->dma_dir; + + ring->pp = page_pool_create(&pp); + if (!ring->pp) goto err_ring; + if (xdp_rxq_info_reg(&ring->xdp_rxq, priv->dev, queue_index, 0) < 0) + goto err_pp; + + err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_PAGE_POOL, + ring->pp); + if (err) + goto err_xdp_info; + tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS * sizeof(struct mlx4_en_rx_alloc)); ring->rx_info = kvzalloc_node(tmp, GFP_KERNEL, node); @@ -319,6 +338,8 @@ err_info: ring->rx_info = NULL; err_xdp_info: xdp_rxq_info_unreg(&ring->xdp_rxq); +err_pp: + page_pool_destroy(ring->pp); err_ring: kfree(ring); *pring = NULL; @@ -445,6 +466,7 @@ void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, xdp_rxq_info_unreg(&ring->xdp_rxq); mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE); kvfree(ring->rx_info); + page_pool_destroy(ring->pp); ring->rx_info = NULL; kfree(ring); *pring = NULL; diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h index 28b70dcc652e..29f48e63081b 100644 --- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h +++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h @@ -335,10 +335,11 @@ struct mlx4_en_rx_ring { u16 stride; u16 log_stride; u16 cqn; /* index of port CQ associated with this ring */ + u8 fcs_del; u32 prod; u32 cons; u32 buf_size; - u8 fcs_del; + struct page_pool *pp; void *buf; void *rx_info; struct bpf_prog __rcu *xdp_prog; -- 2.50.1