xdp: get rid of xdp_frame::mem.id

Initially, xdp_frame::mem.id was used to search for the corresponding
&page_pool to return the page correctly.
However, after that struct page was extended to have a direct pointer
to its PP (netmem has it as well), further keeping of this field makes
no sense. xdp_return_frame_bulk() still used it to do a lookup, and
this leftover is now removed.
Remove xdp_frame::mem and replace it with ::mem_type, as only memory
type still matters and we need to know it to be able to free the frame
correctly.
As a cute side effect, we can now make every scalar field in &xdp_frame
of 4 byte width, speeding up accesses to them.

Reviewed-by: Toke Høiland-Jørgensen <toke@redhat.com>
Signed-off-by: Alexander Lobakin <aleksander.lobakin@intel.com>
Link: https://patch.msgid.link/20241211172649.761483-3-aleksander.lobakin@intel.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
index bf5baef5c3e065a4b22b9050aa1c760de3db3cf4..4948b4906584e099515b1e1fc46428f6f0a56d1b 100644 (file)
--- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
+++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
@@ -2281,7 +2281,7 @@ static int dpaa_a050385_wa_xdpf(struct dpaa_priv *priv,
        new_xdpf->len = xdpf->len;
        new_xdpf->headroom = priv->tx_headroom;
        new_xdpf->frame_sz = DPAA_BP_RAW_SIZE;
-       new_xdpf->mem.type = MEM_TYPE_PAGE_ORDER0;
+       new_xdpf->mem_type = MEM_TYPE_PAGE_ORDER0;
 
        /* Release the initial buffer */
        xdp_return_frame_rx_napi(xdpf);

diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index 07ebb800edf170e6fa662a44e26715fef2308930..01251868a9c27592f0dfbcdc32c0afbd7e9baafc 100644 (file)
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -634,7 +634,7 @@ static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq,
                        break;
                case XDP_TX:
                        orig_frame = *frame;
-                       xdp->rxq->mem = frame->mem;
+                       xdp->rxq->mem.type = frame->mem_type;
                        if (unlikely(veth_xdp_tx(rq, xdp, bq) < 0)) {
                                trace_xdp_exception(rq->dev, xdp_prog, act);
                                frame = &orig_frame;
@@ -646,7 +646,7 @@ static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq,
                        goto xdp_xmit;
                case XDP_REDIRECT:
                        orig_frame = *frame;
-                       xdp->rxq->mem = frame->mem;
+                       xdp->rxq->mem.type = frame->mem_type;
                        if (xdp_do_redirect(rq->dev, xdp, xdp_prog)) {
                                frame = &orig_frame;
                                stats->rx_drops++;

diff --git a/include/net/xdp.h b/include/net/xdp.h
index 9e7eb82235133755d8fabd30de12cecec6acc331..1c260869a353911c3752d8a377182af22daf77fe 100644 (file)
--- a/include/net/xdp.h
+++ b/include/net/xdp.h
@@ -169,13 +169,13 @@ out:
 
 struct xdp_frame {
        void *data;
-       u16 len;
-       u16 headroom;
+       u32 len;
+       u32 headroom;
        u32 metasize; /* uses lower 8-bits */
        /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time,
-        * while mem info is valid on remote CPU.
+        * while mem_type is valid on remote CPU.
         */
-       struct xdp_mem_info mem;
+       enum xdp_mem_type mem_type:32;
        struct net_device *dev_rx; /* used by cpumap */
        u32 frame_sz;
        u32 flags; /* supported values defined in xdp_buff_flags */
@@ -306,13 +306,13 @@ struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp)
        if (unlikely(xdp_update_frame_from_buff(xdp, xdp_frame) < 0))
                return NULL;
 
-       /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */
-       xdp_frame->mem = xdp->rxq->mem;
+       /* rxq only valid until napi_schedule ends, convert to xdp_mem_type */
+       xdp_frame->mem_type = xdp->rxq->mem.type;
 
        return xdp_frame;
 }
 
-void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
+void __xdp_return(void *data, enum xdp_mem_type mem_type, bool napi_direct,
                  struct xdp_buff *xdp);
 void xdp_return_frame(struct xdp_frame *xdpf);
 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);

diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index a2f46785ac3b3c54a69b19641cc463055c2978d9..774accbd4a223b612f751e4f0e0605d6a24558f4 100644 (file)
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -190,7 +190,7 @@ static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu,
                int err;
 
                rxq.dev = xdpf->dev_rx;
-               rxq.mem = xdpf->mem;
+               rxq.mem.type = xdpf->mem_type;
                /* TODO: report queue_index to xdp_rxq_info */
 
                xdp_convert_frame_to_buff(xdpf, &xdp);

diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
index 501ec4249fedc3d34fe39aff50eea66f82b88a11..9ae2a7f1738b3559e8f7fd53b6e07e93022146e8 100644 (file)
--- a/net/bpf/test_run.c
+++ b/net/bpf/test_run.c
@@ -153,7 +153,7 @@ static void xdp_test_run_init_page(netmem_ref netmem, void *arg)
        new_ctx->data = new_ctx->data_meta + meta_len;
 
        xdp_update_frame_from_buff(new_ctx, frm);
-       frm->mem = new_ctx->rxq->mem;
+       frm->mem_type = new_ctx->rxq->mem.type;
 
        memcpy(&head->orig_ctx, new_ctx, sizeof(head->orig_ctx));
 }
@@ -246,7 +246,7 @@ static void reset_ctx(struct xdp_page_head *head)
        head->ctx.data_meta = head->orig_ctx.data_meta;
        head->ctx.data_end = head->orig_ctx.data_end;
        xdp_update_frame_from_buff(&head->ctx, head->frame);
-       head->frame->mem = head->orig_ctx.rxq->mem;
+       head->frame->mem_type = head->orig_ctx.rxq->mem.type;
 }
 
 static int xdp_recv_frames(struct xdp_frame **frames, int nframes,

diff --git a/net/core/filter.c b/net/core/filter.c
index fac245065b0a433f913a2de9b7c3bf76e2fe1696..6c036708634bb09213179b2fc6476f63249b4777 100644 (file)
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -4119,13 +4119,13 @@ static int bpf_xdp_frags_increase_tail(struct xdp_buff *xdp, int offset)
 }
 
 static void bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink,
-                                  struct xdp_mem_info *mem_info, bool release)
+                                  enum xdp_mem_type mem_type, bool release)
 {
        struct xdp_buff *zc_frag = xsk_buff_get_tail(xdp);
 
        if (release) {
                xsk_buff_del_tail(zc_frag);
-               __xdp_return(NULL, mem_info, false, zc_frag);
+               __xdp_return(NULL, mem_type, false, zc_frag);
        } else {
                zc_frag->data_end -= shrink;
        }
@@ -4134,18 +4134,18 @@ static void bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink,
 static bool bpf_xdp_shrink_data(struct xdp_buff *xdp, skb_frag_t *frag,
                                int shrink)
 {
-       struct xdp_mem_info *mem_info = &xdp->rxq->mem;
+       enum xdp_mem_type mem_type = xdp->rxq->mem.type;
        bool release = skb_frag_size(frag) == shrink;
 
-       if (mem_info->type == MEM_TYPE_XSK_BUFF_POOL) {
-               bpf_xdp_shrink_data_zc(xdp, shrink, mem_info, release);
+       if (mem_type == MEM_TYPE_XSK_BUFF_POOL) {
+               bpf_xdp_shrink_data_zc(xdp, shrink, mem_type, release);
                goto out;
        }
 
        if (release) {
                struct page *page = skb_frag_page(frag);
 
-               __xdp_return(page_address(page), mem_info, false, NULL);
+               __xdp_return(page_address(page), mem_type, false, NULL);
        }
 
 out:

diff --git a/net/core/xdp.c b/net/core/xdp.c
index 56127e8ec85fbbbd08747c4c669a46e577127f75..d367571c583863811c81440cc43b4f10fadcf332 100644 (file)
--- a/net/core/xdp.c
+++ b/net/core/xdp.c
@@ -430,12 +430,12 @@ EXPORT_SYMBOL_GPL(xdp_rxq_info_attach_page_pool);
  * is used for those calls sites.  Thus, allowing for faster recycling
  * of xdp_frames/pages in those cases.
  */
-void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
+void __xdp_return(void *data, enum xdp_mem_type mem_type, bool napi_direct,
                  struct xdp_buff *xdp)
 {
        struct page *page;
 
-       switch (mem->type) {
+       switch (mem_type) {
        case MEM_TYPE_PAGE_POOL:
                page = virt_to_head_page(data);
                if (napi_direct && xdp_return_frame_no_direct())
@@ -458,7 +458,7 @@ void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
                break;
        default:
                /* Not possible, checked in xdp_rxq_info_reg_mem_model() */
-               WARN(1, "Incorrect XDP memory type (%d) usage", mem->type);
+               WARN(1, "Incorrect XDP memory type (%d) usage", mem_type);
                break;
        }
 }
@@ -475,10 +475,10 @@ void xdp_return_frame(struct xdp_frame *xdpf)
        for (i = 0; i < sinfo->nr_frags; i++) {
                struct page *page = skb_frag_page(&sinfo->frags[i]);
 
-               __xdp_return(page_address(page), &xdpf->mem, false, NULL);
+               __xdp_return(page_address(page), xdpf->mem_type, false, NULL);
        }
 out:
-       __xdp_return(xdpf->data, &xdpf->mem, false, NULL);
+       __xdp_return(xdpf->data, xdpf->mem_type, false, NULL);
 }
 EXPORT_SYMBOL_GPL(xdp_return_frame);
 
@@ -494,10 +494,10 @@ void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
        for (i = 0; i < sinfo->nr_frags; i++) {
                struct page *page = skb_frag_page(&sinfo->frags[i]);
 
-               __xdp_return(page_address(page), &xdpf->mem, true, NULL);
+               __xdp_return(page_address(page), xdpf->mem_type, true, NULL);
        }
 out:
-       __xdp_return(xdpf->data, &xdpf->mem, true, NULL);
+       __xdp_return(xdpf->data, xdpf->mem_type, true, NULL);
 }
 EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
 
@@ -516,7 +516,7 @@ EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
 void xdp_return_frame_bulk(struct xdp_frame *xdpf,
                           struct xdp_frame_bulk *bq)
 {
-       if (xdpf->mem.type != MEM_TYPE_PAGE_POOL) {
+       if (xdpf->mem_type != MEM_TYPE_PAGE_POOL) {
                xdp_return_frame(xdpf);
                return;
        }
@@ -553,10 +553,11 @@ void xdp_return_buff(struct xdp_buff *xdp)
        for (i = 0; i < sinfo->nr_frags; i++) {
                struct page *page = skb_frag_page(&sinfo->frags[i]);
 
-               __xdp_return(page_address(page), &xdp->rxq->mem, true, xdp);
+               __xdp_return(page_address(page), xdp->rxq->mem.type, true,
+                            xdp);
        }
 out:
-       __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp);
+       __xdp_return(xdp->data, xdp->rxq->mem.type, true, xdp);
 }
 EXPORT_SYMBOL_GPL(xdp_return_buff);
 
@@ -602,7 +603,7 @@ struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp)
        xdpf->headroom = 0;
        xdpf->metasize = metasize;
        xdpf->frame_sz = PAGE_SIZE;
-       xdpf->mem.type = MEM_TYPE_PAGE_ORDER0;
+       xdpf->mem_type = MEM_TYPE_PAGE_ORDER0;
 
        xsk_buff_free(xdp);
        return xdpf;
@@ -672,7 +673,7 @@ struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,
         * - RX ring dev queue index    (skb_record_rx_queue)
         */
 
-       if (xdpf->mem.type == MEM_TYPE_PAGE_POOL)
+       if (xdpf->mem_type == MEM_TYPE_PAGE_POOL)
                skb_mark_for_recycle(skb);
 
        /* Allow SKB to reuse area used by xdp_frame */
@@ -719,8 +720,7 @@ struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf)
        nxdpf = addr;
        nxdpf->data = addr + headroom;
        nxdpf->frame_sz = PAGE_SIZE;
-       nxdpf->mem.type = MEM_TYPE_PAGE_ORDER0;
-       nxdpf->mem.id = 0;
+       nxdpf->mem_type = MEM_TYPE_PAGE_ORDER0;
 
        return nxdpf;
 }