xdp: make __xdp_return() MP-agnostic

Currently, __xdp_return() takes pointer to the virtual memory to free
a buffer. Apart from that this sometimes provokes redundant
data <--> page conversions, taking data pointer effectively prevents
lots of XDP code to support non-page-backed buffers, as there's no
mapping for the non-host memory (data is always NULL).
Just convert it to always take netmem reference. For
xdp_return_{buff,frame*}(), this chops off one page_address() per each
frag and adds one virt_to_netmem() (same as virt_to_page()) per header
buffer. For __xdp_return() itself, it removes one virt_to_page() for
MEM_TYPE_PAGE_POOL and another one for MEM_TYPE_PAGE_ORDER0, adding
one page_address() for [not really common nowadays]
MEM_TYPE_PAGE_SHARED, but the main effect is that the abovementioned
functions won't die or memleak anymore if the frame has non-host memory
attached and will correctly free those.

Signed-off-by: Alexander Lobakin <aleksander.lobakin@intel.com>
Link: https://patch.msgid.link/20241211172649.761483-4-aleksander.lobakin@intel.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/net/xdp.h b/include/net/xdp.h
index 1c260869a353911c3752d8a377182af22daf77fe..d2089cfecefd5f102e1c7e32c2d0e95837530cae 100644 (file)
--- a/include/net/xdp.h
+++ b/include/net/xdp.h
@@ -312,8 +312,8 @@ struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp)
        return xdp_frame;
 }
 
-void __xdp_return(void *data, enum xdp_mem_type mem_type, bool napi_direct,
-                 struct xdp_buff *xdp);
+void __xdp_return(netmem_ref netmem, enum xdp_mem_type mem_type,
+                 bool napi_direct, struct xdp_buff *xdp);
 void xdp_return_frame(struct xdp_frame *xdpf);
 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
 void xdp_return_buff(struct xdp_buff *xdp);

diff --git a/net/core/filter.c b/net/core/filter.c
index 6c036708634bb09213179b2fc6476f63249b4777..5fea874025d3a31f60877550936a077ca94d684b 100644 (file)
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -4125,7 +4125,7 @@ static void bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink,
 
        if (release) {
                xsk_buff_del_tail(zc_frag);
-               __xdp_return(NULL, mem_type, false, zc_frag);
+               __xdp_return(0, mem_type, false, zc_frag);
        } else {
                zc_frag->data_end -= shrink;
        }
@@ -4142,11 +4142,8 @@ static bool bpf_xdp_shrink_data(struct xdp_buff *xdp, skb_frag_t *frag,
                goto out;
        }
 
-       if (release) {
-               struct page *page = skb_frag_page(frag);
-
-               __xdp_return(page_address(page), mem_type, false, NULL);
-       }
+       if (release)
+               __xdp_return(skb_frag_netmem(frag), mem_type, false, NULL);
 
 out:
        return release;

diff --git a/net/core/xdp.c b/net/core/xdp.c
index d367571c583863811c81440cc43b4f10fadcf332..f1165a35411bcae72c5a78d5f5480ac0e8c69bbe 100644 (file)
--- a/net/core/xdp.c
+++ b/net/core/xdp.c
@@ -430,27 +430,25 @@ EXPORT_SYMBOL_GPL(xdp_rxq_info_attach_page_pool);
  * is used for those calls sites.  Thus, allowing for faster recycling
  * of xdp_frames/pages in those cases.
  */
-void __xdp_return(void *data, enum xdp_mem_type mem_type, bool napi_direct,
-                 struct xdp_buff *xdp)
+void __xdp_return(netmem_ref netmem, enum xdp_mem_type mem_type,
+                 bool napi_direct, struct xdp_buff *xdp)
 {
-       struct page *page;
-
        switch (mem_type) {
        case MEM_TYPE_PAGE_POOL:
-               page = virt_to_head_page(data);
+               netmem = netmem_compound_head(netmem);
                if (napi_direct && xdp_return_frame_no_direct())
                        napi_direct = false;
                /* No need to check ((page->pp_magic & ~0x3UL) == PP_SIGNATURE)
                 * as mem->type knows this a page_pool page
                 */
-               page_pool_put_full_page(page->pp, page, napi_direct);
+               page_pool_put_full_netmem(netmem_get_pp(netmem), netmem,
+                                         napi_direct);
                break;
        case MEM_TYPE_PAGE_SHARED:
-               page_frag_free(data);
+               page_frag_free(__netmem_address(netmem));
                break;
        case MEM_TYPE_PAGE_ORDER0:
-               page = virt_to_page(data); /* Assumes order0 page*/
-               put_page(page);
+               put_page(__netmem_to_page(netmem));
                break;
        case MEM_TYPE_XSK_BUFF_POOL:
                /* NB! Only valid from an xdp_buff! */
@@ -466,38 +464,34 @@ void __xdp_return(void *data, enum xdp_mem_type mem_type, bool napi_direct,
 void xdp_return_frame(struct xdp_frame *xdpf)
 {
        struct skb_shared_info *sinfo;
-       int i;
 
        if (likely(!xdp_frame_has_frags(xdpf)))
                goto out;
 
        sinfo = xdp_get_shared_info_from_frame(xdpf);
-       for (i = 0; i < sinfo->nr_frags; i++) {
-               struct page *page = skb_frag_page(&sinfo->frags[i]);
+       for (u32 i = 0; i < sinfo->nr_frags; i++)
+               __xdp_return(skb_frag_netmem(&sinfo->frags[i]), xdpf->mem_type,
+                            false, NULL);
 
-               __xdp_return(page_address(page), xdpf->mem_type, false, NULL);
-       }
 out:
-       __xdp_return(xdpf->data, xdpf->mem_type, false, NULL);
+       __xdp_return(virt_to_netmem(xdpf->data), xdpf->mem_type, false, NULL);
 }
 EXPORT_SYMBOL_GPL(xdp_return_frame);
 
 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
 {
        struct skb_shared_info *sinfo;
-       int i;
 
        if (likely(!xdp_frame_has_frags(xdpf)))
                goto out;
 
        sinfo = xdp_get_shared_info_from_frame(xdpf);
-       for (i = 0; i < sinfo->nr_frags; i++) {
-               struct page *page = skb_frag_page(&sinfo->frags[i]);
+       for (u32 i = 0; i < sinfo->nr_frags; i++)
+               __xdp_return(skb_frag_netmem(&sinfo->frags[i]), xdpf->mem_type,
+                            true, NULL);
 
-               __xdp_return(page_address(page), xdpf->mem_type, true, NULL);
-       }
 out:
-       __xdp_return(xdpf->data, xdpf->mem_type, true, NULL);
+       __xdp_return(virt_to_netmem(xdpf->data), xdpf->mem_type, true, NULL);
 }
 EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
 
@@ -544,20 +538,17 @@ EXPORT_SYMBOL_GPL(xdp_return_frame_bulk);
 void xdp_return_buff(struct xdp_buff *xdp)
 {
        struct skb_shared_info *sinfo;
-       int i;
 
        if (likely(!xdp_buff_has_frags(xdp)))
                goto out;
 
        sinfo = xdp_get_shared_info_from_buff(xdp);
-       for (i = 0; i < sinfo->nr_frags; i++) {
-               struct page *page = skb_frag_page(&sinfo->frags[i]);
+       for (u32 i = 0; i < sinfo->nr_frags; i++)
+               __xdp_return(skb_frag_netmem(&sinfo->frags[i]),
+                            xdp->rxq->mem.type, true, xdp);
 
-               __xdp_return(page_address(page), xdp->rxq->mem.type, true,
-                            xdp);
-       }
 out:
-       __xdp_return(xdp->data, xdp->rxq->mem.type, true, xdp);
+       __xdp_return(virt_to_netmem(xdp->data), xdp->rxq->mem.type, true, xdp);
 }
 EXPORT_SYMBOL_GPL(xdp_return_buff);