io_uring/zcrx: split out memory holders from area

In the data path users of struct io_zcrx_area don't need to know what
kind of memory it's backed by. Only keep there generic bits in there and
and split out memory type dependent fields into a new structure. It also
logically separates the step that actually imports the memory, e.g.
pinning user pages, from the generic area initialisation.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/b60fc09c76921bf69e77eb17e07eb4decedb3bf4.1746097431.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index b5335dd4f5b192fc85353c18d15e4e7351f08d98..8d4cfd957e38d7d841523d7b58f06197bda211fc 100644 (file)
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -26,6 +26,8 @@
 #include "zcrx.h"
 #include "rsrc.h"
 
+#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
+
 static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp)
 {
        return pp->mp_priv;
@@ -42,10 +44,43 @@ static inline struct page *io_zcrx_iov_page(const struct net_iov *niov)
 {
        struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
 
-       return area->pages[net_iov_idx(niov)];
+       return area->mem.pages[net_iov_idx(niov)];
 }
 
-#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
+static void io_release_area_mem(struct io_zcrx_mem *mem)
+{
+       if (mem->pages) {
+               unpin_user_pages(mem->pages, mem->nr_folios);
+               kvfree(mem->pages);
+       }
+}
+
+static int io_import_area(struct io_zcrx_ifq *ifq,
+                         struct io_zcrx_mem *mem,
+                         struct io_uring_zcrx_area_reg *area_reg)
+{
+       struct page **pages;
+       int nr_pages;
+       int ret;
+
+       ret = io_validate_user_buf_range(area_reg->addr, area_reg->len);
+       if (ret)
+               return ret;
+       if (!area_reg->addr)
+               return -EFAULT;
+       if (area_reg->addr & ~PAGE_MASK || area_reg->len & ~PAGE_MASK)
+               return -EINVAL;
+
+       pages = io_pin_pages((unsigned long)area_reg->addr, area_reg->len,
+                                  &nr_pages);
+       if (IS_ERR(pages))
+               return PTR_ERR(pages);
+
+       mem->pages = pages;
+       mem->nr_folios = nr_pages;
+       mem->size = area_reg->len;
+       return 0;
+}
 
 static void __io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
                                 struct io_zcrx_area *area, int nr_mapped)
@@ -84,8 +119,8 @@ static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area)
                struct net_iov *niov = &area->nia.niovs[i];
                dma_addr_t dma;
 
-               dma = dma_map_page_attrs(ifq->dev, area->pages[i], 0, PAGE_SIZE,
-                                        DMA_FROM_DEVICE, IO_DMA_ATTR);
+               dma = dma_map_page_attrs(ifq->dev, area->mem.pages[i], 0,
+                                        PAGE_SIZE, DMA_FROM_DEVICE, IO_DMA_ATTR);
                if (dma_mapping_error(ifq->dev, dma))
                        break;
                if (net_mp_niov_set_dma_addr(niov, dma)) {
@@ -192,14 +227,11 @@ static void io_free_rbuf_ring(struct io_zcrx_ifq *ifq)
 static void io_zcrx_free_area(struct io_zcrx_area *area)
 {
        io_zcrx_unmap_area(area->ifq, area);
+       io_release_area_mem(&area->mem);
 
        kvfree(area->freelist);
        kvfree(area->nia.niovs);
        kvfree(area->user_refs);
-       if (area->pages) {
-               unpin_user_pages(area->pages, area->nr_folios);
-               kvfree(area->pages);
-       }
        kfree(area);
 }
 
@@ -208,36 +240,27 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,
                               struct io_uring_zcrx_area_reg *area_reg)
 {
        struct io_zcrx_area *area;
-       int i, ret, nr_pages, nr_iovs;
+       unsigned nr_iovs;
+       int i, ret;
 
        if (area_reg->flags || area_reg->rq_area_token)
                return -EINVAL;
        if (area_reg->__resv1 || area_reg->__resv2[0] || area_reg->__resv2[1])
                return -EINVAL;
-       if (area_reg->addr & ~PAGE_MASK || area_reg->len & ~PAGE_MASK)
-               return -EINVAL;
-
-       ret = io_validate_user_buf_range(area_reg->addr, area_reg->len);
-       if (ret)
-               return ret;
-       if (!area_reg->addr)
-               return -EFAULT;
 
        ret = -ENOMEM;
        area = kzalloc(sizeof(*area), GFP_KERNEL);
        if (!area)
                goto err;
 
-       area->pages = io_pin_pages((unsigned long)area_reg->addr, area_reg->len,
-                                  &nr_pages);
-       if (IS_ERR(area->pages)) {
-               ret = PTR_ERR(area->pages);
-               area->pages = NULL;
+       ret = io_import_area(ifq, &area->mem, area_reg);
+       if (ret)
                goto err;
-       }
-       area->nr_folios = nr_iovs = nr_pages;
+
+       nr_iovs = area->mem.size >> PAGE_SHIFT;
        area->nia.num_niovs = nr_iovs;
 
+       ret = -ENOMEM;
        area->nia.niovs = kvmalloc_array(nr_iovs, sizeof(area->nia.niovs[0]),
                                         GFP_KERNEL | __GFP_ZERO);
        if (!area->nia.niovs)

diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h
index e3c7c4e647f11e111ea4c70c7c2d398d946999c8..9c22807af8072730de61aebf9a952a0c46c6409b 100644 (file)
--- a/io_uring/zcrx.h
+++ b/io_uring/zcrx.h
@@ -7,6 +7,13 @@
 #include <net/page_pool/types.h>
 #include <net/net_trackers.h>
 
+struct io_zcrx_mem {
+       unsigned long                   size;
+
+       struct page                     **pages;
+       unsigned long                   nr_folios;
+};
+
 struct io_zcrx_area {
        struct net_iov_area     nia;
        struct io_zcrx_ifq      *ifq;
@@ -14,13 +21,13 @@ struct io_zcrx_area {
 
        bool                    is_mapped;
        u16                     area_id;
-       struct page             **pages;
-       unsigned long           nr_folios;
 
        /* freelist */
        spinlock_t              freelist_lock ____cacheline_aligned_in_smp;
        u32                     free_count;
        u32                     *freelist;
+
+       struct io_zcrx_mem      mem;
 };
 
 struct io_zcrx_ifq {