xsk: Support tx_metadata_len

For zerocopy mode, tx_desc->addr can point to an arbitrary offset
and carry some TX metadata in the headroom. For copy mode, there
is no way currently to populate skb metadata.

Introduce new tx_metadata_len umem config option that indicates how many
bytes to treat as metadata. Metadata bytes come prior to tx_desc address
(same as in RX case).

The size of the metadata has mostly the same constraints as XDP:
- less than 256 bytes
- 8-byte aligned (compared to 4-byte alignment on xdp, due to 8-byte
  timestamp in the completion)
- non-zero

This data is not interpreted in any way right now.

Reviewed-by: Song Yoong Siang <yoong.siang.song@intel.com>
Signed-off-by: Stanislav Fomichev <sdf@google.com>
Reviewed-by: Jakub Kicinski <kuba@kernel.org>
Link: https://lore.kernel.org/r/20231127190319.1190813-2-sdf@google.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
index f83128007fb054d2220b59bae2189b4fd1ce961d..bcf765124f72de5efa7bafae255eea0dd84bc4e7 100644 (file)
--- a/include/net/xdp_sock.h
+++ b/include/net/xdp_sock.h
@@ -30,6 +30,7 @@ struct xdp_umem {
        struct user_struct *user;
        refcount_t users;
        u8 flags;
+       u8 tx_metadata_len;
        bool zc;
        struct page **pgs;
        int id;

diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h
index b0bdff26fc882ccafe8bdd3c2727366198eabb94..1985ffaf9b0c6d8ae91ae28a263a94a8c0f08105 100644 (file)
--- a/include/net/xsk_buff_pool.h
+++ b/include/net/xsk_buff_pool.h
@@ -77,6 +77,7 @@ struct xsk_buff_pool {
        u32 chunk_size;
        u32 chunk_shift;
        u32 frame_len;
+       u8 tx_metadata_len; /* inherited from umem */
        u8 cached_need_wakeup;
        bool uses_need_wakeup;
        bool dma_need_sync;

diff --git a/include/uapi/linux/if_xdp.h b/include/uapi/linux/if_xdp.h
index 8d48863472b922c290ecb36f935b4a35dde41563..2ecf79282c267e4bd016b26c30379d48a5662ae4 100644 (file)
--- a/include/uapi/linux/if_xdp.h
+++ b/include/uapi/linux/if_xdp.h
@@ -76,6 +76,7 @@ struct xdp_umem_reg {
        __u32 chunk_size;
        __u32 headroom;
        __u32 flags;
+       __u32 tx_metadata_len;
 };
 
 struct xdp_statistics {

diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c
index 06cead2b8e349859c5bc238ede27946cf803f1fe..946a687fb8e83eb9491d39df3bed99f525cb8144 100644 (file)
--- a/net/xdp/xdp_umem.c
+++ b/net/xdp/xdp_umem.c
@@ -199,6 +199,9 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
        if (headroom >= chunk_size - XDP_PACKET_HEADROOM)
                return -EINVAL;
 
+       if (mr->tx_metadata_len >= 256 || mr->tx_metadata_len % 8)
+               return -EINVAL;
+
        umem->size = size;
        umem->headroom = headroom;
        umem->chunk_size = chunk_size;
@@ -207,6 +210,7 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
        umem->pgs = NULL;
        umem->user = NULL;
        umem->flags = mr->flags;
+       umem->tx_metadata_len = mr->tx_metadata_len;
 
        INIT_LIST_HEAD(&umem->xsk_dma_list);
        refcount_set(&umem->users, 1);

diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index ae9f8cb611f6ca9b1bd0693fd5771198f7555d8f..c904356e2800e1859fa32e88183bc34c2cbfd7b3 100644 (file)
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -1283,6 +1283,14 @@ struct xdp_umem_reg_v1 {
        __u32 headroom;
 };
 
+struct xdp_umem_reg_v2 {
+       __u64 addr; /* Start of packet data area */
+       __u64 len; /* Length of packet data area */
+       __u32 chunk_size;
+       __u32 headroom;
+       __u32 flags;
+};
+
 static int xsk_setsockopt(struct socket *sock, int level, int optname,
                          sockptr_t optval, unsigned int optlen)
 {
@@ -1326,8 +1334,10 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname,
 
                if (optlen < sizeof(struct xdp_umem_reg_v1))
                        return -EINVAL;
-               else if (optlen < sizeof(mr))
+               else if (optlen < sizeof(struct xdp_umem_reg_v2))
                        mr_size = sizeof(struct xdp_umem_reg_v1);
+               else if (optlen < sizeof(mr))
+                       mr_size = sizeof(struct xdp_umem_reg_v2);
 
                if (copy_from_sockptr(&mr, optval, mr_size))
                        return -EFAULT;

diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c
index 49cb9f9a09beedb8f029da2325ea1fad1d338e0c..386eddcdf8374e12ec6b910ad938ba798ce7bb53 100644 (file)
--- a/net/xdp/xsk_buff_pool.c
+++ b/net/xdp/xsk_buff_pool.c
@@ -85,6 +85,7 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs,
                XDP_PACKET_HEADROOM;
        pool->umem = umem;
        pool->addrs = umem->addrs;
+       pool->tx_metadata_len = umem->tx_metadata_len;
        INIT_LIST_HEAD(&pool->free_list);
        INIT_LIST_HEAD(&pool->xskb_list);
        INIT_LIST_HEAD(&pool->xsk_tx_list);

diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h
index 13354a1e428011a2685174b1f20cf6f640999f46..c74a1372bcb94696ce57fc28fa17f602e1b1d1fe 100644 (file)
--- a/net/xdp/xsk_queue.h
+++ b/net/xdp/xsk_queue.h
@@ -143,15 +143,17 @@ static inline bool xp_unused_options_set(u32 options)
 static inline bool xp_aligned_validate_desc(struct xsk_buff_pool *pool,
                                            struct xdp_desc *desc)
 {
-       u64 offset = desc->addr & (pool->chunk_size - 1);
+       u64 addr = desc->addr - pool->tx_metadata_len;
+       u64 len = desc->len + pool->tx_metadata_len;
+       u64 offset = addr & (pool->chunk_size - 1);
 
        if (!desc->len)
                return false;
 
-       if (offset + desc->len > pool->chunk_size)
+       if (offset + len > pool->chunk_size)
                return false;
 
-       if (desc->addr >= pool->addrs_cnt)
+       if (addr >= pool->addrs_cnt)
                return false;
 
        if (xp_unused_options_set(desc->options))
@@ -162,16 +164,17 @@ static inline bool xp_aligned_validate_desc(struct xsk_buff_pool *pool,
 static inline bool xp_unaligned_validate_desc(struct xsk_buff_pool *pool,
                                              struct xdp_desc *desc)
 {
-       u64 addr = xp_unaligned_add_offset_to_addr(desc->addr);
+       u64 addr = xp_unaligned_add_offset_to_addr(desc->addr) - pool->tx_metadata_len;
+       u64 len = desc->len + pool->tx_metadata_len;
 
        if (!desc->len)
                return false;
 
-       if (desc->len > pool->chunk_size)
+       if (len > pool->chunk_size)
                return false;
 
-       if (addr >= pool->addrs_cnt || addr + desc->len > pool->addrs_cnt ||
-           xp_desc_crosses_non_contig_pg(pool, addr, desc->len))
+       if (addr >= pool->addrs_cnt || addr + len > pool->addrs_cnt ||
+           xp_desc_crosses_non_contig_pg(pool, addr, len))
                return false;
 
        if (xp_unused_options_set(desc->options))

diff --git a/tools/include/uapi/linux/if_xdp.h b/tools/include/uapi/linux/if_xdp.h
index 73a47da885dc1b47e12f9ed6134bdec10c6f0233..34411a2e5b6c4d25fe2fa2d29ba7e7a5999bc1a6 100644 (file)
--- a/tools/include/uapi/linux/if_xdp.h
+++ b/tools/include/uapi/linux/if_xdp.h
@@ -76,6 +76,7 @@ struct xdp_umem_reg {
        __u32 chunk_size;
        __u32 headroom;
        __u32 flags;
+       __u32 tx_metadata_len;
 };
 
 struct xdp_statistics {