net: add callback for setting a ubuf_info to skb

At the moment an skb can only have one ubuf_info associated with it,
which might be a performance problem for zerocopy sends in cases like
TCP via io_uring. Add a callback for assigning ubuf_info to skb, this
way we will implement smarter assignment later like linking ubuf_info
together.

Note, it's an optional callback, which should be compatible with
skb_zcopy_set(), that's because the net stack might potentially decide
to clone an skb and take another reference to ubuf_info whenever it
wishes. Also, a correct implementation should always be able to bind to
an skb without prior ubuf_info, otherwise we could end up in a situation
when the send would not be able to progress.

Reviewed-by: Jens Axboe <axboe@kernel.dk>
Reviewed-by: David Ahern <dsahern@kernel.org>
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Link: https://lore.kernel.org/all/b7918aadffeb787c84c9e72e34c729dc04f3a45d.1713369317.git.asml.silence@gmail.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index a110e97e074a0c9eafaa90ddb6fc165675a95711..ced69f37977f375dd6d10cced97c74602db67038 100644 (file)
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -530,6 +530,8 @@ enum {
 struct ubuf_info_ops {
        void (*complete)(struct sk_buff *, struct ubuf_info *,
                         bool zerocopy_success);
+       /* has to be compatible with skb_zcopy_set() */
+       int (*link_skb)(struct sk_buff *skb, struct ubuf_info *uarg);
 };
 
 /*

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 9d36750459f394750cd29061adae8de5db1d5aaa..4a0c10685cd2607f1ec1e96d9fb032ae96f04235 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -1880,11 +1880,18 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
        struct ubuf_info *orig_uarg = skb_zcopy(skb);
        int err, orig_len = skb->len;
 
-       /* An skb can only point to one uarg. This edge case happens when
-        * TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
-        */
-       if (orig_uarg && uarg != orig_uarg)
-               return -EEXIST;
+       if (uarg->ops->link_skb) {
+               err = uarg->ops->link_skb(skb, uarg);
+               if (err)
+                       return err;
+       } else {
+               /* An skb can only point to one uarg. This edge case happens
+                * when TCP appends to an skb, but zerocopy_realloc triggered
+                * a new alloc.
+                */
+               if (orig_uarg && uarg != orig_uarg)
+                       return -EEXIST;
+       }
 
        err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len);
        if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
@@ -1898,7 +1905,8 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
                return err;
        }
 
-       skb_zcopy_set(skb, uarg, NULL);
+       if (!uarg->ops->link_skb)
+               skb_zcopy_set(skb, uarg, NULL);
        return skb->len - orig_len;
 }
 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);