net: add SO_DEVMEM_DONTNEED setsockopt to release RX frags

Add an interface for the user to notify the kernel that it is done
reading the devmem dmabuf frags returned as cmsg. The kernel will
drop the reference on the frags to make them available for reuse.

Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: Kaiyuan Zhang <kaiyuanz@google.com>
Signed-off-by: Mina Almasry <almasrymina@google.com>
Reviewed-by: Pavel Begunkov <asml.silence@gmail.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Link: https://patch.msgid.link/20240910171458.219195-11-almasrymina@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/arch/alpha/include/uapi/asm/socket.h b/arch/alpha/include/uapi/asm/socket.h
index ef4656a41058ad982bbcf6d7f5b27020a9798226..251b73c5481eaa669873ff551787e9c7c453c537 100644 (file)
--- a/arch/alpha/include/uapi/asm/socket.h
+++ b/arch/alpha/include/uapi/asm/socket.h
@@ -144,6 +144,7 @@
 #define SCM_DEVMEM_LINEAR      SO_DEVMEM_LINEAR
 #define SO_DEVMEM_DMABUF       79
 #define SCM_DEVMEM_DMABUF      SO_DEVMEM_DMABUF
+#define SO_DEVMEM_DONTNEED     80
 
 #if !defined(__KERNEL__)
 

diff --git a/arch/mips/include/uapi/asm/socket.h b/arch/mips/include/uapi/asm/socket.h
index 414807d55e33fac4ab6bff40cfe585e70e1bd0e1..8ab7582291abf2cf82216abc3d3b340990f372bd 100644 (file)
--- a/arch/mips/include/uapi/asm/socket.h
+++ b/arch/mips/include/uapi/asm/socket.h
@@ -155,6 +155,7 @@
 #define SCM_DEVMEM_LINEAR      SO_DEVMEM_LINEAR
 #define SO_DEVMEM_DMABUF       79
 #define SCM_DEVMEM_DMABUF      SO_DEVMEM_DMABUF
+#define SO_DEVMEM_DONTNEED     80
 
 #if !defined(__KERNEL__)
 

diff --git a/arch/parisc/include/uapi/asm/socket.h b/arch/parisc/include/uapi/asm/socket.h
index 2b817efd454442f81c4db4ff4ec8c923e1917c43..38fc0b188e08424054b50ddc4a661b8bef74042f 100644 (file)
--- a/arch/parisc/include/uapi/asm/socket.h
+++ b/arch/parisc/include/uapi/asm/socket.h
@@ -136,6 +136,7 @@
 #define SCM_DEVMEM_LINEAR      SO_DEVMEM_LINEAR
 #define SO_DEVMEM_DMABUF       79
 #define SCM_DEVMEM_DMABUF      SO_DEVMEM_DMABUF
+#define SO_DEVMEM_DONTNEED     80
 
 #if !defined(__KERNEL__)
 

diff --git a/arch/sparc/include/uapi/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h
index 00248fc68977380206db3c4aaa1c1eca92a08e98..57084ed2f3c4ec1325397cfe8ea2356bb67b0ffd 100644 (file)
--- a/arch/sparc/include/uapi/asm/socket.h
+++ b/arch/sparc/include/uapi/asm/socket.h
@@ -137,6 +137,7 @@
 #define SCM_DEVMEM_LINEAR        SO_DEVMEM_LINEAR
 #define SO_DEVMEM_DMABUF         0x0058
 #define SCM_DEVMEM_DMABUF        SO_DEVMEM_DMABUF
+#define SO_DEVMEM_DONTNEED       0x0059
 
 #if !defined(__KERNEL__)
 

diff --git a/include/uapi/asm-generic/socket.h b/include/uapi/asm-generic/socket.h
index e993edc9c0ee69824f0c60b033494fe4df7d75b2..3b4e3e8156025f305fea4639bb108264b5316bfa 100644 (file)
--- a/include/uapi/asm-generic/socket.h
+++ b/include/uapi/asm-generic/socket.h
@@ -139,6 +139,7 @@
 #define SCM_DEVMEM_LINEAR      SO_DEVMEM_LINEAR
 #define SO_DEVMEM_DMABUF       79
 #define SCM_DEVMEM_DMABUF      SO_DEVMEM_DMABUF
+#define SO_DEVMEM_DONTNEED     80
 
 #if !defined(__KERNEL__)
 

diff --git a/include/uapi/linux/uio.h b/include/uapi/linux/uio.h
index 3a22ddae376a2eaf28e63dcfddbd12b37a954582..649739e0c40488d540358bea131473aaf8138c92 100644 (file)
--- a/include/uapi/linux/uio.h
+++ b/include/uapi/linux/uio.h
@@ -33,6 +33,11 @@ struct dmabuf_cmsg {
                                 */
 };
 
+struct dmabuf_token {
+       __u32 token_start;
+       __u32 token_count;
+};
+
 /*
  *     UIO_MAXIOV shall be at least 16 1003.1g (5.4.1.1)
  */

diff --git a/net/core/sock.c b/net/core/sock.c
index 468b1239606c69cb2c26a3dc74088660b4fa0367..bbb57b5af0b193a895b0b4114839cd9d486732b7 100644 (file)
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -124,6 +124,7 @@
 #include <linux/netdevice.h>
 #include <net/protocol.h>
 #include <linux/skbuff.h>
+#include <linux/skbuff_ref.h>
 #include <net/net_namespace.h>
 #include <net/request_sock.h>
 #include <net/sock.h>
@@ -1049,6 +1050,69 @@ static int sock_reserve_memory(struct sock *sk, int bytes)
        return 0;
 }
 
+#ifdef CONFIG_PAGE_POOL
+
+/* This is the number of tokens that the user can SO_DEVMEM_DONTNEED in
+ * 1 syscall. The limit exists to limit the amount of memory the kernel
+ * allocates to copy these tokens.
+ */
+#define MAX_DONTNEED_TOKENS 128
+
+static noinline_for_stack int
+sock_devmem_dontneed(struct sock *sk, sockptr_t optval, unsigned int optlen)
+{
+       unsigned int num_tokens, i, j, k, netmem_num = 0;
+       struct dmabuf_token *tokens;
+       netmem_ref netmems[16];
+       int ret = 0;
+
+       if (!sk_is_tcp(sk))
+               return -EBADF;
+
+       if (optlen % sizeof(struct dmabuf_token) ||
+           optlen > sizeof(*tokens) * MAX_DONTNEED_TOKENS)
+               return -EINVAL;
+
+       tokens = kvmalloc_array(optlen, sizeof(*tokens), GFP_KERNEL);
+       if (!tokens)
+               return -ENOMEM;
+
+       num_tokens = optlen / sizeof(struct dmabuf_token);
+       if (copy_from_sockptr(tokens, optval, optlen)) {
+               kvfree(tokens);
+               return -EFAULT;
+       }
+
+       xa_lock_bh(&sk->sk_user_frags);
+       for (i = 0; i < num_tokens; i++) {
+               for (j = 0; j < tokens[i].token_count; j++) {
+                       netmem_ref netmem = (__force netmem_ref)__xa_erase(
+                               &sk->sk_user_frags, tokens[i].token_start + j);
+
+                       if (netmem &&
+                           !WARN_ON_ONCE(!netmem_is_net_iov(netmem))) {
+                               netmems[netmem_num++] = netmem;
+                               if (netmem_num == ARRAY_SIZE(netmems)) {
+                                       xa_unlock_bh(&sk->sk_user_frags);
+                                       for (k = 0; k < netmem_num; k++)
+                                               WARN_ON_ONCE(!napi_pp_put_page(netmems[k]));
+                                       netmem_num = 0;
+                                       xa_lock_bh(&sk->sk_user_frags);
+                               }
+                               ret++;
+                       }
+               }
+       }
+
+       xa_unlock_bh(&sk->sk_user_frags);
+       for (k = 0; k < netmem_num; k++)
+               WARN_ON_ONCE(!napi_pp_put_page(netmems[k]));
+
+       kvfree(tokens);
+       return ret;
+}
+#endif
+
 void sockopt_lock_sock(struct sock *sk)
 {
        /* When current->bpf_ctx is set, the setsockopt is called from
@@ -1211,6 +1275,10 @@ int sk_setsockopt(struct sock *sk, int level, int optname,
                        ret = -EOPNOTSUPP;
                return ret;
                }
+#ifdef CONFIG_PAGE_POOL
+       case SO_DEVMEM_DONTNEED:
+               return sock_devmem_dontneed(sk, optval, optlen);
+#endif
        }
 
        sockopt_lock_sock(sk);