udp: avoid calling sock_def_readable() if possible

sock_def_readable() is quite expensive (particularly
when ep_poll_callback() is in the picture).

We must call sk->sk_data_ready() when :

- receive queue was empty, or
- SO_PEEK_OFF is enabled on the socket, or
- sk->sk_data_ready is not sock_def_readable.

We still need to call sk_wake_async().

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Acked-by: Paolo Abeni <pabeni@redhat.com>
Link: https://lore.kernel.org/r/20240328144032.1864988-4-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 19d7db4563acd02222aead860e0fa8f22c8ada2e..143043cd2dcbdcdd1bab47ea9e5746b322bae1c2 100644 (file)
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -1492,6 +1492,7 @@ int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb)
        struct sk_buff_head *list = &sk->sk_receive_queue;
        int rmem, err = -ENOMEM;
        spinlock_t *busy = NULL;
+       bool becomes_readable;
        int size, rcvbuf;
 
        /* Immediately drop when the receive queue is full.
@@ -1532,12 +1533,19 @@ int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb)
         */
        sock_skb_set_dropcount(sk, skb);
 
+       becomes_readable = skb_queue_empty(list);
        __skb_queue_tail(list, skb);
        spin_unlock(&list->lock);
 
-       if (!sock_flag(sk, SOCK_DEAD))
-               INDIRECT_CALL_1(sk->sk_data_ready, sock_def_readable, sk);
-
+       if (!sock_flag(sk, SOCK_DEAD)) {
+               if (becomes_readable ||
+                   sk->sk_data_ready != sock_def_readable ||
+                   READ_ONCE(sk->sk_peek_off) >= 0)
+                       INDIRECT_CALL_1(sk->sk_data_ready,
+                                       sock_def_readable, sk);
+               else
+                       sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
+       }
        busylock_release(busy);
        return 0;