net: add __sock_wfree() helper

Hosts sending lot of ACK packets exhibit high sock_wfree() cost
because of cache line miss to test SOCK_USE_WRITE_QUEUE

We could move this flag close to sk_wmem_alloc but it is better
to perform the atomic_sub_and_test() on a clean cache line,
as it avoid one extra bus transaction.

skb_orphan_partial() can also have a fast track for packets that either
are TCP acks, or already went through another skb_orphan_partial()

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/sock.h b/include/net/sock.h
index 1dbb1f9f7c1b444b50cb3acb708517920126b773..45f5b492c65883cd22e2f615e019fe0d0ba31167 100644 (file)
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -1445,6 +1445,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority);
 
 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
                             gfp_t priority);
+void __sock_wfree(struct sk_buff *skb);
 void sock_wfree(struct sk_buff *skb);
 void skb_orphan_partial(struct sk_buff *skb);
 void sock_rfree(struct sk_buff *skb);

diff --git a/net/core/sock.c b/net/core/sock.c
index f615e93911707e8e4b9bbdf9f5454e84da9c887b..08bf97eceeb3827d0b237d8c01910e5e0a0f5d6a 100644 (file)
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1655,6 +1655,17 @@ void sock_wfree(struct sk_buff *skb)
 }
 EXPORT_SYMBOL(sock_wfree);
 
+/* This variant of sock_wfree() is used by TCP,
+ * since it sets SOCK_USE_WRITE_QUEUE.
+ */
+void __sock_wfree(struct sk_buff *skb)
+{
+       struct sock *sk = skb->sk;
+
+       if (atomic_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
+               __sk_free(sk);
+}
+
 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
 {
        skb_orphan(skb);
@@ -1677,8 +1688,21 @@ void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
 }
 EXPORT_SYMBOL(skb_set_owner_w);
 
+/* This helper is used by netem, as it can hold packets in its
+ * delay queue. We want to allow the owner socket to send more
+ * packets, as if they were already TX completed by a typical driver.
+ * But we also want to keep skb->sk set because some packet schedulers
+ * rely on it (sch_fq for example). So we set skb->truesize to a small
+ * amount (1) and decrease sk_wmem_alloc accordingly.
+ */
 void skb_orphan_partial(struct sk_buff *skb)
 {
+       /* If this skb is a TCP pure ACK or already went here,
+        * we have nothing to do. 2 is already a very small truesize.
+        */
+       if (skb->truesize <= 2)
+               return;
+
        /* TCP stack sets skb->ooo_okay based on sk_wmem_alloc,
         * so we do not completely orphan skb, but transfert all
         * accounted bytes but one, to avoid unexpected reorders.

diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index 25d527922b18d88c34381d433aea08fdee3f42b0..8daefd8b1b495f23e45b6c9e6b70525715498c08 100644 (file)
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -949,7 +949,7 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
 
        skb_orphan(skb);
        skb->sk = sk;
-       skb->destructor = skb_is_tcp_pure_ack(skb) ? sock_wfree : tcp_wfree;
+       skb->destructor = skb_is_tcp_pure_ack(skb) ? __sock_wfree : tcp_wfree;
        skb_set_hash_from_sk(skb, sk);
        atomic_add(skb->truesize, &sk->sk_wmem_alloc);