net: release dst entry in dev_hard_start_xmit()

One point of contention in high network loads is the dst_release() performed
when a transmited skb is freed. This is because NIC tx completion calls
dev_kree_skb() long after original call to dev_queue_xmit(skb).

CPU cache is cold and the atomic op in dst_release() stalls. On SMP, this is
quite visible if one CPU is 100% handling softirqs for a network device,
since dst_clone() is done by other cpus, involving cache line ping pongs.

It seems right place to release dst is in dev_hard_start_xmit(), for most
devices but ones that are virtual, and some exceptions.

David Miller suggested to define a new device flag, set in alloc_netdev_mq()
(so that most devices set it at init time), and carefuly unset in devices
which dont want a NULL skb->dst in their ndo_start_xmit().

List of devices that must clear this flag is :

- loopback device, because it calls netif_rx() and quoting Patrick :
    "ip_route_input() doesn't accept loopback addresses, so loopback packets
     already need to have a dst_entry attached."
- appletalk/ipddp.c : needs skb->dst in its xmit function

- And all devices that call again dev_queue_xmit() from their xmit function
(as some classifiers need skb->dst) : bonding, vlan, macvlan, eql, ifb, hdlc_fr

Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/drivers/net/appletalk/ipddp.c b/drivers/net/appletalk/ipddp.c
index da64ba88d7f8f362ac785570fa73c3c4ecf16935..f939e92fcf8aa3d8daf1cded2e6f34f0c31a9911 100644 (file)
--- a/drivers/net/appletalk/ipddp.c
+++ b/drivers/net/appletalk/ipddp.c
@@ -71,6 +71,7 @@ static struct net_device * __init ipddp_init(void)
        if (!dev)
                return ERR_PTR(-ENOMEM);
 
+       dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
        strcpy(dev->name, "ipddp%d");
 
        if (version_printed++ == 0)

diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index 96d7689995cdde6351a4ddd708093fa35cf0d9bc..92a9d69c5650e91c6c8c536d345172d35b8acbdc 100644 (file)
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -5148,6 +5148,7 @@ int bond_create(char *name, struct bond_params *params)
                goto out_rtnl;
        }
 
+       bond_dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
        if (!name) {
                res = dev_alloc_name(bond_dev, "bond%d");
                if (res < 0)

diff --git a/drivers/net/eql.c b/drivers/net/eql.c
index 5210bb1027cce43d168d5f09d391064af2fac812..19b7dd983944d84add883ee3f0d50abf270e0b80 100644 (file)
--- a/drivers/net/eql.c
+++ b/drivers/net/eql.c
@@ -194,6 +194,7 @@ static void __init eql_setup(struct net_device *dev)
 
        dev->type               = ARPHRD_SLIP;
        dev->tx_queue_len       = 5;            /* Hands them off fast */
+       dev->priv_flags        &= ~IFF_XMIT_DST_RELEASE;
 }
 
 static int eql_open(struct net_device *dev)

diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
index 60a263001933a53e48e32d28f7adc7dc8914d437..96713ef0629848dd1d529246a7b8fe1941f43816 100644 (file)
--- a/drivers/net/ifb.c
+++ b/drivers/net/ifb.c
@@ -156,6 +156,7 @@ static void ifb_setup(struct net_device *dev)
 
        dev->flags |= IFF_NOARP;
        dev->flags &= ~IFF_MULTICAST;
+       dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
        random_ether_addr(dev->dev_addr);
 }
 

diff --git a/drivers/net/loopback.c b/drivers/net/loopback.c
index 6f71157bea8e48f0058faefc19847bb9347d1315..da472c687481ab23ce0e6a9d0afa9c8733fd4ab8 100644 (file)
--- a/drivers/net/loopback.c
+++ b/drivers/net/loopback.c
@@ -170,6 +170,7 @@ static void loopback_setup(struct net_device *dev)
        dev->tx_queue_len       = 0;
        dev->type               = ARPHRD_LOOPBACK;      /* 0x0001*/
        dev->flags              = IFF_LOOPBACK;
+       dev->priv_flags        &= ~IFF_XMIT_DST_RELEASE;
        dev->features           = NETIF_F_SG | NETIF_F_FRAGLIST
                | NETIF_F_TSO
                | NETIF_F_NO_CSUM

diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index 329cd50d0e2968a6a992e8a5461a570d3b638abb..d5334b41e4b4eee895d1fd7d37d931956d56be1a 100644 (file)
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -414,6 +414,7 @@ static void macvlan_setup(struct net_device *dev)
 {
        ether_setup(dev);
 
+       dev->priv_flags        &= ~IFF_XMIT_DST_RELEASE;
        dev->netdev_ops         = &macvlan_netdev_ops;
        dev->destructor         = free_netdev;
        dev->header_ops         = &macvlan_hard_header_ops,

diff --git a/drivers/net/wan/hdlc_fr.c b/drivers/net/wan/hdlc_fr.c
index 800530101093a10901ee16956bf75040a6ec971a..bfa0161a02d32001f8f22d5f6beb2b9c5d2a2c93 100644 (file)
--- a/drivers/net/wan/hdlc_fr.c
+++ b/drivers/net/wan/hdlc_fr.c
@@ -1054,6 +1054,7 @@ static void pvc_setup(struct net_device *dev)
        dev->flags = IFF_POINTOPOINT;
        dev->hard_header_len = 10;
        dev->addr_len = 2;
+       dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
 }
 
 static const struct net_device_ops pvc_ops = {

diff --git a/include/linux/if.h b/include/linux/if.h
index 1108f3e099e3bcb7a80b89c9ee2f43fa5dcefa88..b9a6229f3be7751d58ea43bc938da8afecfc42b0 100644 (file)
--- a/include/linux/if.h
+++ b/include/linux/if.h
@@ -67,6 +67,9 @@
 #define IFF_ISATAP     0x80            /* ISATAP interface (RFC4214)   */
 #define IFF_MASTER_ARPMON 0x100                /* bonding master, ARP mon in use */
 #define IFF_WAN_HDLC   0x200           /* WAN HDLC device              */
+#define IFF_XMIT_DST_RELEASE 0x400     /* dev_hard_start_xmit() is allowed to
+                                        * release skb->dst
+                                        */
 
 #define IF_GET_IFACE   0x0001          /* for querying only */
 #define IF_GET_PROTO   0x0002

diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c
index 8faacee686336cfac9fcdf78fe90eec52c5c2bcc..ff7572ac548173e8e97b61a9cc1e8e5f673e7c9f 100644 (file)
--- a/net/8021q/vlan_dev.c
+++ b/net/8021q/vlan_dev.c
@@ -739,6 +739,7 @@ void vlan_setup(struct net_device *dev)
        ether_setup(dev);
 
        dev->priv_flags         |= IFF_802_1Q_VLAN;
+       dev->priv_flags         &= ~IFF_XMIT_DST_RELEASE;
        dev->tx_queue_len       = 0;
 
        dev->netdev_ops         = &vlan_netdev_ops;

diff --git a/net/core/dev.c b/net/core/dev.c
index 6d3630d16271cfa6f3a988c79e8b4575e60374fa..92ebeca29901fd803daba7f2cc979a6a68b890d2 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -1688,6 +1688,14 @@ int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
                                goto gso;
                }
 
+               /*
+                * If device doesnt need skb->dst, release it right now while
+                * its hot in this cpu cache
+                */
+               if ((dev->priv_flags & IFF_XMIT_DST_RELEASE) && skb->dst) {
+                       dst_release(skb->dst);
+                       skb->dst = NULL;
+               }
                rc = ops->ndo_start_xmit(skb, dev);
                /*
                 * TODO: if skb_orphan() was called by
@@ -5045,6 +5053,7 @@ struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
        netdev_init_queues(dev);
 
        INIT_LIST_HEAD(&dev->napi_list);
+       dev->priv_flags = IFF_XMIT_DST_RELEASE;
        setup(dev);
        strcpy(dev->name, name);
        return dev;