net: Pre-COW metrics for TCP.

TCP is going to record metrics for the connection,
so pre-COW the route metrics at route cache entry
creation time.

This avoids several atomic operations that have to
occur if we COW the metrics after the entry reaches
global visibility.

Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/flow.h b/include/net/flow.h
index 240b7f356c7105c5ded9baa91c6c41cbdbc85261..1ae901f244368415e369996a14914a601cc00644 100644 (file)
--- a/include/net/flow.h
+++ b/include/net/flow.h
@@ -48,7 +48,8 @@ struct flowi {
 
        __u8    proto;
        __u8    flags;
-#define FLOWI_FLAG_ANYSRC 0x01
+#define FLOWI_FLAG_ANYSRC              0x01
+#define FLOWI_FLAG_PRECOW_METRICS      0x02
        union {
                struct {
                        __be16  sport;

diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h
index 8181498fa96ca5334cbff5f10a36a41abd8882dd..6e6dfd7576827cec28e15f4034b9424b9c011723 100644 (file)
--- a/include/net/inet_sock.h
+++ b/include/net/inet_sock.h
@@ -219,7 +219,13 @@ static inline struct request_sock *inet_reqsk_alloc(struct request_sock_ops *ops
 
 static inline __u8 inet_sk_flowi_flags(const struct sock *sk)
 {
-       return inet_sk(sk)->transparent ? FLOWI_FLAG_ANYSRC : 0;
+       __u8 flags = 0;
+
+       if (inet_sk(sk)->transparent)
+               flags |= FLOWI_FLAG_ANYSRC;
+       if (sk->sk_protocol == IPPROTO_TCP)
+               flags |= FLOWI_FLAG_PRECOW_METRICS;
+       return flags;
 }
 
 #endif /* _INET_SOCK_H */

diff --git a/include/net/route.h b/include/net/route.h
index 5677cbf0c6e659f58cf3c7b9befed68be2e15f0e..e5864658dc76a21f1a87e0a4a87ffc9d241a757d 100644 (file)
--- a/include/net/route.h
+++ b/include/net/route.h
@@ -182,6 +182,8 @@ static inline int ip_route_connect(struct rtable **rp, __be32 dst,
 
        if (inet_sk(sk)->transparent)
                fl.flags |= FLOWI_FLAG_ANYSRC;
+       if (protocol == IPPROTO_TCP)
+               fl.flags |= FLOWI_FLAG_PRECOW_METRICS;
 
        if (!dst || !src) {
                err = __ip_route_output_key(net, rp, &fl);
@@ -209,6 +211,8 @@ static inline int ip_route_newports(struct rtable **rp, u8 protocol,
                fl.proto = protocol;
                if (inet_sk(sk)->transparent)
                        fl.flags |= FLOWI_FLAG_ANYSRC;
+               if (protocol == IPPROTO_TCP)
+                       fl.flags |= FLOWI_FLAG_PRECOW_METRICS;
                ip_rt_put(*rp);
                *rp = NULL;
                security_sk_classify_flow(sk, &fl);

diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index 68cee358d9a38a6c02f8f9edf15e208b54bccd93..dd57f489673611de363b14d75e4261df064d520d 100644 (file)
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -1857,6 +1857,28 @@ static unsigned int ipv4_default_mtu(const struct dst_entry *dst)
        return mtu;
 }
 
+static void rt_init_metrics(struct rtable *rt, struct fib_info *fi)
+{
+       if (!(rt->fl.flags & FLOWI_FLAG_PRECOW_METRICS)) {
+       no_cow:
+               rt->fi = fi;
+               atomic_inc(&fi->fib_clntref);
+               dst_init_metrics(&rt->dst, fi->fib_metrics, true);
+       } else {
+               struct inet_peer *peer;
+
+               if (!rt->peer)
+                       rt_bind_peer(rt, 1);
+               peer = rt->peer;
+               if (!peer)
+                       goto no_cow;
+               if (inet_metrics_new(peer))
+                       memcpy(peer->metrics, fi->fib_metrics,
+                              sizeof(u32) * RTAX_MAX);
+               dst_init_metrics(&rt->dst, peer->metrics, false);
+       }
+}
+
 static void rt_set_nexthop(struct rtable *rt, struct fib_result *res, u32 itag)
 {
        struct dst_entry *dst = &rt->dst;
@@ -1866,9 +1888,7 @@ static void rt_set_nexthop(struct rtable *rt, struct fib_result *res, u32 itag)
                if (FIB_RES_GW(*res) &&
                    FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK)
                        rt->rt_gateway = FIB_RES_GW(*res);
-               rt->fi = fi;
-               atomic_inc(&fi->fib_clntref);
-               dst_init_metrics(dst, fi->fib_metrics, true);
+               rt_init_metrics(rt, fi);
 #ifdef CONFIG_IP_ROUTE_CLASSID
                dst->tclassid = FIB_RES_NH(*res).nh_tclassid;
 #endif