net/ipv6: Make from in rt6_info rcu protected

When a dst entry is created from a fib entry, the 'from' in rt6_info
is set to the fib entry. The 'from' reference is used most notably for
cookie checking - making sure stale dst entries are updated if the
fib entry is changed.

When a fib entry is deleted, the pcpu routes on it are walked releasing
the fib6_info reference. This is needed for the fib6_info cleanup to
happen and to make sure all device references are released in a timely
manner.

There is a race window when a FIB entry is deleted and the 'from' on the
pcpu route is dropped and the pcpu route hits a cookie check. Handle
this race using rcu on from.

Signed-off-by: David Ahern <dsahern@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/ip6_fib.h b/include/net/ip6_fib.h
index dc3505fb62b38d3d79e6cf81b56d1e847dc9143d..1af450d4e9233c531d154644d8b7b94cbdb611c4 100644 (file)
--- a/include/net/ip6_fib.h
+++ b/include/net/ip6_fib.h
@@ -174,7 +174,7 @@ struct fib6_info {
 
 struct rt6_info {
        struct dst_entry                dst;
-       struct fib6_info                *from;
+       struct fib6_info __rcu          *from;
 
        struct rt6key                   rt6i_dst;
        struct rt6key                   rt6i_src;
@@ -248,13 +248,15 @@ static inline bool fib6_get_cookie_safe(const struct fib6_info *f6i,
 
 static inline u32 rt6_get_cookie(const struct rt6_info *rt)
 {
+       struct fib6_info *from;
        u32 cookie = 0;
 
        rcu_read_lock();
 
-       if (rt->rt6i_flags & RTF_PCPU ||
-           (unlikely(!list_empty(&rt->rt6i_uncached)) && rt->from))
-               fib6_get_cookie_safe(rt->from, &cookie);
+       from = rcu_dereference(rt->from);
+       if (from && (rt->rt6i_flags & RTF_PCPU ||
+           unlikely(!list_empty(&rt->rt6i_uncached))))
+               fib6_get_cookie_safe(from, &cookie);
 
        rcu_read_unlock();
 

diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c
index 64ca02b7745f67cd5b99847067b60a57b19f5abf..6421c893466e7df6896e39c4c462c19eeacf1b65 100644 (file)
--- a/net/ipv6/ip6_fib.c
+++ b/net/ipv6/ip6_fib.c
@@ -875,8 +875,12 @@ static void fib6_drop_pcpu_from(struct fib6_info *f6i,
                ppcpu_rt = per_cpu_ptr(f6i->rt6i_pcpu, cpu);
                pcpu_rt = *ppcpu_rt;
                if (pcpu_rt) {
-                       fib6_info_release(pcpu_rt->from);
-                       pcpu_rt->from = NULL;
+                       struct fib6_info *from;
+
+                       from = rcu_dereference_protected(pcpu_rt->from,
+                                            lockdep_is_held(&table->tb6_lock));
+                       rcu_assign_pointer(pcpu_rt->from, NULL);
+                       fib6_info_release(from);
                }
        }
 }

diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
index 3db47986ef3869257374d8feca6ec5f82967c917..6a477d54f8c72e5b502c202c7af0a04a552ee26e 100644 (file)
--- a/net/ipv6/ip6_output.c
+++ b/net/ipv6/ip6_output.c
@@ -962,16 +962,21 @@ static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk,
         * that's why we try it again later.
         */
        if (ipv6_addr_any(&fl6->saddr) && (!*dst || !(*dst)->error)) {
+               struct fib6_info *from;
                struct rt6_info *rt;
                bool had_dst = *dst != NULL;
 
                if (!had_dst)
                        *dst = ip6_route_output(net, sk, fl6);
                rt = (*dst)->error ? NULL : (struct rt6_info *)*dst;
-               err = ip6_route_get_saddr(net, rt ? rt->from : NULL,
-                                         &fl6->daddr,
+
+               rcu_read_lock();
+               from = rt ? rcu_dereference(rt->from) : NULL;
+               err = ip6_route_get_saddr(net, from, &fl6->daddr,
                                          sk ? inet6_sk(sk)->srcprefs : 0,
                                          &fl6->saddr);
+               rcu_read_unlock();
+
                if (err)
                        goto out_err_release;
 

diff --git a/net/ipv6/route.c b/net/ipv6/route.c
index 810a37ac043b1e67bd80224aecf5faaf3877509c..004d00fe2fe5e04189a654b539913854aa437505 100644 (file)
--- a/net/ipv6/route.c
+++ b/net/ipv6/route.c
@@ -359,7 +359,7 @@ EXPORT_SYMBOL(ip6_dst_alloc);
 static void ip6_dst_destroy(struct dst_entry *dst)
 {
        struct rt6_info *rt = (struct rt6_info *)dst;
-       struct fib6_info *from = rt->from;
+       struct fib6_info *from;
        struct inet6_dev *idev;
 
        dst_destroy_metrics_generic(dst);
@@ -371,8 +371,11 @@ static void ip6_dst_destroy(struct dst_entry *dst)
                in6_dev_put(idev);
        }
 
-       rt->from = NULL;
+       rcu_read_lock();
+       from = rcu_dereference(rt->from);
+       rcu_assign_pointer(rt->from, NULL);
        fib6_info_release(from);
+       rcu_read_unlock();
 }
 
 static void ip6_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
@@ -402,12 +405,16 @@ static bool __rt6_check_expired(const struct rt6_info *rt)
 
 static bool rt6_check_expired(const struct rt6_info *rt)
 {
+       struct fib6_info *from;
+
+       from = rcu_dereference(rt->from);
+
        if (rt->rt6i_flags & RTF_EXPIRES) {
                if (time_after(jiffies, rt->dst.expires))
                        return true;
-       } else if (rt->from) {
+       } else if (from) {
                return rt->dst.obsolete != DST_OBSOLETE_FORCE_CHK ||
-                       fib6_check_expired(rt->from);
+                       fib6_check_expired(from);
        }
        return false;
 }
@@ -963,7 +970,7 @@ static void rt6_set_from(struct rt6_info *rt, struct fib6_info *from)
 {
        rt->rt6i_flags &= ~RTF_EXPIRES;
        fib6_info_hold(from);
-       rt->from = from;
+       rcu_assign_pointer(rt->from, from);
        dst_init_metrics(&rt->dst, from->fib6_metrics->metrics, true);
        if (from->fib6_metrics != &dst_default_metrics) {
                rt->dst._metrics |= DST_METRICS_REFCOUNTED;
@@ -2133,11 +2140,13 @@ static bool fib6_check(struct fib6_info *f6i, u32 cookie)
        return true;
 }
 
-static struct dst_entry *rt6_check(struct rt6_info *rt, u32 cookie)
+static struct dst_entry *rt6_check(struct rt6_info *rt,
+                                  struct fib6_info *from,
+                                  u32 cookie)
 {
        u32 rt_cookie = 0;
 
-       if ((rt->from && !fib6_get_cookie_safe(rt->from, &rt_cookie)) ||
+       if ((from && !fib6_get_cookie_safe(from, &rt_cookie)) ||
            rt_cookie != cookie)
                return NULL;
 
@@ -2147,11 +2156,13 @@ static struct dst_entry *rt6_check(struct rt6_info *rt, u32 cookie)
        return &rt->dst;
 }
 
-static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt, u32 cookie)
+static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt,
+                                           struct fib6_info *from,
+                                           u32 cookie)
 {
        if (!__rt6_check_expired(rt) &&
            rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
-           fib6_check(rt->from, cookie))
+           fib6_check(from, cookie))
                return &rt->dst;
        else
                return NULL;
@@ -2160,6 +2171,7 @@ static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt, u32 cookie)
 static struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie)
 {
        struct dst_entry *dst_ret;
+       struct fib6_info *from;
        struct rt6_info *rt;
 
        rt = container_of(dst, struct rt6_info, dst);
@@ -2171,11 +2183,13 @@ static struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie)
         * into this function always.
         */
 
-       if (rt->rt6i_flags & RTF_PCPU ||
-           (unlikely(!list_empty(&rt->rt6i_uncached)) && rt->from))
-               dst_ret = rt6_dst_from_check(rt, cookie);
+       from = rcu_dereference(rt->from);
+
+       if (from && (rt->rt6i_flags & RTF_PCPU ||
+           unlikely(!list_empty(&rt->rt6i_uncached))))
+               dst_ret = rt6_dst_from_check(rt, from, cookie);
        else
-               dst_ret = rt6_check(rt, cookie);
+               dst_ret = rt6_check(rt, from, cookie);
 
        rcu_read_unlock();
 
@@ -2211,13 +2225,17 @@ static void ip6_link_failure(struct sk_buff *skb)
                if (rt->rt6i_flags & RTF_CACHE) {
                        if (dst_hold_safe(&rt->dst))
                                rt6_remove_exception_rt(rt);
-               } else if (rt->from) {
+               } else {
+                       struct fib6_info *from;
                        struct fib6_node *fn;
 
                        rcu_read_lock();
-                       fn = rcu_dereference(rt->from->fib6_node);
-                       if (fn && (rt->rt6i_flags & RTF_DEFAULT))
-                               fn->fn_sernum = -1;
+                       from = rcu_dereference(rt->from);
+                       if (from) {
+                               fn = rcu_dereference(from->fib6_node);
+                               if (fn && (rt->rt6i_flags & RTF_DEFAULT))
+                                       fn->fn_sernum = -1;
+                       }
                        rcu_read_unlock();
                }
        }
@@ -2225,8 +2243,15 @@ static void ip6_link_failure(struct sk_buff *skb)
 
 static void rt6_update_expires(struct rt6_info *rt0, int timeout)
 {
-       if (!(rt0->rt6i_flags & RTF_EXPIRES) && rt0->from)
-               rt0->dst.expires = rt0->from->expires;
+       if (!(rt0->rt6i_flags & RTF_EXPIRES)) {
+               struct fib6_info *from;
+
+               rcu_read_lock();
+               from = rcu_dereference(rt0->from);
+               if (from)
+                       rt0->dst.expires = from->expires;
+               rcu_read_unlock();
+       }
 
        dst_set_expires(&rt0->dst, timeout);
        rt0->rt6i_flags |= RTF_EXPIRES;
@@ -2243,8 +2268,14 @@ static void rt6_do_update_pmtu(struct rt6_info *rt, u32 mtu)
 
 static bool rt6_cache_allowed_for_pmtu(const struct rt6_info *rt)
 {
+       bool from_set;
+
+       rcu_read_lock();
+       from_set = !!rcu_dereference(rt->from);
+       rcu_read_unlock();
+
        return !(rt->rt6i_flags & RTF_CACHE) &&
-               (rt->rt6i_flags & RTF_PCPU || rt->from);
+               (rt->rt6i_flags & RTF_PCPU || from_set);
 }
 
 static void __ip6_rt_update_pmtu(struct dst_entry *dst, const struct sock *sk,
@@ -2280,16 +2311,18 @@ static void __ip6_rt_update_pmtu(struct dst_entry *dst, const struct sock *sk,
                if (rt6->rt6i_flags & RTF_CACHE)
                        rt6_update_exception_stamp_rt(rt6);
        } else if (daddr) {
+               struct fib6_info *from;
                struct rt6_info *nrt6;
 
                rcu_read_lock();
-               nrt6 = ip6_rt_cache_alloc(rt6->from, daddr, saddr);
-               rcu_read_unlock();
+               from = rcu_dereference(rt6->from);
+               nrt6 = ip6_rt_cache_alloc(from, daddr, saddr);
                if (nrt6) {
                        rt6_do_update_pmtu(nrt6, mtu);
-                       if (rt6_insert_exception(nrt6, rt6->from))
+                       if (rt6_insert_exception(nrt6, from))
                                dst_release_immediate(&nrt6->dst);
                }
+               rcu_read_unlock();
        }
 }
 
@@ -3222,6 +3255,7 @@ static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_bu
        struct ndisc_options ndopts;
        struct inet6_dev *in6_dev;
        struct neighbour *neigh;
+       struct fib6_info *from;
        struct rd_msg *msg;
        int optlen, on_link;
        u8 *lladdr;
@@ -3304,7 +3338,8 @@ static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_bu
                     NDISC_REDIRECT, &ndopts);
 
        rcu_read_lock();
-       nrt = ip6_rt_cache_alloc(rt->from, &msg->dest, NULL);
+       from = rcu_dereference(rt->from);
+       nrt = ip6_rt_cache_alloc(from, &msg->dest, NULL);
        rcu_read_unlock();
        if (!nrt)
                goto out;
@@ -4687,6 +4722,7 @@ static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
        struct net *net = sock_net(in_skb->sk);
        struct nlattr *tb[RTA_MAX+1];
        int err, iif = 0, oif = 0;
+       struct fib6_info *from;
        struct dst_entry *dst;
        struct rt6_info *rt;
        struct sk_buff *skb;
@@ -4783,15 +4819,21 @@ static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
        }
 
        skb_dst_set(skb, &rt->dst);
+
+       rcu_read_lock();
+       from = rcu_dereference(rt->from);
+
        if (fibmatch)
-               err = rt6_fill_node(net, skb, rt->from, NULL, NULL, NULL, iif,
+               err = rt6_fill_node(net, skb, from, NULL, NULL, NULL, iif,
                                    RTM_NEWROUTE, NETLINK_CB(in_skb).portid,
                                    nlh->nlmsg_seq, 0);
        else
-               err = rt6_fill_node(net, skb, rt->from, dst,
-                                   &fl6.daddr, &fl6.saddr, iif, RTM_NEWROUTE,
+               err = rt6_fill_node(net, skb, from, dst, &fl6.daddr,
+                                   &fl6.saddr, iif, RTM_NEWROUTE,
                                    NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
                                    0);
+       rcu_read_unlock();
+
        if (err < 0) {
                kfree_skb(skb);
                goto errout;