mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 02:30:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

net/ipv6: separate handling of FIB entries from dst based routes

Browse source 
Last step before flipping the data type for FIB entries:
- use fib6_info_alloc to create FIB entries in ip6_route_info_create
  and addrconf_dst_alloc
- use fib6_info_release in place of dst_release, ip6_rt_put and
  rt6_release
- remove the dst_hold before calling __ip6_ins_rt or ip6_del_rt
- when purging routes, drop per-cpu routes
- replace inc and dec of rt6i_ref with fib6_info_hold and fib6_info_release
- use rt->from since it points to the FIB entry
- drop references to exception bucket, fib6_metrics and per-cpu from
  dst entries (those are relevant for fib entries only)

Signed-off-by: David Ahern <dsahern@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David Ahern 2018-04-17 17:33:25 -07:00 committed by David S. Miller
parent a64efe142f
commit 93531c6743
8 changed files with 115 additions and 152 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

4
include/net/ip6_fib.h
View file

@ -314,9 +314,7 @@ static inline u32 rt6_get_cookie(const struct rt6_info *rt)
if (rt->rt6i_flags & RTF_PCPU || if (rt->rt6i_flags & RTF_PCPU ||
(unlikely(!list_empty(&rt->rt6i_uncached)) && rt->from)) (unlikely(!list_empty(&rt->rt6i_uncached)) && rt->from))
rt = rt->from; rt6_get_cookie_safe(rt->from, &cookie);
rt6_get_cookie_safe(rt, &cookie);
return cookie; return cookie;
} }

3
include/net/ip6_route.h
View file

@ -114,8 +114,7 @@ static inline int ip6_route_get_saddr(struct net *net, struct rt6_info *rt,
unsigned int prefs, unsigned int prefs,
struct in6_addr *saddr) struct in6_addr *saddr)
{ {
struct inet6_dev *idev = struct inet6_dev *idev = rt ? rt->rt6i_idev : NULL;
rt ? ip6_dst_idev((struct dst_entry *)rt) : NULL;
int err = 0; int err = 0;
if (rt && rt->rt6i_prefsrc.plen) if (rt && rt->rt6i_prefsrc.plen)

18
net/ipv6/addrconf.c
View file

@ -916,7 +916,6 @@ void inet6_ifa_finish_destroy(struct inet6_ifaddr *ifp)
pr_warn("Freeing alive inet6 address %p\n", ifp); pr_warn("Freeing alive inet6 address %p\n", ifp);
return; return;
} }
ip6_rt_put(ifp->rt);
kfree_rcu(ifp, rcu); kfree_rcu(ifp, rcu);
} }
@ -1102,8 +1101,8 @@ ipv6_add_addr(struct inet6_dev *idev, const struct in6_addr *addr,
inet6addr_notifier_call_chain(NETDEV_UP, ifa); inet6addr_notifier_call_chain(NETDEV_UP, ifa);
out: out:
if (unlikely(err < 0)) { if (unlikely(err < 0)) {
if (rt) fib6_info_release(rt);
ip6_rt_put(rt);
if (ifa) { if (ifa) {
if (ifa->idev) if (ifa->idev)
in6_dev_put(ifa->idev); in6_dev_put(ifa->idev);
@ -1191,7 +1190,7 @@ cleanup_prefix_route(struct inet6_ifaddr *ifp, unsigned long expires, bool del_r
else { else {
if (!(rt->rt6i_flags & RTF_EXPIRES)) if (!(rt->rt6i_flags & RTF_EXPIRES))
fib6_set_expires(rt, expires); fib6_set_expires(rt, expires);
ip6_rt_put(rt); fib6_info_release(rt);
} }
} }
} }
@ -2375,8 +2374,7 @@ static struct rt6_info *addrconf_get_prefix_route(const struct in6_addr *pfx,
continue; continue;
if ((rt->rt6i_flags & noflags) != 0) if ((rt->rt6i_flags & noflags) != 0)
continue; continue;
if (!dst_hold_safe(&rt->dst)) fib6_info_hold(rt);
rt = NULL;
break; break;
} }
out: out:
@ -2687,7 +2685,7 @@ void addrconf_prefix_rcv(struct net_device *dev, u8 *opt, int len, bool sllao)
addrconf_prefix_route(&pinfo->prefix, pinfo->prefix_len, addrconf_prefix_route(&pinfo->prefix, pinfo->prefix_len,
dev, expires, flags, GFP_ATOMIC); dev, expires, flags, GFP_ATOMIC);
} }
ip6_rt_put(rt); fib6_info_release(rt);
} }
/* Try to figure out our local address for this prefix */ /* Try to figure out our local address for this prefix */
@ -3361,7 +3359,7 @@ static int fixup_permanent_addr(struct net *net,
ifp->rt = rt; ifp->rt = rt;
spin_unlock(&ifp->lock); spin_unlock(&ifp->lock);
ip6_rt_put(prev); fib6_info_release(prev);
} }
if (!(ifp->flags & IFA_F_NOPREFIXROUTE)) { if (!(ifp->flags & IFA_F_NOPREFIXROUTE)) {
@ -5636,8 +5634,8 @@ static void __ipv6_ifa_notify(int event, struct inet6_ifaddr *ifp)
ip6_del_rt(net, rt); ip6_del_rt(net, rt);
} }
if (ifp->rt) { if (ifp->rt) {
if (dst_hold_safe(&ifp->rt->dst)) ip6_del_rt(net, ifp->rt);
ip6_del_rt(net, ifp->rt); ifp->rt = NULL;
} }
rt_genid_bump_ipv6(net); rt_genid_bump_ipv6(net);
break; break;

7
net/ipv6/anycast.c
View file

@ -213,7 +213,7 @@ static void aca_put(struct ifacaddr6 *ac)
{ {
if (refcount_dec_and_test(&ac->aca_refcnt)) { if (refcount_dec_and_test(&ac->aca_refcnt)) {
in6_dev_put(ac->aca_idev); in6_dev_put(ac->aca_idev);
dst_release(&ac->aca_rt->dst); fib6_info_release(ac->aca_rt);
kfree(ac); kfree(ac);
} }
} }
@ -231,6 +231,7 @@ static struct ifacaddr6 *aca_alloc(struct rt6_info *rt,
aca->aca_addr = *addr; aca->aca_addr = *addr;
in6_dev_hold(idev); in6_dev_hold(idev);
aca->aca_idev = idev; aca->aca_idev = idev;
fib6_info_hold(rt);
aca->aca_rt = rt; aca->aca_rt = rt;
aca->aca_users = 1; aca->aca_users = 1;
/* aca_tstamp should be updated upon changes */ /* aca_tstamp should be updated upon changes */
@ -274,7 +275,7 @@ int __ipv6_dev_ac_inc(struct inet6_dev *idev, const struct in6_addr *addr)
} }
aca = aca_alloc(rt, addr); aca = aca_alloc(rt, addr);
if (!aca) { if (!aca) {
ip6_rt_put(rt); fib6_info_release(rt);
err = -ENOMEM; err = -ENOMEM;
goto out; goto out;
} }
@ -330,7 +331,6 @@ int __ipv6_dev_ac_dec(struct inet6_dev *idev, const struct in6_addr *addr)
write_unlock_bh(&idev->lock); write_unlock_bh(&idev->lock);
addrconf_leave_solict(idev, &aca->aca_addr); addrconf_leave_solict(idev, &aca->aca_addr);
dst_hold(&aca->aca_rt->dst);
ip6_del_rt(dev_net(idev->dev), aca->aca_rt); ip6_del_rt(dev_net(idev->dev), aca->aca_rt);
aca_put(aca); aca_put(aca);
@ -358,7 +358,6 @@ void ipv6_ac_destroy_dev(struct inet6_dev *idev)
addrconf_leave_solict(idev, &aca->aca_addr); addrconf_leave_solict(idev, &aca->aca_addr);
dst_hold(&aca->aca_rt->dst);
ip6_del_rt(dev_net(idev->dev), aca->aca_rt); ip6_del_rt(dev_net(idev->dev), aca->aca_rt);
aca_put(aca); aca_put(aca);

55
net/ipv6/ip6_fib.c
View file

@ -170,6 +170,7 @@ struct rt6_info *fib6_info_alloc(gfp_t gfp_flags)
void fib6_info_destroy(struct rt6_info *f6i) void fib6_info_destroy(struct rt6_info *f6i)
{ {
struct rt6_exception_bucket *bucket; struct rt6_exception_bucket *bucket;
struct dst_metrics *m;
WARN_ON(f6i->rt6i_node); WARN_ON(f6i->rt6i_node);
@ -201,6 +202,10 @@ void fib6_info_destroy(struct rt6_info *f6i)
if (f6i->fib6_nh.nh_dev) if (f6i->fib6_nh.nh_dev)
dev_put(f6i->fib6_nh.nh_dev); dev_put(f6i->fib6_nh.nh_dev);
m = f6i->fib6_metrics;
if (m != &dst_default_metrics && refcount_dec_and_test(&m->refcnt))
kfree(m);
kfree(f6i); kfree(f6i);
} }
EXPORT_SYMBOL_GPL(fib6_info_destroy); EXPORT_SYMBOL_GPL(fib6_info_destroy);
@ -714,7 +719,7 @@ static struct fib6_node *fib6_add_1(struct net *net,
/* clean up an intermediate node */ /* clean up an intermediate node */
if (!(fn->fn_flags & RTN_RTINFO)) { if (!(fn->fn_flags & RTN_RTINFO)) {
RCU_INIT_POINTER(fn->leaf, NULL); RCU_INIT_POINTER(fn->leaf, NULL);
rt6_release(leaf); fib6_info_release(leaf);
/* remove null_entry in the root node */ /* remove null_entry in the root node */
} else if (fn->fn_flags & RTN_TL_ROOT && } else if (fn->fn_flags & RTN_TL_ROOT &&
rcu_access_pointer(fn->leaf) == rcu_access_pointer(fn->leaf) ==
@ -898,12 +903,32 @@ static void fib6_purge_rt(struct rt6_info *rt, struct fib6_node *fn,
if (!(fn->fn_flags & RTN_RTINFO) && leaf == rt) { if (!(fn->fn_flags & RTN_RTINFO) && leaf == rt) {
new_leaf = fib6_find_prefix(net, table, fn); new_leaf = fib6_find_prefix(net, table, fn);
atomic_inc(&new_leaf->rt6i_ref); atomic_inc(&new_leaf->rt6i_ref);
rcu_assign_pointer(fn->leaf, new_leaf); rcu_assign_pointer(fn->leaf, new_leaf);
rt6_release(rt); fib6_info_release(rt);
} }
fn = rcu_dereference_protected(fn->parent, fn = rcu_dereference_protected(fn->parent,
lockdep_is_held(&table->tb6_lock)); lockdep_is_held(&table->tb6_lock));
} }
if (rt->rt6i_pcpu) {
int cpu;
/* release the reference to this fib entry from
* all of its cached pcpu routes
*/
for_each_possible_cpu(cpu) {
struct rt6_info **ppcpu_rt;
struct rt6_info *pcpu_rt;
ppcpu_rt = per_cpu_ptr(rt->rt6i_pcpu, cpu);
pcpu_rt = *ppcpu_rt;
if (pcpu_rt) {
fib6_info_release(pcpu_rt->from);
pcpu_rt->from = NULL;
}
}
}
} }
} }
@ -1099,7 +1124,7 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct rt6_info *rt,
fib6_purge_rt(iter, fn, info->nl_net); fib6_purge_rt(iter, fn, info->nl_net);
if (rcu_access_pointer(fn->rr_ptr) == iter) if (rcu_access_pointer(fn->rr_ptr) == iter)
fn->rr_ptr = NULL; fn->rr_ptr = NULL;
rt6_release(iter); fib6_info_release(iter);
if (nsiblings) { if (nsiblings) {
/* Replacing an ECMP route, remove all siblings */ /* Replacing an ECMP route, remove all siblings */
@ -1115,7 +1140,7 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct rt6_info *rt,
fib6_purge_rt(iter, fn, info->nl_net); fib6_purge_rt(iter, fn, info->nl_net);
if (rcu_access_pointer(fn->rr_ptr) == iter) if (rcu_access_pointer(fn->rr_ptr) == iter)
fn->rr_ptr = NULL; fn->rr_ptr = NULL;
rt6_release(iter); fib6_info_release(iter);
nsiblings--; nsiblings--;
info->nl_net->ipv6.rt6_stats->fib_rt_entries--; info->nl_net->ipv6.rt6_stats->fib_rt_entries--;
} else { } else {
@ -1183,9 +1208,6 @@ int fib6_add(struct fib6_node *root, struct rt6_info *rt,
int replace_required = 0; int replace_required = 0;
int sernum = fib6_new_sernum(info->nl_net); int sernum = fib6_new_sernum(info->nl_net);
if (WARN_ON_ONCE(!atomic_read(&rt->dst.__refcnt)))
return -EINVAL;
if (info->nlh) { if (info->nlh) {
if (!(info->nlh->nlmsg_flags & NLM_F_CREATE)) if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
allow_create = 0; allow_create = 0;
@ -1300,7 +1322,7 @@ int fib6_add(struct fib6_node *root, struct rt6_info *rt,
if (pn_leaf == rt) { if (pn_leaf == rt) {
pn_leaf = NULL; pn_leaf = NULL;
RCU_INIT_POINTER(pn->leaf, NULL); RCU_INIT_POINTER(pn->leaf, NULL);
atomic_dec(&rt->rt6i_ref); fib6_info_release(rt);
} }
if (!pn_leaf && !(pn->fn_flags & RTN_RTINFO)) { if (!pn_leaf && !(pn->fn_flags & RTN_RTINFO)) {
pn_leaf = fib6_find_prefix(info->nl_net, table, pn_leaf = fib6_find_prefix(info->nl_net, table,
@ -1312,7 +1334,7 @@ int fib6_add(struct fib6_node *root, struct rt6_info *rt,
info->nl_net->ipv6.fib6_null_entry; info->nl_net->ipv6.fib6_null_entry;
} }
#endif #endif
atomic_inc(&pn_leaf->rt6i_ref); fib6_info_hold(pn_leaf);
rcu_assign_pointer(pn->leaf, pn_leaf); rcu_assign_pointer(pn->leaf, pn_leaf);
} }
} }
@ -1334,10 +1356,6 @@ int fib6_add(struct fib6_node *root, struct rt6_info *rt,
(fn->fn_flags & RTN_TL_ROOT && (fn->fn_flags & RTN_TL_ROOT &&
!rcu_access_pointer(fn->leaf)))) !rcu_access_pointer(fn->leaf))))
fib6_repair_tree(info->nl_net, table, fn); fib6_repair_tree(info->nl_net, table, fn);
/* Always release dst as dst->__refcnt is guaranteed
* to be taken before entering this function
*/
dst_release_immediate(&rt->dst);
return err; return err;
} }
@ -1637,7 +1655,7 @@ static struct fib6_node *fib6_repair_tree(struct net *net,
new_fn_leaf = net->ipv6.fib6_null_entry; new_fn_leaf = net->ipv6.fib6_null_entry;
} }
#endif #endif
atomic_inc(&new_fn_leaf->rt6i_ref); fib6_info_hold(new_fn_leaf);
rcu_assign_pointer(fn->leaf, new_fn_leaf); rcu_assign_pointer(fn->leaf, new_fn_leaf);
return pn; return pn;
} }
@ -1693,7 +1711,7 @@ static struct fib6_node *fib6_repair_tree(struct net *net,
return pn; return pn;
RCU_INIT_POINTER(pn->leaf, NULL); RCU_INIT_POINTER(pn->leaf, NULL);
rt6_release(pn_leaf); fib6_info_release(pn_leaf);
fn = pn; fn = pn;
} }
} }
@ -1763,7 +1781,7 @@ static void fib6_del_route(struct fib6_table *table, struct fib6_node *fn,
call_fib6_entry_notifiers(net, FIB_EVENT_ENTRY_DEL, rt, NULL); call_fib6_entry_notifiers(net, FIB_EVENT_ENTRY_DEL, rt, NULL);
if (!info->skip_notify) if (!info->skip_notify)
inet6_rt_notify(RTM_DELROUTE, rt, info, 0); inet6_rt_notify(RTM_DELROUTE, rt, info, 0);
rt6_release(rt); fib6_info_release(rt);
} }
/* Need to own table->tb6_lock */ /* Need to own table->tb6_lock */
@ -2261,9 +2279,8 @@ static int ipv6_route_seq_show(struct seq_file *seq, void *v)
dev = rt->fib6_nh.nh_dev; dev = rt->fib6_nh.nh_dev;
seq_printf(seq, " %08x %08x %08x %08x %8s\n", seq_printf(seq, " %08x %08x %08x %08x %8s\n",
rt->rt6i_metric, atomic_read(&rt->dst.__refcnt), rt->rt6i_metric, atomic_read(&rt->rt6i_ref), 0,
rt->dst.__use, rt->rt6i_flags, rt->rt6i_flags, dev ? dev->name : "");
dev ? dev->name : "");
iter->w.leaf = NULL; iter->w.leaf = NULL;
return 0; return 0;
} }

3
net/ipv6/ip6_output.c
View file

@ -968,7 +968,8 @@ static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk,
if (!had_dst) if (!had_dst)
*dst = ip6_route_output(net, sk, fl6); *dst = ip6_route_output(net, sk, fl6);
rt = (*dst)->error ? NULL : (struct rt6_info *)*dst; rt = (*dst)->error ? NULL : (struct rt6_info *)*dst;
err = ip6_route_get_saddr(net, rt, &fl6->daddr, err = ip6_route_get_saddr(net, rt ? rt->from : NULL,
&fl6->daddr,
sk ? inet6_sk(sk)->srcprefs : 0, sk ? inet6_sk(sk)->srcprefs : 0,
&fl6->saddr); &fl6->saddr);
if (err) if (err)

6
net/ipv6/ndisc.c
View file

@ -1283,7 +1283,7 @@ static void ndisc_router_discovery(struct sk_buff *skb)
ND_PRINTK(0, err, ND_PRINTK(0, err,
"RA: %s got default router without neighbour\n", "RA: %s got default router without neighbour\n",
__func__); __func__);
ip6_rt_put(rt); fib6_info_release(rt);
return; return;
} }
} }
@ -1313,7 +1313,7 @@ static void ndisc_router_discovery(struct sk_buff *skb)
ND_PRINTK(0, err, ND_PRINTK(0, err,
"RA: %s got default router without neighbour\n", "RA: %s got default router without neighbour\n",
__func__); __func__);
ip6_rt_put(rt); fib6_info_release(rt);
return; return;
} }
neigh->flags |= NTF_ROUTER; neigh->flags |= NTF_ROUTER;
@ -1499,7 +1499,7 @@ static void ndisc_router_discovery(struct sk_buff *skb)
ND_PRINTK(2, warn, "RA: invalid RA options\n"); ND_PRINTK(2, warn, "RA: invalid RA options\n");
} }
out: out:
ip6_rt_put(rt); fib6_info_release(rt);
if (neigh) if (neigh)
neigh_release(neigh); neigh_release(neigh);
} }

171
net/ipv6/route.c
View file

@ -351,13 +351,11 @@ static void rt6_info_init(struct rt6_info *rt)
memset(dst + 1, 0, sizeof(*rt) - sizeof(*dst)); memset(dst + 1, 0, sizeof(*rt) - sizeof(*dst));
INIT_LIST_HEAD(&rt->rt6i_siblings); INIT_LIST_HEAD(&rt->rt6i_siblings);
INIT_LIST_HEAD(&rt->rt6i_uncached); INIT_LIST_HEAD(&rt->rt6i_uncached);
rt->fib6_metrics = (struct dst_metrics *)&dst_default_metrics;
} }
/* allocate dst with ip6_dst_ops */ /* allocate dst with ip6_dst_ops */
static struct rt6_info *__ip6_dst_alloc(struct net *net, struct rt6_info *ip6_dst_alloc(struct net *net, struct net_device *dev,
struct net_device *dev, int flags)
int flags)
{ {
struct rt6_info *rt = dst_alloc(&net->ipv6.ip6_dst_ops, dev, struct rt6_info *rt = dst_alloc(&net->ipv6.ip6_dst_ops, dev,
1, DST_OBSOLETE_FORCE_CHK, flags); 1, DST_OBSOLETE_FORCE_CHK, flags);
@ -369,35 +367,15 @@ static struct rt6_info *__ip6_dst_alloc(struct net *net,
return rt; return rt;
} }
struct rt6_info *ip6_dst_alloc(struct net *net,
struct net_device *dev,
int flags)
{
struct rt6_info *rt = __ip6_dst_alloc(net, dev, flags);
if (rt) {
rt->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, GFP_ATOMIC);
if (!rt->rt6i_pcpu) {
dst_release_immediate(&rt->dst);
return NULL;
}
}
return rt;
}
EXPORT_SYMBOL(ip6_dst_alloc); EXPORT_SYMBOL(ip6_dst_alloc);
static void ip6_dst_destroy(struct dst_entry *dst) static void ip6_dst_destroy(struct dst_entry *dst)
{ {
struct rt6_info *rt = (struct rt6_info *)dst; struct rt6_info *rt = (struct rt6_info *)dst;
struct rt6_exception_bucket *bucket;
struct rt6_info *from = rt->from; struct rt6_info *from = rt->from;
struct inet6_dev *idev; struct inet6_dev *idev;
struct dst_metrics *m;
dst_destroy_metrics_generic(dst); dst_destroy_metrics_generic(dst);
free_percpu(rt->rt6i_pcpu);
rt6_uncached_list_del(rt); rt6_uncached_list_del(rt);
idev = rt->rt6i_idev; idev = rt->rt6i_idev;
@ -405,18 +383,9 @@ static void ip6_dst_destroy(struct dst_entry *dst)
rt->rt6i_idev = NULL; rt->rt6i_idev = NULL;
in6_dev_put(idev); in6_dev_put(idev);
} }
bucket = rcu_dereference_protected(rt->rt6i_exception_bucket, 1);
if (bucket) {
rt->rt6i_exception_bucket = NULL;
kfree(bucket);
}
m = rt->fib6_metrics;
if (m != &dst_default_metrics && refcount_dec_and_test(&m->refcnt))
kfree(m);
rt->from = NULL; rt->from = NULL;
dst_release(&from->dst); fib6_info_release(from);
} }
static void ip6_dst_ifdown(struct dst_entry *dst, struct net_device *dev, static void ip6_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
@ -891,7 +860,7 @@ int rt6_route_rcv(struct net_device *dev, u8 *opt, int len,
else else
fib6_set_expires(rt, jiffies + HZ * lifetime); fib6_set_expires(rt, jiffies + HZ * lifetime);
ip6_rt_put(rt); fib6_info_release(rt);
} }
return 0; return 0;
} }
@ -1010,11 +979,9 @@ static void ip6_rt_init_dst(struct rt6_info *rt, struct rt6_info *ort)
static void rt6_set_from(struct rt6_info *rt, struct rt6_info *from) static void rt6_set_from(struct rt6_info *rt, struct rt6_info *from)
{ {
BUG_ON(from->from);
rt->rt6i_flags &= ~RTF_EXPIRES; rt->rt6i_flags &= ~RTF_EXPIRES;
if (dst_hold_safe(&from->dst)) fib6_info_hold(from);
rt->from = from; rt->from = from;
dst_init_metrics(&rt->dst, from->fib6_metrics->metrics, true); dst_init_metrics(&rt->dst, from->fib6_metrics->metrics, true);
if (from->fib6_metrics != &dst_default_metrics) { if (from->fib6_metrics != &dst_default_metrics) {
rt->dst._metrics |= DST_METRICS_REFCOUNTED; rt->dst._metrics |= DST_METRICS_REFCOUNTED;
@ -1084,7 +1051,7 @@ static struct rt6_info *ip6_create_rt_rcu(struct rt6_info *rt)
struct net_device *dev = rt->fib6_nh.nh_dev; struct net_device *dev = rt->fib6_nh.nh_dev;
struct rt6_info *nrt; struct rt6_info *nrt;
nrt = __ip6_dst_alloc(dev_net(dev), dev, flags); nrt = ip6_dst_alloc(dev_net(dev), dev, flags);
if (nrt) if (nrt)
ip6_rt_copy_init(nrt, rt); ip6_rt_copy_init(nrt, rt);
@ -1203,8 +1170,6 @@ int ip6_ins_rt(struct net *net, struct rt6_info *rt)
{ {
struct nl_info info = { .nl_net = net, }; struct nl_info info = { .nl_net = net, };
/* Hold dst to account for the reference from the fib6 tree */
dst_hold(&rt->dst);
return __ip6_ins_rt(rt, &info, NULL); return __ip6_ins_rt(rt, &info, NULL);
} }
@ -1221,7 +1186,7 @@ static struct rt6_info *ip6_rt_cache_alloc(struct rt6_info *ort,
rcu_read_lock(); rcu_read_lock();
dev = ip6_rt_get_dev_rcu(ort); dev = ip6_rt_get_dev_rcu(ort);
rt = __ip6_dst_alloc(dev_net(dev), dev, 0); rt = ip6_dst_alloc(dev_net(dev), dev, 0);
rcu_read_unlock(); rcu_read_unlock();
if (!rt) if (!rt)
return NULL; return NULL;
@ -1256,7 +1221,7 @@ static struct rt6_info *ip6_rt_pcpu_alloc(struct rt6_info *rt)
rcu_read_lock(); rcu_read_lock();
dev = ip6_rt_get_dev_rcu(rt); dev = ip6_rt_get_dev_rcu(rt);
pcpu_rt = __ip6_dst_alloc(dev_net(dev), dev, flags); pcpu_rt = ip6_dst_alloc(dev_net(dev), dev, flags);
rcu_read_unlock(); rcu_read_unlock();
if (!pcpu_rt) if (!pcpu_rt)
return NULL; return NULL;
@ -1317,7 +1282,7 @@ static void rt6_remove_exception(struct rt6_exception_bucket *bucket,
net = dev_net(rt6_ex->rt6i->dst.dev); net = dev_net(rt6_ex->rt6i->dst.dev);
rt6_ex->rt6i->rt6i_node = NULL; rt6_ex->rt6i->rt6i_node = NULL;
hlist_del_rcu(&rt6_ex->hlist); hlist_del_rcu(&rt6_ex->hlist);
rt6_release(rt6_ex->rt6i); ip6_rt_put(rt6_ex->rt6i);
kfree_rcu(rt6_ex, rcu); kfree_rcu(rt6_ex, rcu);
WARN_ON_ONCE(!bucket->depth); WARN_ON_ONCE(!bucket->depth);
bucket->depth--; bucket->depth--;
@ -1907,17 +1872,11 @@ struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table,
struct rt6_info *uncached_rt; struct rt6_info *uncached_rt;
if (ip6_hold_safe(net, &f6i, true)) { fib6_info_hold(f6i);
dst_use_noref(&f6i->dst, jiffies);
} else {
rcu_read_unlock();
uncached_rt = f6i;
goto uncached_rt_out;
}
rcu_read_unlock(); rcu_read_unlock();
uncached_rt = ip6_rt_cache_alloc(f6i, &fl6->daddr, NULL); uncached_rt = ip6_rt_cache_alloc(f6i, &fl6->daddr, NULL);
dst_release(&rt->dst); fib6_info_release(f6i);
if (uncached_rt) { if (uncached_rt) {
/* Uncached_rt's refcnt is taken during ip6_rt_cache_alloc() /* Uncached_rt's refcnt is taken during ip6_rt_cache_alloc()
@ -1930,7 +1889,6 @@ struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table,
dst_hold(&uncached_rt->dst); dst_hold(&uncached_rt->dst);
} }
uncached_rt_out:
trace_fib6_table_lookup(net, uncached_rt, table, fl6); trace_fib6_table_lookup(net, uncached_rt, table, fl6);
return uncached_rt; return uncached_rt;
@ -1939,24 +1897,12 @@ struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table,
struct rt6_info *pcpu_rt; struct rt6_info *pcpu_rt;
dst_use_noref(&f6i->dst, jiffies);
local_bh_disable(); local_bh_disable();
pcpu_rt = rt6_get_pcpu_route(f6i); pcpu_rt = rt6_get_pcpu_route(f6i);
if (!pcpu_rt) { if (!pcpu_rt)
/* atomic_inc_not_zero() is needed when using rcu */ pcpu_rt = rt6_make_pcpu_route(net, f6i);
if (atomic_inc_not_zero(&f6i->rt6i_ref)) {
/* No dst_hold() on rt is needed because grabbing
* rt->rt6i_ref makes sure rt can't be released.
*/
pcpu_rt = rt6_make_pcpu_route(net, f6i);
rt6_release(f6i);
} else {
/* rt is already removed from tree */
pcpu_rt = net->ipv6.ip6_null_entry;
dst_hold(&pcpu_rt->dst);
}
}
local_bh_enable(); local_bh_enable();
rcu_read_unlock(); rcu_read_unlock();
trace_fib6_table_lookup(net, pcpu_rt, table, fl6); trace_fib6_table_lookup(net, pcpu_rt, table, fl6);
@ -2193,11 +2139,26 @@ struct dst_entry *ip6_blackhole_route(struct net *net, struct dst_entry *dst_ori
* Destination cache support functions * Destination cache support functions
*/ */
static bool fib6_check(struct rt6_info *f6i, u32 cookie)
{
u32 rt_cookie = 0;
if ((f6i && !rt6_get_cookie_safe(f6i, &rt_cookie)) ||
rt_cookie != cookie)
return false;
if (fib6_check_expired(f6i))
return false;
return true;
}
static struct dst_entry *rt6_check(struct rt6_info *rt, u32 cookie) static struct dst_entry *rt6_check(struct rt6_info *rt, u32 cookie)
{ {
u32 rt_cookie = 0; u32 rt_cookie = 0;
if (!rt6_get_cookie_safe(rt, &rt_cookie) || rt_cookie != cookie) if ((rt->from && !rt6_get_cookie_safe(rt->from, &rt_cookie)) ||
rt_cookie != cookie)
return NULL; return NULL;
if (rt6_check_expired(rt)) if (rt6_check_expired(rt))
@ -2210,7 +2171,7 @@ static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt, u32 cookie)
{ {
if (!__rt6_check_expired(rt) && if (!__rt6_check_expired(rt) &&
rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK && rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
rt6_check(rt->from, cookie)) fib6_check(rt->from, cookie))
return &rt->dst; return &rt->dst;
else else
return NULL; return NULL;
@ -2241,7 +2202,7 @@ static struct dst_entry *ip6_negative_advice(struct dst_entry *dst)
if (rt) { if (rt) {
if (rt->rt6i_flags & RTF_CACHE) { if (rt->rt6i_flags & RTF_CACHE) {
if (rt6_check_expired(rt)) { if (rt6_check_expired(rt)) {
ip6_del_rt(dev_net(dst->dev), rt); rt6_remove_exception_rt(rt);
dst = NULL; dst = NULL;
} }
} else { } else {
@ -2262,12 +2223,12 @@ static void ip6_link_failure(struct sk_buff *skb)
if (rt) { if (rt) {
if (rt->rt6i_flags & RTF_CACHE) { if (rt->rt6i_flags & RTF_CACHE) {
if (dst_hold_safe(&rt->dst)) if (dst_hold_safe(&rt->dst))
ip6_del_rt(dev_net(rt->dst.dev), rt); rt6_remove_exception_rt(rt);
} else { } else if (rt->from) {
struct fib6_node *fn; struct fib6_node *fn;
rcu_read_lock(); rcu_read_lock();
fn = rcu_dereference(rt->rt6i_node); fn = rcu_dereference(rt->from->rt6i_node);
if (fn && (rt->rt6i_flags & RTF_DEFAULT)) if (fn && (rt->rt6i_flags & RTF_DEFAULT))
fn->fn_sernum = -1; fn->fn_sernum = -1;
rcu_read_unlock(); rcu_read_unlock();
@ -2949,13 +2910,13 @@ static struct rt6_info *ip6_route_info_create(struct fib6_config *cfg,
if (!table) if (!table)
goto out; goto out;
rt = ip6_dst_alloc(net, NULL, err = -ENOMEM;
(cfg->fc_flags & RTF_ADDRCONF) ? 0 : DST_NOCOUNT); rt = fib6_info_alloc(gfp_flags);
if (!rt)
if (!rt) {
err = -ENOMEM;
goto out; goto out;
}
if (cfg->fc_flags & RTF_ADDRCONF)
rt->dst_nocount = true;
err = ip6_convert_metrics(net, rt, cfg); err = ip6_convert_metrics(net, rt, cfg);
if (err < 0) if (err < 0)
@ -3029,7 +2990,7 @@ static struct rt6_info *ip6_route_info_create(struct fib6_config *cfg,
if (err) if (err)
goto out; goto out;
rt->fib6_nh.nh_gw = rt->rt6i_gateway = cfg->fc_gateway; rt->fib6_nh.nh_gw = cfg->fc_gateway;
} }
err = -ENODEV; err = -ENODEV;
@ -3066,7 +3027,7 @@ static struct rt6_info *ip6_route_info_create(struct fib6_config *cfg,
!netif_carrier_ok(dev)) !netif_carrier_ok(dev))
rt->fib6_nh.nh_flags |= RTNH_F_LINKDOWN; rt->fib6_nh.nh_flags |= RTNH_F_LINKDOWN;
rt->fib6_nh.nh_flags |= (cfg->fc_flags & RTNH_F_ONLINK); rt->fib6_nh.nh_flags |= (cfg->fc_flags & RTNH_F_ONLINK);
rt->fib6_nh.nh_dev = rt->dst.dev = dev; rt->fib6_nh.nh_dev = dev;
rt->rt6i_idev = idev; rt->rt6i_idev = idev;
rt->rt6i_table = table; rt->rt6i_table = table;
@ -3078,9 +3039,8 @@ static struct rt6_info *ip6_route_info_create(struct fib6_config *cfg,
dev_put(dev); dev_put(dev);
if (idev) if (idev)
in6_dev_put(idev); in6_dev_put(idev);
if (rt)
dst_release_immediate(&rt->dst);
fib6_info_release(rt);
return ERR_PTR(err); return ERR_PTR(err);
} }
@ -3095,6 +3055,7 @@ int ip6_route_add(struct fib6_config *cfg, gfp_t gfp_flags,
return PTR_ERR(rt); return PTR_ERR(rt);
err = __ip6_ins_rt(rt, &cfg->fc_nlinfo, extack); err = __ip6_ins_rt(rt, &cfg->fc_nlinfo, extack);
fib6_info_release(rt);
return err; return err;
} }
@ -3116,7 +3077,7 @@ static int __ip6_del_rt(struct rt6_info *rt, struct nl_info *info)
spin_unlock_bh(&table->tb6_lock); spin_unlock_bh(&table->tb6_lock);
out: out:
ip6_rt_put(rt); fib6_info_release(rt);
return err; return err;
} }
@ -3170,7 +3131,7 @@ static int __ip6_del_rt_siblings(struct rt6_info *rt, struct fib6_config *cfg)
out_unlock: out_unlock:
spin_unlock_bh(&table->tb6_lock); spin_unlock_bh(&table->tb6_lock);
out_put: out_put:
ip6_rt_put(rt); fib6_info_release(rt);
if (skb) { if (skb) {
rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
@ -3241,8 +3202,7 @@ static int ip6_route_del(struct fib6_config *cfg,
continue; continue;
if (cfg->fc_protocol && cfg->fc_protocol != rt->rt6i_protocol) if (cfg->fc_protocol && cfg->fc_protocol != rt->rt6i_protocol)
continue; continue;
if (!dst_hold_safe(&rt->dst)) fib6_info_hold(rt);
break;
rcu_read_unlock(); rcu_read_unlock();
/* if gateway was specified only delete the one hop */ /* if gateway was specified only delete the one hop */
@ -3510,12 +3470,9 @@ static void __rt6_purge_dflt_routers(struct net *net,
for_each_fib6_node_rt_rcu(&table->tb6_root) { for_each_fib6_node_rt_rcu(&table->tb6_root) {
if (rt->rt6i_flags & (RTF_DEFAULT | RTF_ADDRCONF) && if (rt->rt6i_flags & (RTF_DEFAULT | RTF_ADDRCONF) &&
(!rt->rt6i_idev || rt->rt6i_idev->cnf.accept_ra != 2)) { (!rt->rt6i_idev || rt->rt6i_idev->cnf.accept_ra != 2)) {
if (dst_hold_safe(&rt->dst)) { fib6_info_hold(rt);
rcu_read_unlock(); rcu_read_unlock();
ip6_del_rt(net, rt); ip6_del_rt(net, rt);
} else {
rcu_read_unlock();
}
goto restart; goto restart;
} }
} }
@ -3666,7 +3623,7 @@ struct rt6_info *addrconf_dst_alloc(struct net *net,
struct net_device *dev = idev->dev; struct net_device *dev = idev->dev;
struct rt6_info *rt; struct rt6_info *rt;
rt = ip6_dst_alloc(net, dev, DST_NOCOUNT); rt = fib6_info_alloc(gfp_flags);
if (!rt) if (!rt)
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
@ -3687,8 +3644,8 @@ struct rt6_info *addrconf_dst_alloc(struct net *net,
} }
rt->fib6_nh.nh_gw = *addr; rt->fib6_nh.nh_gw = *addr;
dev_hold(dev);
rt->fib6_nh.nh_dev = dev; rt->fib6_nh.nh_dev = dev;
rt->rt6i_gateway = *addr;
rt->rt6i_dst.addr = *addr; rt->rt6i_dst.addr = *addr;
rt->rt6i_dst.plen = 128; rt->rt6i_dst.plen = 128;
tb_id = l3mdev_fib_table(idev->dev) ? : RT6_TABLE_LOCAL; tb_id = l3mdev_fib_table(idev->dev) ? : RT6_TABLE_LOCAL;
@ -4325,7 +4282,7 @@ static int ip6_route_multipath_add(struct fib6_config *cfg,
err = ip6_route_info_append(info->nl_net, &rt6_nh_list, err = ip6_route_info_append(info->nl_net, &rt6_nh_list,
rt, &r_cfg); rt, &r_cfg);
if (err) { if (err) {
dst_release_immediate(&rt->dst); fib6_info_release(rt);
goto cleanup; goto cleanup;
} }
@ -4342,6 +4299,8 @@ static int ip6_route_multipath_add(struct fib6_config *cfg,
list_for_each_entry(nh, &rt6_nh_list, next) { list_for_each_entry(nh, &rt6_nh_list, next) {
rt_last = nh->rt6_info; rt_last = nh->rt6_info;
err = __ip6_ins_rt(nh->rt6_info, info, extack); err = __ip6_ins_rt(nh->rt6_info, info, extack);
fib6_info_release(nh->rt6_info);
/* save reference to first route for notification */ /* save reference to first route for notification */
if (!rt_notif && !err) if (!rt_notif && !err)
rt_notif = nh->rt6_info; rt_notif = nh->rt6_info;
@ -4389,7 +4348,7 @@ static int ip6_route_multipath_add(struct fib6_config *cfg,
cleanup: cleanup:
list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) { list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) {
if (nh->rt6_info) if (nh->rt6_info)
dst_release_immediate(&nh->rt6_info->dst); fib6_info_release(nh->rt6_info);
list_del(&nh->next); list_del(&nh->next);
kfree(nh); kfree(nh);
} }
@ -4814,14 +4773,6 @@ static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
goto errout; goto errout;
} }
if (fibmatch && rt->from) {
struct rt6_info *ort = rt->from;
dst_hold(&ort->dst);
ip6_rt_put(rt);
rt = ort;
}
skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
if (!skb) { if (!skb) {
ip6_rt_put(rt); ip6_rt_put(rt);
@ -4831,12 +4782,12 @@ static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
skb_dst_set(skb, &rt->dst); skb_dst_set(skb, &rt->dst);
if (fibmatch) if (fibmatch)
err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, iif, err = rt6_fill_node(net, skb, rt->from, NULL, NULL, NULL, iif,
RTM_NEWROUTE, NETLINK_CB(in_skb).portid, RTM_NEWROUTE, NETLINK_CB(in_skb).portid,
nlh->nlmsg_seq, 0); nlh->nlmsg_seq, 0);
else else
err = rt6_fill_node(net, skb, rt, dst, &fl6.daddr, &fl6.saddr, err = rt6_fill_node(net, skb, rt->from, dst,
iif, RTM_NEWROUTE, &fl6.daddr, &fl6.saddr, iif, RTM_NEWROUTE,
NETLINK_CB(in_skb).portid, nlh->nlmsg_seq, NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
0); 0);
if (err < 0) { if (err < 0) {