ipv6: Move pcpu cached routes to fib6_nh

rt6_info are specific instances of a fib entry and are tied to a
device and gateway - ie., a nexthop. Before nexthop objects, IPv6 fib
entries have separate fib6_info for each nexthop in a multipath route,
so the location of the pcpu cache in the fib6_info struct worked.
However, with nexthop objects a fib6_info can point to a set of nexthops
(yet another alignment of ipv6 with ipv4). Accordingly, the pcpu
cache needs to be moved to the fib6_nh struct so the cached entries
are local to the nexthop specification used to create the rt6_info.

Initialization and free of the pcpu entries moved to fib6_nh_init and
fib6_nh_release.

Change in location only, from fib6_info down to fib6_nh; no other
functional change intended.

Signed-off-by: David Ahern <dsahern@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/ip6_fib.h b/include/net/ip6_fib.h
index 0d0d06b..38e87ef 100644 (file)
--- a/include/net/ip6_fib.h
+++ b/include/net/ip6_fib.h
@@ -131,6 +131,8 @@ struct fib6_nh {
 #ifdef CONFIG_IPV6_ROUTER_PREF
        unsigned long           last_probe;
 #endif
+
+       struct rt6_info * __percpu *rt6i_pcpu;
 };
 
 struct fib6_info {
@@ -156,7 +158,6 @@ struct fib6_info {
        struct rt6key                   fib6_src;
        struct rt6key                   fib6_prefsrc;
 
-       struct rt6_info * __percpu      *rt6i_pcpu;
        struct rt6_exception_bucket __rcu *rt6i_exception_bucket;
 
        u32                             fib6_metric;

diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c
index f96d1de..4bc35dd 100644 (file)
--- a/net/ipv6/addrconf.c
+++ b/net/ipv6/addrconf.c
@@ -6341,16 +6341,16 @@ void addrconf_disable_policy_idev(struct inet6_dev *idev, int val)
        list_for_each_entry(ifa, &idev->addr_list, if_list) {
                spin_lock(&ifa->lock);
                if (ifa->rt) {
-                       struct fib6_info *rt = ifa->rt;
+                       struct fib6_nh *nh = &ifa->rt->fib6_nh;
                        int cpu;
 
                        rcu_read_lock();
                        ifa->rt->dst_nopolicy = val ? true : false;
-                       if (rt->rt6i_pcpu) {
+                       if (nh->rt6i_pcpu) {
                                for_each_possible_cpu(cpu) {
                                        struct rt6_info **rtp;
 
-                                       rtp = per_cpu_ptr(rt->rt6i_pcpu, cpu);
+                                       rtp = per_cpu_ptr(nh->rt6i_pcpu, cpu);
                                        addrconf_set_nopolicy(*rtp, val);
                                }
                        }

diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c
index 7958cf9..274f124 100644 (file)
--- a/net/ipv6/ip6_fib.c
+++ b/net/ipv6/ip6_fib.c
@@ -155,12 +155,6 @@ struct fib6_info *fib6_info_alloc(gfp_t gfp_flags)
        if (!f6i)
                return NULL;
 
-       f6i->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags);
-       if (!f6i->rt6i_pcpu) {
-               kfree(f6i);
-               return NULL;
-       }
-
        INIT_LIST_HEAD(&f6i->fib6_siblings);
        refcount_set(&f6i->fib6_ref, 1);
 
@@ -177,25 +171,6 @@ void fib6_info_destroy_rcu(struct rcu_head *head)
        bucket = rcu_dereference_protected(f6i->rt6i_exception_bucket, 1);
        kfree(bucket);
 
-       if (f6i->rt6i_pcpu) {
-               int cpu;
-
-               for_each_possible_cpu(cpu) {
-                       struct rt6_info **ppcpu_rt;
-                       struct rt6_info *pcpu_rt;
-
-                       ppcpu_rt = per_cpu_ptr(f6i->rt6i_pcpu, cpu);
-                       pcpu_rt = *ppcpu_rt;
-                       if (pcpu_rt) {
-                               dst_dev_put(&pcpu_rt->dst);
-                               dst_release(&pcpu_rt->dst);
-                               *ppcpu_rt = NULL;
-                       }
-               }
-
-               free_percpu(f6i->rt6i_pcpu);
-       }
-
        fib6_nh_release(&f6i->fib6_nh);
 
        ip_fib_metrics_put(f6i->fib6_metrics);
@@ -902,8 +877,12 @@ insert_above:
 static void fib6_drop_pcpu_from(struct fib6_info *f6i,
                                const struct fib6_table *table)
 {
+       struct fib6_nh *fib6_nh = &f6i->fib6_nh;
        int cpu;
 
+       if (!fib6_nh->rt6i_pcpu)
+               return;
+
        /* Make sure rt6_make_pcpu_route() wont add other percpu routes
         * while we are cleaning them here.
         */
@@ -917,7 +896,7 @@ static void fib6_drop_pcpu_from(struct fib6_info *f6i,
                struct rt6_info **ppcpu_rt;
                struct rt6_info *pcpu_rt;
 
-               ppcpu_rt = per_cpu_ptr(f6i->rt6i_pcpu, cpu);
+               ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu);
                pcpu_rt = *ppcpu_rt;
                if (pcpu_rt) {
                        struct fib6_info *from;
@@ -933,8 +912,7 @@ static void fib6_purge_rt(struct fib6_info *rt, struct fib6_node *fn,
 {
        struct fib6_table *table = rt->fib6_table;
 
-       if (rt->rt6i_pcpu)
-               fib6_drop_pcpu_from(rt, table);
+       fib6_drop_pcpu_from(rt, table);
 
        if (refcount_read(&rt->fib6_ref) != 1) {
                /* This route is used as dummy address holder in some split

diff --git a/net/ipv6/route.c b/net/ipv6/route.c
index 5f0661c..e404813 100644 (file)
--- a/net/ipv6/route.c
+++ b/net/ipv6/route.c
@@ -1270,7 +1270,7 @@ static struct rt6_info *rt6_get_pcpu_route(const struct fib6_result *res)
 {
        struct rt6_info *pcpu_rt, **p;
 
-       p = this_cpu_ptr(res->f6i->rt6i_pcpu);
+       p = this_cpu_ptr(res->nh->rt6i_pcpu);
        pcpu_rt = *p;
 
        if (pcpu_rt)
@@ -1291,7 +1291,7 @@ static struct rt6_info *rt6_make_pcpu_route(struct net *net,
        }
 
        dst_hold(&pcpu_rt->dst);
-       p = this_cpu_ptr(res->f6i->rt6i_pcpu);
+       p = this_cpu_ptr(res->nh->rt6i_pcpu);
        prev = cmpxchg(p, NULL, pcpu_rt);
        BUG_ON(prev);
 
@@ -3068,6 +3068,12 @@ int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh,
            !netif_carrier_ok(dev))
                fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;
 
+       fib6_nh->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags);
+       if (!fib6_nh->rt6i_pcpu) {
+               err = -ENOMEM;
+               goto out;
+       }
+
        err = fib_nh_common_init(&fib6_nh->nh_common, cfg->fc_encap,
                                 cfg->fc_encap_type, cfg, gfp_flags, extack);
        if (err)
@@ -3092,6 +3098,25 @@ out:
 
 void fib6_nh_release(struct fib6_nh *fib6_nh)
 {
+       if (fib6_nh->rt6i_pcpu) {
+               int cpu;
+
+               for_each_possible_cpu(cpu) {
+                       struct rt6_info **ppcpu_rt;
+                       struct rt6_info *pcpu_rt;
+
+                       ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu);
+                       pcpu_rt = *ppcpu_rt;
+                       if (pcpu_rt) {
+                               dst_dev_put(&pcpu_rt->dst);
+                               dst_release(&pcpu_rt->dst);
+                               *ppcpu_rt = NULL;
+                       }
+               }
+
+               free_percpu(fib6_nh->rt6i_pcpu);
+       }
+
        fib_nh_common_release(&fib6_nh->nh_common);
 }