net/ipv4/nexthop.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Generic nexthop implementation
   3  *
   4  * Copyright (c) 2017-19 Cumulus Networks
   5  * Copyright (c) 2017-19 David Ahern <dsa@cumulusnetworks.com>
   6  */
   7
   8 #include <linux/nexthop.h>
   9 #include <linux/rtnetlink.h>
  10 #include <linux/slab.h>
  11 #include <net/arp.h>
  12 #include <net/ipv6_stubs.h>
  13 #include <net/lwtunnel.h>
  14 #include <net/ndisc.h>
  15 #include <net/nexthop.h>
  16 #include <net/route.h>
  17 #include <net/sock.h>
  18
  19 static void remove_nexthop(struct net *net, struct nexthop *nh,
  20                            struct nl_info *nlinfo);
  21
  22 #define NH_DEV_HASHBITS  8
  23 #define NH_DEV_HASHSIZE (1U << NH_DEV_HASHBITS)
  24
  25 static const struct nla_policy rtm_nh_policy[NHA_MAX + 1] = {
  26         [NHA_UNSPEC]            = { .strict_start_type = NHA_UNSPEC + 1 },
  27         [NHA_ID]                = { .type = NLA_U32 },
  28         [NHA_GROUP]             = { .type = NLA_BINARY },
  29         [NHA_GROUP_TYPE]        = { .type = NLA_U16 },
  30         [NHA_BLACKHOLE]         = { .type = NLA_FLAG },
  31         [NHA_OIF]               = { .type = NLA_U32 },
  32         [NHA_GATEWAY]           = { .type = NLA_BINARY },
  33         [NHA_ENCAP_TYPE]        = { .type = NLA_U16 },
  34         [NHA_ENCAP]             = { .type = NLA_NESTED },
  35         [NHA_GROUPS]            = { .type = NLA_FLAG },
  36         [NHA_MASTER]            = { .type = NLA_U32 },
  37 };
  38
  39 static unsigned int nh_dev_hashfn(unsigned int val)
  40 {
  41         unsigned int mask = NH_DEV_HASHSIZE - 1;
  42
  43         return (val ^
  44                 (val >> NH_DEV_HASHBITS) ^
  45                 (val >> (NH_DEV_HASHBITS * 2))) & mask;
  46 }
  47
  48 static void nexthop_devhash_add(struct net *net, struct nh_info *nhi)
  49 {
  50         struct net_device *dev = nhi->fib_nhc.nhc_dev;
  51         struct hlist_head *head;
  52         unsigned int hash;
  53
  54         WARN_ON(!dev);
  55
  56         hash = nh_dev_hashfn(dev->ifindex);
  57         head = &net->nexthop.devhash[hash];
  58         hlist_add_head(&nhi->dev_hash, head);
  59 }
  60
  61 static void nexthop_free_mpath(struct nexthop *nh)
  62 {
  63         struct nh_group *nhg;
  64         int i;
  65
  66         nhg = rcu_dereference_raw(nh->nh_grp);
  67         for (i = 0; i < nhg->num_nh; ++i)
  68                 WARN_ON(nhg->nh_entries[i].nh);
  69
  70         kfree(nhg);
  71 }
  72
  73 static void nexthop_free_single(struct nexthop *nh)
  74 {
  75         struct nh_info *nhi;
  76
  77         nhi = rcu_dereference_raw(nh->nh_info);
  78         switch (nhi->family) {
  79         case AF_INET:
  80                 fib_nh_release(nh->net, &nhi->fib_nh);
  81                 break;
  82         case AF_INET6:
  83                 ipv6_stub->fib6_nh_release(&nhi->fib6_nh);
  84                 break;
  85         }
  86         kfree(nhi);
  87 }
  88
  89 void nexthop_free_rcu(struct rcu_head *head)
  90 {
  91         struct nexthop *nh = container_of(head, struct nexthop, rcu);
  92
  93         if (nh->is_group)
  94                 nexthop_free_mpath(nh);
  95         else
  96                 nexthop_free_single(nh);
  97
  98         kfree(nh);
  99 }
 100 EXPORT_SYMBOL_GPL(nexthop_free_rcu);
 101
 102 static struct nexthop *nexthop_alloc(void)
 103 {
 104         struct nexthop *nh;
 105
 106         nh = kzalloc(sizeof(struct nexthop), GFP_KERNEL);
 107         if (nh) {
 108                 INIT_LIST_HEAD(&nh->fi_list);
 109                 INIT_LIST_HEAD(&nh->f6i_list);
 110                 INIT_LIST_HEAD(&nh->grp_list);
 111         }
 112         return nh;
 113 }
 114
 115 static struct nh_group *nexthop_grp_alloc(u16 num_nh)
 116 {
 117         size_t sz = offsetof(struct nexthop, nh_grp)
 118                     + sizeof(struct nh_group)
 119                     + sizeof(struct nh_grp_entry) * num_nh;
 120         struct nh_group *nhg;
 121
 122         nhg = kzalloc(sz, GFP_KERNEL);
 123         if (nhg)
 124                 nhg->num_nh = num_nh;
 125
 126         return nhg;
 127 }
 128
 129 static void nh_base_seq_inc(struct net *net)
 130 {
 131         while (++net->nexthop.seq == 0)
 132                 ;
 133 }
 134
 135 /* no reference taken; rcu lock or rtnl must be held */
 136 struct nexthop *nexthop_find_by_id(struct net *net, u32 id)
 137 {
 138         struct rb_node **pp, *parent = NULL, *next;
 139
 140         pp = &net->nexthop.rb_root.rb_node;
 141         while (1) {
 142                 struct nexthop *nh;
 143
 144                 next = rcu_dereference_raw(*pp);
 145                 if (!next)
 146                         break;
 147                 parent = next;
 148
 149                 nh = rb_entry(parent, struct nexthop, rb_node);
 150                 if (id < nh->id)
 151                         pp = &next->rb_left;
 152                 else if (id > nh->id)
 153                         pp = &next->rb_right;
 154                 else
 155                         return nh;
 156         }
 157         return NULL;
 158 }
 159 EXPORT_SYMBOL_GPL(nexthop_find_by_id);
 160
 161 /* used for auto id allocation; called with rtnl held */
 162 static u32 nh_find_unused_id(struct net *net)
 163 {
 164         u32 id_start = net->nexthop.last_id_allocated;
 165
 166         while (1) {
 167                 net->nexthop.last_id_allocated++;
 168                 if (net->nexthop.last_id_allocated == id_start)
 169                         break;
 170
 171                 if (!nexthop_find_by_id(net, net->nexthop.last_id_allocated))
 172                         return net->nexthop.last_id_allocated;
 173         }
 174         return 0;
 175 }
 176
 177 static int nla_put_nh_group(struct sk_buff *skb, struct nh_group *nhg)
 178 {
 179         struct nexthop_grp *p;
 180         size_t len = nhg->num_nh * sizeof(*p);
 181         struct nlattr *nla;
 182         u16 group_type = 0;
 183         int i;
 184
 185         if (nhg->mpath)
 186                 group_type = NEXTHOP_GRP_TYPE_MPATH;
 187
 188         if (nla_put_u16(skb, NHA_GROUP_TYPE, group_type))
 189                 goto nla_put_failure;
 190
 191         nla = nla_reserve(skb, NHA_GROUP, len);
 192         if (!nla)
 193                 goto nla_put_failure;
 194
 195         p = nla_data(nla);
 196         for (i = 0; i < nhg->num_nh; ++i) {
 197                 p->id = nhg->nh_entries[i].nh->id;
 198                 p->weight = nhg->nh_entries[i].weight - 1;
 199                 p += 1;
 200         }
 201
 202         return 0;
 203
 204 nla_put_failure:
 205         return -EMSGSIZE;
 206 }
 207
 208 static int nh_fill_node(struct sk_buff *skb, struct nexthop *nh,
 209                         int event, u32 portid, u32 seq, unsigned int nlflags)
 210 {
 211         struct fib6_nh *fib6_nh;
 212         struct fib_nh *fib_nh;
 213         struct nlmsghdr *nlh;
 214         struct nh_info *nhi;
 215         struct nhmsg *nhm;
 216
 217         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*nhm), nlflags);
 218         if (!nlh)
 219                 return -EMSGSIZE;
 220
 221         nhm = nlmsg_data(nlh);
 222         nhm->nh_family = AF_UNSPEC;
 223         nhm->nh_flags = nh->nh_flags;
 224         nhm->nh_protocol = nh->protocol;
 225         nhm->nh_scope = 0;
 226         nhm->resvd = 0;
 227
 228         if (nla_put_u32(skb, NHA_ID, nh->id))
 229                 goto nla_put_failure;
 230
 231         if (nh->is_group) {
 232                 struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 233
 234                 if (nla_put_nh_group(skb, nhg))
 235                         goto nla_put_failure;
 236                 goto out;
 237         }
 238
 239         nhi = rtnl_dereference(nh->nh_info);
 240         nhm->nh_family = nhi->family;
 241         if (nhi->reject_nh) {
 242                 if (nla_put_flag(skb, NHA_BLACKHOLE))
 243                         goto nla_put_failure;
 244                 goto out;
 245         } else {
 246                 const struct net_device *dev;
 247
 248                 dev = nhi->fib_nhc.nhc_dev;
 249                 if (dev && nla_put_u32(skb, NHA_OIF, dev->ifindex))
 250                         goto nla_put_failure;
 251         }
 252
 253         nhm->nh_scope = nhi->fib_nhc.nhc_scope;
 254         switch (nhi->family) {
 255         case AF_INET:
 256                 fib_nh = &nhi->fib_nh;
 257                 if (fib_nh->fib_nh_gw_family &&
 258                     nla_put_u32(skb, NHA_GATEWAY, fib_nh->fib_nh_gw4))
 259                         goto nla_put_failure;
 260                 break;
 261
 262         case AF_INET6:
 263                 fib6_nh = &nhi->fib6_nh;
 264                 if (fib6_nh->fib_nh_gw_family &&
 265                     nla_put_in6_addr(skb, NHA_GATEWAY, &fib6_nh->fib_nh_gw6))
 266                         goto nla_put_failure;
 267                 break;
 268         }
 269
 270         if (nhi->fib_nhc.nhc_lwtstate &&
 271             lwtunnel_fill_encap(skb, nhi->fib_nhc.nhc_lwtstate,
 272                                 NHA_ENCAP, NHA_ENCAP_TYPE) < 0)
 273                 goto nla_put_failure;
 274
 275 out:
 276         nlmsg_end(skb, nlh);
 277         return 0;
 278
 279 nla_put_failure:
 280         return -EMSGSIZE;
 281 }
 282
 283 static size_t nh_nlmsg_size_grp(struct nexthop *nh)
 284 {
 285         struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 286         size_t sz = sizeof(struct nexthop_grp) * nhg->num_nh;
 287
 288         return nla_total_size(sz) +
 289                nla_total_size(2);  /* NHA_GROUP_TYPE */
 290 }
 291
 292 static size_t nh_nlmsg_size_single(struct nexthop *nh)
 293 {
 294         struct nh_info *nhi = rtnl_dereference(nh->nh_info);
 295         size_t sz;
 296
 297         /* covers NHA_BLACKHOLE since NHA_OIF and BLACKHOLE
 298          * are mutually exclusive
 299          */
 300         sz = nla_total_size(4);  /* NHA_OIF */
 301
 302         switch (nhi->family) {
 303         case AF_INET:
 304                 if (nhi->fib_nh.fib_nh_gw_family)
 305                         sz += nla_total_size(4);  /* NHA_GATEWAY */
 306                 break;
 307
 308         case AF_INET6:
 309                 /* NHA_GATEWAY */
 310                 if (nhi->fib6_nh.fib_nh_gw_family)
 311                         sz += nla_total_size(sizeof(const struct in6_addr));
 312                 break;
 313         }
 314
 315         if (nhi->fib_nhc.nhc_lwtstate) {
 316                 sz += lwtunnel_get_encap_size(nhi->fib_nhc.nhc_lwtstate);
 317                 sz += nla_total_size(2);  /* NHA_ENCAP_TYPE */
 318         }
 319
 320         return sz;
 321 }
 322
 323 static size_t nh_nlmsg_size(struct nexthop *nh)
 324 {
 325         size_t sz = nla_total_size(4);    /* NHA_ID */
 326
 327         if (nh->is_group)
 328                 sz += nh_nlmsg_size_grp(nh);
 329         else
 330                 sz += nh_nlmsg_size_single(nh);
 331
 332         return sz;
 333 }
 334
 335 static void nexthop_notify(int event, struct nexthop *nh, struct nl_info *info)
 336 {
 337         unsigned int nlflags = info->nlh ? info->nlh->nlmsg_flags : 0;
 338         u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
 339         struct sk_buff *skb;
 340         int err = -ENOBUFS;
 341
 342         skb = nlmsg_new(nh_nlmsg_size(nh), gfp_any());
 343         if (!skb)
 344                 goto errout;
 345
 346         err = nh_fill_node(skb, nh, event, info->portid, seq, nlflags);
 347         if (err < 0) {
 348                 /* -EMSGSIZE implies BUG in nh_nlmsg_size() */
 349                 WARN_ON(err == -EMSGSIZE);
 350                 kfree_skb(skb);
 351                 goto errout;
 352         }
 353
 354         rtnl_notify(skb, info->nl_net, info->portid, RTNLGRP_NEXTHOP,
 355                     info->nlh, gfp_any());
 356         return;
 357 errout:
 358         if (err < 0)
 359                 rtnl_set_sk_err(info->nl_net, RTNLGRP_NEXTHOP, err);
 360 }
 361
 362 static bool valid_group_nh(struct nexthop *nh, unsigned int npaths,
 363                            struct netlink_ext_ack *extack)
 364 {
 365         if (nh->is_group) {
 366                 struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 367
 368                 /* nested multipath (group within a group) is not
 369                  * supported
 370                  */
 371                 if (nhg->mpath) {
 372                         NL_SET_ERR_MSG(extack,
 373                                        "Multipath group can not be a nexthop within a group");
 374                         return false;
 375                 }
 376         } else {
 377                 struct nh_info *nhi = rtnl_dereference(nh->nh_info);
 378
 379                 if (nhi->reject_nh && npaths > 1) {
 380                         NL_SET_ERR_MSG(extack,
 381                                        "Blackhole nexthop can not be used in a group with more than 1 path");
 382                         return false;
 383                 }
 384         }
 385
 386         return true;
 387 }
 388
 389 static int nh_check_attr_group(struct net *net, struct nlattr *tb[],
 390                                struct netlink_ext_ack *extack)
 391 {
 392         unsigned int len = nla_len(tb[NHA_GROUP]);
 393         struct nexthop_grp *nhg;
 394         unsigned int i, j;
 395
 396         if (len & (sizeof(struct nexthop_grp) - 1)) {
 397                 NL_SET_ERR_MSG(extack,
 398                                "Invalid length for nexthop group attribute");
 399                 return -EINVAL;
 400         }
 401
 402         /* convert len to number of nexthop ids */
 403         len /= sizeof(*nhg);
 404
 405         nhg = nla_data(tb[NHA_GROUP]);
 406         for (i = 0; i < len; ++i) {
 407                 if (nhg[i].resvd1 || nhg[i].resvd2) {
 408                         NL_SET_ERR_MSG(extack, "Reserved fields in nexthop_grp must be 0");
 409                         return -EINVAL;
 410                 }
 411                 if (nhg[i].weight > 254) {
 412                         NL_SET_ERR_MSG(extack, "Invalid value for weight");
 413                         return -EINVAL;
 414                 }
 415                 for (j = i + 1; j < len; ++j) {
 416                         if (nhg[i].id == nhg[j].id) {
 417                                 NL_SET_ERR_MSG(extack, "Nexthop id can not be used twice in a group");
 418                                 return -EINVAL;
 419                         }
 420                 }
 421         }
 422
 423         nhg = nla_data(tb[NHA_GROUP]);
 424         for (i = 0; i < len; ++i) {
 425                 struct nexthop *nh;
 426
 427                 nh = nexthop_find_by_id(net, nhg[i].id);
 428                 if (!nh) {
 429                         NL_SET_ERR_MSG(extack, "Invalid nexthop id");
 430                         return -EINVAL;
 431                 }
 432                 if (!valid_group_nh(nh, len, extack))
 433                         return -EINVAL;
 434         }
 435         for (i = NHA_GROUP + 1; i < __NHA_MAX; ++i) {
 436                 if (!tb[i])
 437                         continue;
 438
 439                 NL_SET_ERR_MSG(extack,
 440                                "No other attributes can be set in nexthop groups");
 441                 return -EINVAL;
 442         }
 443
 444         return 0;
 445 }
 446
 447 static bool ipv6_good_nh(const struct fib6_nh *nh)
 448 {
 449         int state = NUD_REACHABLE;
 450         struct neighbour *n;
 451
 452         rcu_read_lock_bh();
 453
 454         n = __ipv6_neigh_lookup_noref_stub(nh->fib_nh_dev, &nh->fib_nh_gw6);
 455         if (n)
 456                 state = n->nud_state;
 457
 458         rcu_read_unlock_bh();
 459
 460         return !!(state & NUD_VALID);
 461 }
 462
 463 static bool ipv4_good_nh(const struct fib_nh *nh)
 464 {
 465         int state = NUD_REACHABLE;
 466         struct neighbour *n;
 467
 468         rcu_read_lock_bh();
 469
 470         n = __ipv4_neigh_lookup_noref(nh->fib_nh_dev,
 471                                       (__force u32)nh->fib_nh_gw4);
 472         if (n)
 473                 state = n->nud_state;
 474
 475         rcu_read_unlock_bh();
 476
 477         return !!(state & NUD_VALID);
 478 }
 479
 480 struct nexthop *nexthop_select_path(struct nexthop *nh, int hash)
 481 {
 482         struct nexthop *rc = NULL;
 483         struct nh_group *nhg;
 484         int i;
 485
 486         if (!nh->is_group)
 487                 return nh;
 488
 489         nhg = rcu_dereference(nh->nh_grp);
 490         for (i = 0; i < nhg->num_nh; ++i) {
 491                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 492                 struct nh_info *nhi;
 493
 494                 if (hash > atomic_read(&nhge->upper_bound))
 495                         continue;
 496
 497                 /* nexthops always check if it is good and does
 498                  * not rely on a sysctl for this behavior
 499                  */
 500                 nhi = rcu_dereference(nhge->nh->nh_info);
 501                 switch (nhi->family) {
 502                 case AF_INET:
 503                         if (ipv4_good_nh(&nhi->fib_nh))
 504                                 return nhge->nh;
 505                         break;
 506                 case AF_INET6:
 507                         if (ipv6_good_nh(&nhi->fib6_nh))
 508                                 return nhge->nh;
 509                         break;
 510                 }
 511
 512                 if (!rc)
 513                         rc = nhge->nh;
 514         }
 515
 516         return rc;
 517 }
 518 EXPORT_SYMBOL_GPL(nexthop_select_path);
 519
 520 int nexthop_for_each_fib6_nh(struct nexthop *nh,
 521                              int (*cb)(struct fib6_nh *nh, void *arg),
 522                              void *arg)
 523 {
 524         struct nh_info *nhi;
 525         int err;
 526
 527         if (nh->is_group) {
 528                 struct nh_group *nhg;
 529                 int i;
 530
 531                 nhg = rcu_dereference_rtnl(nh->nh_grp);
 532                 for (i = 0; i < nhg->num_nh; i++) {
 533                         struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 534
 535                         nhi = rcu_dereference_rtnl(nhge->nh->nh_info);
 536                         err = cb(&nhi->fib6_nh, arg);
 537                         if (err)
 538                                 return err;
 539                 }
 540         } else {
 541                 nhi = rcu_dereference_rtnl(nh->nh_info);
 542                 err = cb(&nhi->fib6_nh, arg);
 543                 if (err)
 544                         return err;
 545         }
 546
 547         return 0;
 548 }
 549 EXPORT_SYMBOL_GPL(nexthop_for_each_fib6_nh);
 550
 551 int fib6_check_nexthop(struct nexthop *nh, struct fib6_config *cfg,
 552                        struct netlink_ext_ack *extack)
 553 {
 554         struct nh_info *nhi;
 555
 556         /* fib6_src is unique to a fib6_info and limits the ability to cache
 557          * routes in fib6_nh within a nexthop that is potentially shared
 558          * across multiple fib entries. If the config wants to use source
 559          * routing it can not use nexthop objects. mlxsw also does not allow
 560          * fib6_src on routes.
 561          */
 562         if (!ipv6_addr_any(&cfg->fc_src)) {
 563                 NL_SET_ERR_MSG(extack, "IPv6 routes using source address can not use nexthop objects");
 564                 return -EINVAL;
 565         }
 566
 567         if (nh->is_group) {
 568                 struct nh_group *nhg;
 569
 570                 nhg = rtnl_dereference(nh->nh_grp);
 571                 if (nhg->has_v4)
 572                         goto no_v4_nh;
 573         } else {
 574                 nhi = rtnl_dereference(nh->nh_info);
 575                 if (nhi->family == AF_INET)
 576                         goto no_v4_nh;
 577         }
 578
 579         return 0;
 580 no_v4_nh:
 581         NL_SET_ERR_MSG(extack, "IPv6 routes can not use an IPv4 nexthop");
 582         return -EINVAL;
 583 }
 584 EXPORT_SYMBOL_GPL(fib6_check_nexthop);
 585
 586 static int nexthop_check_scope(struct nexthop *nh, u8 scope,
 587                                struct netlink_ext_ack *extack)
 588 {
 589         struct nh_info *nhi;
 590
 591         nhi = rtnl_dereference(nh->nh_info);
 592         if (scope == RT_SCOPE_HOST && nhi->fib_nhc.nhc_gw_family) {
 593                 NL_SET_ERR_MSG(extack,
 594                                "Route with host scope can not have a gateway");
 595                 return -EINVAL;
 596         }
 597
 598         if (nhi->fib_nhc.nhc_flags & RTNH_F_ONLINK && scope >= RT_SCOPE_LINK) {
 599                 NL_SET_ERR_MSG(extack, "Scope mismatch with nexthop");
 600                 return -EINVAL;
 601         }
 602
 603         return 0;
 604 }
 605
 606 /* Invoked by fib add code to verify nexthop by id is ok with
 607  * config for prefix; parts of fib_check_nh not done when nexthop
 608  * object is used.
 609  */
 610 int fib_check_nexthop(struct nexthop *nh, u8 scope,
 611                       struct netlink_ext_ack *extack)
 612 {
 613         int err = 0;
 614
 615         if (nh->is_group) {
 616                 struct nh_group *nhg;
 617
 618                 if (scope == RT_SCOPE_HOST) {
 619                         NL_SET_ERR_MSG(extack, "Route with host scope can not have multiple nexthops");
 620                         err = -EINVAL;
 621                         goto out;
 622                 }
 623
 624                 nhg = rtnl_dereference(nh->nh_grp);
 625                 /* all nexthops in a group have the same scope */
 626                 err = nexthop_check_scope(nhg->nh_entries[0].nh, scope, extack);
 627         } else {
 628                 err = nexthop_check_scope(nh, scope, extack);
 629         }
 630 out:
 631         return err;
 632 }
 633
 634 static void nh_group_rebalance(struct nh_group *nhg)
 635 {
 636         int total = 0;
 637         int w = 0;
 638         int i;
 639
 640         for (i = 0; i < nhg->num_nh; ++i)
 641                 total += nhg->nh_entries[i].weight;
 642
 643         for (i = 0; i < nhg->num_nh; ++i) {
 644                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 645                 int upper_bound;
 646
 647                 w += nhge->weight;
 648                 upper_bound = DIV_ROUND_CLOSEST_ULL((u64)w << 31, total) - 1;
 649                 atomic_set(&nhge->upper_bound, upper_bound);
 650         }
 651 }
 652
 653 static void remove_nh_grp_entry(struct nh_grp_entry *nhge,
 654                                 struct nh_group *nhg,
 655                                 struct nl_info *nlinfo)
 656 {
 657         struct nexthop *nh = nhge->nh;
 658         struct nh_grp_entry *nhges;
 659         bool found = false;
 660         int i;
 661
 662         WARN_ON(!nh);
 663
 664         nhges = nhg->nh_entries;
 665         for (i = 0; i < nhg->num_nh; ++i) {
 666                 if (found) {
 667                         nhges[i-1].nh = nhges[i].nh;
 668                         nhges[i-1].weight = nhges[i].weight;
 669                         list_del(&nhges[i].nh_list);
 670                         list_add(&nhges[i-1].nh_list, &nhges[i-1].nh->grp_list);
 671                 } else if (nhg->nh_entries[i].nh == nh) {
 672                         found = true;
 673                 }
 674         }
 675
 676         if (WARN_ON(!found))
 677                 return;
 678
 679         nhg->num_nh--;
 680         nhg->nh_entries[nhg->num_nh].nh = NULL;
 681
 682         nh_group_rebalance(nhg);
 683
 684         nexthop_put(nh);
 685
 686         if (nlinfo)
 687                 nexthop_notify(RTM_NEWNEXTHOP, nhge->nh_parent, nlinfo);
 688 }
 689
 690 static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh,
 691                                        struct nl_info *nlinfo)
 692 {
 693         struct nh_grp_entry *nhge, *tmp;
 694
 695         list_for_each_entry_safe(nhge, tmp, &nh->grp_list, nh_list) {
 696                 struct nh_group *nhg;
 697
 698                 list_del(&nhge->nh_list);
 699                 nhg = rtnl_dereference(nhge->nh_parent->nh_grp);
 700                 remove_nh_grp_entry(nhge, nhg, nlinfo);
 701
 702                 /* if this group has no more entries then remove it */
 703                 if (!nhg->num_nh)
 704                         remove_nexthop(net, nhge->nh_parent, nlinfo);
 705         }
 706 }
 707
 708 static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo)
 709 {
 710         struct nh_group *nhg = rcu_dereference_rtnl(nh->nh_grp);
 711         int i, num_nh = nhg->num_nh;
 712
 713         for (i = 0; i < num_nh; ++i) {
 714                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 715
 716                 if (WARN_ON(!nhge->nh))
 717                         continue;
 718
 719                 list_del(&nhge->nh_list);
 720                 nexthop_put(nhge->nh);
 721                 nhge->nh = NULL;
 722                 nhg->num_nh--;
 723         }
 724 }
 725
 726 static void __remove_nexthop_fib(struct net *net, struct nexthop *nh)
 727 {
 728         struct fib6_info *f6i, *tmp;
 729         bool do_flush = false;
 730         struct fib_info *fi;
 731
 732         list_for_each_entry(fi, &nh->fi_list, nh_list) {
 733                 fi->fib_flags |= RTNH_F_DEAD;
 734                 do_flush = true;
 735         }
 736         if (do_flush)
 737                 fib_flush(net);
 738
 739         /* ip6_del_rt removes the entry from this list hence the _safe */
 740         list_for_each_entry_safe(f6i, tmp, &nh->f6i_list, nh_list) {
 741                 /* __ip6_del_rt does a release, so do a hold here */
 742                 fib6_info_hold(f6i);
 743                 ipv6_stub->ip6_del_rt(net, f6i);
 744         }
 745 }
 746
 747 static void __remove_nexthop(struct net *net, struct nexthop *nh,
 748                              struct nl_info *nlinfo)
 749 {
 750         __remove_nexthop_fib(net, nh);
 751
 752         if (nh->is_group) {
 753                 remove_nexthop_group(nh, nlinfo);
 754         } else {
 755                 struct nh_info *nhi;
 756
 757                 nhi = rtnl_dereference(nh->nh_info);
 758                 if (nhi->fib_nhc.nhc_dev)
 759                         hlist_del(&nhi->dev_hash);
 760
 761                 remove_nexthop_from_groups(net, nh, nlinfo);
 762         }
 763 }
 764
 765 static void remove_nexthop(struct net *net, struct nexthop *nh,
 766                            struct nl_info *nlinfo)
 767 {
 768         /* remove from the tree */
 769         rb_erase(&nh->rb_node, &net->nexthop.rb_root);
 770
 771         if (nlinfo)
 772                 nexthop_notify(RTM_DELNEXTHOP, nh, nlinfo);
 773
 774         __remove_nexthop(net, nh, nlinfo);
 775         nh_base_seq_inc(net);
 776
 777         nexthop_put(nh);
 778 }
 779
 780 static int replace_nexthop(struct net *net, struct nexthop *old,
 781                            struct nexthop *new, struct netlink_ext_ack *extack)
 782 {
 783         return -EEXIST;
 784 }
 785
 786 /* called with rtnl_lock held */
 787 static int insert_nexthop(struct net *net, struct nexthop *new_nh,
 788                           struct nh_config *cfg, struct netlink_ext_ack *extack)
 789 {
 790         struct rb_node **pp, *parent = NULL, *next;
 791         struct rb_root *root = &net->nexthop.rb_root;
 792         bool replace = !!(cfg->nlflags & NLM_F_REPLACE);
 793         bool create = !!(cfg->nlflags & NLM_F_CREATE);
 794         u32 new_id = new_nh->id;
 795         int rc = -EEXIST;
 796
 797         pp = &root->rb_node;
 798         while (1) {
 799                 struct nexthop *nh;
 800
 801                 next = rtnl_dereference(*pp);
 802                 if (!next)
 803                         break;
 804
 805                 parent = next;
 806
 807                 nh = rb_entry(parent, struct nexthop, rb_node);
 808                 if (new_id < nh->id) {
 809                         pp = &next->rb_left;
 810                 } else if (new_id > nh->id) {
 811                         pp = &next->rb_right;
 812                 } else if (replace) {
 813                         rc = replace_nexthop(net, nh, new_nh, extack);
 814                         if (!rc)
 815                                 new_nh = nh; /* send notification with old nh */
 816                         goto out;
 817                 } else {
 818                         /* id already exists and not a replace */
 819                         goto out;
 820                 }
 821         }
 822
 823         if (replace && !create) {
 824                 NL_SET_ERR_MSG(extack, "Replace specified without create and no entry exists");
 825                 rc = -ENOENT;
 826                 goto out;
 827         }
 828
 829         rb_link_node_rcu(&new_nh->rb_node, parent, pp);
 830         rb_insert_color(&new_nh->rb_node, root);
 831         rc = 0;
 832 out:
 833         if (!rc) {
 834                 nh_base_seq_inc(net);
 835                 nexthop_notify(RTM_NEWNEXTHOP, new_nh, &cfg->nlinfo);
 836         }
 837
 838         return rc;
 839 }
 840
 841 /* rtnl */
 842 /* remove all nexthops tied to a device being deleted */
 843 static void nexthop_flush_dev(struct net_device *dev)
 844 {
 845         unsigned int hash = nh_dev_hashfn(dev->ifindex);
 846         struct net *net = dev_net(dev);
 847         struct hlist_head *head = &net->nexthop.devhash[hash];
 848         struct hlist_node *n;
 849         struct nh_info *nhi;
 850
 851         hlist_for_each_entry_safe(nhi, n, head, dev_hash) {
 852                 if (nhi->fib_nhc.nhc_dev != dev)
 853                         continue;
 854
 855                 remove_nexthop(net, nhi->nh_parent, NULL);
 856         }
 857 }
 858
 859 /* rtnl; called when net namespace is deleted */
 860 static void flush_all_nexthops(struct net *net)
 861 {
 862         struct rb_root *root = &net->nexthop.rb_root;
 863         struct rb_node *node;
 864         struct nexthop *nh;
 865
 866         while ((node = rb_first(root))) {
 867                 nh = rb_entry(node, struct nexthop, rb_node);
 868                 remove_nexthop(net, nh, NULL);
 869                 cond_resched();
 870         }
 871 }
 872
 873 static struct nexthop *nexthop_create_group(struct net *net,
 874                                             struct nh_config *cfg)
 875 {
 876         struct nlattr *grps_attr = cfg->nh_grp;
 877         struct nexthop_grp *entry = nla_data(grps_attr);
 878         struct nh_group *nhg;
 879         struct nexthop *nh;
 880         int i;
 881
 882         nh = nexthop_alloc();
 883         if (!nh)
 884                 return ERR_PTR(-ENOMEM);
 885
 886         nh->is_group = 1;
 887
 888         nhg = nexthop_grp_alloc(nla_len(grps_attr) / sizeof(*entry));
 889         if (!nhg) {
 890                 kfree(nh);
 891                 return ERR_PTR(-ENOMEM);
 892         }
 893
 894         for (i = 0; i < nhg->num_nh; ++i) {
 895                 struct nexthop *nhe;
 896                 struct nh_info *nhi;
 897
 898                 nhe = nexthop_find_by_id(net, entry[i].id);
 899                 if (!nexthop_get(nhe))
 900                         goto out_no_nh;
 901
 902                 nhi = rtnl_dereference(nhe->nh_info);
 903                 if (nhi->family == AF_INET)
 904                         nhg->has_v4 = true;
 905
 906                 nhg->nh_entries[i].nh = nhe;
 907                 nhg->nh_entries[i].weight = entry[i].weight + 1;
 908                 list_add(&nhg->nh_entries[i].nh_list, &nhe->grp_list);
 909                 nhg->nh_entries[i].nh_parent = nh;
 910         }
 911
 912         if (cfg->nh_grp_type == NEXTHOP_GRP_TYPE_MPATH) {
 913                 nhg->mpath = 1;
 914                 nh_group_rebalance(nhg);
 915         }
 916
 917         rcu_assign_pointer(nh->nh_grp, nhg);
 918
 919         return nh;
 920
 921 out_no_nh:
 922         for (; i >= 0; --i)
 923                 nexthop_put(nhg->nh_entries[i].nh);
 924
 925         kfree(nhg);
 926         kfree(nh);
 927
 928         return ERR_PTR(-ENOENT);
 929 }
 930
 931 static int nh_create_ipv4(struct net *net, struct nexthop *nh,
 932                           struct nh_info *nhi, struct nh_config *cfg,
 933                           struct netlink_ext_ack *extack)
 934 {
 935         struct fib_nh *fib_nh = &nhi->fib_nh;
 936         struct fib_config fib_cfg = {
 937                 .fc_oif   = cfg->nh_ifindex,
 938                 .fc_gw4   = cfg->gw.ipv4,
 939                 .fc_gw_family = cfg->gw.ipv4 ? AF_INET : 0,
 940                 .fc_flags = cfg->nh_flags,
 941                 .fc_encap = cfg->nh_encap,
 942                 .fc_encap_type = cfg->nh_encap_type,
 943         };
 944         u32 tb_id = l3mdev_fib_table(cfg->dev);
 945         int err = -EINVAL;
 946
 947         err = fib_nh_init(net, fib_nh, &fib_cfg, 1, extack);
 948         if (err) {
 949                 fib_nh_release(net, fib_nh);
 950                 goto out;
 951         }
 952
 953         /* sets nh_dev if successful */
 954         err = fib_check_nh(net, fib_nh, tb_id, 0, extack);
 955         if (!err) {
 956                 nh->nh_flags = fib_nh->fib_nh_flags;
 957                 fib_info_update_nhc_saddr(net, &fib_nh->nh_common,
 958                                           fib_nh->fib_nh_scope);
 959         } else {
 960                 fib_nh_release(net, fib_nh);
 961         }
 962 out:
 963         return err;
 964 }
 965
 966 static int nh_create_ipv6(struct net *net,  struct nexthop *nh,
 967                           struct nh_info *nhi, struct nh_config *cfg,
 968                           struct netlink_ext_ack *extack)
 969 {
 970         struct fib6_nh *fib6_nh = &nhi->fib6_nh;
 971         struct fib6_config fib6_cfg = {
 972                 .fc_table = l3mdev_fib_table(cfg->dev),
 973                 .fc_ifindex = cfg->nh_ifindex,
 974                 .fc_gateway = cfg->gw.ipv6,
 975                 .fc_flags = cfg->nh_flags,
 976                 .fc_encap = cfg->nh_encap,
 977                 .fc_encap_type = cfg->nh_encap_type,
 978         };
 979         int err;
 980
 981         if (!ipv6_addr_any(&cfg->gw.ipv6))
 982                 fib6_cfg.fc_flags |= RTF_GATEWAY;
 983
 984         /* sets nh_dev if successful */
 985         err = ipv6_stub->fib6_nh_init(net, fib6_nh, &fib6_cfg, GFP_KERNEL,
 986                                       extack);
 987         if (err)
 988                 ipv6_stub->fib6_nh_release(fib6_nh);
 989         else
 990                 nh->nh_flags = fib6_nh->fib_nh_flags;
 991
 992         return err;
 993 }
 994
 995 static struct nexthop *nexthop_create(struct net *net, struct nh_config *cfg,
 996                                       struct netlink_ext_ack *extack)
 997 {
 998         struct nh_info *nhi;
 999         struct nexthop *nh;
1000         int err = 0;
1001
1002         nh = nexthop_alloc();
1003         if (!nh)
1004                 return ERR_PTR(-ENOMEM);
1005
1006         nhi = kzalloc(sizeof(*nhi), GFP_KERNEL);
1007         if (!nhi) {
1008                 kfree(nh);
1009                 return ERR_PTR(-ENOMEM);
1010         }
1011
1012         nh->nh_flags = cfg->nh_flags;
1013         nh->net = net;
1014
1015         nhi->nh_parent = nh;
1016         nhi->family = cfg->nh_family;
1017         nhi->fib_nhc.nhc_scope = RT_SCOPE_LINK;
1018
1019         if (cfg->nh_blackhole) {
1020                 nhi->reject_nh = 1;
1021                 cfg->nh_ifindex = net->loopback_dev->ifindex;
1022         }
1023
1024         switch (cfg->nh_family) {
1025         case AF_INET:
1026                 err = nh_create_ipv4(net, nh, nhi, cfg, extack);
1027                 break;
1028         case AF_INET6:
1029                 err = nh_create_ipv6(net, nh, nhi, cfg, extack);
1030                 break;
1031         }
1032
1033         if (err) {
1034                 kfree(nhi);
1035                 kfree(nh);
1036                 return ERR_PTR(err);
1037         }
1038
1039         /* add the entry to the device based hash */
1040         nexthop_devhash_add(net, nhi);
1041
1042         rcu_assign_pointer(nh->nh_info, nhi);
1043
1044         return nh;
1045 }
1046
1047 /* called with rtnl lock held */
1048 static struct nexthop *nexthop_add(struct net *net, struct nh_config *cfg,
1049                                    struct netlink_ext_ack *extack)
1050 {
1051         struct nexthop *nh;
1052         int err;
1053
1054         if (cfg->nlflags & NLM_F_REPLACE && !cfg->nh_id) {
1055                 NL_SET_ERR_MSG(extack, "Replace requires nexthop id");
1056                 return ERR_PTR(-EINVAL);
1057         }
1058
1059         if (!cfg->nh_id) {
1060                 cfg->nh_id = nh_find_unused_id(net);
1061                 if (!cfg->nh_id) {
1062                         NL_SET_ERR_MSG(extack, "No unused id");
1063                         return ERR_PTR(-EINVAL);
1064                 }
1065         }
1066
1067         if (cfg->nh_grp)
1068                 nh = nexthop_create_group(net, cfg);
1069         else
1070                 nh = nexthop_create(net, cfg, extack);
1071
1072         if (IS_ERR(nh))
1073                 return nh;
1074
1075         refcount_set(&nh->refcnt, 1);
1076         nh->id = cfg->nh_id;
1077         nh->protocol = cfg->nh_protocol;
1078         nh->net = net;
1079
1080         err = insert_nexthop(net, nh, cfg, extack);
1081         if (err) {
1082                 __remove_nexthop(net, nh, NULL);
1083                 nexthop_put(nh);
1084                 nh = ERR_PTR(err);
1085         }
1086
1087         return nh;
1088 }
1089
1090 static int rtm_to_nh_config(struct net *net, struct sk_buff *skb,
1091                             struct nlmsghdr *nlh, struct nh_config *cfg,
1092                             struct netlink_ext_ack *extack)
1093 {
1094         struct nhmsg *nhm = nlmsg_data(nlh);
1095         struct nlattr *tb[NHA_MAX + 1];
1096         int err;
1097
1098         err = nlmsg_parse(nlh, sizeof(*nhm), tb, NHA_MAX, rtm_nh_policy,
1099                           extack);
1100         if (err < 0)
1101                 return err;
1102
1103         err = -EINVAL;
1104         if (nhm->resvd || nhm->nh_scope) {
1105                 NL_SET_ERR_MSG(extack, "Invalid values in ancillary header");
1106                 goto out;
1107         }
1108         if (nhm->nh_flags & ~NEXTHOP_VALID_USER_FLAGS) {
1109                 NL_SET_ERR_MSG(extack, "Invalid nexthop flags in ancillary header");
1110                 goto out;
1111         }
1112
1113         switch (nhm->nh_family) {
1114         case AF_INET:
1115         case AF_INET6:
1116                 break;
1117         case AF_UNSPEC:
1118                 if (tb[NHA_GROUP])
1119                         break;
1120                 /* fallthrough */
1121         default:
1122                 NL_SET_ERR_MSG(extack, "Invalid address family");
1123                 goto out;
1124         }
1125
1126         if (tb[NHA_GROUPS] || tb[NHA_MASTER]) {
1127                 NL_SET_ERR_MSG(extack, "Invalid attributes in request");
1128                 goto out;
1129         }
1130
1131         memset(cfg, 0, sizeof(*cfg));
1132         cfg->nlflags = nlh->nlmsg_flags;
1133         cfg->nlinfo.portid = NETLINK_CB(skb).portid;
1134         cfg->nlinfo.nlh = nlh;
1135         cfg->nlinfo.nl_net = net;
1136
1137         cfg->nh_family = nhm->nh_family;
1138         cfg->nh_protocol = nhm->nh_protocol;
1139         cfg->nh_flags = nhm->nh_flags;
1140
1141         if (tb[NHA_ID])
1142                 cfg->nh_id = nla_get_u32(tb[NHA_ID]);
1143
1144         if (tb[NHA_GROUP]) {
1145                 if (nhm->nh_family != AF_UNSPEC) {
1146                         NL_SET_ERR_MSG(extack, "Invalid family for group");
1147                         goto out;
1148                 }
1149                 cfg->nh_grp = tb[NHA_GROUP];
1150
1151                 cfg->nh_grp_type = NEXTHOP_GRP_TYPE_MPATH;
1152                 if (tb[NHA_GROUP_TYPE])
1153                         cfg->nh_grp_type = nla_get_u16(tb[NHA_GROUP_TYPE]);
1154
1155                 if (cfg->nh_grp_type > NEXTHOP_GRP_TYPE_MAX) {
1156                         NL_SET_ERR_MSG(extack, "Invalid group type");
1157                         goto out;
1158                 }
1159                 err = nh_check_attr_group(net, tb, extack);
1160
1161                 /* no other attributes should be set */
1162                 goto out;
1163         }
1164
1165         if (tb[NHA_BLACKHOLE]) {
1166                 if (tb[NHA_GATEWAY] || tb[NHA_OIF] ||
1167                     tb[NHA_ENCAP]   || tb[NHA_ENCAP_TYPE]) {
1168                         NL_SET_ERR_MSG(extack, "Blackhole attribute can not be used with gateway or oif");
1169                         goto out;
1170                 }
1171
1172                 cfg->nh_blackhole = 1;
1173                 err = 0;
1174                 goto out;
1175         }
1176
1177         if (!tb[NHA_OIF]) {
1178                 NL_SET_ERR_MSG(extack, "Device attribute required for non-blackhole nexthops");
1179                 goto out;
1180         }
1181
1182         cfg->nh_ifindex = nla_get_u32(tb[NHA_OIF]);
1183         if (cfg->nh_ifindex)
1184                 cfg->dev = __dev_get_by_index(net, cfg->nh_ifindex);
1185
1186         if (!cfg->dev) {
1187                 NL_SET_ERR_MSG(extack, "Invalid device index");
1188                 goto out;
1189         } else if (!(cfg->dev->flags & IFF_UP)) {
1190                 NL_SET_ERR_MSG(extack, "Nexthop device is not up");
1191                 err = -ENETDOWN;
1192                 goto out;
1193         } else if (!netif_carrier_ok(cfg->dev)) {
1194                 NL_SET_ERR_MSG(extack, "Carrier for nexthop device is down");
1195                 err = -ENETDOWN;
1196                 goto out;
1197         }
1198
1199         err = -EINVAL;
1200         if (tb[NHA_GATEWAY]) {
1201                 struct nlattr *gwa = tb[NHA_GATEWAY];
1202
1203                 switch (cfg->nh_family) {
1204                 case AF_INET:
1205                         if (nla_len(gwa) != sizeof(u32)) {
1206                                 NL_SET_ERR_MSG(extack, "Invalid gateway");
1207                                 goto out;
1208                         }
1209                         cfg->gw.ipv4 = nla_get_be32(gwa);
1210                         break;
1211                 case AF_INET6:
1212                         if (nla_len(gwa) != sizeof(struct in6_addr)) {
1213                                 NL_SET_ERR_MSG(extack, "Invalid gateway");
1214                                 goto out;
1215                         }
1216                         cfg->gw.ipv6 = nla_get_in6_addr(gwa);
1217                         break;
1218                 default:
1219                         NL_SET_ERR_MSG(extack,
1220                                        "Unknown address family for gateway");
1221                         goto out;
1222                 }
1223         } else {
1224                 /* device only nexthop (no gateway) */
1225                 if (cfg->nh_flags & RTNH_F_ONLINK) {
1226                         NL_SET_ERR_MSG(extack,
1227                                        "ONLINK flag can not be set for nexthop without a gateway");
1228                         goto out;
1229                 }
1230         }
1231
1232         if (tb[NHA_ENCAP]) {
1233                 cfg->nh_encap = tb[NHA_ENCAP];
1234
1235                 if (!tb[NHA_ENCAP_TYPE]) {
1236                         NL_SET_ERR_MSG(extack, "LWT encapsulation type is missing");
1237                         goto out;
1238                 }
1239
1240                 cfg->nh_encap_type = nla_get_u16(tb[NHA_ENCAP_TYPE]);
1241                 err = lwtunnel_valid_encap_type(cfg->nh_encap_type, extack);
1242                 if (err < 0)
1243                         goto out;
1244
1245         } else if (tb[NHA_ENCAP_TYPE]) {
1246                 NL_SET_ERR_MSG(extack, "LWT encapsulation attribute is missing");
1247                 goto out;
1248         }
1249
1250
1251         err = 0;
1252 out:
1253         return err;
1254 }
1255
1256 /* rtnl */
1257 static int rtm_new_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
1258                            struct netlink_ext_ack *extack)
1259 {
1260         struct net *net = sock_net(skb->sk);
1261         struct nh_config cfg;
1262         struct nexthop *nh;
1263         int err;
1264
1265         err = rtm_to_nh_config(net, skb, nlh, &cfg, extack);
1266         if (!err) {
1267                 nh = nexthop_add(net, &cfg, extack);
1268                 if (IS_ERR(nh))
1269                         err = PTR_ERR(nh);
1270         }
1271
1272         return err;
1273 }
1274
1275 static int nh_valid_get_del_req(struct nlmsghdr *nlh, u32 *id,
1276                                 struct netlink_ext_ack *extack)
1277 {
1278         struct nhmsg *nhm = nlmsg_data(nlh);
1279         struct nlattr *tb[NHA_MAX + 1];
1280         int err, i;
1281
1282         err = nlmsg_parse(nlh, sizeof(*nhm), tb, NHA_MAX, rtm_nh_policy,
1283                           extack);
1284         if (err < 0)
1285                 return err;
1286
1287         err = -EINVAL;
1288         for (i = 0; i < __NHA_MAX; ++i) {
1289                 if (!tb[i])
1290                         continue;
1291
1292                 switch (i) {
1293                 case NHA_ID:
1294                         break;
1295                 default:
1296                         NL_SET_ERR_MSG_ATTR(extack, tb[i],
1297                                             "Unexpected attribute in request");
1298                         goto out;
1299                 }
1300         }
1301         if (nhm->nh_protocol || nhm->resvd || nhm->nh_scope || nhm->nh_flags) {
1302                 NL_SET_ERR_MSG(extack, "Invalid values in header");
1303                 goto out;
1304         }
1305
1306         if (!tb[NHA_ID]) {
1307                 NL_SET_ERR_MSG(extack, "Nexthop id is missing");
1308                 goto out;
1309         }
1310
1311         *id = nla_get_u32(tb[NHA_ID]);
1312         if (!(*id))
1313                 NL_SET_ERR_MSG(extack, "Invalid nexthop id");
1314         else
1315                 err = 0;
1316 out:
1317         return err;
1318 }
1319
1320 /* rtnl */
1321 static int rtm_del_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
1322                            struct netlink_ext_ack *extack)
1323 {
1324         struct net *net = sock_net(skb->sk);
1325         struct nl_info nlinfo = {
1326                 .nlh = nlh,
1327                 .nl_net = net,
1328                 .portid = NETLINK_CB(skb).portid,
1329         };
1330         struct nexthop *nh;
1331         int err;
1332         u32 id;
1333
1334         err = nh_valid_get_del_req(nlh, &id, extack);
1335         if (err)
1336                 return err;
1337
1338         nh = nexthop_find_by_id(net, id);
1339         if (!nh)
1340                 return -ENOENT;
1341
1342         remove_nexthop(net, nh, &nlinfo);
1343
1344         return 0;
1345 }
1346
1347 /* rtnl */
1348 static int rtm_get_nexthop(struct sk_buff *in_skb, struct nlmsghdr *nlh,
1349                            struct netlink_ext_ack *extack)
1350 {
1351         struct net *net = sock_net(in_skb->sk);
1352         struct sk_buff *skb = NULL;
1353         struct nexthop *nh;
1354         int err;
1355         u32 id;
1356
1357         err = nh_valid_get_del_req(nlh, &id, extack);
1358         if (err)
1359                 return err;
1360
1361         err = -ENOBUFS;
1362         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1363         if (!skb)
1364                 goto out;
1365
1366         err = -ENOENT;
1367         nh = nexthop_find_by_id(net, id);
1368         if (!nh)
1369                 goto errout_free;
1370
1371         err = nh_fill_node(skb, nh, RTM_NEWNEXTHOP, NETLINK_CB(in_skb).portid,
1372                            nlh->nlmsg_seq, 0);
1373         if (err < 0) {
1374                 WARN_ON(err == -EMSGSIZE);
1375                 goto errout_free;
1376         }
1377
1378         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
1379 out:
1380         return err;
1381 errout_free:
1382         kfree_skb(skb);
1383         goto out;
1384 }
1385
1386 static bool nh_dump_filtered(struct nexthop *nh, int dev_idx, int master_idx,
1387                              bool group_filter, u8 family)
1388 {
1389         const struct net_device *dev;
1390         const struct nh_info *nhi;
1391
1392         if (group_filter && !nh->is_group)
1393                 return true;
1394
1395         if (!dev_idx && !master_idx && !family)
1396                 return false;
1397
1398         if (nh->is_group)
1399                 return true;
1400
1401         nhi = rtnl_dereference(nh->nh_info);
1402         if (family && nhi->family != family)
1403                 return true;
1404
1405         dev = nhi->fib_nhc.nhc_dev;
1406         if (dev_idx && (!dev || dev->ifindex != dev_idx))
1407                 return true;
1408
1409         if (master_idx) {
1410                 struct net_device *master;
1411
1412                 if (!dev)
1413                         return true;
1414
1415                 master = netdev_master_upper_dev_get((struct net_device *)dev);
1416                 if (!master || master->ifindex != master_idx)
1417                         return true;
1418         }
1419
1420         return false;
1421 }
1422
1423 static int nh_valid_dump_req(const struct nlmsghdr *nlh, int *dev_idx,
1424                              int *master_idx, bool *group_filter,
1425                              struct netlink_callback *cb)
1426 {
1427         struct netlink_ext_ack *extack = cb->extack;
1428         struct nlattr *tb[NHA_MAX + 1];
1429         struct nhmsg *nhm;
1430         int err, i;
1431         u32 idx;
1432
1433         err = nlmsg_parse(nlh, sizeof(*nhm), tb, NHA_MAX, rtm_nh_policy,
1434                           NULL);
1435         if (err < 0)
1436                 return err;
1437
1438         for (i = 0; i <= NHA_MAX; ++i) {
1439                 if (!tb[i])
1440                         continue;
1441
1442                 switch (i) {
1443                 case NHA_OIF:
1444                         idx = nla_get_u32(tb[i]);
1445                         if (idx > INT_MAX) {
1446                                 NL_SET_ERR_MSG(extack, "Invalid device index");
1447                                 return -EINVAL;
1448                         }
1449                         *dev_idx = idx;
1450                         break;
1451                 case NHA_MASTER:
1452                         idx = nla_get_u32(tb[i]);
1453                         if (idx > INT_MAX) {
1454                                 NL_SET_ERR_MSG(extack, "Invalid master device index");
1455                                 return -EINVAL;
1456                         }
1457                         *master_idx = idx;
1458                         break;
1459                 case NHA_GROUPS:
1460                         *group_filter = true;
1461                         break;
1462                 default:
1463                         NL_SET_ERR_MSG(extack, "Unsupported attribute in dump request");
1464                         return -EINVAL;
1465                 }
1466         }
1467
1468         nhm = nlmsg_data(nlh);
1469         if (nhm->nh_protocol || nhm->resvd || nhm->nh_scope || nhm->nh_flags) {
1470                 NL_SET_ERR_MSG(extack, "Invalid values in header for nexthop dump request");
1471                 return -EINVAL;
1472         }
1473
1474         return 0;
1475 }
1476
1477 /* rtnl */
1478 static int rtm_dump_nexthop(struct sk_buff *skb, struct netlink_callback *cb)
1479 {
1480         struct nhmsg *nhm = nlmsg_data(cb->nlh);
1481         int dev_filter_idx = 0, master_idx = 0;
1482         struct net *net = sock_net(skb->sk);
1483         struct rb_root *root = &net->nexthop.rb_root;
1484         bool group_filter = false;
1485         struct rb_node *node;
1486         int idx = 0, s_idx;
1487         int err;
1488
1489         err = nh_valid_dump_req(cb->nlh, &dev_filter_idx, &master_idx,
1490                                 &group_filter, cb);
1491         if (err < 0)
1492                 return err;
1493
1494         s_idx = cb->args[0];
1495         for (node = rb_first(root); node; node = rb_next(node)) {
1496                 struct nexthop *nh;
1497
1498                 if (idx < s_idx)
1499                         goto cont;
1500
1501                 nh = rb_entry(node, struct nexthop, rb_node);
1502                 if (nh_dump_filtered(nh, dev_filter_idx, master_idx,
1503                                      group_filter, nhm->nh_family))
1504                         goto cont;
1505
1506                 err = nh_fill_node(skb, nh, RTM_NEWNEXTHOP,
1507                                    NETLINK_CB(cb->skb).portid,
1508                                    cb->nlh->nlmsg_seq, NLM_F_MULTI);
1509                 if (err < 0) {
1510                         if (likely(skb->len))
1511                                 goto out;
1512
1513                         goto out_err;
1514                 }
1515 cont:
1516                 idx++;
1517         }
1518
1519 out:
1520         err = skb->len;
1521 out_err:
1522         cb->args[0] = idx;
1523         cb->seq = net->nexthop.seq;
1524         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1525
1526         return err;
1527 }
1528
1529 static void nexthop_sync_mtu(struct net_device *dev, u32 orig_mtu)
1530 {
1531         unsigned int hash = nh_dev_hashfn(dev->ifindex);
1532         struct net *net = dev_net(dev);
1533         struct hlist_head *head = &net->nexthop.devhash[hash];
1534         struct hlist_node *n;
1535         struct nh_info *nhi;
1536
1537         hlist_for_each_entry_safe(nhi, n, head, dev_hash) {
1538                 if (nhi->fib_nhc.nhc_dev == dev) {
1539                         if (nhi->family == AF_INET)
1540                                 fib_nhc_update_mtu(&nhi->fib_nhc, dev->mtu,
1541                                                    orig_mtu);
1542                 }
1543         }
1544 }
1545
1546 /* rtnl */
1547 static int nh_netdev_event(struct notifier_block *this,
1548                            unsigned long event, void *ptr)
1549 {
1550         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1551         struct netdev_notifier_info_ext *info_ext;
1552
1553         switch (event) {
1554         case NETDEV_DOWN:
1555         case NETDEV_UNREGISTER:
1556                 nexthop_flush_dev(dev);
1557                 break;
1558         case NETDEV_CHANGE:
1559                 if (!(dev_get_flags(dev) & (IFF_RUNNING | IFF_LOWER_UP)))
1560                         nexthop_flush_dev(dev);
1561                 break;
1562         case NETDEV_CHANGEMTU:
1563                 info_ext = ptr;
1564                 nexthop_sync_mtu(dev, info_ext->ext.mtu);
1565                 rt_cache_flush(dev_net(dev));
1566                 break;
1567         }
1568         return NOTIFY_DONE;
1569 }
1570
1571 static struct notifier_block nh_netdev_notifier = {
1572         .notifier_call = nh_netdev_event,
1573 };
1574
1575 static void __net_exit nexthop_net_exit(struct net *net)
1576 {
1577         rtnl_lock();
1578         flush_all_nexthops(net);
1579         rtnl_unlock();
1580         kfree(net->nexthop.devhash);
1581 }
1582
1583 static int __net_init nexthop_net_init(struct net *net)
1584 {
1585         size_t sz = sizeof(struct hlist_head) * NH_DEV_HASHSIZE;
1586
1587         net->nexthop.rb_root = RB_ROOT;
1588         net->nexthop.devhash = kzalloc(sz, GFP_KERNEL);
1589         if (!net->nexthop.devhash)
1590                 return -ENOMEM;
1591
1592         return 0;
1593 }
1594
1595 static struct pernet_operations nexthop_net_ops = {
1596         .init = nexthop_net_init,
1597         .exit = nexthop_net_exit,
1598 };
1599
1600 static int __init nexthop_init(void)
1601 {
1602         register_pernet_subsys(&nexthop_net_ops);
1603
1604         register_netdevice_notifier(&nh_netdev_notifier);
1605
1606         rtnl_register(PF_UNSPEC, RTM_NEWNEXTHOP, rtm_new_nexthop, NULL, 0);
1607         rtnl_register(PF_UNSPEC, RTM_DELNEXTHOP, rtm_del_nexthop, NULL, 0);
1608         rtnl_register(PF_UNSPEC, RTM_GETNEXTHOP, rtm_get_nexthop,
1609                       rtm_dump_nexthop, 0);
1610
1611         rtnl_register(PF_INET, RTM_NEWNEXTHOP, rtm_new_nexthop, NULL, 0);
1612         rtnl_register(PF_INET, RTM_GETNEXTHOP, NULL, rtm_dump_nexthop, 0);
1613
1614         rtnl_register(PF_INET6, RTM_NEWNEXTHOP, rtm_new_nexthop, NULL, 0);
1615         rtnl_register(PF_INET6, RTM_GETNEXTHOP, NULL, rtm_dump_nexthop, 0);
1616
1617         return 0;
1618 }
1619 subsys_initcall(nexthop_init);