drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the names of the copyright holders nor the names of its
  17  *    contributors may be used to endorse or promote products derived from
  18  *    this software without specific prior written permission.
  19  *
  20  * Alternatively, this software may be distributed under the terms of the
  21  * GNU General Public License ("GPL") version 2 as published by the Free
  22  * Software Foundation.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 #include <linux/kernel.h>
  38 #include <linux/types.h>
  39 #include <linux/rhashtable.h>
  40 #include <linux/bitops.h>
  41 #include <linux/in6.h>
  42 #include <linux/notifier.h>
  43 #include <linux/inetdevice.h>
  44 #include <net/netevent.h>
  45 #include <net/neighbour.h>
  46 #include <net/arp.h>
  47 #include <net/ip_fib.h>
  48 #include <net/fib_rules.h>
  49
  50 #include "spectrum.h"
  51 #include "core.h"
  52 #include "reg.h"
  53
  54 struct mlxsw_sp_rif {
  55         struct list_head nexthop_list;
  56         struct list_head neigh_list;
  57         struct net_device *dev;
  58         struct mlxsw_sp_fid *f;
  59         unsigned char addr[ETH_ALEN];
  60         int mtu;
  61         u16 rif;
  62         u16 vr_id;
  63 };
  64
  65 static struct mlxsw_sp_rif *
  66 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
  67                          const struct net_device *dev);
  68
  69 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
  70         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
  71
  72 static bool
  73 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
  74                              struct mlxsw_sp_prefix_usage *prefix_usage2)
  75 {
  76         unsigned char prefix;
  77
  78         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
  79                 if (!test_bit(prefix, prefix_usage2->b))
  80                         return false;
  81         }
  82         return true;
  83 }
  84
  85 static bool
  86 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
  87                          struct mlxsw_sp_prefix_usage *prefix_usage2)
  88 {
  89         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
  90 }
  91
  92 static bool
  93 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
  94 {
  95         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
  96
  97         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
  98 }
  99
 100 static void
 101 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 102                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 103 {
 104         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 105 }
 106
 107 static void
 108 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 109                           unsigned char prefix_len)
 110 {
 111         set_bit(prefix_len, prefix_usage->b);
 112 }
 113
 114 static void
 115 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 116                             unsigned char prefix_len)
 117 {
 118         clear_bit(prefix_len, prefix_usage->b);
 119 }
 120
 121 struct mlxsw_sp_fib_key {
 122         unsigned char addr[sizeof(struct in6_addr)];
 123         unsigned char prefix_len;
 124 };
 125
 126 enum mlxsw_sp_fib_entry_type {
 127         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 128         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 129         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 130 };
 131
 132 struct mlxsw_sp_nexthop_group;
 133
 134 struct mlxsw_sp_fib_node {
 135         struct list_head entry_list;
 136         struct list_head list;
 137         struct rhash_head ht_node;
 138         struct mlxsw_sp_fib *fib;
 139         struct mlxsw_sp_fib_key key;
 140 };
 141
 142 struct mlxsw_sp_fib_entry_params {
 143         u32 tb_id;
 144         u32 prio;
 145         u8 tos;
 146         u8 type;
 147 };
 148
 149 struct mlxsw_sp_fib_entry {
 150         struct list_head list;
 151         struct mlxsw_sp_fib_node *fib_node;
 152         enum mlxsw_sp_fib_entry_type type;
 153         struct list_head nexthop_group_node;
 154         struct mlxsw_sp_nexthop_group *nh_group;
 155         struct mlxsw_sp_fib_entry_params params;
 156         bool offloaded;
 157 };
 158
 159 struct mlxsw_sp_fib {
 160         struct rhashtable ht;
 161         struct list_head node_list;
 162         struct mlxsw_sp_vr *vr;
 163         struct mlxsw_sp_lpm_tree *lpm_tree;
 164         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 165         struct mlxsw_sp_prefix_usage prefix_usage;
 166         enum mlxsw_sp_l3proto proto;
 167 };
 168
 169 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 170
 171 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
 172                                                 enum mlxsw_sp_l3proto proto)
 173 {
 174         struct mlxsw_sp_fib *fib;
 175         int err;
 176
 177         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 178         if (!fib)
 179                 return ERR_PTR(-ENOMEM);
 180         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 181         if (err)
 182                 goto err_rhashtable_init;
 183         INIT_LIST_HEAD(&fib->node_list);
 184         fib->proto = proto;
 185         fib->vr = vr;
 186         return fib;
 187
 188 err_rhashtable_init:
 189         kfree(fib);
 190         return ERR_PTR(err);
 191 }
 192
 193 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 194 {
 195         WARN_ON(!list_empty(&fib->node_list));
 196         WARN_ON(fib->lpm_tree);
 197         rhashtable_destroy(&fib->ht);
 198         kfree(fib);
 199 }
 200
 201 static struct mlxsw_sp_lpm_tree *
 202 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 203 {
 204         static struct mlxsw_sp_lpm_tree *lpm_tree;
 205         int i;
 206
 207         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 208                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 209                 if (lpm_tree->ref_count == 0)
 210                         return lpm_tree;
 211         }
 212         return NULL;
 213 }
 214
 215 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 216                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 217 {
 218         char ralta_pl[MLXSW_REG_RALTA_LEN];
 219
 220         mlxsw_reg_ralta_pack(ralta_pl, true,
 221                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 222                              lpm_tree->id);
 223         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 224 }
 225
 226 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 227                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 228 {
 229         char ralta_pl[MLXSW_REG_RALTA_LEN];
 230
 231         mlxsw_reg_ralta_pack(ralta_pl, false,
 232                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 233                              lpm_tree->id);
 234         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 235 }
 236
 237 static int
 238 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 239                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 240                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 241 {
 242         char ralst_pl[MLXSW_REG_RALST_LEN];
 243         u8 root_bin = 0;
 244         u8 prefix;
 245         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 246
 247         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 248                 root_bin = prefix;
 249
 250         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 251         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 252                 if (prefix == 0)
 253                         continue;
 254                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 255                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 256                 last_prefix = prefix;
 257         }
 258         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 259 }
 260
 261 static struct mlxsw_sp_lpm_tree *
 262 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 263                          struct mlxsw_sp_prefix_usage *prefix_usage,
 264                          enum mlxsw_sp_l3proto proto)
 265 {
 266         struct mlxsw_sp_lpm_tree *lpm_tree;
 267         int err;
 268
 269         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 270         if (!lpm_tree)
 271                 return ERR_PTR(-EBUSY);
 272         lpm_tree->proto = proto;
 273         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 274         if (err)
 275                 return ERR_PTR(err);
 276
 277         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 278                                                 lpm_tree);
 279         if (err)
 280                 goto err_left_struct_set;
 281         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 282                sizeof(lpm_tree->prefix_usage));
 283         return lpm_tree;
 284
 285 err_left_struct_set:
 286         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 287         return ERR_PTR(err);
 288 }
 289
 290 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 291                                      struct mlxsw_sp_lpm_tree *lpm_tree)
 292 {
 293         return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 294 }
 295
 296 static struct mlxsw_sp_lpm_tree *
 297 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 298                       struct mlxsw_sp_prefix_usage *prefix_usage,
 299                       enum mlxsw_sp_l3proto proto)
 300 {
 301         struct mlxsw_sp_lpm_tree *lpm_tree;
 302         int i;
 303
 304         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 305                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 306                 if (lpm_tree->ref_count != 0 &&
 307                     lpm_tree->proto == proto &&
 308                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 309                                              prefix_usage))
 310                         goto inc_ref_count;
 311         }
 312         lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
 313                                             proto);
 314         if (IS_ERR(lpm_tree))
 315                 return lpm_tree;
 316
 317 inc_ref_count:
 318         lpm_tree->ref_count++;
 319         return lpm_tree;
 320 }
 321
 322 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 323                                  struct mlxsw_sp_lpm_tree *lpm_tree)
 324 {
 325         if (--lpm_tree->ref_count == 0)
 326                 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 327         return 0;
 328 }
 329
 330 static void mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 331 {
 332         struct mlxsw_sp_lpm_tree *lpm_tree;
 333         int i;
 334
 335         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 336                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 337                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 338         }
 339 }
 340
 341 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 342 {
 343         return !!vr->fib4;
 344 }
 345
 346 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 347 {
 348         struct mlxsw_sp_vr *vr;
 349         int i;
 350
 351         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 352                 vr = &mlxsw_sp->router.vrs[i];
 353                 if (!mlxsw_sp_vr_is_used(vr))
 354                         return vr;
 355         }
 356         return NULL;
 357 }
 358
 359 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 360                                      const struct mlxsw_sp_fib *fib)
 361 {
 362         char raltb_pl[MLXSW_REG_RALTB_LEN];
 363
 364         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 365                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 366                              fib->lpm_tree->id);
 367         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 368 }
 369
 370 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 371                                        const struct mlxsw_sp_fib *fib)
 372 {
 373         char raltb_pl[MLXSW_REG_RALTB_LEN];
 374
 375         /* Bind to tree 0 which is default */
 376         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 377                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 378         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 379 }
 380
 381 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 382 {
 383         /* For our purpose, squash main and local table into one */
 384         if (tb_id == RT_TABLE_LOCAL)
 385                 tb_id = RT_TABLE_MAIN;
 386         return tb_id;
 387 }
 388
 389 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 390                                             u32 tb_id)
 391 {
 392         struct mlxsw_sp_vr *vr;
 393         int i;
 394
 395         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 396
 397         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 398                 vr = &mlxsw_sp->router.vrs[i];
 399                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 400                         return vr;
 401         }
 402         return NULL;
 403 }
 404
 405 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 406                                             enum mlxsw_sp_l3proto proto)
 407 {
 408         switch (proto) {
 409         case MLXSW_SP_L3_PROTO_IPV4:
 410                 return vr->fib4;
 411         case MLXSW_SP_L3_PROTO_IPV6:
 412                 BUG_ON(1);
 413         }
 414         return NULL;
 415 }
 416
 417 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 418                                               u32 tb_id)
 419 {
 420         struct mlxsw_sp_vr *vr;
 421
 422         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 423         if (!vr)
 424                 return ERR_PTR(-EBUSY);
 425         vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
 426         if (IS_ERR(vr->fib4))
 427                 return ERR_CAST(vr->fib4);
 428         vr->tb_id = tb_id;
 429         return vr;
 430 }
 431
 432 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
 433 {
 434         mlxsw_sp_fib_destroy(vr->fib4);
 435         vr->fib4 = NULL;
 436 }
 437
 438 static int
 439 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib,
 440                            struct mlxsw_sp_prefix_usage *req_prefix_usage)
 441 {
 442         struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree;
 443         struct mlxsw_sp_lpm_tree *new_tree;
 444         int err;
 445
 446         if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
 447                 return 0;
 448
 449         new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
 450                                          fib->proto);
 451         if (IS_ERR(new_tree)) {
 452                 /* We failed to get a tree according to the required
 453                  * prefix usage. However, the current tree might be still good
 454                  * for us if our requirement is subset of the prefixes used
 455                  * in the tree.
 456                  */
 457                 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
 458                                                  &lpm_tree->prefix_usage))
 459                         return 0;
 460                 return PTR_ERR(new_tree);
 461         }
 462
 463         /* Prevent packet loss by overwriting existing binding */
 464         fib->lpm_tree = new_tree;
 465         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
 466         if (err)
 467                 goto err_tree_bind;
 468         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 469
 470         return 0;
 471
 472 err_tree_bind:
 473         fib->lpm_tree = lpm_tree;
 474         mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 475         return err;
 476 }
 477
 478 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
 479 {
 480         struct mlxsw_sp_vr *vr;
 481
 482         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 483         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 484         if (!vr)
 485                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
 486         return vr;
 487 }
 488
 489 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
 490 {
 491         if (!vr->rif_count && list_empty(&vr->fib4->node_list))
 492                 mlxsw_sp_vr_destroy(vr);
 493 }
 494
 495 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 496 {
 497         struct mlxsw_sp_vr *vr;
 498         u64 max_vrs;
 499         int i;
 500
 501         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 502                 return -EIO;
 503
 504         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 505         mlxsw_sp->router.vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 506                                        GFP_KERNEL);
 507         if (!mlxsw_sp->router.vrs)
 508                 return -ENOMEM;
 509
 510         for (i = 0; i < max_vrs; i++) {
 511                 vr = &mlxsw_sp->router.vrs[i];
 512                 vr->id = i;
 513         }
 514
 515         return 0;
 516 }
 517
 518 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 519
 520 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 521 {
 522         /* At this stage we're guaranteed not to have new incoming
 523          * FIB notifications and the work queue is free from FIBs
 524          * sitting on top of mlxsw netdevs. However, we can still
 525          * have other FIBs queued. Flush the queue before flushing
 526          * the device's tables. No need for locks, as we're the only
 527          * writer.
 528          */
 529         mlxsw_core_flush_owq();
 530         mlxsw_sp_router_fib_flush(mlxsw_sp);
 531         kfree(mlxsw_sp->router.vrs);
 532 }
 533
 534 struct mlxsw_sp_neigh_key {
 535         struct neighbour *n;
 536 };
 537
 538 struct mlxsw_sp_neigh_entry {
 539         struct list_head rif_list_node;
 540         struct rhash_head ht_node;
 541         struct mlxsw_sp_neigh_key key;
 542         u16 rif;
 543         bool connected;
 544         unsigned char ha[ETH_ALEN];
 545         struct list_head nexthop_list; /* list of nexthops using
 546                                         * this neigh entry
 547                                         */
 548         struct list_head nexthop_neighs_list_node;
 549 };
 550
 551 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
 552         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
 553         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
 554         .key_len = sizeof(struct mlxsw_sp_neigh_key),
 555 };
 556
 557 static struct mlxsw_sp_neigh_entry *
 558 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
 559                            u16 rif)
 560 {
 561         struct mlxsw_sp_neigh_entry *neigh_entry;
 562
 563         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
 564         if (!neigh_entry)
 565                 return NULL;
 566
 567         neigh_entry->key.n = n;
 568         neigh_entry->rif = rif;
 569         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
 570
 571         return neigh_entry;
 572 }
 573
 574 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
 575 {
 576         kfree(neigh_entry);
 577 }
 578
 579 static int
 580 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
 581                             struct mlxsw_sp_neigh_entry *neigh_entry)
 582 {
 583         return rhashtable_insert_fast(&mlxsw_sp->router.neigh_ht,
 584                                       &neigh_entry->ht_node,
 585                                       mlxsw_sp_neigh_ht_params);
 586 }
 587
 588 static void
 589 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
 590                             struct mlxsw_sp_neigh_entry *neigh_entry)
 591 {
 592         rhashtable_remove_fast(&mlxsw_sp->router.neigh_ht,
 593                                &neigh_entry->ht_node,
 594                                mlxsw_sp_neigh_ht_params);
 595 }
 596
 597 static struct mlxsw_sp_neigh_entry *
 598 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 599 {
 600         struct mlxsw_sp_neigh_entry *neigh_entry;
 601         struct mlxsw_sp_rif *r;
 602         int err;
 603
 604         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
 605         if (!r)
 606                 return ERR_PTR(-EINVAL);
 607
 608         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, r->rif);
 609         if (!neigh_entry)
 610                 return ERR_PTR(-ENOMEM);
 611
 612         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
 613         if (err)
 614                 goto err_neigh_entry_insert;
 615
 616         list_add(&neigh_entry->rif_list_node, &r->neigh_list);
 617
 618         return neigh_entry;
 619
 620 err_neigh_entry_insert:
 621         mlxsw_sp_neigh_entry_free(neigh_entry);
 622         return ERR_PTR(err);
 623 }
 624
 625 static void
 626 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
 627                              struct mlxsw_sp_neigh_entry *neigh_entry)
 628 {
 629         list_del(&neigh_entry->rif_list_node);
 630         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
 631         mlxsw_sp_neigh_entry_free(neigh_entry);
 632 }
 633
 634 static struct mlxsw_sp_neigh_entry *
 635 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 636 {
 637         struct mlxsw_sp_neigh_key key;
 638
 639         key.n = n;
 640         return rhashtable_lookup_fast(&mlxsw_sp->router.neigh_ht,
 641                                       &key, mlxsw_sp_neigh_ht_params);
 642 }
 643
 644 static void
 645 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
 646 {
 647         unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
 648
 649         mlxsw_sp->router.neighs_update.interval = jiffies_to_msecs(interval);
 650 }
 651
 652 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 653                                                    char *rauhtd_pl,
 654                                                    int ent_index)
 655 {
 656         struct net_device *dev;
 657         struct neighbour *n;
 658         __be32 dipn;
 659         u32 dip;
 660         u16 rif;
 661
 662         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
 663
 664         if (!mlxsw_sp->rifs[rif]) {
 665                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
 666                 return;
 667         }
 668
 669         dipn = htonl(dip);
 670         dev = mlxsw_sp->rifs[rif]->dev;
 671         n = neigh_lookup(&arp_tbl, &dipn, dev);
 672         if (!n) {
 673                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
 674                            &dip);
 675                 return;
 676         }
 677
 678         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
 679         neigh_event_send(n, NULL);
 680         neigh_release(n);
 681 }
 682
 683 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 684                                                    char *rauhtd_pl,
 685                                                    int rec_index)
 686 {
 687         u8 num_entries;
 688         int i;
 689
 690         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 691                                                                 rec_index);
 692         /* Hardware starts counting at 0, so add 1. */
 693         num_entries++;
 694
 695         /* Each record consists of several neighbour entries. */
 696         for (i = 0; i < num_entries; i++) {
 697                 int ent_index;
 698
 699                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
 700                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
 701                                                        ent_index);
 702         }
 703
 704 }
 705
 706 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
 707                                               char *rauhtd_pl, int rec_index)
 708 {
 709         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
 710         case MLXSW_REG_RAUHTD_TYPE_IPV4:
 711                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
 712                                                        rec_index);
 713                 break;
 714         case MLXSW_REG_RAUHTD_TYPE_IPV6:
 715                 WARN_ON_ONCE(1);
 716                 break;
 717         }
 718 }
 719
 720 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
 721 {
 722         u8 num_rec, last_rec_index, num_entries;
 723
 724         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
 725         last_rec_index = num_rec - 1;
 726
 727         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
 728                 return false;
 729         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
 730             MLXSW_REG_RAUHTD_TYPE_IPV6)
 731                 return true;
 732
 733         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 734                                                                 last_rec_index);
 735         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
 736                 return true;
 737         return false;
 738 }
 739
 740 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
 741 {
 742         char *rauhtd_pl;
 743         u8 num_rec;
 744         int i, err;
 745
 746         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
 747         if (!rauhtd_pl)
 748                 return -ENOMEM;
 749
 750         /* Make sure the neighbour's netdev isn't removed in the
 751          * process.
 752          */
 753         rtnl_lock();
 754         do {
 755                 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
 756                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
 757                                       rauhtd_pl);
 758                 if (err) {
 759                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
 760                         break;
 761                 }
 762                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
 763                 for (i = 0; i < num_rec; i++)
 764                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
 765                                                           i);
 766         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
 767         rtnl_unlock();
 768
 769         kfree(rauhtd_pl);
 770         return err;
 771 }
 772
 773 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
 774 {
 775         struct mlxsw_sp_neigh_entry *neigh_entry;
 776
 777         /* Take RTNL mutex here to prevent lists from changes */
 778         rtnl_lock();
 779         list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
 780                             nexthop_neighs_list_node)
 781                 /* If this neigh have nexthops, make the kernel think this neigh
 782                  * is active regardless of the traffic.
 783                  */
 784                 neigh_event_send(neigh_entry->key.n, NULL);
 785         rtnl_unlock();
 786 }
 787
 788 static void
 789 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
 790 {
 791         unsigned long interval = mlxsw_sp->router.neighs_update.interval;
 792
 793         mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw,
 794                                msecs_to_jiffies(interval));
 795 }
 796
 797 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
 798 {
 799         struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
 800                                                  router.neighs_update.dw.work);
 801         int err;
 802
 803         err = mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp);
 804         if (err)
 805                 dev_err(mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
 806
 807         mlxsw_sp_router_neighs_update_nh(mlxsw_sp);
 808
 809         mlxsw_sp_router_neighs_update_work_schedule(mlxsw_sp);
 810 }
 811
 812 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
 813 {
 814         struct mlxsw_sp_neigh_entry *neigh_entry;
 815         struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
 816                                                  router.nexthop_probe_dw.work);
 817
 818         /* Iterate over nexthop neighbours, find those who are unresolved and
 819          * send arp on them. This solves the chicken-egg problem when
 820          * the nexthop wouldn't get offloaded until the neighbor is resolved
 821          * but it wouldn't get resolved ever in case traffic is flowing in HW
 822          * using different nexthop.
 823          *
 824          * Take RTNL mutex here to prevent lists from changes.
 825          */
 826         rtnl_lock();
 827         list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
 828                             nexthop_neighs_list_node)
 829                 if (!neigh_entry->connected)
 830                         neigh_event_send(neigh_entry->key.n, NULL);
 831         rtnl_unlock();
 832
 833         mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw,
 834                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
 835 }
 836
 837 static void
 838 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
 839                               struct mlxsw_sp_neigh_entry *neigh_entry,
 840                               bool removing);
 841
 842 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
 843 {
 844         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
 845                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
 846 }
 847
 848 static void
 849 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
 850                                 struct mlxsw_sp_neigh_entry *neigh_entry,
 851                                 enum mlxsw_reg_rauht_op op)
 852 {
 853         struct neighbour *n = neigh_entry->key.n;
 854         u32 dip = ntohl(*((__be32 *) n->primary_key));
 855         char rauht_pl[MLXSW_REG_RAUHT_LEN];
 856
 857         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
 858                               dip);
 859         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
 860 }
 861
 862 static void
 863 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
 864                             struct mlxsw_sp_neigh_entry *neigh_entry,
 865                             bool adding)
 866 {
 867         if (!adding && !neigh_entry->connected)
 868                 return;
 869         neigh_entry->connected = adding;
 870         if (neigh_entry->key.n->tbl == &arp_tbl)
 871                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
 872                                                 mlxsw_sp_rauht_op(adding));
 873         else
 874                 WARN_ON_ONCE(1);
 875 }
 876
 877 struct mlxsw_sp_neigh_event_work {
 878         struct work_struct work;
 879         struct mlxsw_sp *mlxsw_sp;
 880         struct neighbour *n;
 881 };
 882
 883 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
 884 {
 885         struct mlxsw_sp_neigh_event_work *neigh_work =
 886                 container_of(work, struct mlxsw_sp_neigh_event_work, work);
 887         struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
 888         struct mlxsw_sp_neigh_entry *neigh_entry;
 889         struct neighbour *n = neigh_work->n;
 890         unsigned char ha[ETH_ALEN];
 891         bool entry_connected;
 892         u8 nud_state, dead;
 893
 894         /* If these parameters are changed after we release the lock,
 895          * then we are guaranteed to receive another event letting us
 896          * know about it.
 897          */
 898         read_lock_bh(&n->lock);
 899         memcpy(ha, n->ha, ETH_ALEN);
 900         nud_state = n->nud_state;
 901         dead = n->dead;
 902         read_unlock_bh(&n->lock);
 903
 904         rtnl_lock();
 905         entry_connected = nud_state & NUD_VALID && !dead;
 906         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
 907         if (!entry_connected && !neigh_entry)
 908                 goto out;
 909         if (!neigh_entry) {
 910                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
 911                 if (IS_ERR(neigh_entry))
 912                         goto out;
 913         }
 914
 915         memcpy(neigh_entry->ha, ha, ETH_ALEN);
 916         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
 917         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
 918
 919         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
 920                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
 921
 922 out:
 923         rtnl_unlock();
 924         neigh_release(n);
 925         kfree(neigh_work);
 926 }
 927
 928 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
 929                                    unsigned long event, void *ptr)
 930 {
 931         struct mlxsw_sp_neigh_event_work *neigh_work;
 932         struct mlxsw_sp_port *mlxsw_sp_port;
 933         struct mlxsw_sp *mlxsw_sp;
 934         unsigned long interval;
 935         struct neigh_parms *p;
 936         struct neighbour *n;
 937
 938         switch (event) {
 939         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
 940                 p = ptr;
 941
 942                 /* We don't care about changes in the default table. */
 943                 if (!p->dev || p->tbl != &arp_tbl)
 944                         return NOTIFY_DONE;
 945
 946                 /* We are in atomic context and can't take RTNL mutex,
 947                  * so use RCU variant to walk the device chain.
 948                  */
 949                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
 950                 if (!mlxsw_sp_port)
 951                         return NOTIFY_DONE;
 952
 953                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 954                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
 955                 mlxsw_sp->router.neighs_update.interval = interval;
 956
 957                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
 958                 break;
 959         case NETEVENT_NEIGH_UPDATE:
 960                 n = ptr;
 961
 962                 if (n->tbl != &arp_tbl)
 963                         return NOTIFY_DONE;
 964
 965                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
 966                 if (!mlxsw_sp_port)
 967                         return NOTIFY_DONE;
 968
 969                 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
 970                 if (!neigh_work) {
 971                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
 972                         return NOTIFY_BAD;
 973                 }
 974
 975                 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
 976                 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 977                 neigh_work->n = n;
 978
 979                 /* Take a reference to ensure the neighbour won't be
 980                  * destructed until we drop the reference in delayed
 981                  * work.
 982                  */
 983                 neigh_clone(n);
 984                 mlxsw_core_schedule_work(&neigh_work->work);
 985                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
 986                 break;
 987         }
 988
 989         return NOTIFY_DONE;
 990 }
 991
 992 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
 993 {
 994         int err;
 995
 996         err = rhashtable_init(&mlxsw_sp->router.neigh_ht,
 997                               &mlxsw_sp_neigh_ht_params);
 998         if (err)
 999                 return err;
1000
1001         /* Initialize the polling interval according to the default
1002          * table.
1003          */
1004         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1005
1006         /* Create the delayed works for the activity_update */
1007         INIT_DELAYED_WORK(&mlxsw_sp->router.neighs_update.dw,
1008                           mlxsw_sp_router_neighs_update_work);
1009         INIT_DELAYED_WORK(&mlxsw_sp->router.nexthop_probe_dw,
1010                           mlxsw_sp_router_probe_unresolved_nexthops);
1011         mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw, 0);
1012         mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw, 0);
1013         return 0;
1014 }
1015
1016 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1017 {
1018         cancel_delayed_work_sync(&mlxsw_sp->router.neighs_update.dw);
1019         cancel_delayed_work_sync(&mlxsw_sp->router.nexthop_probe_dw);
1020         rhashtable_destroy(&mlxsw_sp->router.neigh_ht);
1021 }
1022
1023 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp,
1024                                     const struct mlxsw_sp_rif *r)
1025 {
1026         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1027
1028         mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL,
1029                              r->rif, r->addr);
1030         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1031 }
1032
1033 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1034                                          struct mlxsw_sp_rif *r)
1035 {
1036         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1037
1038         mlxsw_sp_neigh_rif_flush(mlxsw_sp, r);
1039         list_for_each_entry_safe(neigh_entry, tmp, &r->neigh_list,
1040                                  rif_list_node)
1041                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1042 }
1043
1044 struct mlxsw_sp_nexthop_key {
1045         struct fib_nh *fib_nh;
1046 };
1047
1048 struct mlxsw_sp_nexthop {
1049         struct list_head neigh_list_node; /* member of neigh entry list */
1050         struct list_head rif_list_node;
1051         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1052                                                 * this belongs to
1053                                                 */
1054         struct rhash_head ht_node;
1055         struct mlxsw_sp_nexthop_key key;
1056         struct mlxsw_sp_rif *r;
1057         u8 should_offload:1, /* set indicates this neigh is connected and
1058                               * should be put to KVD linear area of this group.
1059                               */
1060            offloaded:1, /* set in case the neigh is actually put into
1061                          * KVD linear area of this group.
1062                          */
1063            update:1; /* set indicates that MAC of this neigh should be
1064                       * updated in HW
1065                       */
1066         struct mlxsw_sp_neigh_entry *neigh_entry;
1067 };
1068
1069 struct mlxsw_sp_nexthop_group_key {
1070         struct fib_info *fi;
1071 };
1072
1073 struct mlxsw_sp_nexthop_group {
1074         struct rhash_head ht_node;
1075         struct list_head fib_list; /* list of fib entries that use this group */
1076         struct mlxsw_sp_nexthop_group_key key;
1077         u8 adj_index_valid:1,
1078            gateway:1; /* routes using the group use a gateway */
1079         u32 adj_index;
1080         u16 ecmp_size;
1081         u16 count;
1082         struct mlxsw_sp_nexthop nexthops[0];
1083 #define nh_rif  nexthops[0].r
1084 };
1085
1086 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1087         .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
1088         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1089         .key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
1090 };
1091
1092 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1093                                          struct mlxsw_sp_nexthop_group *nh_grp)
1094 {
1095         return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_group_ht,
1096                                       &nh_grp->ht_node,
1097                                       mlxsw_sp_nexthop_group_ht_params);
1098 }
1099
1100 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1101                                           struct mlxsw_sp_nexthop_group *nh_grp)
1102 {
1103         rhashtable_remove_fast(&mlxsw_sp->router.nexthop_group_ht,
1104                                &nh_grp->ht_node,
1105                                mlxsw_sp_nexthop_group_ht_params);
1106 }
1107
1108 static struct mlxsw_sp_nexthop_group *
1109 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
1110                               struct mlxsw_sp_nexthop_group_key key)
1111 {
1112         return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_group_ht, &key,
1113                                       mlxsw_sp_nexthop_group_ht_params);
1114 }
1115
1116 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1117         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1118         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1119         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1120 };
1121
1122 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1123                                    struct mlxsw_sp_nexthop *nh)
1124 {
1125         return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_ht,
1126                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1127 }
1128
1129 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1130                                     struct mlxsw_sp_nexthop *nh)
1131 {
1132         rhashtable_remove_fast(&mlxsw_sp->router.nexthop_ht, &nh->ht_node,
1133                                mlxsw_sp_nexthop_ht_params);
1134 }
1135
1136 static struct mlxsw_sp_nexthop *
1137 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1138                         struct mlxsw_sp_nexthop_key key)
1139 {
1140         return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_ht, &key,
1141                                       mlxsw_sp_nexthop_ht_params);
1142 }
1143
1144 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1145                                              const struct mlxsw_sp_fib *fib,
1146                                              u32 adj_index, u16 ecmp_size,
1147                                              u32 new_adj_index,
1148                                              u16 new_ecmp_size)
1149 {
1150         char raleu_pl[MLXSW_REG_RALEU_LEN];
1151
1152         mlxsw_reg_raleu_pack(raleu_pl,
1153                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
1154                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
1155                              new_ecmp_size);
1156         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1157 }
1158
1159 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1160                                           struct mlxsw_sp_nexthop_group *nh_grp,
1161                                           u32 old_adj_index, u16 old_ecmp_size)
1162 {
1163         struct mlxsw_sp_fib_entry *fib_entry;
1164         struct mlxsw_sp_fib *fib = NULL;
1165         int err;
1166
1167         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1168                 if (fib == fib_entry->fib_node->fib)
1169                         continue;
1170                 fib = fib_entry->fib_node->fib;
1171                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
1172                                                         old_adj_index,
1173                                                         old_ecmp_size,
1174                                                         nh_grp->adj_index,
1175                                                         nh_grp->ecmp_size);
1176                 if (err)
1177                         return err;
1178         }
1179         return 0;
1180 }
1181
1182 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1183                                        struct mlxsw_sp_nexthop *nh)
1184 {
1185         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1186         char ratr_pl[MLXSW_REG_RATR_LEN];
1187
1188         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1189                             true, adj_index, neigh_entry->rif);
1190         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1191         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1192 }
1193
1194 static int
1195 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1196                                   struct mlxsw_sp_nexthop_group *nh_grp,
1197                                   bool reallocate)
1198 {
1199         u32 adj_index = nh_grp->adj_index; /* base */
1200         struct mlxsw_sp_nexthop *nh;
1201         int i;
1202         int err;
1203
1204         for (i = 0; i < nh_grp->count; i++) {
1205                 nh = &nh_grp->nexthops[i];
1206
1207                 if (!nh->should_offload) {
1208                         nh->offloaded = 0;
1209                         continue;
1210                 }
1211
1212                 if (nh->update || reallocate) {
1213                         err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1214                                                           adj_index, nh);
1215                         if (err)
1216                                 return err;
1217                         nh->update = 0;
1218                         nh->offloaded = 1;
1219                 }
1220                 adj_index++;
1221         }
1222         return 0;
1223 }
1224
1225 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1226                                      struct mlxsw_sp_fib_entry *fib_entry);
1227
1228 static int
1229 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1230                                     struct mlxsw_sp_nexthop_group *nh_grp)
1231 {
1232         struct mlxsw_sp_fib_entry *fib_entry;
1233         int err;
1234
1235         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1236                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1237                 if (err)
1238                         return err;
1239         }
1240         return 0;
1241 }
1242
1243 static void
1244 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1245                                struct mlxsw_sp_nexthop_group *nh_grp)
1246 {
1247         struct mlxsw_sp_nexthop *nh;
1248         bool offload_change = false;
1249         u32 adj_index;
1250         u16 ecmp_size = 0;
1251         bool old_adj_index_valid;
1252         u32 old_adj_index;
1253         u16 old_ecmp_size;
1254         int ret;
1255         int i;
1256         int err;
1257
1258         if (!nh_grp->gateway) {
1259                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1260                 return;
1261         }
1262
1263         for (i = 0; i < nh_grp->count; i++) {
1264                 nh = &nh_grp->nexthops[i];
1265
1266                 if (nh->should_offload ^ nh->offloaded) {
1267                         offload_change = true;
1268                         if (nh->should_offload)
1269                                 nh->update = 1;
1270                 }
1271                 if (nh->should_offload)
1272                         ecmp_size++;
1273         }
1274         if (!offload_change) {
1275                 /* Nothing was added or removed, so no need to reallocate. Just
1276                  * update MAC on existing adjacency indexes.
1277                  */
1278                 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1279                                                         false);
1280                 if (err) {
1281                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1282                         goto set_trap;
1283                 }
1284                 return;
1285         }
1286         if (!ecmp_size)
1287                 /* No neigh of this group is connected so we just set
1288                  * the trap and let everthing flow through kernel.
1289                  */
1290                 goto set_trap;
1291
1292         ret = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size);
1293         if (ret < 0) {
1294                 /* We ran out of KVD linear space, just set the
1295                  * trap and let everything flow through kernel.
1296                  */
1297                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1298                 goto set_trap;
1299         }
1300         adj_index = ret;
1301         old_adj_index_valid = nh_grp->adj_index_valid;
1302         old_adj_index = nh_grp->adj_index;
1303         old_ecmp_size = nh_grp->ecmp_size;
1304         nh_grp->adj_index_valid = 1;
1305         nh_grp->adj_index = adj_index;
1306         nh_grp->ecmp_size = ecmp_size;
1307         err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
1308         if (err) {
1309                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1310                 goto set_trap;
1311         }
1312
1313         if (!old_adj_index_valid) {
1314                 /* The trap was set for fib entries, so we have to call
1315                  * fib entry update to unset it and use adjacency index.
1316                  */
1317                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1318                 if (err) {
1319                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1320                         goto set_trap;
1321                 }
1322                 return;
1323         }
1324
1325         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1326                                              old_adj_index, old_ecmp_size);
1327         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1328         if (err) {
1329                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1330                 goto set_trap;
1331         }
1332         return;
1333
1334 set_trap:
1335         old_adj_index_valid = nh_grp->adj_index_valid;
1336         nh_grp->adj_index_valid = 0;
1337         for (i = 0; i < nh_grp->count; i++) {
1338                 nh = &nh_grp->nexthops[i];
1339                 nh->offloaded = 0;
1340         }
1341         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1342         if (err)
1343                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1344         if (old_adj_index_valid)
1345                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1346 }
1347
1348 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1349                                             bool removing)
1350 {
1351         if (!removing && !nh->should_offload)
1352                 nh->should_offload = 1;
1353         else if (removing && nh->offloaded)
1354                 nh->should_offload = 0;
1355         nh->update = 1;
1356 }
1357
1358 static void
1359 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1360                               struct mlxsw_sp_neigh_entry *neigh_entry,
1361                               bool removing)
1362 {
1363         struct mlxsw_sp_nexthop *nh;
1364
1365         list_for_each_entry(nh, &neigh_entry->nexthop_list,
1366                             neigh_list_node) {
1367                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1368                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1369         }
1370 }
1371
1372 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
1373                                       struct mlxsw_sp_rif *r)
1374 {
1375         if (nh->r)
1376                 return;
1377
1378         nh->r = r;
1379         list_add(&nh->rif_list_node, &r->nexthop_list);
1380 }
1381
1382 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
1383 {
1384         if (!nh->r)
1385                 return;
1386
1387         list_del(&nh->rif_list_node);
1388         nh->r = NULL;
1389 }
1390
1391 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
1392                                        struct mlxsw_sp_nexthop *nh)
1393 {
1394         struct mlxsw_sp_neigh_entry *neigh_entry;
1395         struct fib_nh *fib_nh = nh->key.fib_nh;
1396         struct neighbour *n;
1397         u8 nud_state, dead;
1398         int err;
1399
1400         if (!nh->nh_grp->gateway || nh->neigh_entry)
1401                 return 0;
1402
1403         /* Take a reference of neigh here ensuring that neigh would
1404          * not be detructed before the nexthop entry is finished.
1405          * The reference is taken either in neigh_lookup() or
1406          * in neigh_create() in case n is not found.
1407          */
1408         n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1409         if (!n) {
1410                 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1411                 if (IS_ERR(n))
1412                         return PTR_ERR(n);
1413                 neigh_event_send(n, NULL);
1414         }
1415         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1416         if (!neigh_entry) {
1417                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1418                 if (IS_ERR(neigh_entry)) {
1419                         err = -EINVAL;
1420                         goto err_neigh_entry_create;
1421                 }
1422         }
1423
1424         /* If that is the first nexthop connected to that neigh, add to
1425          * nexthop_neighs_list
1426          */
1427         if (list_empty(&neigh_entry->nexthop_list))
1428                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1429                               &mlxsw_sp->router.nexthop_neighs_list);
1430
1431         nh->neigh_entry = neigh_entry;
1432         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1433         read_lock_bh(&n->lock);
1434         nud_state = n->nud_state;
1435         dead = n->dead;
1436         read_unlock_bh(&n->lock);
1437         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
1438
1439         return 0;
1440
1441 err_neigh_entry_create:
1442         neigh_release(n);
1443         return err;
1444 }
1445
1446 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
1447                                         struct mlxsw_sp_nexthop *nh)
1448 {
1449         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1450         struct neighbour *n;
1451
1452         if (!neigh_entry)
1453                 return;
1454         n = neigh_entry->key.n;
1455
1456         __mlxsw_sp_nexthop_neigh_update(nh, true);
1457         list_del(&nh->neigh_list_node);
1458         nh->neigh_entry = NULL;
1459
1460         /* If that is the last nexthop connected to that neigh, remove from
1461          * nexthop_neighs_list
1462          */
1463         if (list_empty(&neigh_entry->nexthop_list))
1464                 list_del(&neigh_entry->nexthop_neighs_list_node);
1465
1466         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1467                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1468
1469         neigh_release(n);
1470 }
1471
1472 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1473                                  struct mlxsw_sp_nexthop_group *nh_grp,
1474                                  struct mlxsw_sp_nexthop *nh,
1475                                  struct fib_nh *fib_nh)
1476 {
1477         struct net_device *dev = fib_nh->nh_dev;
1478         struct in_device *in_dev;
1479         struct mlxsw_sp_rif *r;
1480         int err;
1481
1482         nh->nh_grp = nh_grp;
1483         nh->key.fib_nh = fib_nh;
1484         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
1485         if (err)
1486                 return err;
1487
1488         if (!dev)
1489                 return 0;
1490
1491         in_dev = __in_dev_get_rtnl(dev);
1492         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
1493             fib_nh->nh_flags & RTNH_F_LINKDOWN)
1494                 return 0;
1495
1496         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
1497         if (!r)
1498                 return 0;
1499         mlxsw_sp_nexthop_rif_init(nh, r);
1500
1501         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1502         if (err)
1503                 goto err_nexthop_neigh_init;
1504
1505         return 0;
1506
1507 err_nexthop_neigh_init:
1508         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1509         return err;
1510 }
1511
1512 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1513                                   struct mlxsw_sp_nexthop *nh)
1514 {
1515         mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1516         mlxsw_sp_nexthop_rif_fini(nh);
1517         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1518 }
1519
1520 static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp,
1521                                    unsigned long event, struct fib_nh *fib_nh)
1522 {
1523         struct mlxsw_sp_nexthop_key key;
1524         struct mlxsw_sp_nexthop *nh;
1525         struct mlxsw_sp_rif *r;
1526
1527         if (mlxsw_sp->router.aborted)
1528                 return;
1529
1530         key.fib_nh = fib_nh;
1531         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
1532         if (WARN_ON_ONCE(!nh))
1533                 return;
1534
1535         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
1536         if (!r)
1537                 return;
1538
1539         switch (event) {
1540         case FIB_EVENT_NH_ADD:
1541                 mlxsw_sp_nexthop_rif_init(nh, r);
1542                 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1543                 break;
1544         case FIB_EVENT_NH_DEL:
1545                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1546                 mlxsw_sp_nexthop_rif_fini(nh);
1547                 break;
1548         }
1549
1550         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1551 }
1552
1553 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1554                                            struct mlxsw_sp_rif *r)
1555 {
1556         struct mlxsw_sp_nexthop *nh, *tmp;
1557
1558         list_for_each_entry_safe(nh, tmp, &r->nexthop_list, rif_list_node) {
1559                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1560                 mlxsw_sp_nexthop_rif_fini(nh);
1561                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1562         }
1563 }
1564
1565 static struct mlxsw_sp_nexthop_group *
1566 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1567 {
1568         struct mlxsw_sp_nexthop_group *nh_grp;
1569         struct mlxsw_sp_nexthop *nh;
1570         struct fib_nh *fib_nh;
1571         size_t alloc_size;
1572         int i;
1573         int err;
1574
1575         alloc_size = sizeof(*nh_grp) +
1576                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1577         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1578         if (!nh_grp)
1579                 return ERR_PTR(-ENOMEM);
1580         INIT_LIST_HEAD(&nh_grp->fib_list);
1581         nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
1582         nh_grp->count = fi->fib_nhs;
1583         nh_grp->key.fi = fi;
1584         for (i = 0; i < nh_grp->count; i++) {
1585                 nh = &nh_grp->nexthops[i];
1586                 fib_nh = &fi->fib_nh[i];
1587                 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1588                 if (err)
1589                         goto err_nexthop_init;
1590         }
1591         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
1592         if (err)
1593                 goto err_nexthop_group_insert;
1594         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1595         return nh_grp;
1596
1597 err_nexthop_group_insert:
1598 err_nexthop_init:
1599         for (i--; i >= 0; i--) {
1600                 nh = &nh_grp->nexthops[i];
1601                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1602         }
1603         kfree(nh_grp);
1604         return ERR_PTR(err);
1605 }
1606
1607 static void
1608 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1609                                struct mlxsw_sp_nexthop_group *nh_grp)
1610 {
1611         struct mlxsw_sp_nexthop *nh;
1612         int i;
1613
1614         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
1615         for (i = 0; i < nh_grp->count; i++) {
1616                 nh = &nh_grp->nexthops[i];
1617                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1618         }
1619         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1620         WARN_ON_ONCE(nh_grp->adj_index_valid);
1621         kfree(nh_grp);
1622 }
1623
1624 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1625                                       struct mlxsw_sp_fib_entry *fib_entry,
1626                                       struct fib_info *fi)
1627 {
1628         struct mlxsw_sp_nexthop_group_key key;
1629         struct mlxsw_sp_nexthop_group *nh_grp;
1630
1631         key.fi = fi;
1632         nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
1633         if (!nh_grp) {
1634                 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1635                 if (IS_ERR(nh_grp))
1636                         return PTR_ERR(nh_grp);
1637         }
1638         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1639         fib_entry->nh_group = nh_grp;
1640         return 0;
1641 }
1642
1643 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1644                                        struct mlxsw_sp_fib_entry *fib_entry)
1645 {
1646         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1647
1648         list_del(&fib_entry->nexthop_group_node);
1649         if (!list_empty(&nh_grp->fib_list))
1650                 return;
1651         mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1652 }
1653
1654 static bool
1655 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
1656 {
1657         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
1658
1659         if (fib_entry->params.tos)
1660                 return false;
1661
1662         switch (fib_entry->type) {
1663         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1664                 return !!nh_group->adj_index_valid;
1665         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1666                 return !!nh_group->nh_rif;
1667         default:
1668                 return false;
1669         }
1670 }
1671
1672 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
1673 {
1674         fib_entry->offloaded = true;
1675
1676         switch (fib_entry->fib_node->fib->proto) {
1677         case MLXSW_SP_L3_PROTO_IPV4:
1678                 fib_info_offload_inc(fib_entry->nh_group->key.fi);
1679                 break;
1680         case MLXSW_SP_L3_PROTO_IPV6:
1681                 WARN_ON_ONCE(1);
1682         }
1683 }
1684
1685 static void
1686 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
1687 {
1688         switch (fib_entry->fib_node->fib->proto) {
1689         case MLXSW_SP_L3_PROTO_IPV4:
1690                 fib_info_offload_dec(fib_entry->nh_group->key.fi);
1691                 break;
1692         case MLXSW_SP_L3_PROTO_IPV6:
1693                 WARN_ON_ONCE(1);
1694         }
1695
1696         fib_entry->offloaded = false;
1697 }
1698
1699 static void
1700 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
1701                                    enum mlxsw_reg_ralue_op op, int err)
1702 {
1703         switch (op) {
1704         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
1705                 if (!fib_entry->offloaded)
1706                         return;
1707                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
1708         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
1709                 if (err)
1710                         return;
1711                 if (mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1712                     !fib_entry->offloaded)
1713                         mlxsw_sp_fib_entry_offload_set(fib_entry);
1714                 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1715                          fib_entry->offloaded)
1716                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
1717                 return;
1718         default:
1719                 return;
1720         }
1721 }
1722
1723 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
1724                                          struct mlxsw_sp_fib_entry *fib_entry,
1725                                          enum mlxsw_reg_ralue_op op)
1726 {
1727         char ralue_pl[MLXSW_REG_RALUE_LEN];
1728         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1729         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1730         enum mlxsw_reg_ralue_trap_action trap_action;
1731         u16 trap_id = 0;
1732         u32 adjacency_index = 0;
1733         u16 ecmp_size = 0;
1734
1735         /* In case the nexthop group adjacency index is valid, use it
1736          * with provided ECMP size. Otherwise, setup trap and pass
1737          * traffic to kernel.
1738          */
1739         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
1740                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1741                 adjacency_index = fib_entry->nh_group->adj_index;
1742                 ecmp_size = fib_entry->nh_group->ecmp_size;
1743         } else {
1744                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1745                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1746         }
1747
1748         mlxsw_reg_ralue_pack4(ralue_pl,
1749                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1750                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
1751                               *p_dip);
1752         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
1753                                         adjacency_index, ecmp_size);
1754         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1755 }
1756
1757 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
1758                                         struct mlxsw_sp_fib_entry *fib_entry,
1759                                         enum mlxsw_reg_ralue_op op)
1760 {
1761         struct mlxsw_sp_rif *r = fib_entry->nh_group->nh_rif;
1762         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1763         enum mlxsw_reg_ralue_trap_action trap_action;
1764         char ralue_pl[MLXSW_REG_RALUE_LEN];
1765         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1766         u16 trap_id = 0;
1767         u16 rif = 0;
1768
1769         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
1770                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1771                 rif = r->rif;
1772         } else {
1773                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1774                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1775         }
1776
1777         mlxsw_reg_ralue_pack4(ralue_pl,
1778                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1779                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
1780                               *p_dip);
1781         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, rif);
1782         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1783 }
1784
1785 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
1786                                        struct mlxsw_sp_fib_entry *fib_entry,
1787                                        enum mlxsw_reg_ralue_op op)
1788 {
1789         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1790         char ralue_pl[MLXSW_REG_RALUE_LEN];
1791         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1792
1793         mlxsw_reg_ralue_pack4(ralue_pl,
1794                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1795                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
1796                               *p_dip);
1797         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
1798         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1799 }
1800
1801 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
1802                                   struct mlxsw_sp_fib_entry *fib_entry,
1803                                   enum mlxsw_reg_ralue_op op)
1804 {
1805         switch (fib_entry->type) {
1806         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1807                 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
1808         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1809                 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
1810         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
1811                 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
1812         }
1813         return -EINVAL;
1814 }
1815
1816 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
1817                                  struct mlxsw_sp_fib_entry *fib_entry,
1818                                  enum mlxsw_reg_ralue_op op)
1819 {
1820         int err = -EINVAL;
1821
1822         switch (fib_entry->fib_node->fib->proto) {
1823         case MLXSW_SP_L3_PROTO_IPV4:
1824                 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
1825                 break;
1826         case MLXSW_SP_L3_PROTO_IPV6:
1827                 return err;
1828         }
1829         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
1830         return err;
1831 }
1832
1833 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1834                                      struct mlxsw_sp_fib_entry *fib_entry)
1835 {
1836         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1837                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
1838 }
1839
1840 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
1841                                   struct mlxsw_sp_fib_entry *fib_entry)
1842 {
1843         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1844                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
1845 }
1846
1847 static int
1848 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
1849                              const struct fib_entry_notifier_info *fen_info,
1850                              struct mlxsw_sp_fib_entry *fib_entry)
1851 {
1852         struct fib_info *fi = fen_info->fi;
1853
1854         switch (fen_info->type) {
1855         case RTN_BROADCAST: /* fall through */
1856         case RTN_LOCAL:
1857                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1858                 return 0;
1859         case RTN_UNREACHABLE: /* fall through */
1860         case RTN_BLACKHOLE: /* fall through */
1861         case RTN_PROHIBIT:
1862                 /* Packets hitting these routes need to be trapped, but
1863                  * can do so with a lower priority than packets directed
1864                  * at the host, so use action type local instead of trap.
1865                  */
1866                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1867                 return 0;
1868         case RTN_UNICAST:
1869                 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
1870                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1871                 else
1872                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
1873                 return 0;
1874         default:
1875                 return -EINVAL;
1876         }
1877 }
1878
1879 static struct mlxsw_sp_fib_entry *
1880 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
1881                            struct mlxsw_sp_fib_node *fib_node,
1882                            const struct fib_entry_notifier_info *fen_info)
1883 {
1884         struct mlxsw_sp_fib_entry *fib_entry;
1885         int err;
1886
1887         fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
1888         if (!fib_entry) {
1889                 err = -ENOMEM;
1890                 goto err_fib_entry_alloc;
1891         }
1892
1893         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
1894         if (err)
1895                 goto err_fib4_entry_type_set;
1896
1897         err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi);
1898         if (err)
1899                 goto err_nexthop_group_get;
1900
1901         fib_entry->params.prio = fen_info->fi->fib_priority;
1902         fib_entry->params.tb_id = fen_info->tb_id;
1903         fib_entry->params.type = fen_info->type;
1904         fib_entry->params.tos = fen_info->tos;
1905
1906         fib_entry->fib_node = fib_node;
1907
1908         return fib_entry;
1909
1910 err_nexthop_group_get:
1911 err_fib4_entry_type_set:
1912         kfree(fib_entry);
1913 err_fib_entry_alloc:
1914         return ERR_PTR(err);
1915 }
1916
1917 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1918                                         struct mlxsw_sp_fib_entry *fib_entry)
1919 {
1920         mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
1921         kfree(fib_entry);
1922 }
1923
1924 static struct mlxsw_sp_fib_node *
1925 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
1926                        const struct fib_entry_notifier_info *fen_info);
1927
1928 static struct mlxsw_sp_fib_entry *
1929 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
1930                            const struct fib_entry_notifier_info *fen_info)
1931 {
1932         struct mlxsw_sp_fib_entry *fib_entry;
1933         struct mlxsw_sp_fib_node *fib_node;
1934
1935         fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
1936         if (IS_ERR(fib_node))
1937                 return NULL;
1938
1939         list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
1940                 if (fib_entry->params.tb_id == fen_info->tb_id &&
1941                     fib_entry->params.tos == fen_info->tos &&
1942                     fib_entry->params.type == fen_info->type &&
1943                     fib_entry->nh_group->key.fi == fen_info->fi) {
1944                         return fib_entry;
1945                 }
1946         }
1947
1948         return NULL;
1949 }
1950
1951 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
1952         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
1953         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
1954         .key_len = sizeof(struct mlxsw_sp_fib_key),
1955         .automatic_shrinking = true,
1956 };
1957
1958 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
1959                                     struct mlxsw_sp_fib_node *fib_node)
1960 {
1961         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
1962                                       mlxsw_sp_fib_ht_params);
1963 }
1964
1965 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
1966                                      struct mlxsw_sp_fib_node *fib_node)
1967 {
1968         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
1969                                mlxsw_sp_fib_ht_params);
1970 }
1971
1972 static struct mlxsw_sp_fib_node *
1973 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1974                          size_t addr_len, unsigned char prefix_len)
1975 {
1976         struct mlxsw_sp_fib_key key;
1977
1978         memset(&key, 0, sizeof(key));
1979         memcpy(key.addr, addr, addr_len);
1980         key.prefix_len = prefix_len;
1981         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
1982 }
1983
1984 static struct mlxsw_sp_fib_node *
1985 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
1986                          size_t addr_len, unsigned char prefix_len)
1987 {
1988         struct mlxsw_sp_fib_node *fib_node;
1989
1990         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
1991         if (!fib_node)
1992                 return NULL;
1993
1994         INIT_LIST_HEAD(&fib_node->entry_list);
1995         list_add(&fib_node->list, &fib->node_list);
1996         memcpy(fib_node->key.addr, addr, addr_len);
1997         fib_node->key.prefix_len = prefix_len;
1998
1999         return fib_node;
2000 }
2001
2002 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2003 {
2004         list_del(&fib_node->list);
2005         WARN_ON(!list_empty(&fib_node->entry_list));
2006         kfree(fib_node);
2007 }
2008
2009 static bool
2010 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2011                                  const struct mlxsw_sp_fib_entry *fib_entry)
2012 {
2013         return list_first_entry(&fib_node->entry_list,
2014                                 struct mlxsw_sp_fib_entry, list) == fib_entry;
2015 }
2016
2017 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
2018 {
2019         unsigned char prefix_len = fib_node->key.prefix_len;
2020         struct mlxsw_sp_fib *fib = fib_node->fib;
2021
2022         if (fib->prefix_ref_count[prefix_len]++ == 0)
2023                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
2024 }
2025
2026 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
2027 {
2028         unsigned char prefix_len = fib_node->key.prefix_len;
2029         struct mlxsw_sp_fib *fib = fib_node->fib;
2030
2031         if (--fib->prefix_ref_count[prefix_len] == 0)
2032                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
2033 }
2034
2035 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
2036                                   struct mlxsw_sp_fib_node *fib_node,
2037                                   struct mlxsw_sp_fib *fib)
2038 {
2039         struct mlxsw_sp_prefix_usage req_prefix_usage;
2040         struct mlxsw_sp_lpm_tree *lpm_tree;
2041         int err;
2042
2043         err = mlxsw_sp_fib_node_insert(fib, fib_node);
2044         if (err)
2045                 return err;
2046         fib_node->fib = fib;
2047
2048         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage);
2049         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2050
2051         if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2052                 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib,
2053                                                  &req_prefix_usage);
2054                 if (err)
2055                         goto err_tree_check;
2056         } else {
2057                 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2058                                                  fib->proto);
2059                 if (IS_ERR(lpm_tree))
2060                         return PTR_ERR(lpm_tree);
2061                 fib->lpm_tree = lpm_tree;
2062                 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
2063                 if (err)
2064                         goto err_tree_bind;
2065         }
2066
2067         mlxsw_sp_fib_node_prefix_inc(fib_node);
2068
2069         return 0;
2070
2071 err_tree_bind:
2072         fib->lpm_tree = NULL;
2073         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2074 err_tree_check:
2075         fib_node->fib = NULL;
2076         mlxsw_sp_fib_node_remove(fib, fib_node);
2077         return err;
2078 }
2079
2080 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
2081                                    struct mlxsw_sp_fib_node *fib_node)
2082 {
2083         struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
2084         struct mlxsw_sp_fib *fib = fib_node->fib;
2085
2086         mlxsw_sp_fib_node_prefix_dec(fib_node);
2087
2088         if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2089                 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
2090                 fib->lpm_tree = NULL;
2091                 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2092         } else {
2093                 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage);
2094         }
2095
2096         fib_node->fib = NULL;
2097         mlxsw_sp_fib_node_remove(fib, fib_node);
2098 }
2099
2100 static struct mlxsw_sp_fib_node *
2101 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2102                        const struct fib_entry_notifier_info *fen_info)
2103 {
2104         struct mlxsw_sp_fib_node *fib_node;
2105         struct mlxsw_sp_fib *fib;
2106         struct mlxsw_sp_vr *vr;
2107         int err;
2108
2109         vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->tb_id);
2110         if (IS_ERR(vr))
2111                 return ERR_CAST(vr);
2112         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
2113
2114         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2115                                             sizeof(fen_info->dst),
2116                                             fen_info->dst_len);
2117         if (fib_node)
2118                 return fib_node;
2119
2120         fib_node = mlxsw_sp_fib_node_create(fib, &fen_info->dst,
2121                                             sizeof(fen_info->dst),
2122                                             fen_info->dst_len);
2123         if (!fib_node) {
2124                 err = -ENOMEM;
2125                 goto err_fib_node_create;
2126         }
2127
2128         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
2129         if (err)
2130                 goto err_fib_node_init;
2131
2132         return fib_node;
2133
2134 err_fib_node_init:
2135         mlxsw_sp_fib_node_destroy(fib_node);
2136 err_fib_node_create:
2137         mlxsw_sp_vr_put(vr);
2138         return ERR_PTR(err);
2139 }
2140
2141 static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp,
2142                                    struct mlxsw_sp_fib_node *fib_node)
2143 {
2144         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
2145
2146         if (!list_empty(&fib_node->entry_list))
2147                 return;
2148         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
2149         mlxsw_sp_fib_node_destroy(fib_node);
2150         mlxsw_sp_vr_put(vr);
2151 }
2152
2153 static struct mlxsw_sp_fib_entry *
2154 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
2155                               const struct mlxsw_sp_fib_entry_params *params)
2156 {
2157         struct mlxsw_sp_fib_entry *fib_entry;
2158
2159         list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2160                 if (fib_entry->params.tb_id > params->tb_id)
2161                         continue;
2162                 if (fib_entry->params.tb_id != params->tb_id)
2163                         break;
2164                 if (fib_entry->params.tos > params->tos)
2165                         continue;
2166                 if (fib_entry->params.prio >= params->prio ||
2167                     fib_entry->params.tos < params->tos)
2168                         return fib_entry;
2169         }
2170
2171         return NULL;
2172 }
2173
2174 static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry,
2175                                           struct mlxsw_sp_fib_entry *new_entry)
2176 {
2177         struct mlxsw_sp_fib_node *fib_node;
2178
2179         if (WARN_ON(!fib_entry))
2180                 return -EINVAL;
2181
2182         fib_node = fib_entry->fib_node;
2183         list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) {
2184                 if (fib_entry->params.tb_id != new_entry->params.tb_id ||
2185                     fib_entry->params.tos != new_entry->params.tos ||
2186                     fib_entry->params.prio != new_entry->params.prio)
2187                         break;
2188         }
2189
2190         list_add_tail(&new_entry->list, &fib_entry->list);
2191         return 0;
2192 }
2193
2194 static int
2195 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node,
2196                                struct mlxsw_sp_fib_entry *new_entry,
2197                                bool replace, bool append)
2198 {
2199         struct mlxsw_sp_fib_entry *fib_entry;
2200
2201         fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params);
2202
2203         if (append)
2204                 return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry);
2205         if (replace && WARN_ON(!fib_entry))
2206                 return -EINVAL;
2207
2208         /* Insert new entry before replaced one, so that we can later
2209          * remove the second.
2210          */
2211         if (fib_entry) {
2212                 list_add_tail(&new_entry->list, &fib_entry->list);
2213         } else {
2214                 struct mlxsw_sp_fib_entry *last;
2215
2216                 list_for_each_entry(last, &fib_node->entry_list, list) {
2217                         if (new_entry->params.tb_id > last->params.tb_id)
2218                                 break;
2219                         fib_entry = last;
2220                 }
2221
2222                 if (fib_entry)
2223                         list_add(&new_entry->list, &fib_entry->list);
2224                 else
2225                         list_add(&new_entry->list, &fib_node->entry_list);
2226         }
2227
2228         return 0;
2229 }
2230
2231 static void
2232 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry)
2233 {
2234         list_del(&fib_entry->list);
2235 }
2236
2237 static int
2238 mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp,
2239                              const struct mlxsw_sp_fib_node *fib_node,
2240                              struct mlxsw_sp_fib_entry *fib_entry)
2241 {
2242         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2243                 return 0;
2244
2245         /* To prevent packet loss, overwrite the previously offloaded
2246          * entry.
2247          */
2248         if (!list_is_singular(&fib_node->entry_list)) {
2249                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2250                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2251
2252                 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
2253         }
2254
2255         return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2256 }
2257
2258 static void
2259 mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp,
2260                              const struct mlxsw_sp_fib_node *fib_node,
2261                              struct mlxsw_sp_fib_entry *fib_entry)
2262 {
2263         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2264                 return;
2265
2266         /* Promote the next entry by overwriting the deleted entry */
2267         if (!list_is_singular(&fib_node->entry_list)) {
2268                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2269                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2270
2271                 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
2272                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2273                 return;
2274         }
2275
2276         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
2277 }
2278
2279 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
2280                                          struct mlxsw_sp_fib_entry *fib_entry,
2281                                          bool replace, bool append)
2282 {
2283         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2284         int err;
2285
2286         err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace,
2287                                              append);
2288         if (err)
2289                 return err;
2290
2291         err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry);
2292         if (err)
2293                 goto err_fib4_node_entry_add;
2294
2295         return 0;
2296
2297 err_fib4_node_entry_add:
2298         mlxsw_sp_fib4_node_list_remove(fib_entry);
2299         return err;
2300 }
2301
2302 static void
2303 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
2304                                 struct mlxsw_sp_fib_entry *fib_entry)
2305 {
2306         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2307
2308         mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry);
2309         mlxsw_sp_fib4_node_list_remove(fib_entry);
2310 }
2311
2312 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
2313                                         struct mlxsw_sp_fib_entry *fib_entry,
2314                                         bool replace)
2315 {
2316         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2317         struct mlxsw_sp_fib_entry *replaced;
2318
2319         if (!replace)
2320                 return;
2321
2322         /* We inserted the new entry before replaced one */
2323         replaced = list_next_entry(fib_entry, list);
2324
2325         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
2326         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
2327         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2328 }
2329
2330 static int
2331 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
2332                          const struct fib_entry_notifier_info *fen_info,
2333                          bool replace, bool append)
2334 {
2335         struct mlxsw_sp_fib_entry *fib_entry;
2336         struct mlxsw_sp_fib_node *fib_node;
2337         int err;
2338
2339         if (mlxsw_sp->router.aborted)
2340                 return 0;
2341
2342         fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2343         if (IS_ERR(fib_node)) {
2344                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
2345                 return PTR_ERR(fib_node);
2346         }
2347
2348         fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
2349         if (IS_ERR(fib_entry)) {
2350                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
2351                 err = PTR_ERR(fib_entry);
2352                 goto err_fib4_entry_create;
2353         }
2354
2355         err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace,
2356                                             append);
2357         if (err) {
2358                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
2359                 goto err_fib4_node_entry_link;
2360         }
2361
2362         mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace);
2363
2364         return 0;
2365
2366 err_fib4_node_entry_link:
2367         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2368 err_fib4_entry_create:
2369         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2370         return err;
2371 }
2372
2373 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
2374                                      struct fib_entry_notifier_info *fen_info)
2375 {
2376         struct mlxsw_sp_fib_entry *fib_entry;
2377         struct mlxsw_sp_fib_node *fib_node;
2378
2379         if (mlxsw_sp->router.aborted)
2380                 return;
2381
2382         fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
2383         if (WARN_ON(!fib_entry))
2384                 return;
2385         fib_node = fib_entry->fib_node;
2386
2387         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2388         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2389         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2390 }
2391
2392 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
2393 {
2394         char ralta_pl[MLXSW_REG_RALTA_LEN];
2395         char ralst_pl[MLXSW_REG_RALST_LEN];
2396         int i, err;
2397
2398         mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
2399                              MLXSW_SP_LPM_TREE_MIN);
2400         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
2401         if (err)
2402                 return err;
2403
2404         mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
2405         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
2406         if (err)
2407                 return err;
2408
2409         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2410                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i];
2411                 char raltb_pl[MLXSW_REG_RALTB_LEN];
2412                 char ralue_pl[MLXSW_REG_RALUE_LEN];
2413
2414                 if (!mlxsw_sp_vr_is_used(vr))
2415                         continue;
2416
2417                 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
2418                                      MLXSW_REG_RALXX_PROTOCOL_IPV4,
2419                                      MLXSW_SP_LPM_TREE_MIN);
2420                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
2421                                       raltb_pl);
2422                 if (err)
2423                         return err;
2424
2425                 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
2426                                       MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0,
2427                                       0);
2428                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2429                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
2430                                       ralue_pl);
2431                 if (err)
2432                         return err;
2433         }
2434
2435         return 0;
2436 }
2437
2438 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
2439                                      struct mlxsw_sp_fib_node *fib_node)
2440 {
2441         struct mlxsw_sp_fib_entry *fib_entry, *tmp;
2442
2443         list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) {
2444                 bool do_break = &tmp->list == &fib_node->entry_list;
2445
2446                 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2447                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2448                 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2449                 /* Break when entry list is empty and node was freed.
2450                  * Otherwise, we'll access freed memory in the next
2451                  * iteration.
2452                  */
2453                 if (do_break)
2454                         break;
2455         }
2456 }
2457
2458 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
2459                                     struct mlxsw_sp_fib_node *fib_node)
2460 {
2461         switch (fib_node->fib->proto) {
2462         case MLXSW_SP_L3_PROTO_IPV4:
2463                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
2464                 break;
2465         case MLXSW_SP_L3_PROTO_IPV6:
2466                 WARN_ON_ONCE(1);
2467                 break;
2468         }
2469 }
2470
2471 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
2472                                   struct mlxsw_sp_vr *vr,
2473                                   enum mlxsw_sp_l3proto proto)
2474 {
2475         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
2476         struct mlxsw_sp_fib_node *fib_node, *tmp;
2477
2478         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
2479                 bool do_break = &tmp->list == &fib->node_list;
2480
2481                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
2482                 if (do_break)
2483                         break;
2484         }
2485 }
2486
2487 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
2488 {
2489         int i;
2490
2491         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2492                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i];
2493
2494                 if (!mlxsw_sp_vr_is_used(vr))
2495                         continue;
2496                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
2497         }
2498 }
2499
2500 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
2501 {
2502         int err;
2503
2504         if (mlxsw_sp->router.aborted)
2505                 return;
2506         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
2507         mlxsw_sp_router_fib_flush(mlxsw_sp);
2508         mlxsw_sp->router.aborted = true;
2509         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
2510         if (err)
2511                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
2512 }
2513
2514 struct mlxsw_sp_fib_event_work {
2515         struct work_struct work;
2516         union {
2517                 struct fib_entry_notifier_info fen_info;
2518                 struct fib_rule_notifier_info fr_info;
2519                 struct fib_nh_notifier_info fnh_info;
2520         };
2521         struct mlxsw_sp *mlxsw_sp;
2522         unsigned long event;
2523 };
2524
2525 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
2526 {
2527         struct mlxsw_sp_fib_event_work *fib_work =
2528                 container_of(work, struct mlxsw_sp_fib_event_work, work);
2529         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
2530         struct fib_rule *rule;
2531         bool replace, append;
2532         int err;
2533
2534         /* Protect internal structures from changes */
2535         rtnl_lock();
2536         switch (fib_work->event) {
2537         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2538         case FIB_EVENT_ENTRY_APPEND: /* fall through */
2539         case FIB_EVENT_ENTRY_ADD:
2540                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
2541                 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
2542                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
2543                                                replace, append);
2544                 if (err)
2545                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
2546                 fib_info_put(fib_work->fen_info.fi);
2547                 break;
2548         case FIB_EVENT_ENTRY_DEL:
2549                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
2550                 fib_info_put(fib_work->fen_info.fi);
2551                 break;
2552         case FIB_EVENT_RULE_ADD: /* fall through */
2553         case FIB_EVENT_RULE_DEL:
2554                 rule = fib_work->fr_info.rule;
2555                 if (!fib4_rule_default(rule))
2556                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
2557                 fib_rule_put(rule);
2558                 break;
2559         case FIB_EVENT_NH_ADD: /* fall through */
2560         case FIB_EVENT_NH_DEL:
2561                 mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event,
2562                                        fib_work->fnh_info.fib_nh);
2563                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
2564                 break;
2565         }
2566         rtnl_unlock();
2567         kfree(fib_work);
2568 }
2569
2570 /* Called with rcu_read_lock() */
2571 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
2572                                      unsigned long event, void *ptr)
2573 {
2574         struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
2575         struct mlxsw_sp_fib_event_work *fib_work;
2576         struct fib_notifier_info *info = ptr;
2577
2578         if (!net_eq(info->net, &init_net))
2579                 return NOTIFY_DONE;
2580
2581         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
2582         if (WARN_ON(!fib_work))
2583                 return NOTIFY_BAD;
2584
2585         INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work);
2586         fib_work->mlxsw_sp = mlxsw_sp;
2587         fib_work->event = event;
2588
2589         switch (event) {
2590         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2591         case FIB_EVENT_ENTRY_APPEND: /* fall through */
2592         case FIB_EVENT_ENTRY_ADD: /* fall through */
2593         case FIB_EVENT_ENTRY_DEL:
2594                 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
2595                 /* Take referece on fib_info to prevent it from being
2596                  * freed while work is queued. Release it afterwards.
2597                  */
2598                 fib_info_hold(fib_work->fen_info.fi);
2599                 break;
2600         case FIB_EVENT_RULE_ADD: /* fall through */
2601         case FIB_EVENT_RULE_DEL:
2602                 memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info));
2603                 fib_rule_get(fib_work->fr_info.rule);
2604                 break;
2605         case FIB_EVENT_NH_ADD: /* fall through */
2606         case FIB_EVENT_NH_DEL:
2607                 memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info));
2608                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
2609                 break;
2610         }
2611
2612         mlxsw_core_schedule_work(&fib_work->work);
2613
2614         return NOTIFY_DONE;
2615 }
2616
2617 static struct mlxsw_sp_rif *
2618 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
2619                          const struct net_device *dev)
2620 {
2621         int i;
2622
2623         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2624                 if (mlxsw_sp->rifs[i] && mlxsw_sp->rifs[i]->dev == dev)
2625                         return mlxsw_sp->rifs[i];
2626
2627         return NULL;
2628 }
2629
2630 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
2631 {
2632         char ritr_pl[MLXSW_REG_RITR_LEN];
2633         int err;
2634
2635         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
2636         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2637         if (WARN_ON_ONCE(err))
2638                 return err;
2639
2640         mlxsw_reg_ritr_enable_set(ritr_pl, false);
2641         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2642 }
2643
2644 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2645                                           struct mlxsw_sp_rif *r)
2646 {
2647         mlxsw_sp_router_rif_disable(mlxsw_sp, r->rif);
2648         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, r);
2649         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, r);
2650 }
2651
2652 static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *r,
2653                                        const struct in_device *in_dev,
2654                                        unsigned long event)
2655 {
2656         switch (event) {
2657         case NETDEV_UP:
2658                 if (!r)
2659                         return true;
2660                 return false;
2661         case NETDEV_DOWN:
2662                 if (r && !in_dev->ifa_list)
2663                         return true;
2664                 /* It is possible we already removed the RIF ourselves
2665                  * if it was assigned to a netdev that is now a bridge
2666                  * or LAG slave.
2667                  */
2668                 return false;
2669         }
2670
2671         return false;
2672 }
2673
2674 #define MLXSW_SP_INVALID_RIF 0xffff
2675 static int mlxsw_sp_avail_rif_get(struct mlxsw_sp *mlxsw_sp)
2676 {
2677         int i;
2678
2679         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2680                 if (!mlxsw_sp->rifs[i])
2681                         return i;
2682
2683         return MLXSW_SP_INVALID_RIF;
2684 }
2685
2686 static void mlxsw_sp_vport_rif_sp_attr_get(struct mlxsw_sp_port *mlxsw_sp_vport,
2687                                            bool *p_lagged, u16 *p_system_port)
2688 {
2689         u8 local_port = mlxsw_sp_vport->local_port;
2690
2691         *p_lagged = mlxsw_sp_vport->lagged;
2692         *p_system_port = *p_lagged ? mlxsw_sp_vport->lag_id : local_port;
2693 }
2694
2695 static int mlxsw_sp_vport_rif_sp_op(struct mlxsw_sp_port *mlxsw_sp_vport,
2696                                     u16 vr_id, struct net_device *l3_dev,
2697                                     u16 rif, bool create)
2698 {
2699         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2700         bool lagged = mlxsw_sp_vport->lagged;
2701         char ritr_pl[MLXSW_REG_RITR_LEN];
2702         u16 system_port;
2703
2704         mlxsw_reg_ritr_pack(ritr_pl, create, MLXSW_REG_RITR_SP_IF, rif, vr_id,
2705                             l3_dev->mtu, l3_dev->dev_addr);
2706
2707         mlxsw_sp_vport_rif_sp_attr_get(mlxsw_sp_vport, &lagged, &system_port);
2708         mlxsw_reg_ritr_sp_if_pack(ritr_pl, lagged, system_port,
2709                                   mlxsw_sp_vport_vid_get(mlxsw_sp_vport));
2710
2711         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2712 }
2713
2714 static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport);
2715
2716 static u16 mlxsw_sp_rif_sp_to_fid(u16 rif)
2717 {
2718         return MLXSW_SP_RFID_BASE + rif;
2719 }
2720
2721 static struct mlxsw_sp_fid *
2722 mlxsw_sp_rfid_alloc(u16 fid, struct net_device *l3_dev)
2723 {
2724         struct mlxsw_sp_fid *f;
2725
2726         f = kzalloc(sizeof(*f), GFP_KERNEL);
2727         if (!f)
2728                 return NULL;
2729
2730         f->leave = mlxsw_sp_vport_rif_sp_leave;
2731         f->ref_count = 0;
2732         f->dev = l3_dev;
2733         f->fid = fid;
2734
2735         return f;
2736 }
2737
2738 static struct mlxsw_sp_rif *
2739 mlxsw_sp_rif_alloc(u16 rif, u16 vr_id, struct net_device *l3_dev,
2740                    struct mlxsw_sp_fid *f)
2741 {
2742         struct mlxsw_sp_rif *r;
2743
2744         r = kzalloc(sizeof(*r), GFP_KERNEL);
2745         if (!r)
2746                 return NULL;
2747
2748         INIT_LIST_HEAD(&r->nexthop_list);
2749         INIT_LIST_HEAD(&r->neigh_list);
2750         ether_addr_copy(r->addr, l3_dev->dev_addr);
2751         r->mtu = l3_dev->mtu;
2752         r->vr_id = vr_id;
2753         r->dev = l3_dev;
2754         r->rif = rif;
2755         r->f = f;
2756
2757         return r;
2758 }
2759
2760 static struct mlxsw_sp_rif *
2761 mlxsw_sp_vport_rif_sp_create(struct mlxsw_sp_port *mlxsw_sp_vport,
2762                              struct net_device *l3_dev)
2763 {
2764         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2765         struct mlxsw_sp_vr *vr;
2766         struct mlxsw_sp_fid *f;
2767         struct mlxsw_sp_rif *r;
2768         u16 fid, rif;
2769         int err;
2770
2771         rif = mlxsw_sp_avail_rif_get(mlxsw_sp);
2772         if (rif == MLXSW_SP_INVALID_RIF)
2773                 return ERR_PTR(-ERANGE);
2774
2775         vr = mlxsw_sp_vr_get(mlxsw_sp, RT_TABLE_MAIN);
2776         if (IS_ERR(vr))
2777                 return ERR_CAST(vr);
2778
2779         err = mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif,
2780                                        true);
2781         if (err)
2782                 goto err_vport_rif_sp_op;
2783
2784         fid = mlxsw_sp_rif_sp_to_fid(rif);
2785         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, true);
2786         if (err)
2787                 goto err_rif_fdb_op;
2788
2789         f = mlxsw_sp_rfid_alloc(fid, l3_dev);
2790         if (!f) {
2791                 err = -ENOMEM;
2792                 goto err_rfid_alloc;
2793         }
2794
2795         r = mlxsw_sp_rif_alloc(rif, vr->id, l3_dev, f);
2796         if (!r) {
2797                 err = -ENOMEM;
2798                 goto err_rif_alloc;
2799         }
2800
2801         f->r = r;
2802         mlxsw_sp->rifs[rif] = r;
2803         vr->rif_count++;
2804
2805         return r;
2806
2807 err_rif_alloc:
2808         kfree(f);
2809 err_rfid_alloc:
2810         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
2811 err_rif_fdb_op:
2812         mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif, false);
2813 err_vport_rif_sp_op:
2814         mlxsw_sp_vr_put(vr);
2815         return ERR_PTR(err);
2816 }
2817
2818 static void mlxsw_sp_vport_rif_sp_destroy(struct mlxsw_sp_port *mlxsw_sp_vport,
2819                                           struct mlxsw_sp_rif *r)
2820 {
2821         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2822         struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[r->vr_id];
2823         struct net_device *l3_dev = r->dev;
2824         struct mlxsw_sp_fid *f = r->f;
2825         u16 fid = f->fid;
2826         u16 rif = r->rif;
2827
2828         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, r);
2829
2830         vr->rif_count--;
2831         mlxsw_sp->rifs[rif] = NULL;
2832         f->r = NULL;
2833
2834         kfree(r);
2835
2836         kfree(f);
2837
2838         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
2839
2840         mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif, false);
2841
2842         mlxsw_sp_vr_put(vr);
2843 }
2844
2845 static int mlxsw_sp_vport_rif_sp_join(struct mlxsw_sp_port *mlxsw_sp_vport,
2846                                       struct net_device *l3_dev)
2847 {
2848         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2849         struct mlxsw_sp_rif *r;
2850
2851         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
2852         if (!r) {
2853                 r = mlxsw_sp_vport_rif_sp_create(mlxsw_sp_vport, l3_dev);
2854                 if (IS_ERR(r))
2855                         return PTR_ERR(r);
2856         }
2857
2858         mlxsw_sp_vport_fid_set(mlxsw_sp_vport, r->f);
2859         r->f->ref_count++;
2860
2861         netdev_dbg(mlxsw_sp_vport->dev, "Joined FID=%d\n", r->f->fid);
2862
2863         return 0;
2864 }
2865
2866 static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport)
2867 {
2868         struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
2869
2870         netdev_dbg(mlxsw_sp_vport->dev, "Left FID=%d\n", f->fid);
2871
2872         mlxsw_sp_vport_fid_set(mlxsw_sp_vport, NULL);
2873         if (--f->ref_count == 0)
2874                 mlxsw_sp_vport_rif_sp_destroy(mlxsw_sp_vport, f->r);
2875 }
2876
2877 static int mlxsw_sp_inetaddr_vport_event(struct net_device *l3_dev,
2878                                          struct net_device *port_dev,
2879                                          unsigned long event, u16 vid)
2880 {
2881         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
2882         struct mlxsw_sp_port *mlxsw_sp_vport;
2883
2884         mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, vid);
2885         if (WARN_ON(!mlxsw_sp_vport))
2886                 return -EINVAL;
2887
2888         switch (event) {
2889         case NETDEV_UP:
2890                 return mlxsw_sp_vport_rif_sp_join(mlxsw_sp_vport, l3_dev);
2891         case NETDEV_DOWN:
2892                 mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
2893                 break;
2894         }
2895
2896         return 0;
2897 }
2898
2899 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
2900                                         unsigned long event)
2901 {
2902         if (netif_is_bridge_port(port_dev) || netif_is_lag_port(port_dev))
2903                 return 0;
2904
2905         return mlxsw_sp_inetaddr_vport_event(port_dev, port_dev, event, 1);
2906 }
2907
2908 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
2909                                          struct net_device *lag_dev,
2910                                          unsigned long event, u16 vid)
2911 {
2912         struct net_device *port_dev;
2913         struct list_head *iter;
2914         int err;
2915
2916         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
2917                 if (mlxsw_sp_port_dev_check(port_dev)) {
2918                         err = mlxsw_sp_inetaddr_vport_event(l3_dev, port_dev,
2919                                                             event, vid);
2920                         if (err)
2921                                 return err;
2922                 }
2923         }
2924
2925         return 0;
2926 }
2927
2928 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
2929                                        unsigned long event)
2930 {
2931         if (netif_is_bridge_port(lag_dev))
2932                 return 0;
2933
2934         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
2935 }
2936
2937 static struct mlxsw_sp_fid *mlxsw_sp_bridge_fid_get(struct mlxsw_sp *mlxsw_sp,
2938                                                     struct net_device *l3_dev)
2939 {
2940         u16 fid;
2941
2942         if (is_vlan_dev(l3_dev))
2943                 fid = vlan_dev_vlan_id(l3_dev);
2944         else if (mlxsw_sp->master_bridge.dev == l3_dev)
2945                 fid = 1;
2946         else
2947                 return mlxsw_sp_vfid_find(mlxsw_sp, l3_dev);
2948
2949         return mlxsw_sp_fid_find(mlxsw_sp, fid);
2950 }
2951
2952 static enum mlxsw_flood_table_type mlxsw_sp_flood_table_type_get(u16 fid)
2953 {
2954         return mlxsw_sp_fid_is_vfid(fid) ? MLXSW_REG_SFGC_TABLE_TYPE_FID :
2955                MLXSW_REG_SFGC_TABLE_TYPE_FID_OFFEST;
2956 }
2957
2958 static u16 mlxsw_sp_flood_table_index_get(u16 fid)
2959 {
2960         return mlxsw_sp_fid_is_vfid(fid) ? mlxsw_sp_fid_to_vfid(fid) : fid;
2961 }
2962
2963 static int mlxsw_sp_router_port_flood_set(struct mlxsw_sp *mlxsw_sp, u16 fid,
2964                                           bool set)
2965 {
2966         enum mlxsw_flood_table_type table_type;
2967         char *sftr_pl;
2968         u16 index;
2969         int err;
2970
2971         sftr_pl = kmalloc(MLXSW_REG_SFTR_LEN, GFP_KERNEL);
2972         if (!sftr_pl)
2973                 return -ENOMEM;
2974
2975         table_type = mlxsw_sp_flood_table_type_get(fid);
2976         index = mlxsw_sp_flood_table_index_get(fid);
2977         mlxsw_reg_sftr_pack(sftr_pl, MLXSW_SP_FLOOD_TABLE_BC, index, table_type,
2978                             1, MLXSW_PORT_ROUTER_PORT, set);
2979         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sftr), sftr_pl);
2980
2981         kfree(sftr_pl);
2982         return err;
2983 }
2984
2985 static enum mlxsw_reg_ritr_if_type mlxsw_sp_rif_type_get(u16 fid)
2986 {
2987         if (mlxsw_sp_fid_is_vfid(fid))
2988                 return MLXSW_REG_RITR_FID_IF;
2989         else
2990                 return MLXSW_REG_RITR_VLAN_IF;
2991 }
2992
2993 static int mlxsw_sp_rif_bridge_op(struct mlxsw_sp *mlxsw_sp, u16 vr_id,
2994                                   struct net_device *l3_dev,
2995                                   u16 fid, u16 rif,
2996                                   bool create)
2997 {
2998         enum mlxsw_reg_ritr_if_type rif_type;
2999         char ritr_pl[MLXSW_REG_RITR_LEN];
3000
3001         rif_type = mlxsw_sp_rif_type_get(fid);
3002         mlxsw_reg_ritr_pack(ritr_pl, create, rif_type, rif, vr_id, l3_dev->mtu,
3003                             l3_dev->dev_addr);
3004         mlxsw_reg_ritr_fid_set(ritr_pl, rif_type, fid);
3005
3006         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3007 }
3008
3009 static int mlxsw_sp_rif_bridge_create(struct mlxsw_sp *mlxsw_sp,
3010                                       struct net_device *l3_dev,
3011                                       struct mlxsw_sp_fid *f)
3012 {
3013         struct mlxsw_sp_vr *vr;
3014         struct mlxsw_sp_rif *r;
3015         u16 rif;
3016         int err;
3017
3018         rif = mlxsw_sp_avail_rif_get(mlxsw_sp);
3019         if (rif == MLXSW_SP_INVALID_RIF)
3020                 return -ERANGE;
3021
3022         vr = mlxsw_sp_vr_get(mlxsw_sp, RT_TABLE_MAIN);
3023         if (IS_ERR(vr))
3024                 return PTR_ERR(vr);
3025
3026         err = mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, true);
3027         if (err)
3028                 goto err_port_flood_set;
3029
3030         err = mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif,
3031                                      true);
3032         if (err)
3033                 goto err_rif_bridge_op;
3034
3035         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, true);
3036         if (err)
3037                 goto err_rif_fdb_op;
3038
3039         r = mlxsw_sp_rif_alloc(rif, vr->id, l3_dev, f);
3040         if (!r) {
3041                 err = -ENOMEM;
3042                 goto err_rif_alloc;
3043         }
3044
3045         f->r = r;
3046         mlxsw_sp->rifs[rif] = r;
3047         vr->rif_count++;
3048
3049         netdev_dbg(l3_dev, "RIF=%d created\n", rif);
3050
3051         return 0;
3052
3053 err_rif_alloc:
3054         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
3055 err_rif_fdb_op:
3056         mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif, false);
3057 err_rif_bridge_op:
3058         mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
3059 err_port_flood_set:
3060         mlxsw_sp_vr_put(vr);
3061         return err;
3062 }
3063
3064 void mlxsw_sp_rif_bridge_destroy(struct mlxsw_sp *mlxsw_sp,
3065                                  struct mlxsw_sp_rif *r)
3066 {
3067         struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[r->vr_id];
3068         struct net_device *l3_dev = r->dev;
3069         struct mlxsw_sp_fid *f = r->f;
3070         u16 rif = r->rif;
3071
3072         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, r);
3073
3074         vr->rif_count--;
3075         mlxsw_sp->rifs[rif] = NULL;
3076         f->r = NULL;
3077
3078         kfree(r);
3079
3080         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
3081
3082         mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif, false);
3083
3084         mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
3085
3086         mlxsw_sp_vr_put(vr);
3087
3088         netdev_dbg(l3_dev, "RIF=%d destroyed\n", rif);
3089 }
3090
3091 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
3092                                           struct net_device *br_dev,
3093                                           unsigned long event)
3094 {
3095         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3096         struct mlxsw_sp_fid *f;
3097
3098         /* FID can either be an actual FID if the L3 device is the
3099          * VLAN-aware bridge or a VLAN device on top. Otherwise, the
3100          * L3 device is a VLAN-unaware bridge and we get a vFID.
3101          */
3102         f = mlxsw_sp_bridge_fid_get(mlxsw_sp, l3_dev);
3103         if (WARN_ON(!f))
3104                 return -EINVAL;
3105
3106         switch (event) {
3107         case NETDEV_UP:
3108                 return mlxsw_sp_rif_bridge_create(mlxsw_sp, l3_dev, f);
3109         case NETDEV_DOWN:
3110                 mlxsw_sp_rif_bridge_destroy(mlxsw_sp, f->r);
3111                 break;
3112         }
3113
3114         return 0;
3115 }
3116
3117 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
3118                                         unsigned long event)
3119 {
3120         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
3121         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(vlan_dev);
3122         u16 vid = vlan_dev_vlan_id(vlan_dev);
3123
3124         if (mlxsw_sp_port_dev_check(real_dev))
3125                 return mlxsw_sp_inetaddr_vport_event(vlan_dev, real_dev, event,
3126                                                      vid);
3127         else if (netif_is_lag_master(real_dev))
3128                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
3129                                                      vid);
3130         else if (netif_is_bridge_master(real_dev) &&
3131                  mlxsw_sp->master_bridge.dev == real_dev)
3132                 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, real_dev,
3133                                                       event);
3134
3135         return 0;
3136 }
3137
3138 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
3139                             unsigned long event, void *ptr)
3140 {
3141         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
3142         struct net_device *dev = ifa->ifa_dev->dev;
3143         struct mlxsw_sp *mlxsw_sp;
3144         struct mlxsw_sp_rif *r;
3145         int err = 0;
3146
3147         mlxsw_sp = mlxsw_sp_lower_get(dev);
3148         if (!mlxsw_sp)
3149                 goto out;
3150
3151         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3152         if (!mlxsw_sp_rif_should_config(r, ifa->ifa_dev, event))
3153                 goto out;
3154
3155         if (mlxsw_sp_port_dev_check(dev))
3156                 err = mlxsw_sp_inetaddr_port_event(dev, event);
3157         else if (netif_is_lag_master(dev))
3158                 err = mlxsw_sp_inetaddr_lag_event(dev, event);
3159         else if (netif_is_bridge_master(dev))
3160                 err = mlxsw_sp_inetaddr_bridge_event(dev, dev, event);
3161         else if (is_vlan_dev(dev))
3162                 err = mlxsw_sp_inetaddr_vlan_event(dev, event);
3163
3164 out:
3165         return notifier_from_errno(err);
3166 }
3167
3168 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif,
3169                              const char *mac, int mtu)
3170 {
3171         char ritr_pl[MLXSW_REG_RITR_LEN];
3172         int err;
3173
3174         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
3175         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3176         if (err)
3177                 return err;
3178
3179         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
3180         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
3181         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
3182         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3183 }
3184
3185 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
3186 {
3187         struct mlxsw_sp *mlxsw_sp;
3188         struct mlxsw_sp_rif *r;
3189         int err;
3190
3191         mlxsw_sp = mlxsw_sp_lower_get(dev);
3192         if (!mlxsw_sp)
3193                 return 0;
3194
3195         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3196         if (!r)
3197                 return 0;
3198
3199         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, r->addr, r->f->fid, false);
3200         if (err)
3201                 return err;
3202
3203         err = mlxsw_sp_rif_edit(mlxsw_sp, r->rif, dev->dev_addr, dev->mtu);
3204         if (err)
3205                 goto err_rif_edit;
3206
3207         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, r->f->fid, true);
3208         if (err)
3209                 goto err_rif_fdb_op;
3210
3211         ether_addr_copy(r->addr, dev->dev_addr);
3212         r->mtu = dev->mtu;
3213
3214         netdev_dbg(dev, "Updated RIF=%d\n", r->rif);
3215
3216         return 0;
3217
3218 err_rif_fdb_op:
3219         mlxsw_sp_rif_edit(mlxsw_sp, r->rif, r->addr, r->mtu);
3220 err_rif_edit:
3221         mlxsw_sp_rif_fdb_op(mlxsw_sp, r->addr, r->f->fid, true);
3222         return err;
3223 }
3224
3225 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
3226 {
3227         struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
3228
3229         /* Flush pending FIB notifications and then flush the device's
3230          * table before requesting another dump. The FIB notification
3231          * block is unregistered, so no need to take RTNL.
3232          */
3233         mlxsw_core_flush_owq();
3234         mlxsw_sp_router_fib_flush(mlxsw_sp);
3235 }
3236
3237 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3238 {
3239         char rgcr_pl[MLXSW_REG_RGCR_LEN];
3240         u64 max_rifs;
3241         int err;
3242
3243         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
3244                 return -EIO;
3245
3246         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3247         mlxsw_sp->rifs = kcalloc(max_rifs, sizeof(struct mlxsw_sp_rif *),
3248                                  GFP_KERNEL);
3249         if (!mlxsw_sp->rifs)
3250                 return -ENOMEM;
3251
3252         mlxsw_reg_rgcr_pack(rgcr_pl, true);
3253         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
3254         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3255         if (err)
3256                 goto err_rgcr_fail;
3257
3258         return 0;
3259
3260 err_rgcr_fail:
3261         kfree(mlxsw_sp->rifs);
3262         return err;
3263 }
3264
3265 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3266 {
3267         char rgcr_pl[MLXSW_REG_RGCR_LEN];
3268         int i;
3269
3270         mlxsw_reg_rgcr_pack(rgcr_pl, false);
3271         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3272
3273         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
3274                 WARN_ON_ONCE(mlxsw_sp->rifs[i]);
3275
3276         kfree(mlxsw_sp->rifs);
3277 }
3278
3279 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3280 {
3281         int err;
3282
3283         INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_neighs_list);
3284         err = __mlxsw_sp_router_init(mlxsw_sp);
3285         if (err)
3286                 return err;
3287
3288         err = rhashtable_init(&mlxsw_sp->router.nexthop_ht,
3289                               &mlxsw_sp_nexthop_ht_params);
3290         if (err)
3291                 goto err_nexthop_ht_init;
3292
3293         err = rhashtable_init(&mlxsw_sp->router.nexthop_group_ht,
3294                               &mlxsw_sp_nexthop_group_ht_params);
3295         if (err)
3296                 goto err_nexthop_group_ht_init;
3297
3298         mlxsw_sp_lpm_init(mlxsw_sp);
3299         err = mlxsw_sp_vrs_init(mlxsw_sp);
3300         if (err)
3301                 goto err_vrs_init;
3302
3303         err = mlxsw_sp_neigh_init(mlxsw_sp);
3304         if (err)
3305                 goto err_neigh_init;
3306
3307         mlxsw_sp->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
3308         err = register_fib_notifier(&mlxsw_sp->fib_nb,
3309                                     mlxsw_sp_router_fib_dump_flush);
3310         if (err)
3311                 goto err_register_fib_notifier;
3312
3313         return 0;
3314
3315 err_register_fib_notifier:
3316         mlxsw_sp_neigh_fini(mlxsw_sp);
3317 err_neigh_init:
3318         mlxsw_sp_vrs_fini(mlxsw_sp);
3319 err_vrs_init:
3320         rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
3321 err_nexthop_group_ht_init:
3322         rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
3323 err_nexthop_ht_init:
3324         __mlxsw_sp_router_fini(mlxsw_sp);
3325         return err;
3326 }
3327
3328 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3329 {
3330         unregister_fib_notifier(&mlxsw_sp->fib_nb);
3331         mlxsw_sp_neigh_fini(mlxsw_sp);
3332         mlxsw_sp_vrs_fini(mlxsw_sp);
3333         rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
3334         rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
3335         __mlxsw_sp_router_fini(mlxsw_sp);
3336 }