drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016-2017 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  * Copyright (c) 2017 Petr Machata <petrm@mellanox.com>
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  * 3. Neither the names of the copyright holders nor the names of its
  18  *    contributors may be used to endorse or promote products derived from
  19  *    this software without specific prior written permission.
  20  *
  21  * Alternatively, this software may be distributed under the terms of the
  22  * GNU General Public License ("GPL") version 2 as published by the Free
  23  * Software Foundation.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  26  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  29  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  30  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  31  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  32  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  33  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  35  * POSSIBILITY OF SUCH DAMAGE.
  36  */
  37
  38 #include <linux/kernel.h>
  39 #include <linux/types.h>
  40 #include <linux/rhashtable.h>
  41 #include <linux/bitops.h>
  42 #include <linux/in6.h>
  43 #include <linux/notifier.h>
  44 #include <linux/inetdevice.h>
  45 #include <linux/netdevice.h>
  46 #include <linux/if_bridge.h>
  47 #include <linux/socket.h>
  48 #include <linux/route.h>
  49 #include <net/netevent.h>
  50 #include <net/neighbour.h>
  51 #include <net/arp.h>
  52 #include <net/ip_fib.h>
  53 #include <net/ip6_fib.h>
  54 #include <net/fib_rules.h>
  55 #include <net/ip_tunnels.h>
  56 #include <net/l3mdev.h>
  57 #include <net/addrconf.h>
  58 #include <net/ndisc.h>
  59 #include <net/ipv6.h>
  60 #include <net/fib_notifier.h>
  61
  62 #include "spectrum.h"
  63 #include "core.h"
  64 #include "reg.h"
  65 #include "spectrum_cnt.h"
  66 #include "spectrum_dpipe.h"
  67 #include "spectrum_ipip.h"
  68 #include "spectrum_router.h"
  69
  70 struct mlxsw_sp_vr;
  71 struct mlxsw_sp_lpm_tree;
  72 struct mlxsw_sp_rif_ops;
  73
  74 struct mlxsw_sp_router {
  75         struct mlxsw_sp *mlxsw_sp;
  76         struct mlxsw_sp_rif **rifs;
  77         struct mlxsw_sp_vr *vrs;
  78         struct rhashtable neigh_ht;
  79         struct rhashtable nexthop_group_ht;
  80         struct rhashtable nexthop_ht;
  81         struct {
  82                 struct mlxsw_sp_lpm_tree *trees;
  83                 unsigned int tree_count;
  84         } lpm;
  85         struct {
  86                 struct delayed_work dw;
  87                 unsigned long interval; /* ms */
  88         } neighs_update;
  89         struct delayed_work nexthop_probe_dw;
  90 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
  91         struct list_head nexthop_neighs_list;
  92         struct list_head ipip_list;
  93         bool aborted;
  94         struct notifier_block fib_nb;
  95         const struct mlxsw_sp_rif_ops **rif_ops_arr;
  96         const struct mlxsw_sp_ipip_ops **ipip_ops_arr;
  97 };
  98
  99 struct mlxsw_sp_rif {
 100         struct list_head nexthop_list;
 101         struct list_head neigh_list;
 102         struct net_device *dev;
 103         struct mlxsw_sp_fid *fid;
 104         unsigned char addr[ETH_ALEN];
 105         int mtu;
 106         u16 rif_index;
 107         u16 vr_id;
 108         const struct mlxsw_sp_rif_ops *ops;
 109         struct mlxsw_sp *mlxsw_sp;
 110
 111         unsigned int counter_ingress;
 112         bool counter_ingress_valid;
 113         unsigned int counter_egress;
 114         bool counter_egress_valid;
 115 };
 116
 117 struct mlxsw_sp_rif_params {
 118         struct net_device *dev;
 119         union {
 120                 u16 system_port;
 121                 u16 lag_id;
 122         };
 123         u16 vid;
 124         bool lag;
 125 };
 126
 127 struct mlxsw_sp_rif_subport {
 128         struct mlxsw_sp_rif common;
 129         union {
 130                 u16 system_port;
 131                 u16 lag_id;
 132         };
 133         u16 vid;
 134         bool lag;
 135 };
 136
 137 struct mlxsw_sp_rif_ipip_lb {
 138         struct mlxsw_sp_rif common;
 139         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 140         u16 ul_vr_id; /* Reserved for Spectrum-2. */
 141 };
 142
 143 struct mlxsw_sp_rif_params_ipip_lb {
 144         struct mlxsw_sp_rif_params common;
 145         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 146 };
 147
 148 struct mlxsw_sp_rif_ops {
 149         enum mlxsw_sp_rif_type type;
 150         size_t rif_size;
 151
 152         void (*setup)(struct mlxsw_sp_rif *rif,
 153                       const struct mlxsw_sp_rif_params *params);
 154         int (*configure)(struct mlxsw_sp_rif *rif);
 155         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 156         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
 157 };
 158
 159 static unsigned int *
 160 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 161                            enum mlxsw_sp_rif_counter_dir dir)
 162 {
 163         switch (dir) {
 164         case MLXSW_SP_RIF_COUNTER_EGRESS:
 165                 return &rif->counter_egress;
 166         case MLXSW_SP_RIF_COUNTER_INGRESS:
 167                 return &rif->counter_ingress;
 168         }
 169         return NULL;
 170 }
 171
 172 static bool
 173 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 174                                enum mlxsw_sp_rif_counter_dir dir)
 175 {
 176         switch (dir) {
 177         case MLXSW_SP_RIF_COUNTER_EGRESS:
 178                 return rif->counter_egress_valid;
 179         case MLXSW_SP_RIF_COUNTER_INGRESS:
 180                 return rif->counter_ingress_valid;
 181         }
 182         return false;
 183 }
 184
 185 static void
 186 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 187                                enum mlxsw_sp_rif_counter_dir dir,
 188                                bool valid)
 189 {
 190         switch (dir) {
 191         case MLXSW_SP_RIF_COUNTER_EGRESS:
 192                 rif->counter_egress_valid = valid;
 193                 break;
 194         case MLXSW_SP_RIF_COUNTER_INGRESS:
 195                 rif->counter_ingress_valid = valid;
 196                 break;
 197         }
 198 }
 199
 200 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 201                                      unsigned int counter_index, bool enable,
 202                                      enum mlxsw_sp_rif_counter_dir dir)
 203 {
 204         char ritr_pl[MLXSW_REG_RITR_LEN];
 205         bool is_egress = false;
 206         int err;
 207
 208         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 209                 is_egress = true;
 210         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 211         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 212         if (err)
 213                 return err;
 214
 215         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 216                                     is_egress);
 217         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 218 }
 219
 220 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 221                                    struct mlxsw_sp_rif *rif,
 222                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 223 {
 224         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 225         unsigned int *p_counter_index;
 226         bool valid;
 227         int err;
 228
 229         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 230         if (!valid)
 231                 return -EINVAL;
 232
 233         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 234         if (!p_counter_index)
 235                 return -EINVAL;
 236         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 237                              MLXSW_REG_RICNT_OPCODE_NOP);
 238         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 239         if (err)
 240                 return err;
 241         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 242         return 0;
 243 }
 244
 245 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 246                                       unsigned int counter_index)
 247 {
 248         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 249
 250         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 251                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 252         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 253 }
 254
 255 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 256                                struct mlxsw_sp_rif *rif,
 257                                enum mlxsw_sp_rif_counter_dir dir)
 258 {
 259         unsigned int *p_counter_index;
 260         int err;
 261
 262         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 263         if (!p_counter_index)
 264                 return -EINVAL;
 265         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 266                                      p_counter_index);
 267         if (err)
 268                 return err;
 269
 270         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 271         if (err)
 272                 goto err_counter_clear;
 273
 274         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 275                                         *p_counter_index, true, dir);
 276         if (err)
 277                 goto err_counter_edit;
 278         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 279         return 0;
 280
 281 err_counter_edit:
 282 err_counter_clear:
 283         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 284                               *p_counter_index);
 285         return err;
 286 }
 287
 288 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 289                                struct mlxsw_sp_rif *rif,
 290                                enum mlxsw_sp_rif_counter_dir dir)
 291 {
 292         unsigned int *p_counter_index;
 293
 294         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 295                 return;
 296
 297         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 298         if (WARN_ON(!p_counter_index))
 299                 return;
 300         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 301                                   *p_counter_index, false, dir);
 302         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 303                               *p_counter_index);
 304         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 305 }
 306
 307 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 308 {
 309         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 310         struct devlink *devlink;
 311
 312         devlink = priv_to_devlink(mlxsw_sp->core);
 313         if (!devlink_dpipe_table_counter_enabled(devlink,
 314                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 315                 return;
 316         mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 317 }
 318
 319 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 320 {
 321         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 322
 323         mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 324 }
 325
 326 static struct mlxsw_sp_rif *
 327 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 328                          const struct net_device *dev);
 329
 330 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
 331
 332 struct mlxsw_sp_prefix_usage {
 333         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 334 };
 335
 336 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 337         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 338
 339 static bool
 340 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 341                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 342 {
 343         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 344 }
 345
 346 static bool
 347 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
 348 {
 349         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
 350
 351         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
 352 }
 353
 354 static void
 355 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 356                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 357 {
 358         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 359 }
 360
 361 static void
 362 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 363                           unsigned char prefix_len)
 364 {
 365         set_bit(prefix_len, prefix_usage->b);
 366 }
 367
 368 static void
 369 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 370                             unsigned char prefix_len)
 371 {
 372         clear_bit(prefix_len, prefix_usage->b);
 373 }
 374
 375 struct mlxsw_sp_fib_key {
 376         unsigned char addr[sizeof(struct in6_addr)];
 377         unsigned char prefix_len;
 378 };
 379
 380 enum mlxsw_sp_fib_entry_type {
 381         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 382         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 383         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 384
 385         /* This is a special case of local delivery, where a packet should be
 386          * decapsulated on reception. Note that there is no corresponding ENCAP,
 387          * because that's a type of next hop, not of FIB entry. (There can be
 388          * several next hops in a REMOTE entry, and some of them may be
 389          * encapsulating entries.)
 390          */
 391         MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
 392 };
 393
 394 struct mlxsw_sp_nexthop_group;
 395 struct mlxsw_sp_fib;
 396
 397 struct mlxsw_sp_fib_node {
 398         struct list_head entry_list;
 399         struct list_head list;
 400         struct rhash_head ht_node;
 401         struct mlxsw_sp_fib *fib;
 402         struct mlxsw_sp_fib_key key;
 403 };
 404
 405 struct mlxsw_sp_fib_entry_decap {
 406         struct mlxsw_sp_ipip_entry *ipip_entry;
 407         u32 tunnel_index;
 408 };
 409
 410 struct mlxsw_sp_fib_entry {
 411         struct list_head list;
 412         struct mlxsw_sp_fib_node *fib_node;
 413         enum mlxsw_sp_fib_entry_type type;
 414         struct list_head nexthop_group_node;
 415         struct mlxsw_sp_nexthop_group *nh_group;
 416         struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
 417 };
 418
 419 struct mlxsw_sp_fib4_entry {
 420         struct mlxsw_sp_fib_entry common;
 421         u32 tb_id;
 422         u32 prio;
 423         u8 tos;
 424         u8 type;
 425 };
 426
 427 struct mlxsw_sp_fib6_entry {
 428         struct mlxsw_sp_fib_entry common;
 429         struct list_head rt6_list;
 430         unsigned int nrt6;
 431 };
 432
 433 struct mlxsw_sp_rt6 {
 434         struct list_head list;
 435         struct rt6_info *rt;
 436 };
 437
 438 struct mlxsw_sp_lpm_tree {
 439         u8 id; /* tree ID */
 440         unsigned int ref_count;
 441         enum mlxsw_sp_l3proto proto;
 442         struct mlxsw_sp_prefix_usage prefix_usage;
 443 };
 444
 445 struct mlxsw_sp_fib {
 446         struct rhashtable ht;
 447         struct list_head node_list;
 448         struct mlxsw_sp_vr *vr;
 449         struct mlxsw_sp_lpm_tree *lpm_tree;
 450         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 451         struct mlxsw_sp_prefix_usage prefix_usage;
 452         enum mlxsw_sp_l3proto proto;
 453 };
 454
 455 struct mlxsw_sp_vr {
 456         u16 id; /* virtual router ID */
 457         u32 tb_id; /* kernel fib table id */
 458         unsigned int rif_count;
 459         struct mlxsw_sp_fib *fib4;
 460         struct mlxsw_sp_fib *fib6;
 461 };
 462
 463 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 464
 465 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
 466                                                 enum mlxsw_sp_l3proto proto)
 467 {
 468         struct mlxsw_sp_fib *fib;
 469         int err;
 470
 471         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 472         if (!fib)
 473                 return ERR_PTR(-ENOMEM);
 474         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 475         if (err)
 476                 goto err_rhashtable_init;
 477         INIT_LIST_HEAD(&fib->node_list);
 478         fib->proto = proto;
 479         fib->vr = vr;
 480         return fib;
 481
 482 err_rhashtable_init:
 483         kfree(fib);
 484         return ERR_PTR(err);
 485 }
 486
 487 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 488 {
 489         WARN_ON(!list_empty(&fib->node_list));
 490         WARN_ON(fib->lpm_tree);
 491         rhashtable_destroy(&fib->ht);
 492         kfree(fib);
 493 }
 494
 495 static struct mlxsw_sp_lpm_tree *
 496 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 497 {
 498         static struct mlxsw_sp_lpm_tree *lpm_tree;
 499         int i;
 500
 501         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 502                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 503                 if (lpm_tree->ref_count == 0)
 504                         return lpm_tree;
 505         }
 506         return NULL;
 507 }
 508
 509 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 510                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 511 {
 512         char ralta_pl[MLXSW_REG_RALTA_LEN];
 513
 514         mlxsw_reg_ralta_pack(ralta_pl, true,
 515                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 516                              lpm_tree->id);
 517         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 518 }
 519
 520 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 521                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 522 {
 523         char ralta_pl[MLXSW_REG_RALTA_LEN];
 524
 525         mlxsw_reg_ralta_pack(ralta_pl, false,
 526                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 527                              lpm_tree->id);
 528         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 529 }
 530
 531 static int
 532 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 533                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 534                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 535 {
 536         char ralst_pl[MLXSW_REG_RALST_LEN];
 537         u8 root_bin = 0;
 538         u8 prefix;
 539         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 540
 541         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 542                 root_bin = prefix;
 543
 544         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 545         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 546                 if (prefix == 0)
 547                         continue;
 548                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 549                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 550                 last_prefix = prefix;
 551         }
 552         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 553 }
 554
 555 static struct mlxsw_sp_lpm_tree *
 556 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 557                          struct mlxsw_sp_prefix_usage *prefix_usage,
 558                          enum mlxsw_sp_l3proto proto)
 559 {
 560         struct mlxsw_sp_lpm_tree *lpm_tree;
 561         int err;
 562
 563         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 564         if (!lpm_tree)
 565                 return ERR_PTR(-EBUSY);
 566         lpm_tree->proto = proto;
 567         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 568         if (err)
 569                 return ERR_PTR(err);
 570
 571         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 572                                                 lpm_tree);
 573         if (err)
 574                 goto err_left_struct_set;
 575         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 576                sizeof(lpm_tree->prefix_usage));
 577         return lpm_tree;
 578
 579 err_left_struct_set:
 580         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 581         return ERR_PTR(err);
 582 }
 583
 584 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 585                                       struct mlxsw_sp_lpm_tree *lpm_tree)
 586 {
 587         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 588 }
 589
 590 static struct mlxsw_sp_lpm_tree *
 591 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 592                       struct mlxsw_sp_prefix_usage *prefix_usage,
 593                       enum mlxsw_sp_l3proto proto)
 594 {
 595         struct mlxsw_sp_lpm_tree *lpm_tree;
 596         int i;
 597
 598         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 599                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 600                 if (lpm_tree->ref_count != 0 &&
 601                     lpm_tree->proto == proto &&
 602                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 603                                              prefix_usage))
 604                         return lpm_tree;
 605         }
 606         return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
 607 }
 608
 609 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
 610 {
 611         lpm_tree->ref_count++;
 612 }
 613
 614 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 615                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 616 {
 617         if (--lpm_tree->ref_count == 0)
 618                 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 619 }
 620
 621 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 622
 623 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 624 {
 625         struct mlxsw_sp_lpm_tree *lpm_tree;
 626         u64 max_trees;
 627         int i;
 628
 629         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 630                 return -EIO;
 631
 632         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 633         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 634         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 635                                              sizeof(struct mlxsw_sp_lpm_tree),
 636                                              GFP_KERNEL);
 637         if (!mlxsw_sp->router->lpm.trees)
 638                 return -ENOMEM;
 639
 640         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 641                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 642                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 643         }
 644
 645         return 0;
 646 }
 647
 648 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 649 {
 650         kfree(mlxsw_sp->router->lpm.trees);
 651 }
 652
 653 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 654 {
 655         return !!vr->fib4 || !!vr->fib6;
 656 }
 657
 658 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 659 {
 660         struct mlxsw_sp_vr *vr;
 661         int i;
 662
 663         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 664                 vr = &mlxsw_sp->router->vrs[i];
 665                 if (!mlxsw_sp_vr_is_used(vr))
 666                         return vr;
 667         }
 668         return NULL;
 669 }
 670
 671 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 672                                      const struct mlxsw_sp_fib *fib, u8 tree_id)
 673 {
 674         char raltb_pl[MLXSW_REG_RALTB_LEN];
 675
 676         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 677                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 678                              tree_id);
 679         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 680 }
 681
 682 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 683                                        const struct mlxsw_sp_fib *fib)
 684 {
 685         char raltb_pl[MLXSW_REG_RALTB_LEN];
 686
 687         /* Bind to tree 0 which is default */
 688         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 689                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 690         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 691 }
 692
 693 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 694 {
 695         /* For our purpose, squash main and local table into one */
 696         if (tb_id == RT_TABLE_LOCAL)
 697                 tb_id = RT_TABLE_MAIN;
 698         return tb_id;
 699 }
 700
 701 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 702                                             u32 tb_id)
 703 {
 704         struct mlxsw_sp_vr *vr;
 705         int i;
 706
 707         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 708
 709         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 710                 vr = &mlxsw_sp->router->vrs[i];
 711                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 712                         return vr;
 713         }
 714         return NULL;
 715 }
 716
 717 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 718                                             enum mlxsw_sp_l3proto proto)
 719 {
 720         switch (proto) {
 721         case MLXSW_SP_L3_PROTO_IPV4:
 722                 return vr->fib4;
 723         case MLXSW_SP_L3_PROTO_IPV6:
 724                 return vr->fib6;
 725         }
 726         return NULL;
 727 }
 728
 729 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 730                                               u32 tb_id)
 731 {
 732         struct mlxsw_sp_vr *vr;
 733         int err;
 734
 735         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 736         if (!vr)
 737                 return ERR_PTR(-EBUSY);
 738         vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
 739         if (IS_ERR(vr->fib4))
 740                 return ERR_CAST(vr->fib4);
 741         vr->fib6 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV6);
 742         if (IS_ERR(vr->fib6)) {
 743                 err = PTR_ERR(vr->fib6);
 744                 goto err_fib6_create;
 745         }
 746         vr->tb_id = tb_id;
 747         return vr;
 748
 749 err_fib6_create:
 750         mlxsw_sp_fib_destroy(vr->fib4);
 751         vr->fib4 = NULL;
 752         return ERR_PTR(err);
 753 }
 754
 755 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
 756 {
 757         mlxsw_sp_fib_destroy(vr->fib6);
 758         vr->fib6 = NULL;
 759         mlxsw_sp_fib_destroy(vr->fib4);
 760         vr->fib4 = NULL;
 761 }
 762
 763 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
 764 {
 765         struct mlxsw_sp_vr *vr;
 766
 767         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 768         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 769         if (!vr)
 770                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
 771         return vr;
 772 }
 773
 774 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
 775 {
 776         if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
 777             list_empty(&vr->fib6->node_list))
 778                 mlxsw_sp_vr_destroy(vr);
 779 }
 780
 781 static bool
 782 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
 783                                     enum mlxsw_sp_l3proto proto, u8 tree_id)
 784 {
 785         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 786
 787         if (!mlxsw_sp_vr_is_used(vr))
 788                 return false;
 789         if (fib->lpm_tree && fib->lpm_tree->id == tree_id)
 790                 return true;
 791         return false;
 792 }
 793
 794 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 795                                         struct mlxsw_sp_fib *fib,
 796                                         struct mlxsw_sp_lpm_tree *new_tree)
 797 {
 798         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 799         int err;
 800
 801         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 802         if (err)
 803                 return err;
 804         fib->lpm_tree = new_tree;
 805         mlxsw_sp_lpm_tree_hold(new_tree);
 806         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 807         return 0;
 808 }
 809
 810 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 811                                          struct mlxsw_sp_fib *fib,
 812                                          struct mlxsw_sp_lpm_tree *new_tree)
 813 {
 814         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 815         enum mlxsw_sp_l3proto proto = fib->proto;
 816         u8 old_id, new_id = new_tree->id;
 817         struct mlxsw_sp_vr *vr;
 818         int i, err;
 819
 820         if (!old_tree)
 821                 goto no_replace;
 822         old_id = old_tree->id;
 823
 824         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 825                 vr = &mlxsw_sp->router->vrs[i];
 826                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
 827                         continue;
 828                 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 829                                                    mlxsw_sp_vr_fib(vr, proto),
 830                                                    new_tree);
 831                 if (err)
 832                         goto err_tree_replace;
 833         }
 834
 835         return 0;
 836
 837 err_tree_replace:
 838         for (i--; i >= 0; i--) {
 839                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
 840                         continue;
 841                 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 842                                              mlxsw_sp_vr_fib(vr, proto),
 843                                              old_tree);
 844         }
 845         return err;
 846
 847 no_replace:
 848         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 849         if (err)
 850                 return err;
 851         fib->lpm_tree = new_tree;
 852         mlxsw_sp_lpm_tree_hold(new_tree);
 853         return 0;
 854 }
 855
 856 static void
 857 mlxsw_sp_vrs_prefixes(struct mlxsw_sp *mlxsw_sp,
 858                       enum mlxsw_sp_l3proto proto,
 859                       struct mlxsw_sp_prefix_usage *req_prefix_usage)
 860 {
 861         int i;
 862
 863         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 864                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
 865                 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 866                 unsigned char prefix;
 867
 868                 if (!mlxsw_sp_vr_is_used(vr))
 869                         continue;
 870                 mlxsw_sp_prefix_usage_for_each(prefix, &fib->prefix_usage)
 871                         mlxsw_sp_prefix_usage_set(req_prefix_usage, prefix);
 872         }
 873 }
 874
 875 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 876 {
 877         struct mlxsw_sp_vr *vr;
 878         u64 max_vrs;
 879         int i;
 880
 881         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 882                 return -EIO;
 883
 884         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 885         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 886                                         GFP_KERNEL);
 887         if (!mlxsw_sp->router->vrs)
 888                 return -ENOMEM;
 889
 890         for (i = 0; i < max_vrs; i++) {
 891                 vr = &mlxsw_sp->router->vrs[i];
 892                 vr->id = i;
 893         }
 894
 895         return 0;
 896 }
 897
 898 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 899
 900 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 901 {
 902         /* At this stage we're guaranteed not to have new incoming
 903          * FIB notifications and the work queue is free from FIBs
 904          * sitting on top of mlxsw netdevs. However, we can still
 905          * have other FIBs queued. Flush the queue before flushing
 906          * the device's tables. No need for locks, as we're the only
 907          * writer.
 908          */
 909         mlxsw_core_flush_owq();
 910         mlxsw_sp_router_fib_flush(mlxsw_sp);
 911         kfree(mlxsw_sp->router->vrs);
 912 }
 913
 914 static struct net_device *
 915 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
 916 {
 917         struct ip_tunnel *tun = netdev_priv(ol_dev);
 918         struct net *net = dev_net(ol_dev);
 919
 920         return __dev_get_by_index(net, tun->parms.link);
 921 }
 922
 923 static u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
 924 {
 925         struct net_device *d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
 926
 927         if (d)
 928                 return l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
 929         else
 930                 return l3mdev_fib_table(ol_dev) ? : RT_TABLE_MAIN;
 931 }
 932
 933 static struct mlxsw_sp_rif *
 934 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
 935                     const struct mlxsw_sp_rif_params *params);
 936
 937 static struct mlxsw_sp_rif_ipip_lb *
 938 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
 939                                 enum mlxsw_sp_ipip_type ipipt,
 940                                 struct net_device *ol_dev)
 941 {
 942         struct mlxsw_sp_rif_params_ipip_lb lb_params;
 943         const struct mlxsw_sp_ipip_ops *ipip_ops;
 944         struct mlxsw_sp_rif *rif;
 945
 946         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
 947         lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
 948                 .common.dev = ol_dev,
 949                 .common.lag = false,
 950                 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
 951         };
 952
 953         rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common);
 954         if (IS_ERR(rif))
 955                 return ERR_CAST(rif);
 956         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
 957 }
 958
 959 static struct mlxsw_sp_ipip_entry *
 960 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
 961                           enum mlxsw_sp_ipip_type ipipt,
 962                           struct net_device *ol_dev)
 963 {
 964         struct mlxsw_sp_ipip_entry *ipip_entry;
 965         struct mlxsw_sp_ipip_entry *ret = NULL;
 966
 967         ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
 968         if (!ipip_entry)
 969                 return ERR_PTR(-ENOMEM);
 970
 971         ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
 972                                                             ol_dev);
 973         if (IS_ERR(ipip_entry->ol_lb)) {
 974                 ret = ERR_CAST(ipip_entry->ol_lb);
 975                 goto err_ol_ipip_lb_create;
 976         }
 977
 978         ipip_entry->ipipt = ipipt;
 979         ipip_entry->ol_dev = ol_dev;
 980
 981         return ipip_entry;
 982
 983 err_ol_ipip_lb_create:
 984         kfree(ipip_entry);
 985         return ret;
 986 }
 987
 988 static void
 989 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp_ipip_entry *ipip_entry)
 990 {
 991         WARN_ON(ipip_entry->ref_count > 0);
 992         mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
 993         kfree(ipip_entry);
 994 }
 995
 996 static __be32
 997 mlxsw_sp_ipip_netdev_saddr4(const struct net_device *ol_dev)
 998 {
 999         struct ip_tunnel *tun = netdev_priv(ol_dev);
1000
1001         return tun->parms.iph.saddr;
1002 }
1003
1004 union mlxsw_sp_l3addr
1005 mlxsw_sp_ipip_netdev_saddr(enum mlxsw_sp_l3proto proto,
1006                            const struct net_device *ol_dev)
1007 {
1008         switch (proto) {
1009         case MLXSW_SP_L3_PROTO_IPV4:
1010                 return (union mlxsw_sp_l3addr) {
1011                         .addr4 = mlxsw_sp_ipip_netdev_saddr4(ol_dev),
1012                 };
1013         case MLXSW_SP_L3_PROTO_IPV6:
1014                 break;
1015         };
1016
1017         WARN_ON(1);
1018         return (union mlxsw_sp_l3addr) {
1019                 .addr4 = 0,
1020         };
1021 }
1022
1023 __be32 mlxsw_sp_ipip_netdev_daddr4(const struct net_device *ol_dev)
1024 {
1025         struct ip_tunnel *tun = netdev_priv(ol_dev);
1026
1027         return tun->parms.iph.daddr;
1028 }
1029
1030 union mlxsw_sp_l3addr
1031 mlxsw_sp_ipip_netdev_daddr(enum mlxsw_sp_l3proto proto,
1032                            const struct net_device *ol_dev)
1033 {
1034         switch (proto) {
1035         case MLXSW_SP_L3_PROTO_IPV4:
1036                 return (union mlxsw_sp_l3addr) {
1037                         .addr4 = mlxsw_sp_ipip_netdev_daddr4(ol_dev),
1038                 };
1039         case MLXSW_SP_L3_PROTO_IPV6:
1040                 break;
1041         };
1042
1043         WARN_ON(1);
1044         return (union mlxsw_sp_l3addr) {
1045                 .addr4 = 0,
1046         };
1047 }
1048
1049 static bool mlxsw_sp_l3addr_eq(const union mlxsw_sp_l3addr *addr1,
1050                                const union mlxsw_sp_l3addr *addr2)
1051 {
1052         return !memcmp(addr1, addr2, sizeof(*addr1));
1053 }
1054
1055 static bool
1056 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1057                                   const enum mlxsw_sp_l3proto ul_proto,
1058                                   union mlxsw_sp_l3addr saddr,
1059                                   u32 ul_tb_id,
1060                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1061 {
1062         u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1063         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1064         union mlxsw_sp_l3addr tun_saddr;
1065
1066         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1067                 return false;
1068
1069         tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1070         return tun_ul_tb_id == ul_tb_id &&
1071                mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1072 }
1073
1074 static int
1075 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1076                               struct mlxsw_sp_fib_entry *fib_entry,
1077                               struct mlxsw_sp_ipip_entry *ipip_entry)
1078 {
1079         u32 tunnel_index;
1080         int err;
1081
1082         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, 1, &tunnel_index);
1083         if (err)
1084                 return err;
1085
1086         ipip_entry->decap_fib_entry = fib_entry;
1087         fib_entry->decap.ipip_entry = ipip_entry;
1088         fib_entry->decap.tunnel_index = tunnel_index;
1089         return 0;
1090 }
1091
1092 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1093                                           struct mlxsw_sp_fib_entry *fib_entry)
1094 {
1095         /* Unlink this node from the IPIP entry that it's the decap entry of. */
1096         fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1097         fib_entry->decap.ipip_entry = NULL;
1098         mlxsw_sp_kvdl_free(mlxsw_sp, fib_entry->decap.tunnel_index);
1099 }
1100
1101 static struct mlxsw_sp_fib_node *
1102 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1103                          size_t addr_len, unsigned char prefix_len);
1104 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1105                                      struct mlxsw_sp_fib_entry *fib_entry);
1106
1107 static void
1108 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1109                                  struct mlxsw_sp_ipip_entry *ipip_entry)
1110 {
1111         struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1112
1113         mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1114         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1115
1116         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1117 }
1118
1119 static void
1120 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1121                                   struct mlxsw_sp_ipip_entry *ipip_entry,
1122                                   struct mlxsw_sp_fib_entry *decap_fib_entry)
1123 {
1124         if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1125                                           ipip_entry))
1126                 return;
1127         decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1128
1129         if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1130                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1131 }
1132
1133 /* Given an IPIP entry, find the corresponding decap route. */
1134 static struct mlxsw_sp_fib_entry *
1135 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1136                                struct mlxsw_sp_ipip_entry *ipip_entry)
1137 {
1138         static struct mlxsw_sp_fib_node *fib_node;
1139         const struct mlxsw_sp_ipip_ops *ipip_ops;
1140         struct mlxsw_sp_fib_entry *fib_entry;
1141         unsigned char saddr_prefix_len;
1142         union mlxsw_sp_l3addr saddr;
1143         struct mlxsw_sp_fib *ul_fib;
1144         struct mlxsw_sp_vr *ul_vr;
1145         const void *saddrp;
1146         size_t saddr_len;
1147         u32 ul_tb_id;
1148         u32 saddr4;
1149
1150         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1151
1152         ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1153         ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1154         if (!ul_vr)
1155                 return NULL;
1156
1157         ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1158         saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1159                                            ipip_entry->ol_dev);
1160
1161         switch (ipip_ops->ul_proto) {
1162         case MLXSW_SP_L3_PROTO_IPV4:
1163                 saddr4 = be32_to_cpu(saddr.addr4);
1164                 saddrp = &saddr4;
1165                 saddr_len = 4;
1166                 saddr_prefix_len = 32;
1167                 break;
1168         case MLXSW_SP_L3_PROTO_IPV6:
1169                 WARN_ON(1);
1170                 return NULL;
1171         }
1172
1173         fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1174                                             saddr_prefix_len);
1175         if (!fib_node || list_empty(&fib_node->entry_list))
1176                 return NULL;
1177
1178         fib_entry = list_first_entry(&fib_node->entry_list,
1179                                      struct mlxsw_sp_fib_entry, list);
1180         if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1181                 return NULL;
1182
1183         return fib_entry;
1184 }
1185
1186 static struct mlxsw_sp_ipip_entry *
1187 mlxsw_sp_ipip_entry_get(struct mlxsw_sp *mlxsw_sp,
1188                         enum mlxsw_sp_ipip_type ipipt,
1189                         struct net_device *ol_dev)
1190 {
1191         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1192         struct mlxsw_sp_router *router = mlxsw_sp->router;
1193         struct mlxsw_sp_fib_entry *decap_fib_entry;
1194         struct mlxsw_sp_ipip_entry *ipip_entry;
1195         enum mlxsw_sp_l3proto ul_proto;
1196         union mlxsw_sp_l3addr saddr;
1197
1198         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1199                             ipip_list_node) {
1200                 if (ipip_entry->ol_dev == ol_dev)
1201                         goto inc_ref_count;
1202
1203                 /* The configuration where several tunnels have the same local
1204                  * address in the same underlay table needs special treatment in
1205                  * the HW. That is currently not implemented in the driver.
1206                  */
1207                 ul_proto = router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1208                 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1209                 if (mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1210                                                       ul_tb_id, ipip_entry))
1211                         return ERR_PTR(-EEXIST);
1212         }
1213
1214         ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1215         if (IS_ERR(ipip_entry))
1216                 return ipip_entry;
1217
1218         decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1219         if (decap_fib_entry)
1220                 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1221                                                   decap_fib_entry);
1222
1223         list_add_tail(&ipip_entry->ipip_list_node,
1224                       &mlxsw_sp->router->ipip_list);
1225
1226 inc_ref_count:
1227         ++ipip_entry->ref_count;
1228         return ipip_entry;
1229 }
1230
1231 static void
1232 mlxsw_sp_ipip_entry_put(struct mlxsw_sp *mlxsw_sp,
1233                         struct mlxsw_sp_ipip_entry *ipip_entry)
1234 {
1235         if (--ipip_entry->ref_count == 0) {
1236                 list_del(&ipip_entry->ipip_list_node);
1237                 if (ipip_entry->decap_fib_entry)
1238                         mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1239                 mlxsw_sp_ipip_entry_destroy(ipip_entry);
1240         }
1241 }
1242
1243 static bool
1244 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1245                                   const struct net_device *ul_dev,
1246                                   enum mlxsw_sp_l3proto ul_proto,
1247                                   union mlxsw_sp_l3addr ul_dip,
1248                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1249 {
1250         u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1251         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1252         struct net_device *ipip_ul_dev;
1253
1254         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1255                 return false;
1256
1257         ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev);
1258         return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1259                                                  ul_tb_id, ipip_entry) &&
1260                (!ipip_ul_dev || ipip_ul_dev == ul_dev);
1261 }
1262
1263 /* Given decap parameters, find the corresponding IPIP entry. */
1264 static struct mlxsw_sp_ipip_entry *
1265 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp,
1266                                   const struct net_device *ul_dev,
1267                                   enum mlxsw_sp_l3proto ul_proto,
1268                                   union mlxsw_sp_l3addr ul_dip)
1269 {
1270         struct mlxsw_sp_ipip_entry *ipip_entry;
1271
1272         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1273                             ipip_list_node)
1274                 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1275                                                       ul_proto, ul_dip,
1276                                                       ipip_entry))
1277                         return ipip_entry;
1278
1279         return NULL;
1280 }
1281
1282 struct mlxsw_sp_neigh_key {
1283         struct neighbour *n;
1284 };
1285
1286 struct mlxsw_sp_neigh_entry {
1287         struct list_head rif_list_node;
1288         struct rhash_head ht_node;
1289         struct mlxsw_sp_neigh_key key;
1290         u16 rif;
1291         bool connected;
1292         unsigned char ha[ETH_ALEN];
1293         struct list_head nexthop_list; /* list of nexthops using
1294                                         * this neigh entry
1295                                         */
1296         struct list_head nexthop_neighs_list_node;
1297         unsigned int counter_index;
1298         bool counter_valid;
1299 };
1300
1301 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
1302         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
1303         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
1304         .key_len = sizeof(struct mlxsw_sp_neigh_key),
1305 };
1306
1307 struct mlxsw_sp_neigh_entry *
1308 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
1309                         struct mlxsw_sp_neigh_entry *neigh_entry)
1310 {
1311         if (!neigh_entry) {
1312                 if (list_empty(&rif->neigh_list))
1313                         return NULL;
1314                 else
1315                         return list_first_entry(&rif->neigh_list,
1316                                                 typeof(*neigh_entry),
1317                                                 rif_list_node);
1318         }
1319         if (neigh_entry->rif_list_node.next == &rif->neigh_list)
1320                 return NULL;
1321         return list_next_entry(neigh_entry, rif_list_node);
1322 }
1323
1324 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
1325 {
1326         return neigh_entry->key.n->tbl->family;
1327 }
1328
1329 unsigned char *
1330 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
1331 {
1332         return neigh_entry->ha;
1333 }
1334
1335 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1336 {
1337         struct neighbour *n;
1338
1339         n = neigh_entry->key.n;
1340         return ntohl(*((__be32 *) n->primary_key));
1341 }
1342
1343 struct in6_addr *
1344 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1345 {
1346         struct neighbour *n;
1347
1348         n = neigh_entry->key.n;
1349         return (struct in6_addr *) &n->primary_key;
1350 }
1351
1352 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
1353                                struct mlxsw_sp_neigh_entry *neigh_entry,
1354                                u64 *p_counter)
1355 {
1356         if (!neigh_entry->counter_valid)
1357                 return -EINVAL;
1358
1359         return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
1360                                          p_counter, NULL);
1361 }
1362
1363 static struct mlxsw_sp_neigh_entry *
1364 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
1365                            u16 rif)
1366 {
1367         struct mlxsw_sp_neigh_entry *neigh_entry;
1368
1369         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
1370         if (!neigh_entry)
1371                 return NULL;
1372
1373         neigh_entry->key.n = n;
1374         neigh_entry->rif = rif;
1375         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
1376
1377         return neigh_entry;
1378 }
1379
1380 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
1381 {
1382         kfree(neigh_entry);
1383 }
1384
1385 static int
1386 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
1387                             struct mlxsw_sp_neigh_entry *neigh_entry)
1388 {
1389         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
1390                                       &neigh_entry->ht_node,
1391                                       mlxsw_sp_neigh_ht_params);
1392 }
1393
1394 static void
1395 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
1396                             struct mlxsw_sp_neigh_entry *neigh_entry)
1397 {
1398         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
1399                                &neigh_entry->ht_node,
1400                                mlxsw_sp_neigh_ht_params);
1401 }
1402
1403 static bool
1404 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
1405                                     struct mlxsw_sp_neigh_entry *neigh_entry)
1406 {
1407         struct devlink *devlink;
1408         const char *table_name;
1409
1410         switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
1411         case AF_INET:
1412                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
1413                 break;
1414         case AF_INET6:
1415                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
1416                 break;
1417         default:
1418                 WARN_ON(1);
1419                 return false;
1420         }
1421
1422         devlink = priv_to_devlink(mlxsw_sp->core);
1423         return devlink_dpipe_table_counter_enabled(devlink, table_name);
1424 }
1425
1426 static void
1427 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
1428                              struct mlxsw_sp_neigh_entry *neigh_entry)
1429 {
1430         if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
1431                 return;
1432
1433         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
1434                 return;
1435
1436         neigh_entry->counter_valid = true;
1437 }
1438
1439 static void
1440 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
1441                             struct mlxsw_sp_neigh_entry *neigh_entry)
1442 {
1443         if (!neigh_entry->counter_valid)
1444                 return;
1445         mlxsw_sp_flow_counter_free(mlxsw_sp,
1446                                    neigh_entry->counter_index);
1447         neigh_entry->counter_valid = false;
1448 }
1449
1450 static struct mlxsw_sp_neigh_entry *
1451 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1452 {
1453         struct mlxsw_sp_neigh_entry *neigh_entry;
1454         struct mlxsw_sp_rif *rif;
1455         int err;
1456
1457         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
1458         if (!rif)
1459                 return ERR_PTR(-EINVAL);
1460
1461         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
1462         if (!neigh_entry)
1463                 return ERR_PTR(-ENOMEM);
1464
1465         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
1466         if (err)
1467                 goto err_neigh_entry_insert;
1468
1469         mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1470         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
1471
1472         return neigh_entry;
1473
1474 err_neigh_entry_insert:
1475         mlxsw_sp_neigh_entry_free(neigh_entry);
1476         return ERR_PTR(err);
1477 }
1478
1479 static void
1480 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1481                              struct mlxsw_sp_neigh_entry *neigh_entry)
1482 {
1483         list_del(&neigh_entry->rif_list_node);
1484         mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1485         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
1486         mlxsw_sp_neigh_entry_free(neigh_entry);
1487 }
1488
1489 static struct mlxsw_sp_neigh_entry *
1490 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1491 {
1492         struct mlxsw_sp_neigh_key key;
1493
1494         key.n = n;
1495         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
1496                                       &key, mlxsw_sp_neigh_ht_params);
1497 }
1498
1499 static void
1500 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
1501 {
1502         unsigned long interval;
1503
1504 #if IS_ENABLED(CONFIG_IPV6)
1505         interval = min_t(unsigned long,
1506                          NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
1507                          NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
1508 #else
1509         interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
1510 #endif
1511         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
1512 }
1513
1514 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1515                                                    char *rauhtd_pl,
1516                                                    int ent_index)
1517 {
1518         struct net_device *dev;
1519         struct neighbour *n;
1520         __be32 dipn;
1521         u32 dip;
1522         u16 rif;
1523
1524         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
1525
1526         if (!mlxsw_sp->router->rifs[rif]) {
1527                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1528                 return;
1529         }
1530
1531         dipn = htonl(dip);
1532         dev = mlxsw_sp->router->rifs[rif]->dev;
1533         n = neigh_lookup(&arp_tbl, &dipn, dev);
1534         if (!n) {
1535                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
1536                            &dip);
1537                 return;
1538         }
1539
1540         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
1541         neigh_event_send(n, NULL);
1542         neigh_release(n);
1543 }
1544
1545 #if IS_ENABLED(CONFIG_IPV6)
1546 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1547                                                    char *rauhtd_pl,
1548                                                    int rec_index)
1549 {
1550         struct net_device *dev;
1551         struct neighbour *n;
1552         struct in6_addr dip;
1553         u16 rif;
1554
1555         mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
1556                                          (char *) &dip);
1557
1558         if (!mlxsw_sp->router->rifs[rif]) {
1559                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1560                 return;
1561         }
1562
1563         dev = mlxsw_sp->router->rifs[rif]->dev;
1564         n = neigh_lookup(&nd_tbl, &dip, dev);
1565         if (!n) {
1566                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI6c\n",
1567                            &dip);
1568                 return;
1569         }
1570
1571         netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
1572         neigh_event_send(n, NULL);
1573         neigh_release(n);
1574 }
1575 #else
1576 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1577                                                    char *rauhtd_pl,
1578                                                    int rec_index)
1579 {
1580 }
1581 #endif
1582
1583 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1584                                                    char *rauhtd_pl,
1585                                                    int rec_index)
1586 {
1587         u8 num_entries;
1588         int i;
1589
1590         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1591                                                                 rec_index);
1592         /* Hardware starts counting at 0, so add 1. */
1593         num_entries++;
1594
1595         /* Each record consists of several neighbour entries. */
1596         for (i = 0; i < num_entries; i++) {
1597                 int ent_index;
1598
1599                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
1600                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
1601                                                        ent_index);
1602         }
1603
1604 }
1605
1606 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1607                                                    char *rauhtd_pl,
1608                                                    int rec_index)
1609 {
1610         /* One record contains one entry. */
1611         mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
1612                                                rec_index);
1613 }
1614
1615 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
1616                                               char *rauhtd_pl, int rec_index)
1617 {
1618         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
1619         case MLXSW_REG_RAUHTD_TYPE_IPV4:
1620                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
1621                                                        rec_index);
1622                 break;
1623         case MLXSW_REG_RAUHTD_TYPE_IPV6:
1624                 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
1625                                                        rec_index);
1626                 break;
1627         }
1628 }
1629
1630 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1631 {
1632         u8 num_rec, last_rec_index, num_entries;
1633
1634         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1635         last_rec_index = num_rec - 1;
1636
1637         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1638                 return false;
1639         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1640             MLXSW_REG_RAUHTD_TYPE_IPV6)
1641                 return true;
1642
1643         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1644                                                                 last_rec_index);
1645         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1646                 return true;
1647         return false;
1648 }
1649
1650 static int
1651 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
1652                                        char *rauhtd_pl,
1653                                        enum mlxsw_reg_rauhtd_type type)
1654 {
1655         int i, num_rec;
1656         int err;
1657
1658         /* Make sure the neighbour's netdev isn't removed in the
1659          * process.
1660          */
1661         rtnl_lock();
1662         do {
1663                 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
1664                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1665                                       rauhtd_pl);
1666                 if (err) {
1667                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1668                         break;
1669                 }
1670                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1671                 for (i = 0; i < num_rec; i++)
1672                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1673                                                           i);
1674         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
1675         rtnl_unlock();
1676
1677         return err;
1678 }
1679
1680 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1681 {
1682         enum mlxsw_reg_rauhtd_type type;
1683         char *rauhtd_pl;
1684         int err;
1685
1686         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1687         if (!rauhtd_pl)
1688                 return -ENOMEM;
1689
1690         type = MLXSW_REG_RAUHTD_TYPE_IPV4;
1691         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1692         if (err)
1693                 goto out;
1694
1695         type = MLXSW_REG_RAUHTD_TYPE_IPV6;
1696         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1697 out:
1698         kfree(rauhtd_pl);
1699         return err;
1700 }
1701
1702 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1703 {
1704         struct mlxsw_sp_neigh_entry *neigh_entry;
1705
1706         /* Take RTNL mutex here to prevent lists from changes */
1707         rtnl_lock();
1708         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
1709                             nexthop_neighs_list_node)
1710                 /* If this neigh have nexthops, make the kernel think this neigh
1711                  * is active regardless of the traffic.
1712                  */
1713                 neigh_event_send(neigh_entry->key.n, NULL);
1714         rtnl_unlock();
1715 }
1716
1717 static void
1718 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1719 {
1720         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
1721
1722         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
1723                                msecs_to_jiffies(interval));
1724 }
1725
1726 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1727 {
1728         struct mlxsw_sp_router *router;
1729         int err;
1730
1731         router = container_of(work, struct mlxsw_sp_router,
1732                               neighs_update.dw.work);
1733         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
1734         if (err)
1735                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
1736
1737         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
1738
1739         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
1740 }
1741
1742 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1743 {
1744         struct mlxsw_sp_neigh_entry *neigh_entry;
1745         struct mlxsw_sp_router *router;
1746
1747         router = container_of(work, struct mlxsw_sp_router,
1748                               nexthop_probe_dw.work);
1749         /* Iterate over nexthop neighbours, find those who are unresolved and
1750          * send arp on them. This solves the chicken-egg problem when
1751          * the nexthop wouldn't get offloaded until the neighbor is resolved
1752          * but it wouldn't get resolved ever in case traffic is flowing in HW
1753          * using different nexthop.
1754          *
1755          * Take RTNL mutex here to prevent lists from changes.
1756          */
1757         rtnl_lock();
1758         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
1759                             nexthop_neighs_list_node)
1760                 if (!neigh_entry->connected)
1761                         neigh_event_send(neigh_entry->key.n, NULL);
1762         rtnl_unlock();
1763
1764         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
1765                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1766 }
1767
1768 static void
1769 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1770                               struct mlxsw_sp_neigh_entry *neigh_entry,
1771                               bool removing);
1772
1773 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1774 {
1775         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1776                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1777 }
1778
1779 static void
1780 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1781                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1782                                 enum mlxsw_reg_rauht_op op)
1783 {
1784         struct neighbour *n = neigh_entry->key.n;
1785         u32 dip = ntohl(*((__be32 *) n->primary_key));
1786         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1787
1788         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1789                               dip);
1790         if (neigh_entry->counter_valid)
1791                 mlxsw_reg_rauht_pack_counter(rauht_pl,
1792                                              neigh_entry->counter_index);
1793         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1794 }
1795
1796 static void
1797 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
1798                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1799                                 enum mlxsw_reg_rauht_op op)
1800 {
1801         struct neighbour *n = neigh_entry->key.n;
1802         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1803         const char *dip = n->primary_key;
1804
1805         mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1806                               dip);
1807         if (neigh_entry->counter_valid)
1808                 mlxsw_reg_rauht_pack_counter(rauht_pl,
1809                                              neigh_entry->counter_index);
1810         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1811 }
1812
1813 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
1814 {
1815         struct neighbour *n = neigh_entry->key.n;
1816
1817         /* Packets with a link-local destination address are trapped
1818          * after LPM lookup and never reach the neighbour table, so
1819          * there is no need to program such neighbours to the device.
1820          */
1821         if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
1822             IPV6_ADDR_LINKLOCAL)
1823                 return true;
1824         return false;
1825 }
1826
1827 static void
1828 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1829                             struct mlxsw_sp_neigh_entry *neigh_entry,
1830                             bool adding)
1831 {
1832         if (!adding && !neigh_entry->connected)
1833                 return;
1834         neigh_entry->connected = adding;
1835         if (neigh_entry->key.n->tbl->family == AF_INET) {
1836                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1837                                                 mlxsw_sp_rauht_op(adding));
1838         } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
1839                 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
1840                         return;
1841                 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
1842                                                 mlxsw_sp_rauht_op(adding));
1843         } else {
1844                 WARN_ON_ONCE(1);
1845         }
1846 }
1847
1848 void
1849 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
1850                                     struct mlxsw_sp_neigh_entry *neigh_entry,
1851                                     bool adding)
1852 {
1853         if (adding)
1854                 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1855         else
1856                 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1857         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
1858 }
1859
1860 struct mlxsw_sp_neigh_event_work {
1861         struct work_struct work;
1862         struct mlxsw_sp *mlxsw_sp;
1863         struct neighbour *n;
1864 };
1865
1866 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1867 {
1868         struct mlxsw_sp_neigh_event_work *neigh_work =
1869                 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1870         struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1871         struct mlxsw_sp_neigh_entry *neigh_entry;
1872         struct neighbour *n = neigh_work->n;
1873         unsigned char ha[ETH_ALEN];
1874         bool entry_connected;
1875         u8 nud_state, dead;
1876
1877         /* If these parameters are changed after we release the lock,
1878          * then we are guaranteed to receive another event letting us
1879          * know about it.
1880          */
1881         read_lock_bh(&n->lock);
1882         memcpy(ha, n->ha, ETH_ALEN);
1883         nud_state = n->nud_state;
1884         dead = n->dead;
1885         read_unlock_bh(&n->lock);
1886
1887         rtnl_lock();
1888         entry_connected = nud_state & NUD_VALID && !dead;
1889         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1890         if (!entry_connected && !neigh_entry)
1891                 goto out;
1892         if (!neigh_entry) {
1893                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1894                 if (IS_ERR(neigh_entry))
1895                         goto out;
1896         }
1897
1898         memcpy(neigh_entry->ha, ha, ETH_ALEN);
1899         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1900         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
1901
1902         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1903                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1904
1905 out:
1906         rtnl_unlock();
1907         neigh_release(n);
1908         kfree(neigh_work);
1909 }
1910
1911 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1912                                    unsigned long event, void *ptr)
1913 {
1914         struct mlxsw_sp_neigh_event_work *neigh_work;
1915         struct mlxsw_sp_port *mlxsw_sp_port;
1916         struct mlxsw_sp *mlxsw_sp;
1917         unsigned long interval;
1918         struct neigh_parms *p;
1919         struct neighbour *n;
1920
1921         switch (event) {
1922         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1923                 p = ptr;
1924
1925                 /* We don't care about changes in the default table. */
1926                 if (!p->dev || (p->tbl->family != AF_INET &&
1927                                 p->tbl->family != AF_INET6))
1928                         return NOTIFY_DONE;
1929
1930                 /* We are in atomic context and can't take RTNL mutex,
1931                  * so use RCU variant to walk the device chain.
1932                  */
1933                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1934                 if (!mlxsw_sp_port)
1935                         return NOTIFY_DONE;
1936
1937                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1938                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1939                 mlxsw_sp->router->neighs_update.interval = interval;
1940
1941                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1942                 break;
1943         case NETEVENT_NEIGH_UPDATE:
1944                 n = ptr;
1945
1946                 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
1947                         return NOTIFY_DONE;
1948
1949                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
1950                 if (!mlxsw_sp_port)
1951                         return NOTIFY_DONE;
1952
1953                 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1954                 if (!neigh_work) {
1955                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
1956                         return NOTIFY_BAD;
1957                 }
1958
1959                 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1960                 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1961                 neigh_work->n = n;
1962
1963                 /* Take a reference to ensure the neighbour won't be
1964                  * destructed until we drop the reference in delayed
1965                  * work.
1966                  */
1967                 neigh_clone(n);
1968                 mlxsw_core_schedule_work(&neigh_work->work);
1969                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1970                 break;
1971         }
1972
1973         return NOTIFY_DONE;
1974 }
1975
1976 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1977 {
1978         int err;
1979
1980         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
1981                               &mlxsw_sp_neigh_ht_params);
1982         if (err)
1983                 return err;
1984
1985         /* Initialize the polling interval according to the default
1986          * table.
1987          */
1988         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1989
1990         /* Create the delayed works for the activity_update */
1991         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
1992                           mlxsw_sp_router_neighs_update_work);
1993         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
1994                           mlxsw_sp_router_probe_unresolved_nexthops);
1995         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1996         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
1997         return 0;
1998 }
1999
2000 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2001 {
2002         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
2003         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
2004         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2005 }
2006
2007 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2008                                          struct mlxsw_sp_rif *rif)
2009 {
2010         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2011
2012         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2013                                  rif_list_node) {
2014                 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2015                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2016         }
2017 }
2018
2019 enum mlxsw_sp_nexthop_type {
2020         MLXSW_SP_NEXTHOP_TYPE_ETH,
2021         MLXSW_SP_NEXTHOP_TYPE_IPIP,
2022 };
2023
2024 struct mlxsw_sp_nexthop_key {
2025         struct fib_nh *fib_nh;
2026 };
2027
2028 struct mlxsw_sp_nexthop {
2029         struct list_head neigh_list_node; /* member of neigh entry list */
2030         struct list_head rif_list_node;
2031         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
2032                                                 * this belongs to
2033                                                 */
2034         struct rhash_head ht_node;
2035         struct mlxsw_sp_nexthop_key key;
2036         unsigned char gw_addr[sizeof(struct in6_addr)];
2037         int ifindex;
2038         struct mlxsw_sp_rif *rif;
2039         u8 should_offload:1, /* set indicates this neigh is connected and
2040                               * should be put to KVD linear area of this group.
2041                               */
2042            offloaded:1, /* set in case the neigh is actually put into
2043                          * KVD linear area of this group.
2044                          */
2045            update:1; /* set indicates that MAC of this neigh should be
2046                       * updated in HW
2047                       */
2048         enum mlxsw_sp_nexthop_type type;
2049         union {
2050                 struct mlxsw_sp_neigh_entry *neigh_entry;
2051                 struct mlxsw_sp_ipip_entry *ipip_entry;
2052         };
2053 };
2054
2055 struct mlxsw_sp_nexthop_group {
2056         void *priv;
2057         struct rhash_head ht_node;
2058         struct list_head fib_list; /* list of fib entries that use this group */
2059         struct neigh_table *neigh_tbl;
2060         u8 adj_index_valid:1,
2061            gateway:1; /* routes using the group use a gateway */
2062         u32 adj_index;
2063         u16 ecmp_size;
2064         u16 count;
2065         struct mlxsw_sp_nexthop nexthops[0];
2066 #define nh_rif  nexthops[0].rif
2067 };
2068
2069 static struct fib_info *
2070 mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp)
2071 {
2072         return nh_grp->priv;
2073 }
2074
2075 struct mlxsw_sp_nexthop_group_cmp_arg {
2076         enum mlxsw_sp_l3proto proto;
2077         union {
2078                 struct fib_info *fi;
2079                 struct mlxsw_sp_fib6_entry *fib6_entry;
2080         };
2081 };
2082
2083 static bool
2084 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
2085                                     const struct in6_addr *gw, int ifindex)
2086 {
2087         int i;
2088
2089         for (i = 0; i < nh_grp->count; i++) {
2090                 const struct mlxsw_sp_nexthop *nh;
2091
2092                 nh = &nh_grp->nexthops[i];
2093                 if (nh->ifindex == ifindex &&
2094                     ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
2095                         return true;
2096         }
2097
2098         return false;
2099 }
2100
2101 static bool
2102 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
2103                             const struct mlxsw_sp_fib6_entry *fib6_entry)
2104 {
2105         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2106
2107         if (nh_grp->count != fib6_entry->nrt6)
2108                 return false;
2109
2110         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2111                 struct in6_addr *gw;
2112                 int ifindex;
2113
2114                 ifindex = mlxsw_sp_rt6->rt->dst.dev->ifindex;
2115                 gw = &mlxsw_sp_rt6->rt->rt6i_gateway;
2116                 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex))
2117                         return false;
2118         }
2119
2120         return true;
2121 }
2122
2123 static int
2124 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
2125 {
2126         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
2127         const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
2128
2129         switch (cmp_arg->proto) {
2130         case MLXSW_SP_L3_PROTO_IPV4:
2131                 return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp);
2132         case MLXSW_SP_L3_PROTO_IPV6:
2133                 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
2134                                                     cmp_arg->fib6_entry);
2135         default:
2136                 WARN_ON(1);
2137                 return 1;
2138         }
2139 }
2140
2141 static int
2142 mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp)
2143 {
2144         return nh_grp->neigh_tbl->family;
2145 }
2146
2147 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
2148 {
2149         const struct mlxsw_sp_nexthop_group *nh_grp = data;
2150         const struct mlxsw_sp_nexthop *nh;
2151         struct fib_info *fi;
2152         unsigned int val;
2153         int i;
2154
2155         switch (mlxsw_sp_nexthop_group_type(nh_grp)) {
2156         case AF_INET:
2157                 fi = mlxsw_sp_nexthop4_group_fi(nh_grp);
2158                 return jhash(&fi, sizeof(fi), seed);
2159         case AF_INET6:
2160                 val = nh_grp->count;
2161                 for (i = 0; i < nh_grp->count; i++) {
2162                         nh = &nh_grp->nexthops[i];
2163                         val ^= nh->ifindex;
2164                 }
2165                 return jhash(&val, sizeof(val), seed);
2166         default:
2167                 WARN_ON(1);
2168                 return 0;
2169         }
2170 }
2171
2172 static u32
2173 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
2174 {
2175         unsigned int val = fib6_entry->nrt6;
2176         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2177         struct net_device *dev;
2178
2179         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2180                 dev = mlxsw_sp_rt6->rt->dst.dev;
2181                 val ^= dev->ifindex;
2182         }
2183
2184         return jhash(&val, sizeof(val), seed);
2185 }
2186
2187 static u32
2188 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
2189 {
2190         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
2191
2192         switch (cmp_arg->proto) {
2193         case MLXSW_SP_L3_PROTO_IPV4:
2194                 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
2195         case MLXSW_SP_L3_PROTO_IPV6:
2196                 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
2197         default:
2198                 WARN_ON(1);
2199                 return 0;
2200         }
2201 }
2202
2203 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
2204         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
2205         .hashfn      = mlxsw_sp_nexthop_group_hash,
2206         .obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
2207         .obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
2208 };
2209
2210 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
2211                                          struct mlxsw_sp_nexthop_group *nh_grp)
2212 {
2213         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2214             !nh_grp->gateway)
2215                 return 0;
2216
2217         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
2218                                       &nh_grp->ht_node,
2219                                       mlxsw_sp_nexthop_group_ht_params);
2220 }
2221
2222 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
2223                                           struct mlxsw_sp_nexthop_group *nh_grp)
2224 {
2225         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2226             !nh_grp->gateway)
2227                 return;
2228
2229         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
2230                                &nh_grp->ht_node,
2231                                mlxsw_sp_nexthop_group_ht_params);
2232 }
2233
2234 static struct mlxsw_sp_nexthop_group *
2235 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
2236                                struct fib_info *fi)
2237 {
2238         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
2239
2240         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4;
2241         cmp_arg.fi = fi;
2242         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
2243                                       &cmp_arg,
2244                                       mlxsw_sp_nexthop_group_ht_params);
2245 }
2246
2247 static struct mlxsw_sp_nexthop_group *
2248 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
2249                                struct mlxsw_sp_fib6_entry *fib6_entry)
2250 {
2251         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
2252
2253         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6;
2254         cmp_arg.fib6_entry = fib6_entry;
2255         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
2256                                       &cmp_arg,
2257                                       mlxsw_sp_nexthop_group_ht_params);
2258 }
2259
2260 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
2261         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
2262         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
2263         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
2264 };
2265
2266 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
2267                                    struct mlxsw_sp_nexthop *nh)
2268 {
2269         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
2270                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
2271 }
2272
2273 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
2274                                     struct mlxsw_sp_nexthop *nh)
2275 {
2276         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
2277                                mlxsw_sp_nexthop_ht_params);
2278 }
2279
2280 static struct mlxsw_sp_nexthop *
2281 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
2282                         struct mlxsw_sp_nexthop_key key)
2283 {
2284         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
2285                                       mlxsw_sp_nexthop_ht_params);
2286 }
2287
2288 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
2289                                              const struct mlxsw_sp_fib *fib,
2290                                              u32 adj_index, u16 ecmp_size,
2291                                              u32 new_adj_index,
2292                                              u16 new_ecmp_size)
2293 {
2294         char raleu_pl[MLXSW_REG_RALEU_LEN];
2295
2296         mlxsw_reg_raleu_pack(raleu_pl,
2297                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
2298                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
2299                              new_ecmp_size);
2300         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
2301 }
2302
2303 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
2304                                           struct mlxsw_sp_nexthop_group *nh_grp,
2305                                           u32 old_adj_index, u16 old_ecmp_size)
2306 {
2307         struct mlxsw_sp_fib_entry *fib_entry;
2308         struct mlxsw_sp_fib *fib = NULL;
2309         int err;
2310
2311         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2312                 if (fib == fib_entry->fib_node->fib)
2313                         continue;
2314                 fib = fib_entry->fib_node->fib;
2315                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
2316                                                         old_adj_index,
2317                                                         old_ecmp_size,
2318                                                         nh_grp->adj_index,
2319                                                         nh_grp->ecmp_size);
2320                 if (err)
2321                         return err;
2322         }
2323         return 0;
2324 }
2325
2326 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
2327                                        struct mlxsw_sp_nexthop *nh)
2328 {
2329         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
2330         char ratr_pl[MLXSW_REG_RATR_LEN];
2331
2332         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
2333                             true, MLXSW_REG_RATR_TYPE_ETHERNET,
2334                             adj_index, neigh_entry->rif);
2335         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
2336         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
2337 }
2338
2339 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
2340                                         u32 adj_index,
2341                                         struct mlxsw_sp_nexthop *nh)
2342 {
2343         const struct mlxsw_sp_ipip_ops *ipip_ops;
2344
2345         ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
2346         return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry);
2347 }
2348
2349 static int
2350 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
2351                               struct mlxsw_sp_nexthop_group *nh_grp,
2352                               bool reallocate)
2353 {
2354         u32 adj_index = nh_grp->adj_index; /* base */
2355         struct mlxsw_sp_nexthop *nh;
2356         int i;
2357         int err;
2358
2359         for (i = 0; i < nh_grp->count; i++) {
2360                 nh = &nh_grp->nexthops[i];
2361
2362                 if (!nh->should_offload) {
2363                         nh->offloaded = 0;
2364                         continue;
2365                 }
2366
2367                 if (nh->update || reallocate) {
2368                         switch (nh->type) {
2369                         case MLXSW_SP_NEXTHOP_TYPE_ETH:
2370                                 err = mlxsw_sp_nexthop_mac_update
2371                                             (mlxsw_sp, adj_index, nh);
2372                                 break;
2373                         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
2374                                 err = mlxsw_sp_nexthop_ipip_update
2375                                             (mlxsw_sp, adj_index, nh);
2376                                 break;
2377                         }
2378                         if (err)
2379                                 return err;
2380                         nh->update = 0;
2381                         nh->offloaded = 1;
2382                 }
2383                 adj_index++;
2384         }
2385         return 0;
2386 }
2387
2388 static bool
2389 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2390                                  const struct mlxsw_sp_fib_entry *fib_entry);
2391
2392 static int
2393 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
2394                                     struct mlxsw_sp_nexthop_group *nh_grp)
2395 {
2396         struct mlxsw_sp_fib_entry *fib_entry;
2397         int err;
2398
2399         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2400                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2401                                                       fib_entry))
2402                         continue;
2403                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2404                 if (err)
2405                         return err;
2406         }
2407         return 0;
2408 }
2409
2410 static void
2411 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
2412                                    enum mlxsw_reg_ralue_op op, int err);
2413
2414 static void
2415 mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp)
2416 {
2417         enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
2418         struct mlxsw_sp_fib_entry *fib_entry;
2419
2420         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2421                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2422                                                       fib_entry))
2423                         continue;
2424                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2425         }
2426 }
2427
2428 static void
2429 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
2430                                struct mlxsw_sp_nexthop_group *nh_grp)
2431 {
2432         struct mlxsw_sp_nexthop *nh;
2433         bool offload_change = false;
2434         u32 adj_index;
2435         u16 ecmp_size = 0;
2436         bool old_adj_index_valid;
2437         u32 old_adj_index;
2438         u16 old_ecmp_size;
2439         int i;
2440         int err;
2441
2442         if (!nh_grp->gateway) {
2443                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2444                 return;
2445         }
2446
2447         for (i = 0; i < nh_grp->count; i++) {
2448                 nh = &nh_grp->nexthops[i];
2449
2450                 if (nh->should_offload != nh->offloaded) {
2451                         offload_change = true;
2452                         if (nh->should_offload)
2453                                 nh->update = 1;
2454                 }
2455                 if (nh->should_offload)
2456                         ecmp_size++;
2457         }
2458         if (!offload_change) {
2459                 /* Nothing was added or removed, so no need to reallocate. Just
2460                  * update MAC on existing adjacency indexes.
2461                  */
2462                 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, false);
2463                 if (err) {
2464                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2465                         goto set_trap;
2466                 }
2467                 return;
2468         }
2469         if (!ecmp_size)
2470                 /* No neigh of this group is connected so we just set
2471                  * the trap and let everthing flow through kernel.
2472                  */
2473                 goto set_trap;
2474
2475         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
2476         if (err) {
2477                 /* We ran out of KVD linear space, just set the
2478                  * trap and let everything flow through kernel.
2479                  */
2480                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
2481                 goto set_trap;
2482         }
2483         old_adj_index_valid = nh_grp->adj_index_valid;
2484         old_adj_index = nh_grp->adj_index;
2485         old_ecmp_size = nh_grp->ecmp_size;
2486         nh_grp->adj_index_valid = 1;
2487         nh_grp->adj_index = adj_index;
2488         nh_grp->ecmp_size = ecmp_size;
2489         err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, true);
2490         if (err) {
2491                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2492                 goto set_trap;
2493         }
2494
2495         if (!old_adj_index_valid) {
2496                 /* The trap was set for fib entries, so we have to call
2497                  * fib entry update to unset it and use adjacency index.
2498                  */
2499                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2500                 if (err) {
2501                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
2502                         goto set_trap;
2503                 }
2504                 return;
2505         }
2506
2507         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
2508                                              old_adj_index, old_ecmp_size);
2509         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
2510         if (err) {
2511                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
2512                 goto set_trap;
2513         }
2514
2515         /* Offload state within the group changed, so update the flags. */
2516         mlxsw_sp_nexthop_fib_entries_refresh(nh_grp);
2517
2518         return;
2519
2520 set_trap:
2521         old_adj_index_valid = nh_grp->adj_index_valid;
2522         nh_grp->adj_index_valid = 0;
2523         for (i = 0; i < nh_grp->count; i++) {
2524                 nh = &nh_grp->nexthops[i];
2525                 nh->offloaded = 0;
2526         }
2527         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2528         if (err)
2529                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
2530         if (old_adj_index_valid)
2531                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
2532 }
2533
2534 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
2535                                             bool removing)
2536 {
2537         if (!removing)
2538                 nh->should_offload = 1;
2539         else if (nh->offloaded)
2540                 nh->should_offload = 0;
2541         nh->update = 1;
2542 }
2543
2544 static void
2545 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2546                               struct mlxsw_sp_neigh_entry *neigh_entry,
2547                               bool removing)
2548 {
2549         struct mlxsw_sp_nexthop *nh;
2550
2551         list_for_each_entry(nh, &neigh_entry->nexthop_list,
2552                             neigh_list_node) {
2553                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
2554                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2555         }
2556 }
2557
2558 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
2559                                       struct mlxsw_sp_rif *rif)
2560 {
2561         if (nh->rif)
2562                 return;
2563
2564         nh->rif = rif;
2565         list_add(&nh->rif_list_node, &rif->nexthop_list);
2566 }
2567
2568 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
2569 {
2570         if (!nh->rif)
2571                 return;
2572
2573         list_del(&nh->rif_list_node);
2574         nh->rif = NULL;
2575 }
2576
2577 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
2578                                        struct mlxsw_sp_nexthop *nh)
2579 {
2580         struct mlxsw_sp_neigh_entry *neigh_entry;
2581         struct neighbour *n;
2582         u8 nud_state, dead;
2583         int err;
2584
2585         if (!nh->nh_grp->gateway || nh->neigh_entry)
2586                 return 0;
2587
2588         /* Take a reference of neigh here ensuring that neigh would
2589          * not be destructed before the nexthop entry is finished.
2590          * The reference is taken either in neigh_lookup() or
2591          * in neigh_create() in case n is not found.
2592          */
2593         n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
2594         if (!n) {
2595                 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
2596                                  nh->rif->dev);
2597                 if (IS_ERR(n))
2598                         return PTR_ERR(n);
2599                 neigh_event_send(n, NULL);
2600         }
2601         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2602         if (!neigh_entry) {
2603                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2604                 if (IS_ERR(neigh_entry)) {
2605                         err = -EINVAL;
2606                         goto err_neigh_entry_create;
2607                 }
2608         }
2609
2610         /* If that is the first nexthop connected to that neigh, add to
2611          * nexthop_neighs_list
2612          */
2613         if (list_empty(&neigh_entry->nexthop_list))
2614                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
2615                               &mlxsw_sp->router->nexthop_neighs_list);
2616
2617         nh->neigh_entry = neigh_entry;
2618         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
2619         read_lock_bh(&n->lock);
2620         nud_state = n->nud_state;
2621         dead = n->dead;
2622         read_unlock_bh(&n->lock);
2623         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
2624
2625         return 0;
2626
2627 err_neigh_entry_create:
2628         neigh_release(n);
2629         return err;
2630 }
2631
2632 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
2633                                         struct mlxsw_sp_nexthop *nh)
2634 {
2635         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
2636         struct neighbour *n;
2637
2638         if (!neigh_entry)
2639                 return;
2640         n = neigh_entry->key.n;
2641
2642         __mlxsw_sp_nexthop_neigh_update(nh, true);
2643         list_del(&nh->neigh_list_node);
2644         nh->neigh_entry = NULL;
2645
2646         /* If that is the last nexthop connected to that neigh, remove from
2647          * nexthop_neighs_list
2648          */
2649         if (list_empty(&neigh_entry->nexthop_list))
2650                 list_del(&neigh_entry->nexthop_neighs_list_node);
2651
2652         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2653                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2654
2655         neigh_release(n);
2656 }
2657
2658 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
2659                                       const struct net_device *dev,
2660                                       enum mlxsw_sp_ipip_type *p_type)
2661 {
2662         struct mlxsw_sp_router *router = mlxsw_sp->router;
2663         const struct mlxsw_sp_ipip_ops *ipip_ops;
2664         enum mlxsw_sp_ipip_type ipipt;
2665
2666         for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
2667                 ipip_ops = router->ipip_ops_arr[ipipt];
2668                 if (dev->type == ipip_ops->dev_type) {
2669                         if (p_type)
2670                                 *p_type = ipipt;
2671                         return true;
2672                 }
2673         }
2674         return false;
2675 }
2676
2677 static int mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
2678                                       enum mlxsw_sp_ipip_type ipipt,
2679                                       struct mlxsw_sp_nexthop *nh,
2680                                       struct net_device *ol_dev)
2681 {
2682         if (!nh->nh_grp->gateway || nh->ipip_entry)
2683                 return 0;
2684
2685         nh->ipip_entry = mlxsw_sp_ipip_entry_get(mlxsw_sp, ipipt, ol_dev);
2686         if (IS_ERR(nh->ipip_entry))
2687                 return PTR_ERR(nh->ipip_entry);
2688
2689         __mlxsw_sp_nexthop_neigh_update(nh, false);
2690         return 0;
2691 }
2692
2693 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
2694                                        struct mlxsw_sp_nexthop *nh)
2695 {
2696         struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
2697
2698         if (!ipip_entry)
2699                 return;
2700
2701         __mlxsw_sp_nexthop_neigh_update(nh, true);
2702         mlxsw_sp_ipip_entry_put(mlxsw_sp, ipip_entry);
2703         nh->ipip_entry = NULL;
2704 }
2705
2706 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
2707                                         const struct fib_nh *fib_nh,
2708                                         enum mlxsw_sp_ipip_type *p_ipipt)
2709 {
2710         struct net_device *dev = fib_nh->nh_dev;
2711
2712         return dev &&
2713                fib_nh->nh_parent->fib_type == RTN_UNICAST &&
2714                mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
2715 }
2716
2717 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
2718                                        struct mlxsw_sp_nexthop *nh)
2719 {
2720         switch (nh->type) {
2721         case MLXSW_SP_NEXTHOP_TYPE_ETH:
2722                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
2723                 mlxsw_sp_nexthop_rif_fini(nh);
2724                 break;
2725         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
2726                 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
2727                 break;
2728         }
2729 }
2730
2731 static int mlxsw_sp_nexthop4_type_init(struct mlxsw_sp *mlxsw_sp,
2732                                        struct mlxsw_sp_nexthop *nh,
2733                                        struct fib_nh *fib_nh)
2734 {
2735         struct mlxsw_sp_router *router = mlxsw_sp->router;
2736         struct net_device *dev = fib_nh->nh_dev;
2737         enum mlxsw_sp_ipip_type ipipt;
2738         struct mlxsw_sp_rif *rif;
2739         int err;
2740
2741         if (mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fib_nh, &ipipt) &&
2742             router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev,
2743                                                      MLXSW_SP_L3_PROTO_IPV4)) {
2744                 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
2745                 return mlxsw_sp_nexthop_ipip_init(mlxsw_sp, ipipt, nh, dev);
2746         }
2747
2748         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
2749         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
2750         if (!rif)
2751                 return 0;
2752
2753         mlxsw_sp_nexthop_rif_init(nh, rif);
2754         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
2755         if (err)
2756                 goto err_neigh_init;
2757
2758         return 0;
2759
2760 err_neigh_init:
2761         mlxsw_sp_nexthop_rif_fini(nh);
2762         return err;
2763 }
2764
2765 static void mlxsw_sp_nexthop4_type_fini(struct mlxsw_sp *mlxsw_sp,
2766                                         struct mlxsw_sp_nexthop *nh)
2767 {
2768         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
2769 }
2770
2771 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
2772                                   struct mlxsw_sp_nexthop_group *nh_grp,
2773                                   struct mlxsw_sp_nexthop *nh,
2774                                   struct fib_nh *fib_nh)
2775 {
2776         struct net_device *dev = fib_nh->nh_dev;
2777         struct in_device *in_dev;
2778         int err;
2779
2780         nh->nh_grp = nh_grp;
2781         nh->key.fib_nh = fib_nh;
2782         memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw));
2783         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
2784         if (err)
2785                 return err;
2786
2787         if (!dev)
2788                 return 0;
2789
2790         in_dev = __in_dev_get_rtnl(dev);
2791         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
2792             fib_nh->nh_flags & RTNH_F_LINKDOWN)
2793                 return 0;
2794
2795         err = mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
2796         if (err)
2797                 goto err_nexthop_neigh_init;
2798
2799         return 0;
2800
2801 err_nexthop_neigh_init:
2802         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2803         return err;
2804 }
2805
2806 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
2807                                    struct mlxsw_sp_nexthop *nh)
2808 {
2809         mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
2810         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2811 }
2812
2813 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
2814                                     unsigned long event, struct fib_nh *fib_nh)
2815 {
2816         struct mlxsw_sp_nexthop_key key;
2817         struct mlxsw_sp_nexthop *nh;
2818
2819         if (mlxsw_sp->router->aborted)
2820                 return;
2821
2822         key.fib_nh = fib_nh;
2823         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
2824         if (WARN_ON_ONCE(!nh))
2825                 return;
2826
2827         switch (event) {
2828         case FIB_EVENT_NH_ADD:
2829                 mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
2830                 break;
2831         case FIB_EVENT_NH_DEL:
2832                 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
2833                 break;
2834         }
2835
2836         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2837 }
2838
2839 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2840                                            struct mlxsw_sp_rif *rif)
2841 {
2842         struct mlxsw_sp_nexthop *nh, *tmp;
2843
2844         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
2845                 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
2846                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2847         }
2848 }
2849
2850 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
2851                                    const struct fib_info *fi)
2852 {
2853         return fi->fib_nh->nh_scope == RT_SCOPE_LINK ||
2854                mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fi->fib_nh, NULL);
2855 }
2856
2857 static struct mlxsw_sp_nexthop_group *
2858 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
2859 {
2860         struct mlxsw_sp_nexthop_group *nh_grp;
2861         struct mlxsw_sp_nexthop *nh;
2862         struct fib_nh *fib_nh;
2863         size_t alloc_size;
2864         int i;
2865         int err;
2866
2867         alloc_size = sizeof(*nh_grp) +
2868                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
2869         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
2870         if (!nh_grp)
2871                 return ERR_PTR(-ENOMEM);
2872         nh_grp->priv = fi;
2873         INIT_LIST_HEAD(&nh_grp->fib_list);
2874         nh_grp->neigh_tbl = &arp_tbl;
2875
2876         nh_grp->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi);
2877         nh_grp->count = fi->fib_nhs;
2878         fib_info_hold(fi);
2879         for (i = 0; i < nh_grp->count; i++) {
2880                 nh = &nh_grp->nexthops[i];
2881                 fib_nh = &fi->fib_nh[i];
2882                 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
2883                 if (err)
2884                         goto err_nexthop4_init;
2885         }
2886         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
2887         if (err)
2888                 goto err_nexthop_group_insert;
2889         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2890         return nh_grp;
2891
2892 err_nexthop_group_insert:
2893 err_nexthop4_init:
2894         for (i--; i >= 0; i--) {
2895                 nh = &nh_grp->nexthops[i];
2896                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2897         }
2898         fib_info_put(fi);
2899         kfree(nh_grp);
2900         return ERR_PTR(err);
2901 }
2902
2903 static void
2904 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
2905                                 struct mlxsw_sp_nexthop_group *nh_grp)
2906 {
2907         struct mlxsw_sp_nexthop *nh;
2908         int i;
2909
2910         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
2911         for (i = 0; i < nh_grp->count; i++) {
2912                 nh = &nh_grp->nexthops[i];
2913                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2914         }
2915         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2916         WARN_ON_ONCE(nh_grp->adj_index_valid);
2917         fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp));
2918         kfree(nh_grp);
2919 }
2920
2921 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
2922                                        struct mlxsw_sp_fib_entry *fib_entry,
2923                                        struct fib_info *fi)
2924 {
2925         struct mlxsw_sp_nexthop_group *nh_grp;
2926
2927         nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
2928         if (!nh_grp) {
2929                 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
2930                 if (IS_ERR(nh_grp))
2931                         return PTR_ERR(nh_grp);
2932         }
2933         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
2934         fib_entry->nh_group = nh_grp;
2935         return 0;
2936 }
2937
2938 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
2939                                         struct mlxsw_sp_fib_entry *fib_entry)
2940 {
2941         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2942
2943         list_del(&fib_entry->nexthop_group_node);
2944         if (!list_empty(&nh_grp->fib_list))
2945                 return;
2946         mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
2947 }
2948
2949 static bool
2950 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2951 {
2952         struct mlxsw_sp_fib4_entry *fib4_entry;
2953
2954         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
2955                                   common);
2956         return !fib4_entry->tos;
2957 }
2958
2959 static bool
2960 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2961 {
2962         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
2963
2964         switch (fib_entry->fib_node->fib->proto) {
2965         case MLXSW_SP_L3_PROTO_IPV4:
2966                 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
2967                         return false;
2968                 break;
2969         case MLXSW_SP_L3_PROTO_IPV6:
2970                 break;
2971         }
2972
2973         switch (fib_entry->type) {
2974         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2975                 return !!nh_group->adj_index_valid;
2976         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2977                 return !!nh_group->nh_rif;
2978         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
2979                 return true;
2980         default:
2981                 return false;
2982         }
2983 }
2984
2985 static struct mlxsw_sp_nexthop *
2986 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
2987                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
2988 {
2989         int i;
2990
2991         for (i = 0; i < nh_grp->count; i++) {
2992                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2993                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
2994
2995                 if (nh->rif && nh->rif->dev == rt->dst.dev &&
2996                     ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
2997                                     &rt->rt6i_gateway))
2998                         return nh;
2999                 continue;
3000         }
3001
3002         return NULL;
3003 }
3004
3005 static void
3006 mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3007 {
3008         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3009         int i;
3010
3011         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL ||
3012             fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP) {
3013                 nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
3014                 return;
3015         }
3016
3017         for (i = 0; i < nh_grp->count; i++) {
3018                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3019
3020                 if (nh->offloaded)
3021                         nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
3022                 else
3023                         nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
3024         }
3025 }
3026
3027 static void
3028 mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3029 {
3030         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3031         int i;
3032
3033         for (i = 0; i < nh_grp->count; i++) {
3034                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3035
3036                 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
3037         }
3038 }
3039
3040 static void
3041 mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3042 {
3043         struct mlxsw_sp_fib6_entry *fib6_entry;
3044         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3045
3046         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
3047                                   common);
3048
3049         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
3050                 list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3051                                  list)->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
3052                 return;
3053         }
3054
3055         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3056                 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3057                 struct mlxsw_sp_nexthop *nh;
3058
3059                 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3060                 if (nh && nh->offloaded)
3061                         mlxsw_sp_rt6->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
3062                 else
3063                         mlxsw_sp_rt6->rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
3064         }
3065 }
3066
3067 static void
3068 mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3069 {
3070         struct mlxsw_sp_fib6_entry *fib6_entry;
3071         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3072
3073         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
3074                                   common);
3075         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3076                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
3077
3078                 rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
3079         }
3080 }
3081
3082 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3083 {
3084         switch (fib_entry->fib_node->fib->proto) {
3085         case MLXSW_SP_L3_PROTO_IPV4:
3086                 mlxsw_sp_fib4_entry_offload_set(fib_entry);
3087                 break;
3088         case MLXSW_SP_L3_PROTO_IPV6:
3089                 mlxsw_sp_fib6_entry_offload_set(fib_entry);
3090                 break;
3091         }
3092 }
3093
3094 static void
3095 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3096 {
3097         switch (fib_entry->fib_node->fib->proto) {
3098         case MLXSW_SP_L3_PROTO_IPV4:
3099                 mlxsw_sp_fib4_entry_offload_unset(fib_entry);
3100                 break;
3101         case MLXSW_SP_L3_PROTO_IPV6:
3102                 mlxsw_sp_fib6_entry_offload_unset(fib_entry);
3103                 break;
3104         }
3105 }
3106
3107 static void
3108 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
3109                                    enum mlxsw_reg_ralue_op op, int err)
3110 {
3111         switch (op) {
3112         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
3113                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
3114         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
3115                 if (err)
3116                         return;
3117                 if (mlxsw_sp_fib_entry_should_offload(fib_entry))
3118                         mlxsw_sp_fib_entry_offload_set(fib_entry);
3119                 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry))
3120                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
3121                 return;
3122         default:
3123                 return;
3124         }
3125 }
3126
3127 static void
3128 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
3129                               const struct mlxsw_sp_fib_entry *fib_entry,
3130                               enum mlxsw_reg_ralue_op op)
3131 {
3132         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
3133         enum mlxsw_reg_ralxx_protocol proto;
3134         u32 *p_dip;
3135
3136         proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
3137
3138         switch (fib->proto) {
3139         case MLXSW_SP_L3_PROTO_IPV4:
3140                 p_dip = (u32 *) fib_entry->fib_node->key.addr;
3141                 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
3142                                       fib_entry->fib_node->key.prefix_len,
3143                                       *p_dip);
3144                 break;
3145         case MLXSW_SP_L3_PROTO_IPV6:
3146                 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
3147                                       fib_entry->fib_node->key.prefix_len,
3148                                       fib_entry->fib_node->key.addr);
3149                 break;
3150         }
3151 }
3152
3153 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
3154                                         struct mlxsw_sp_fib_entry *fib_entry,
3155                                         enum mlxsw_reg_ralue_op op)
3156 {
3157         char ralue_pl[MLXSW_REG_RALUE_LEN];
3158         enum mlxsw_reg_ralue_trap_action trap_action;
3159         u16 trap_id = 0;
3160         u32 adjacency_index = 0;
3161         u16 ecmp_size = 0;
3162
3163         /* In case the nexthop group adjacency index is valid, use it
3164          * with provided ECMP size. Otherwise, setup trap and pass
3165          * traffic to kernel.
3166          */
3167         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
3168                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
3169                 adjacency_index = fib_entry->nh_group->adj_index;
3170                 ecmp_size = fib_entry->nh_group->ecmp_size;
3171         } else {
3172                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
3173                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
3174         }
3175
3176         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3177         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
3178                                         adjacency_index, ecmp_size);
3179         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3180 }
3181
3182 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
3183                                        struct mlxsw_sp_fib_entry *fib_entry,
3184                                        enum mlxsw_reg_ralue_op op)
3185 {
3186         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
3187         enum mlxsw_reg_ralue_trap_action trap_action;
3188         char ralue_pl[MLXSW_REG_RALUE_LEN];
3189         u16 trap_id = 0;
3190         u16 rif_index = 0;
3191
3192         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
3193                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
3194                 rif_index = rif->rif_index;
3195         } else {
3196                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
3197                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
3198         }
3199
3200         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3201         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
3202                                        rif_index);
3203         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3204 }
3205
3206 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
3207                                       struct mlxsw_sp_fib_entry *fib_entry,
3208                                       enum mlxsw_reg_ralue_op op)
3209 {
3210         char ralue_pl[MLXSW_REG_RALUE_LEN];
3211
3212         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3213         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
3214         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3215 }
3216
3217 static int
3218 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
3219                                  struct mlxsw_sp_fib_entry *fib_entry,
3220                                  enum mlxsw_reg_ralue_op op)
3221 {
3222         struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
3223         const struct mlxsw_sp_ipip_ops *ipip_ops;
3224
3225         if (WARN_ON(!ipip_entry))
3226                 return -EINVAL;
3227
3228         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
3229         return ipip_ops->fib_entry_op(mlxsw_sp, ipip_entry, op,
3230                                       fib_entry->decap.tunnel_index);
3231 }
3232
3233 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
3234                                    struct mlxsw_sp_fib_entry *fib_entry,
3235                                    enum mlxsw_reg_ralue_op op)
3236 {
3237         switch (fib_entry->type) {
3238         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
3239                 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
3240         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
3241                 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
3242         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
3243                 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
3244         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
3245                 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp,
3246                                                         fib_entry, op);
3247         }
3248         return -EINVAL;
3249 }
3250
3251 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
3252                                  struct mlxsw_sp_fib_entry *fib_entry,
3253                                  enum mlxsw_reg_ralue_op op)
3254 {
3255         int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
3256
3257         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
3258
3259         return err;
3260 }
3261
3262 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
3263                                      struct mlxsw_sp_fib_entry *fib_entry)
3264 {
3265         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
3266                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
3267 }
3268
3269 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
3270                                   struct mlxsw_sp_fib_entry *fib_entry)
3271 {
3272         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
3273                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
3274 }
3275
3276 static int
3277 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
3278                              const struct fib_entry_notifier_info *fen_info,
3279                              struct mlxsw_sp_fib_entry *fib_entry)
3280 {
3281         union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
3282         struct net_device *dev = fen_info->fi->fib_dev;
3283         struct mlxsw_sp_ipip_entry *ipip_entry;
3284         struct fib_info *fi = fen_info->fi;
3285
3286         switch (fen_info->type) {
3287         case RTN_LOCAL:
3288                 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, dev,
3289                                                  MLXSW_SP_L3_PROTO_IPV4, dip);
3290                 if (ipip_entry) {
3291                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
3292                         return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
3293                                                              fib_entry,
3294                                                              ipip_entry);
3295                 }
3296                 /* fall through */
3297         case RTN_BROADCAST:
3298                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
3299                 return 0;
3300         case RTN_UNREACHABLE: /* fall through */
3301         case RTN_BLACKHOLE: /* fall through */
3302         case RTN_PROHIBIT:
3303                 /* Packets hitting these routes need to be trapped, but
3304                  * can do so with a lower priority than packets directed
3305                  * at the host, so use action type local instead of trap.
3306                  */
3307                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3308                 return 0;
3309         case RTN_UNICAST:
3310                 if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi))
3311                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
3312                 else
3313                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3314                 return 0;
3315         default:
3316                 return -EINVAL;
3317         }
3318 }
3319
3320 static struct mlxsw_sp_fib4_entry *
3321 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
3322                            struct mlxsw_sp_fib_node *fib_node,
3323                            const struct fib_entry_notifier_info *fen_info)
3324 {
3325         struct mlxsw_sp_fib4_entry *fib4_entry;
3326         struct mlxsw_sp_fib_entry *fib_entry;
3327         int err;
3328
3329         fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
3330         if (!fib4_entry)
3331                 return ERR_PTR(-ENOMEM);
3332         fib_entry = &fib4_entry->common;
3333
3334         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
3335         if (err)
3336                 goto err_fib4_entry_type_set;
3337
3338         err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
3339         if (err)
3340                 goto err_nexthop4_group_get;
3341
3342         fib4_entry->prio = fen_info->fi->fib_priority;
3343         fib4_entry->tb_id = fen_info->tb_id;
3344         fib4_entry->type = fen_info->type;
3345         fib4_entry->tos = fen_info->tos;
3346
3347         fib_entry->fib_node = fib_node;
3348
3349         return fib4_entry;
3350
3351 err_nexthop4_group_get:
3352 err_fib4_entry_type_set:
3353         kfree(fib4_entry);
3354         return ERR_PTR(err);
3355 }
3356
3357 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
3358                                         struct mlxsw_sp_fib4_entry *fib4_entry)
3359 {
3360         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
3361         kfree(fib4_entry);
3362 }
3363
3364 static struct mlxsw_sp_fib4_entry *
3365 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
3366                            const struct fib_entry_notifier_info *fen_info)
3367 {
3368         struct mlxsw_sp_fib4_entry *fib4_entry;
3369         struct mlxsw_sp_fib_node *fib_node;
3370         struct mlxsw_sp_fib *fib;
3371         struct mlxsw_sp_vr *vr;
3372
3373         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
3374         if (!vr)
3375                 return NULL;
3376         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
3377
3378         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
3379                                             sizeof(fen_info->dst),
3380                                             fen_info->dst_len);
3381         if (!fib_node)
3382                 return NULL;
3383
3384         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
3385                 if (fib4_entry->tb_id == fen_info->tb_id &&
3386                     fib4_entry->tos == fen_info->tos &&
3387                     fib4_entry->type == fen_info->type &&
3388                     mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) ==
3389                     fen_info->fi) {
3390                         return fib4_entry;
3391                 }
3392         }
3393
3394         return NULL;
3395 }
3396
3397 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
3398         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
3399         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
3400         .key_len = sizeof(struct mlxsw_sp_fib_key),
3401         .automatic_shrinking = true,
3402 };
3403
3404 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
3405                                     struct mlxsw_sp_fib_node *fib_node)
3406 {
3407         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
3408                                       mlxsw_sp_fib_ht_params);
3409 }
3410
3411 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
3412                                      struct mlxsw_sp_fib_node *fib_node)
3413 {
3414         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
3415                                mlxsw_sp_fib_ht_params);
3416 }
3417
3418 static struct mlxsw_sp_fib_node *
3419 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
3420                          size_t addr_len, unsigned char prefix_len)
3421 {
3422         struct mlxsw_sp_fib_key key;
3423
3424         memset(&key, 0, sizeof(key));
3425         memcpy(key.addr, addr, addr_len);
3426         key.prefix_len = prefix_len;
3427         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
3428 }
3429
3430 static struct mlxsw_sp_fib_node *
3431 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
3432                          size_t addr_len, unsigned char prefix_len)
3433 {
3434         struct mlxsw_sp_fib_node *fib_node;
3435
3436         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
3437         if (!fib_node)
3438                 return NULL;
3439
3440         INIT_LIST_HEAD(&fib_node->entry_list);
3441         list_add(&fib_node->list, &fib->node_list);
3442         memcpy(fib_node->key.addr, addr, addr_len);
3443         fib_node->key.prefix_len = prefix_len;
3444
3445         return fib_node;
3446 }
3447
3448 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
3449 {
3450         list_del(&fib_node->list);
3451         WARN_ON(!list_empty(&fib_node->entry_list));
3452         kfree(fib_node);
3453 }
3454
3455 static bool
3456 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
3457                                  const struct mlxsw_sp_fib_entry *fib_entry)
3458 {
3459         return list_first_entry(&fib_node->entry_list,
3460                                 struct mlxsw_sp_fib_entry, list) == fib_entry;
3461 }
3462
3463 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
3464                                       struct mlxsw_sp_fib *fib,
3465                                       struct mlxsw_sp_fib_node *fib_node)
3466 {
3467         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
3468         struct mlxsw_sp_lpm_tree *lpm_tree;
3469         int err;
3470
3471         /* Since the tree is shared between all virtual routers we must
3472          * make sure it contains all the required prefix lengths. This
3473          * can be computed by either adding the new prefix length to the
3474          * existing prefix usage of a bound tree, or by aggregating the
3475          * prefix lengths across all virtual routers and adding the new
3476          * one as well.
3477          */
3478         if (fib->lpm_tree)
3479                 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
3480                                           &fib->lpm_tree->prefix_usage);
3481         else
3482                 mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
3483         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
3484
3485         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
3486                                          fib->proto);
3487         if (IS_ERR(lpm_tree))
3488                 return PTR_ERR(lpm_tree);
3489
3490         if (fib->lpm_tree && fib->lpm_tree->id == lpm_tree->id)
3491                 return 0;
3492
3493         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
3494         if (err)
3495                 return err;
3496
3497         return 0;
3498 }
3499
3500 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
3501                                          struct mlxsw_sp_fib *fib)
3502 {
3503         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
3504         struct mlxsw_sp_lpm_tree *lpm_tree;
3505
3506         /* Aggregate prefix lengths across all virtual routers to make
3507          * sure we only have used prefix lengths in the LPM tree.
3508          */
3509         mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
3510         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
3511                                          fib->proto);
3512         if (IS_ERR(lpm_tree))
3513                 goto err_tree_get;
3514         mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
3515
3516 err_tree_get:
3517         if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage))
3518                 return;
3519         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
3520         mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
3521         fib->lpm_tree = NULL;
3522 }
3523
3524 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
3525 {
3526         unsigned char prefix_len = fib_node->key.prefix_len;
3527         struct mlxsw_sp_fib *fib = fib_node->fib;
3528
3529         if (fib->prefix_ref_count[prefix_len]++ == 0)
3530                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
3531 }
3532
3533 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
3534 {
3535         unsigned char prefix_len = fib_node->key.prefix_len;
3536         struct mlxsw_sp_fib *fib = fib_node->fib;
3537
3538         if (--fib->prefix_ref_count[prefix_len] == 0)
3539                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
3540 }
3541
3542 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
3543                                   struct mlxsw_sp_fib_node *fib_node,
3544                                   struct mlxsw_sp_fib *fib)
3545 {
3546         int err;
3547
3548         err = mlxsw_sp_fib_node_insert(fib, fib_node);
3549         if (err)
3550                 return err;
3551         fib_node->fib = fib;
3552
3553         err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib, fib_node);
3554         if (err)
3555                 goto err_fib_lpm_tree_link;
3556
3557         mlxsw_sp_fib_node_prefix_inc(fib_node);
3558
3559         return 0;
3560
3561 err_fib_lpm_tree_link:
3562         fib_node->fib = NULL;
3563         mlxsw_sp_fib_node_remove(fib, fib_node);
3564         return err;
3565 }
3566
3567 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
3568                                    struct mlxsw_sp_fib_node *fib_node)
3569 {
3570         struct mlxsw_sp_fib *fib = fib_node->fib;
3571
3572         mlxsw_sp_fib_node_prefix_dec(fib_node);
3573         mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib);
3574         fib_node->fib = NULL;
3575         mlxsw_sp_fib_node_remove(fib, fib_node);
3576 }
3577
3578 static struct mlxsw_sp_fib_node *
3579 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
3580                       size_t addr_len, unsigned char prefix_len,
3581                       enum mlxsw_sp_l3proto proto)
3582 {
3583         struct mlxsw_sp_fib_node *fib_node;
3584         struct mlxsw_sp_fib *fib;
3585         struct mlxsw_sp_vr *vr;
3586         int err;
3587
3588         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id);
3589         if (IS_ERR(vr))
3590                 return ERR_CAST(vr);
3591         fib = mlxsw_sp_vr_fib(vr, proto);
3592
3593         fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
3594         if (fib_node)
3595                 return fib_node;
3596
3597         fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
3598         if (!fib_node) {
3599                 err = -ENOMEM;
3600                 goto err_fib_node_create;
3601         }
3602
3603         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
3604         if (err)
3605                 goto err_fib_node_init;
3606
3607         return fib_node;
3608
3609 err_fib_node_init:
3610         mlxsw_sp_fib_node_destroy(fib_node);
3611 err_fib_node_create:
3612         mlxsw_sp_vr_put(vr);
3613         return ERR_PTR(err);
3614 }
3615
3616 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
3617                                   struct mlxsw_sp_fib_node *fib_node)
3618 {
3619         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
3620
3621         if (!list_empty(&fib_node->entry_list))
3622                 return;
3623         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
3624         mlxsw_sp_fib_node_destroy(fib_node);
3625         mlxsw_sp_vr_put(vr);
3626 }
3627
3628 static struct mlxsw_sp_fib4_entry *
3629 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3630                               const struct mlxsw_sp_fib4_entry *new4_entry)
3631 {
3632         struct mlxsw_sp_fib4_entry *fib4_entry;
3633
3634         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
3635                 if (fib4_entry->tb_id > new4_entry->tb_id)
3636                         continue;
3637                 if (fib4_entry->tb_id != new4_entry->tb_id)
3638                         break;
3639                 if (fib4_entry->tos > new4_entry->tos)
3640                         continue;
3641                 if (fib4_entry->prio >= new4_entry->prio ||
3642                     fib4_entry->tos < new4_entry->tos)
3643                         return fib4_entry;
3644         }
3645
3646         return NULL;
3647 }
3648
3649 static int
3650 mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry,
3651                                struct mlxsw_sp_fib4_entry *new4_entry)
3652 {
3653         struct mlxsw_sp_fib_node *fib_node;
3654
3655         if (WARN_ON(!fib4_entry))
3656                 return -EINVAL;
3657
3658         fib_node = fib4_entry->common.fib_node;
3659         list_for_each_entry_from(fib4_entry, &fib_node->entry_list,
3660                                  common.list) {
3661                 if (fib4_entry->tb_id != new4_entry->tb_id ||
3662                     fib4_entry->tos != new4_entry->tos ||
3663                     fib4_entry->prio != new4_entry->prio)
3664                         break;
3665         }
3666
3667         list_add_tail(&new4_entry->common.list, &fib4_entry->common.list);
3668         return 0;
3669 }
3670
3671 static int
3672 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry,
3673                                bool replace, bool append)
3674 {
3675         struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node;
3676         struct mlxsw_sp_fib4_entry *fib4_entry;
3677
3678         fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry);
3679
3680         if (append)
3681                 return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry);
3682         if (replace && WARN_ON(!fib4_entry))
3683                 return -EINVAL;
3684
3685         /* Insert new entry before replaced one, so that we can later
3686          * remove the second.
3687          */
3688         if (fib4_entry) {
3689                 list_add_tail(&new4_entry->common.list,
3690                               &fib4_entry->common.list);
3691         } else {
3692                 struct mlxsw_sp_fib4_entry *last;
3693
3694                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
3695                         if (new4_entry->tb_id > last->tb_id)
3696                                 break;
3697                         fib4_entry = last;
3698                 }
3699
3700                 if (fib4_entry)
3701                         list_add(&new4_entry->common.list,
3702                                  &fib4_entry->common.list);
3703                 else
3704                         list_add(&new4_entry->common.list,
3705                                  &fib_node->entry_list);
3706         }
3707
3708         return 0;
3709 }
3710
3711 static void
3712 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry)
3713 {
3714         list_del(&fib4_entry->common.list);
3715 }
3716
3717 static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp,
3718                                        struct mlxsw_sp_fib_entry *fib_entry)
3719 {
3720         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3721
3722         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3723                 return 0;
3724
3725         /* To prevent packet loss, overwrite the previously offloaded
3726          * entry.
3727          */
3728         if (!list_is_singular(&fib_node->entry_list)) {
3729                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3730                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3731
3732                 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
3733         }
3734
3735         return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3736 }
3737
3738 static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp,
3739                                         struct mlxsw_sp_fib_entry *fib_entry)
3740 {
3741         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3742
3743         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3744                 return;
3745
3746         /* Promote the next entry by overwriting the deleted entry */
3747         if (!list_is_singular(&fib_node->entry_list)) {
3748                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3749                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3750
3751                 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
3752                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
3753                 return;
3754         }
3755
3756         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
3757 }
3758
3759 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
3760                                          struct mlxsw_sp_fib4_entry *fib4_entry,
3761                                          bool replace, bool append)
3762 {
3763         int err;
3764
3765         err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append);
3766         if (err)
3767                 return err;
3768
3769         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common);
3770         if (err)
3771                 goto err_fib_node_entry_add;
3772
3773         return 0;
3774
3775 err_fib_node_entry_add:
3776         mlxsw_sp_fib4_node_list_remove(fib4_entry);
3777         return err;
3778 }
3779
3780 static void
3781 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
3782                                 struct mlxsw_sp_fib4_entry *fib4_entry)
3783 {
3784         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common);
3785         mlxsw_sp_fib4_node_list_remove(fib4_entry);
3786
3787         if (fib4_entry->common.type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP)
3788                 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, &fib4_entry->common);
3789 }
3790
3791 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
3792                                         struct mlxsw_sp_fib4_entry *fib4_entry,
3793                                         bool replace)
3794 {
3795         struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
3796         struct mlxsw_sp_fib4_entry *replaced;
3797
3798         if (!replace)
3799                 return;
3800
3801         /* We inserted the new entry before replaced one */
3802         replaced = list_next_entry(fib4_entry, common.list);
3803
3804         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
3805         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
3806         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3807 }
3808
3809 static int
3810 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
3811                          const struct fib_entry_notifier_info *fen_info,
3812                          bool replace, bool append)
3813 {
3814         struct mlxsw_sp_fib4_entry *fib4_entry;
3815         struct mlxsw_sp_fib_node *fib_node;
3816         int err;
3817
3818         if (mlxsw_sp->router->aborted)
3819                 return 0;
3820
3821         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
3822                                          &fen_info->dst, sizeof(fen_info->dst),
3823                                          fen_info->dst_len,
3824                                          MLXSW_SP_L3_PROTO_IPV4);
3825         if (IS_ERR(fib_node)) {
3826                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
3827                 return PTR_ERR(fib_node);
3828         }
3829
3830         fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
3831         if (IS_ERR(fib4_entry)) {
3832                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
3833                 err = PTR_ERR(fib4_entry);
3834                 goto err_fib4_entry_create;
3835         }
3836
3837         err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace,
3838                                             append);
3839         if (err) {
3840                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
3841                 goto err_fib4_node_entry_link;
3842         }
3843
3844         mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace);
3845
3846         return 0;
3847
3848 err_fib4_node_entry_link:
3849         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3850 err_fib4_entry_create:
3851         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3852         return err;
3853 }
3854
3855 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
3856                                      struct fib_entry_notifier_info *fen_info)
3857 {
3858         struct mlxsw_sp_fib4_entry *fib4_entry;
3859         struct mlxsw_sp_fib_node *fib_node;
3860
3861         if (mlxsw_sp->router->aborted)
3862                 return;
3863
3864         fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
3865         if (WARN_ON(!fib4_entry))
3866                 return;
3867         fib_node = fib4_entry->common.fib_node;
3868
3869         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
3870         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3871         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3872 }
3873
3874 static bool mlxsw_sp_fib6_rt_should_ignore(const struct rt6_info *rt)
3875 {
3876         /* Packets with link-local destination IP arriving to the router
3877          * are trapped to the CPU, so no need to program specific routes
3878          * for them.
3879          */
3880         if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LINKLOCAL)
3881                 return true;
3882
3883         /* Multicast routes aren't supported, so ignore them. Neighbour
3884          * Discovery packets are specifically trapped.
3885          */
3886         if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_MULTICAST)
3887                 return true;
3888
3889         /* Cloned routes are irrelevant in the forwarding path. */
3890         if (rt->rt6i_flags & RTF_CACHE)
3891                 return true;
3892
3893         return false;
3894 }
3895
3896 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct rt6_info *rt)
3897 {
3898         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3899
3900         mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
3901         if (!mlxsw_sp_rt6)
3902                 return ERR_PTR(-ENOMEM);
3903
3904         /* In case of route replace, replaced route is deleted with
3905          * no notification. Take reference to prevent accessing freed
3906          * memory.
3907          */
3908         mlxsw_sp_rt6->rt = rt;
3909         rt6_hold(rt);
3910
3911         return mlxsw_sp_rt6;
3912 }
3913
3914 #if IS_ENABLED(CONFIG_IPV6)
3915 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3916 {
3917         rt6_release(rt);
3918 }
3919 #else
3920 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3921 {
3922 }
3923 #endif
3924
3925 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
3926 {
3927         mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
3928         kfree(mlxsw_sp_rt6);
3929 }
3930
3931 static bool mlxsw_sp_fib6_rt_can_mp(const struct rt6_info *rt)
3932 {
3933         /* RTF_CACHE routes are ignored */
3934         return (rt->rt6i_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY;
3935 }
3936
3937 static struct rt6_info *
3938 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
3939 {
3940         return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3941                                 list)->rt;
3942 }
3943
3944 static struct mlxsw_sp_fib6_entry *
3945 mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3946                                  const struct rt6_info *nrt, bool replace)
3947 {
3948         struct mlxsw_sp_fib6_entry *fib6_entry;
3949
3950         if (!mlxsw_sp_fib6_rt_can_mp(nrt) || replace)
3951                 return NULL;
3952
3953         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3954                 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3955
3956                 /* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same
3957                  * virtual router.
3958                  */
3959                 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
3960                         continue;
3961                 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
3962                         break;
3963                 if (rt->rt6i_metric < nrt->rt6i_metric)
3964                         continue;
3965                 if (rt->rt6i_metric == nrt->rt6i_metric &&
3966                     mlxsw_sp_fib6_rt_can_mp(rt))
3967                         return fib6_entry;
3968                 if (rt->rt6i_metric > nrt->rt6i_metric)
3969                         break;
3970         }
3971
3972         return NULL;
3973 }
3974
3975 static struct mlxsw_sp_rt6 *
3976 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
3977                             const struct rt6_info *rt)
3978 {
3979         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3980
3981         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3982                 if (mlxsw_sp_rt6->rt == rt)
3983                         return mlxsw_sp_rt6;
3984         }
3985
3986         return NULL;
3987 }
3988
3989 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
3990                                         const struct rt6_info *rt,
3991                                         enum mlxsw_sp_ipip_type *ret)
3992 {
3993         return rt->dst.dev &&
3994                mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->dst.dev, ret);
3995 }
3996
3997 static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp,
3998                                        struct mlxsw_sp_nexthop_group *nh_grp,
3999                                        struct mlxsw_sp_nexthop *nh,
4000                                        const struct rt6_info *rt)
4001 {
4002         struct mlxsw_sp_router *router = mlxsw_sp->router;
4003         struct net_device *dev = rt->dst.dev;
4004         enum mlxsw_sp_ipip_type ipipt;
4005         struct mlxsw_sp_rif *rif;
4006         int err;
4007
4008         if (mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, &ipipt) &&
4009             router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev,
4010                                                      MLXSW_SP_L3_PROTO_IPV6)) {
4011                 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
4012                 return mlxsw_sp_nexthop_ipip_init(mlxsw_sp, ipipt, nh, dev);
4013         }
4014
4015         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
4016         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4017         if (!rif)
4018                 return 0;
4019         mlxsw_sp_nexthop_rif_init(nh, rif);
4020
4021         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
4022         if (err)
4023                 goto err_nexthop_neigh_init;
4024
4025         return 0;
4026
4027 err_nexthop_neigh_init:
4028         mlxsw_sp_nexthop_rif_fini(nh);
4029         return err;
4030 }
4031
4032 static void mlxsw_sp_nexthop6_type_fini(struct mlxsw_sp *mlxsw_sp,
4033                                         struct mlxsw_sp_nexthop *nh)
4034 {
4035         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4036 }
4037
4038 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
4039                                   struct mlxsw_sp_nexthop_group *nh_grp,
4040                                   struct mlxsw_sp_nexthop *nh,
4041                                   const struct rt6_info *rt)
4042 {
4043         struct net_device *dev = rt->dst.dev;
4044
4045         nh->nh_grp = nh_grp;
4046         memcpy(&nh->gw_addr, &rt->rt6i_gateway, sizeof(nh->gw_addr));
4047
4048         if (!dev)
4049                 return 0;
4050         nh->ifindex = dev->ifindex;
4051
4052         return mlxsw_sp_nexthop6_type_init(mlxsw_sp, nh_grp, nh, rt);
4053 }
4054
4055 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
4056                                    struct mlxsw_sp_nexthop *nh)
4057 {
4058         mlxsw_sp_nexthop6_type_fini(mlxsw_sp, nh);
4059 }
4060
4061 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
4062                                     const struct rt6_info *rt)
4063 {
4064         return rt->rt6i_flags & RTF_GATEWAY ||
4065                mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
4066 }
4067
4068 static struct mlxsw_sp_nexthop_group *
4069 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
4070                                struct mlxsw_sp_fib6_entry *fib6_entry)
4071 {
4072         struct mlxsw_sp_nexthop_group *nh_grp;
4073         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4074         struct mlxsw_sp_nexthop *nh;
4075         size_t alloc_size;
4076         int i = 0;
4077         int err;
4078
4079         alloc_size = sizeof(*nh_grp) +
4080                      fib6_entry->nrt6 * sizeof(struct mlxsw_sp_nexthop);
4081         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
4082         if (!nh_grp)
4083                 return ERR_PTR(-ENOMEM);
4084         INIT_LIST_HEAD(&nh_grp->fib_list);
4085 #if IS_ENABLED(CONFIG_IPV6)
4086         nh_grp->neigh_tbl = &nd_tbl;
4087 #endif
4088         mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
4089                                         struct mlxsw_sp_rt6, list);
4090         nh_grp->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
4091         nh_grp->count = fib6_entry->nrt6;
4092         for (i = 0; i < nh_grp->count; i++) {
4093                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
4094
4095                 nh = &nh_grp->nexthops[i];
4096                 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
4097                 if (err)
4098                         goto err_nexthop6_init;
4099                 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
4100         }
4101
4102         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
4103         if (err)
4104                 goto err_nexthop_group_insert;
4105
4106         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4107         return nh_grp;
4108
4109 err_nexthop_group_insert:
4110 err_nexthop6_init:
4111         for (i--; i >= 0; i--) {
4112                 nh = &nh_grp->nexthops[i];
4113                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
4114         }
4115         kfree(nh_grp);
4116         return ERR_PTR(err);
4117 }
4118
4119 static void
4120 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
4121                                 struct mlxsw_sp_nexthop_group *nh_grp)
4122 {
4123         struct mlxsw_sp_nexthop *nh;
4124         int i = nh_grp->count;
4125
4126         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
4127         for (i--; i >= 0; i--) {
4128                 nh = &nh_grp->nexthops[i];
4129                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
4130         }
4131         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4132         WARN_ON(nh_grp->adj_index_valid);
4133         kfree(nh_grp);
4134 }
4135
4136 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
4137                                        struct mlxsw_sp_fib6_entry *fib6_entry)
4138 {
4139         struct mlxsw_sp_nexthop_group *nh_grp;
4140
4141         nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
4142         if (!nh_grp) {
4143                 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
4144                 if (IS_ERR(nh_grp))
4145                         return PTR_ERR(nh_grp);
4146         }
4147
4148         list_add_tail(&fib6_entry->common.nexthop_group_node,
4149                       &nh_grp->fib_list);
4150         fib6_entry->common.nh_group = nh_grp;
4151
4152         return 0;
4153 }
4154
4155 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
4156                                         struct mlxsw_sp_fib_entry *fib_entry)
4157 {
4158         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
4159
4160         list_del(&fib_entry->nexthop_group_node);
4161         if (!list_empty(&nh_grp->fib_list))
4162                 return;
4163         mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
4164 }
4165
4166 static int
4167 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
4168                                struct mlxsw_sp_fib6_entry *fib6_entry)
4169 {
4170         struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
4171         int err;
4172
4173         fib6_entry->common.nh_group = NULL;
4174         list_del(&fib6_entry->common.nexthop_group_node);
4175
4176         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
4177         if (err)
4178                 goto err_nexthop6_group_get;
4179
4180         /* In case this entry is offloaded, then the adjacency index
4181          * currently associated with it in the device's table is that
4182          * of the old group. Start using the new one instead.
4183          */
4184         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
4185         if (err)
4186                 goto err_fib_node_entry_add;
4187
4188         if (list_empty(&old_nh_grp->fib_list))
4189                 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
4190
4191         return 0;
4192
4193 err_fib_node_entry_add:
4194         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
4195 err_nexthop6_group_get:
4196         list_add_tail(&fib6_entry->common.nexthop_group_node,
4197                       &old_nh_grp->fib_list);
4198         fib6_entry->common.nh_group = old_nh_grp;
4199         return err;
4200 }
4201
4202 static int
4203 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
4204                                 struct mlxsw_sp_fib6_entry *fib6_entry,
4205                                 struct rt6_info *rt)
4206 {
4207         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4208         int err;
4209
4210         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
4211         if (IS_ERR(mlxsw_sp_rt6))
4212                 return PTR_ERR(mlxsw_sp_rt6);
4213
4214         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
4215         fib6_entry->nrt6++;
4216
4217         err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
4218         if (err)
4219                 goto err_nexthop6_group_update;
4220
4221         return 0;
4222
4223 err_nexthop6_group_update:
4224         fib6_entry->nrt6--;
4225         list_del(&mlxsw_sp_rt6->list);
4226         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4227         return err;
4228 }
4229
4230 static void
4231 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
4232                                 struct mlxsw_sp_fib6_entry *fib6_entry,
4233                                 struct rt6_info *rt)
4234 {
4235         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4236
4237         mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt);
4238         if (WARN_ON(!mlxsw_sp_rt6))
4239                 return;
4240
4241         fib6_entry->nrt6--;
4242         list_del(&mlxsw_sp_rt6->list);
4243         mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
4244         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4245 }
4246
4247 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
4248                                          struct mlxsw_sp_fib_entry *fib_entry,
4249                                          const struct rt6_info *rt)
4250 {
4251         /* Packets hitting RTF_REJECT routes need to be discarded by the
4252          * stack. We can rely on their destination device not having a
4253          * RIF (it's the loopback device) and can thus use action type
4254          * local, which will cause them to be trapped with a lower
4255          * priority than packets that need to be locally received.
4256          */
4257         if (rt->rt6i_flags & (RTF_LOCAL | RTF_ANYCAST))
4258                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
4259         else if (rt->rt6i_flags & RTF_REJECT)
4260                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4261         else if (mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt))
4262                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
4263         else
4264                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4265 }
4266
4267 static void
4268 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
4269 {
4270         struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
4271
4272         list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
4273                                  list) {
4274                 fib6_entry->nrt6--;
4275                 list_del(&mlxsw_sp_rt6->list);
4276                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4277         }
4278 }
4279
4280 static struct mlxsw_sp_fib6_entry *
4281 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
4282                            struct mlxsw_sp_fib_node *fib_node,
4283                            struct rt6_info *rt)
4284 {
4285         struct mlxsw_sp_fib6_entry *fib6_entry;
4286         struct mlxsw_sp_fib_entry *fib_entry;
4287         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4288         int err;
4289
4290         fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
4291         if (!fib6_entry)
4292                 return ERR_PTR(-ENOMEM);
4293         fib_entry = &fib6_entry->common;
4294
4295         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
4296         if (IS_ERR(mlxsw_sp_rt6)) {
4297                 err = PTR_ERR(mlxsw_sp_rt6);
4298                 goto err_rt6_create;
4299         }
4300
4301         mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, mlxsw_sp_rt6->rt);
4302
4303         INIT_LIST_HEAD(&fib6_entry->rt6_list);
4304         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
4305         fib6_entry->nrt6 = 1;
4306         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
4307         if (err)
4308                 goto err_nexthop6_group_get;
4309
4310         fib_entry->fib_node = fib_node;
4311
4312         return fib6_entry;
4313
4314 err_nexthop6_group_get:
4315         list_del(&mlxsw_sp_rt6->list);
4316         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4317 err_rt6_create:
4318         kfree(fib6_entry);
4319         return ERR_PTR(err);
4320 }
4321
4322 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
4323                                         struct mlxsw_sp_fib6_entry *fib6_entry)
4324 {
4325         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
4326         mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
4327         WARN_ON(fib6_entry->nrt6);
4328         kfree(fib6_entry);
4329 }
4330
4331 static struct mlxsw_sp_fib6_entry *
4332 mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
4333                               const struct rt6_info *nrt, bool replace)
4334 {
4335         struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL;
4336
4337         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
4338                 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
4339
4340                 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
4341                         continue;
4342                 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
4343                         break;
4344                 if (replace && rt->rt6i_metric == nrt->rt6i_metric) {
4345                         if (mlxsw_sp_fib6_rt_can_mp(rt) ==
4346                             mlxsw_sp_fib6_rt_can_mp(nrt))
4347                                 return fib6_entry;
4348                         if (mlxsw_sp_fib6_rt_can_mp(nrt))
4349                                 fallback = fallback ?: fib6_entry;
4350                 }
4351                 if (rt->rt6i_metric > nrt->rt6i_metric)
4352                         return fallback ?: fib6_entry;
4353         }
4354
4355         return fallback;
4356 }
4357
4358 static int
4359 mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry,
4360                                bool replace)
4361 {
4362         struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node;
4363         struct rt6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry);
4364         struct mlxsw_sp_fib6_entry *fib6_entry;
4365
4366         fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace);
4367
4368         if (replace && WARN_ON(!fib6_entry))
4369                 return -EINVAL;
4370
4371         if (fib6_entry) {
4372                 list_add_tail(&new6_entry->common.list,
4373                               &fib6_entry->common.list);
4374         } else {
4375                 struct mlxsw_sp_fib6_entry *last;
4376
4377                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
4378                         struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(last);
4379
4380                         if (nrt->rt6i_table->tb6_id > rt->rt6i_table->tb6_id)
4381                                 break;
4382                         fib6_entry = last;
4383                 }
4384
4385                 if (fib6_entry)
4386                         list_add(&new6_entry->common.list,
4387                                  &fib6_entry->common.list);
4388                 else
4389                         list_add(&new6_entry->common.list,
4390                                  &fib_node->entry_list);
4391         }
4392
4393         return 0;
4394 }
4395
4396 static void
4397 mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry)
4398 {
4399         list_del(&fib6_entry->common.list);
4400 }
4401
4402 static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp,
4403                                          struct mlxsw_sp_fib6_entry *fib6_entry,
4404                                          bool replace)
4405 {
4406         int err;
4407
4408         err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace);
4409         if (err)
4410                 return err;
4411
4412         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
4413         if (err)
4414                 goto err_fib_node_entry_add;
4415
4416         return 0;
4417
4418 err_fib_node_entry_add:
4419         mlxsw_sp_fib6_node_list_remove(fib6_entry);
4420         return err;
4421 }
4422
4423 static void
4424 mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
4425                                 struct mlxsw_sp_fib6_entry *fib6_entry)
4426 {
4427         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common);
4428         mlxsw_sp_fib6_node_list_remove(fib6_entry);
4429 }
4430
4431 static struct mlxsw_sp_fib6_entry *
4432 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
4433                            const struct rt6_info *rt)
4434 {
4435         struct mlxsw_sp_fib6_entry *fib6_entry;
4436         struct mlxsw_sp_fib_node *fib_node;
4437         struct mlxsw_sp_fib *fib;
4438         struct mlxsw_sp_vr *vr;
4439
4440         vr = mlxsw_sp_vr_find(mlxsw_sp, rt->rt6i_table->tb6_id);
4441         if (!vr)
4442                 return NULL;
4443         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
4444
4445         fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->rt6i_dst.addr,
4446                                             sizeof(rt->rt6i_dst.addr),
4447                                             rt->rt6i_dst.plen);
4448         if (!fib_node)
4449                 return NULL;
4450
4451         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
4452                 struct rt6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
4453
4454                 if (rt->rt6i_table->tb6_id == iter_rt->rt6i_table->tb6_id &&
4455                     rt->rt6i_metric == iter_rt->rt6i_metric &&
4456                     mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
4457                         return fib6_entry;
4458         }
4459
4460         return NULL;
4461 }
4462
4463 static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp,
4464                                         struct mlxsw_sp_fib6_entry *fib6_entry,
4465                                         bool replace)
4466 {
4467         struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
4468         struct mlxsw_sp_fib6_entry *replaced;
4469
4470         if (!replace)
4471                 return;
4472
4473         replaced = list_next_entry(fib6_entry, common.list);
4474
4475         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced);
4476         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced);
4477         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4478 }
4479
4480 static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
4481                                     struct rt6_info *rt, bool replace)
4482 {
4483         struct mlxsw_sp_fib6_entry *fib6_entry;
4484         struct mlxsw_sp_fib_node *fib_node;
4485         int err;
4486
4487         if (mlxsw_sp->router->aborted)
4488                 return 0;
4489
4490         if (rt->rt6i_src.plen)
4491                 return -EINVAL;
4492
4493         if (mlxsw_sp_fib6_rt_should_ignore(rt))
4494                 return 0;
4495
4496         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->rt6i_table->tb6_id,
4497                                          &rt->rt6i_dst.addr,
4498                                          sizeof(rt->rt6i_dst.addr),
4499                                          rt->rt6i_dst.plen,
4500                                          MLXSW_SP_L3_PROTO_IPV6);
4501         if (IS_ERR(fib_node))
4502                 return PTR_ERR(fib_node);
4503
4504         /* Before creating a new entry, try to append route to an existing
4505          * multipath entry.
4506          */
4507         fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, replace);
4508         if (fib6_entry) {
4509                 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt);
4510                 if (err)
4511                         goto err_fib6_entry_nexthop_add;
4512                 return 0;
4513         }
4514
4515         fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt);
4516         if (IS_ERR(fib6_entry)) {
4517                 err = PTR_ERR(fib6_entry);
4518                 goto err_fib6_entry_create;
4519         }
4520
4521         err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace);
4522         if (err)
4523                 goto err_fib6_node_entry_link;
4524
4525         mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace);
4526
4527         return 0;
4528
4529 err_fib6_node_entry_link:
4530         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4531 err_fib6_entry_create:
4532 err_fib6_entry_nexthop_add:
4533         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4534         return err;
4535 }
4536
4537 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
4538                                      struct rt6_info *rt)
4539 {
4540         struct mlxsw_sp_fib6_entry *fib6_entry;
4541         struct mlxsw_sp_fib_node *fib_node;
4542
4543         if (mlxsw_sp->router->aborted)
4544                 return;
4545
4546         if (mlxsw_sp_fib6_rt_should_ignore(rt))
4547                 return;
4548
4549         fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
4550         if (WARN_ON(!fib6_entry))
4551                 return;
4552
4553         /* If route is part of a multipath entry, but not the last one
4554          * removed, then only reduce its nexthop group.
4555          */
4556         if (!list_is_singular(&fib6_entry->rt6_list)) {
4557                 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt);
4558                 return;
4559         }
4560
4561         fib_node = fib6_entry->common.fib_node;
4562
4563         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4564         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4565         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4566 }
4567
4568 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
4569                                             enum mlxsw_reg_ralxx_protocol proto,
4570                                             u8 tree_id)
4571 {
4572         char ralta_pl[MLXSW_REG_RALTA_LEN];
4573         char ralst_pl[MLXSW_REG_RALST_LEN];
4574         int i, err;
4575
4576         mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id);
4577         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
4578         if (err)
4579                 return err;
4580
4581         mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id);
4582         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
4583         if (err)
4584                 return err;
4585
4586         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
4587                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
4588                 char raltb_pl[MLXSW_REG_RALTB_LEN];
4589                 char ralue_pl[MLXSW_REG_RALUE_LEN];
4590
4591                 mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id);
4592                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
4593                                       raltb_pl);
4594                 if (err)
4595                         return err;
4596
4597                 mlxsw_reg_ralue_pack(ralue_pl, proto,
4598                                      MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0);
4599                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
4600                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
4601                                       ralue_pl);
4602                 if (err)
4603                         return err;
4604         }
4605
4606         return 0;
4607 }
4608
4609 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
4610 {
4611         enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4;
4612         int err;
4613
4614         err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4615                                                MLXSW_SP_LPM_TREE_MIN);
4616         if (err)
4617                 return err;
4618
4619         proto = MLXSW_REG_RALXX_PROTOCOL_IPV6;
4620         return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4621                                                 MLXSW_SP_LPM_TREE_MIN + 1);
4622 }
4623
4624 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
4625                                      struct mlxsw_sp_fib_node *fib_node)
4626 {
4627         struct mlxsw_sp_fib4_entry *fib4_entry, *tmp;
4628
4629         list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list,
4630                                  common.list) {
4631                 bool do_break = &tmp->common.list == &fib_node->entry_list;
4632
4633                 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
4634                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
4635                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4636                 /* Break when entry list is empty and node was freed.
4637                  * Otherwise, we'll access freed memory in the next
4638                  * iteration.
4639                  */
4640                 if (do_break)
4641                         break;
4642         }
4643 }
4644
4645 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
4646                                      struct mlxsw_sp_fib_node *fib_node)
4647 {
4648         struct mlxsw_sp_fib6_entry *fib6_entry, *tmp;
4649
4650         list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list,
4651                                  common.list) {
4652                 bool do_break = &tmp->common.list == &fib_node->entry_list;
4653
4654                 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4655                 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4656                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4657                 if (do_break)
4658                         break;
4659         }
4660 }
4661
4662 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
4663                                     struct mlxsw_sp_fib_node *fib_node)
4664 {
4665         switch (fib_node->fib->proto) {
4666         case MLXSW_SP_L3_PROTO_IPV4:
4667                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
4668                 break;
4669         case MLXSW_SP_L3_PROTO_IPV6:
4670                 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
4671                 break;
4672         }
4673 }
4674
4675 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
4676                                   struct mlxsw_sp_vr *vr,
4677                                   enum mlxsw_sp_l3proto proto)
4678 {
4679         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
4680         struct mlxsw_sp_fib_node *fib_node, *tmp;
4681
4682         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
4683                 bool do_break = &tmp->list == &fib->node_list;
4684
4685                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
4686                 if (do_break)
4687                         break;
4688         }
4689 }
4690
4691 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
4692 {
4693         int i;
4694
4695         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
4696                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
4697
4698                 if (!mlxsw_sp_vr_is_used(vr))
4699                         continue;
4700                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
4701
4702                 /* If virtual router was only used for IPv4, then it's no
4703                  * longer used.
4704                  */
4705                 if (!mlxsw_sp_vr_is_used(vr))
4706                         continue;
4707                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
4708         }
4709 }
4710
4711 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
4712 {
4713         int err;
4714
4715         if (mlxsw_sp->router->aborted)
4716                 return;
4717         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
4718         mlxsw_sp_router_fib_flush(mlxsw_sp);
4719         mlxsw_sp->router->aborted = true;
4720         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
4721         if (err)
4722                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
4723 }
4724
4725 struct mlxsw_sp_fib_event_work {
4726         struct work_struct work;
4727         union {
4728                 struct fib6_entry_notifier_info fen6_info;
4729                 struct fib_entry_notifier_info fen_info;
4730                 struct fib_rule_notifier_info fr_info;
4731                 struct fib_nh_notifier_info fnh_info;
4732         };
4733         struct mlxsw_sp *mlxsw_sp;
4734         unsigned long event;
4735 };
4736
4737 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
4738 {
4739         struct mlxsw_sp_fib_event_work *fib_work =
4740                 container_of(work, struct mlxsw_sp_fib_event_work, work);
4741         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4742         struct fib_rule *rule;
4743         bool replace, append;
4744         int err;
4745
4746         /* Protect internal structures from changes */
4747         rtnl_lock();
4748         switch (fib_work->event) {
4749         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4750         case FIB_EVENT_ENTRY_APPEND: /* fall through */
4751         case FIB_EVENT_ENTRY_ADD:
4752                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4753                 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
4754                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
4755                                                replace, append);
4756                 if (err)
4757                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4758                 fib_info_put(fib_work->fen_info.fi);
4759                 break;
4760         case FIB_EVENT_ENTRY_DEL:
4761                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
4762                 fib_info_put(fib_work->fen_info.fi);
4763                 break;
4764         case FIB_EVENT_RULE_ADD: /* fall through */
4765         case FIB_EVENT_RULE_DEL:
4766                 rule = fib_work->fr_info.rule;
4767                 if (!fib4_rule_default(rule) && !rule->l3mdev)
4768                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4769                 fib_rule_put(rule);
4770                 break;
4771         case FIB_EVENT_NH_ADD: /* fall through */
4772         case FIB_EVENT_NH_DEL:
4773                 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
4774                                         fib_work->fnh_info.fib_nh);
4775                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
4776                 break;
4777         }
4778         rtnl_unlock();
4779         kfree(fib_work);
4780 }
4781
4782 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
4783 {
4784         struct mlxsw_sp_fib_event_work *fib_work =
4785                 container_of(work, struct mlxsw_sp_fib_event_work, work);
4786         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4787         struct fib_rule *rule;
4788         bool replace;
4789         int err;
4790
4791         rtnl_lock();
4792         switch (fib_work->event) {
4793         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4794         case FIB_EVENT_ENTRY_ADD:
4795                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4796                 err = mlxsw_sp_router_fib6_add(mlxsw_sp,
4797                                                fib_work->fen6_info.rt, replace);
4798                 if (err)
4799                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4800                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4801                 break;
4802         case FIB_EVENT_ENTRY_DEL:
4803                 mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt);
4804                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4805                 break;
4806         case FIB_EVENT_RULE_ADD: /* fall through */
4807         case FIB_EVENT_RULE_DEL:
4808                 rule = fib_work->fr_info.rule;
4809                 if (!fib6_rule_default(rule) && !rule->l3mdev)
4810                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4811                 fib_rule_put(rule);
4812                 break;
4813         }
4814         rtnl_unlock();
4815         kfree(fib_work);
4816 }
4817
4818 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
4819                                        struct fib_notifier_info *info)
4820 {
4821         switch (fib_work->event) {
4822         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4823         case FIB_EVENT_ENTRY_APPEND: /* fall through */
4824         case FIB_EVENT_ENTRY_ADD: /* fall through */
4825         case FIB_EVENT_ENTRY_DEL:
4826                 memcpy(&fib_work->fen_info, info, sizeof(fib_work->fen_info));
4827                 /* Take referece on fib_info to prevent it from being
4828                  * freed while work is queued. Release it afterwards.
4829                  */
4830                 fib_info_hold(fib_work->fen_info.fi);
4831                 break;
4832         case FIB_EVENT_RULE_ADD: /* fall through */
4833         case FIB_EVENT_RULE_DEL:
4834                 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4835                 fib_rule_get(fib_work->fr_info.rule);
4836                 break;
4837         case FIB_EVENT_NH_ADD: /* fall through */
4838         case FIB_EVENT_NH_DEL:
4839                 memcpy(&fib_work->fnh_info, info, sizeof(fib_work->fnh_info));
4840                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
4841                 break;
4842         }
4843 }
4844
4845 static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
4846                                        struct fib_notifier_info *info)
4847 {
4848         switch (fib_work->event) {
4849         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4850         case FIB_EVENT_ENTRY_ADD: /* fall through */
4851         case FIB_EVENT_ENTRY_DEL:
4852                 memcpy(&fib_work->fen6_info, info, sizeof(fib_work->fen6_info));
4853                 rt6_hold(fib_work->fen6_info.rt);
4854                 break;
4855         case FIB_EVENT_RULE_ADD: /* fall through */
4856         case FIB_EVENT_RULE_DEL:
4857                 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4858                 fib_rule_get(fib_work->fr_info.rule);
4859                 break;
4860         }
4861 }
4862
4863 /* Called with rcu_read_lock() */
4864 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
4865                                      unsigned long event, void *ptr)
4866 {
4867         struct mlxsw_sp_fib_event_work *fib_work;
4868         struct fib_notifier_info *info = ptr;
4869         struct mlxsw_sp_router *router;
4870
4871         if (!net_eq(info->net, &init_net) ||
4872             (info->family != AF_INET && info->family != AF_INET6))
4873                 return NOTIFY_DONE;
4874
4875         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
4876         if (WARN_ON(!fib_work))
4877                 return NOTIFY_BAD;
4878
4879         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
4880         fib_work->mlxsw_sp = router->mlxsw_sp;
4881         fib_work->event = event;
4882
4883         switch (info->family) {
4884         case AF_INET:
4885                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
4886                 mlxsw_sp_router_fib4_event(fib_work, info);
4887                 break;
4888         case AF_INET6:
4889                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
4890                 mlxsw_sp_router_fib6_event(fib_work, info);
4891                 break;
4892         }
4893
4894         mlxsw_core_schedule_work(&fib_work->work);
4895
4896         return NOTIFY_DONE;
4897 }
4898
4899 static struct mlxsw_sp_rif *
4900 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
4901                          const struct net_device *dev)
4902 {
4903         int i;
4904
4905         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
4906                 if (mlxsw_sp->router->rifs[i] &&
4907                     mlxsw_sp->router->rifs[i]->dev == dev)
4908                         return mlxsw_sp->router->rifs[i];
4909
4910         return NULL;
4911 }
4912
4913 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
4914 {
4915         char ritr_pl[MLXSW_REG_RITR_LEN];
4916         int err;
4917
4918         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
4919         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4920         if (WARN_ON_ONCE(err))
4921                 return err;
4922
4923         mlxsw_reg_ritr_enable_set(ritr_pl, false);
4924         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4925 }
4926
4927 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4928                                           struct mlxsw_sp_rif *rif)
4929 {
4930         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
4931         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
4932         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
4933 }
4934
4935 static bool
4936 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
4937                            unsigned long event)
4938 {
4939         struct inet6_dev *inet6_dev;
4940         bool addr_list_empty = true;
4941         struct in_device *idev;
4942
4943         switch (event) {
4944         case NETDEV_UP:
4945                 return rif == NULL;
4946         case NETDEV_DOWN:
4947                 idev = __in_dev_get_rtnl(dev);
4948                 if (idev && idev->ifa_list)
4949                         addr_list_empty = false;
4950
4951                 inet6_dev = __in6_dev_get(dev);
4952                 if (addr_list_empty && inet6_dev &&
4953                     !list_empty(&inet6_dev->addr_list))
4954                         addr_list_empty = false;
4955
4956                 if (rif && addr_list_empty &&
4957                     !netif_is_l3_slave(rif->dev))
4958                         return true;
4959                 /* It is possible we already removed the RIF ourselves
4960                  * if it was assigned to a netdev that is now a bridge
4961                  * or LAG slave.
4962                  */
4963                 return false;
4964         }
4965
4966         return false;
4967 }
4968
4969 static enum mlxsw_sp_rif_type
4970 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
4971                       const struct net_device *dev)
4972 {
4973         enum mlxsw_sp_fid_type type;
4974
4975         if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
4976                 return MLXSW_SP_RIF_TYPE_IPIP_LB;
4977
4978         /* Otherwise RIF type is derived from the type of the underlying FID. */
4979         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
4980                 type = MLXSW_SP_FID_TYPE_8021Q;
4981         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
4982                 type = MLXSW_SP_FID_TYPE_8021Q;
4983         else if (netif_is_bridge_master(dev))
4984                 type = MLXSW_SP_FID_TYPE_8021D;
4985         else
4986                 type = MLXSW_SP_FID_TYPE_RFID;
4987
4988         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
4989 }
4990
4991 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
4992 {
4993         int i;
4994
4995         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
4996                 if (!mlxsw_sp->router->rifs[i]) {
4997                         *p_rif_index = i;
4998                         return 0;
4999                 }
5000         }
5001
5002         return -ENOBUFS;
5003 }
5004
5005 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
5006                                                u16 vr_id,
5007                                                struct net_device *l3_dev)
5008 {
5009         struct mlxsw_sp_rif *rif;
5010
5011         rif = kzalloc(rif_size, GFP_KERNEL);
5012         if (!rif)
5013                 return NULL;
5014
5015         INIT_LIST_HEAD(&rif->nexthop_list);
5016         INIT_LIST_HEAD(&rif->neigh_list);
5017         ether_addr_copy(rif->addr, l3_dev->dev_addr);
5018         rif->mtu = l3_dev->mtu;
5019         rif->vr_id = vr_id;
5020         rif->dev = l3_dev;
5021         rif->rif_index = rif_index;
5022
5023         return rif;
5024 }
5025
5026 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
5027                                            u16 rif_index)
5028 {
5029         return mlxsw_sp->router->rifs[rif_index];
5030 }
5031
5032 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
5033 {
5034         return rif->rif_index;
5035 }
5036
5037 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
5038 {
5039         return lb_rif->common.rif_index;
5040 }
5041
5042 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
5043 {
5044         return lb_rif->ul_vr_id;
5045 }
5046
5047 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
5048 {
5049         return rif->dev->ifindex;
5050 }
5051
5052 static struct mlxsw_sp_rif *
5053 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
5054                     const struct mlxsw_sp_rif_params *params)
5055 {
5056         u32 tb_id = l3mdev_fib_table(params->dev);
5057         const struct mlxsw_sp_rif_ops *ops;
5058         struct mlxsw_sp_fid *fid = NULL;
5059         enum mlxsw_sp_rif_type type;
5060         struct mlxsw_sp_rif *rif;
5061         struct mlxsw_sp_vr *vr;
5062         u16 rif_index;
5063         int err;
5064
5065         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
5066         ops = mlxsw_sp->router->rif_ops_arr[type];
5067
5068         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
5069         if (IS_ERR(vr))
5070                 return ERR_CAST(vr);
5071
5072         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
5073         if (err)
5074                 goto err_rif_index_alloc;
5075
5076         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
5077         if (!rif) {
5078                 err = -ENOMEM;
5079                 goto err_rif_alloc;
5080         }
5081         rif->mlxsw_sp = mlxsw_sp;
5082         rif->ops = ops;
5083
5084         if (ops->fid_get) {
5085                 fid = ops->fid_get(rif);
5086                 if (IS_ERR(fid)) {
5087                         err = PTR_ERR(fid);
5088                         goto err_fid_get;
5089                 }
5090                 rif->fid = fid;
5091         }
5092
5093         if (ops->setup)
5094                 ops->setup(rif, params);
5095
5096         err = ops->configure(rif);
5097         if (err)
5098                 goto err_configure;
5099
5100         mlxsw_sp_rif_counters_alloc(rif);
5101         mlxsw_sp->router->rifs[rif_index] = rif;
5102         vr->rif_count++;
5103
5104         return rif;
5105
5106 err_configure:
5107         if (fid)
5108                 mlxsw_sp_fid_put(fid);
5109 err_fid_get:
5110         kfree(rif);
5111 err_rif_alloc:
5112 err_rif_index_alloc:
5113         mlxsw_sp_vr_put(vr);
5114         return ERR_PTR(err);
5115 }
5116
5117 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
5118 {
5119         const struct mlxsw_sp_rif_ops *ops = rif->ops;
5120         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5121         struct mlxsw_sp_fid *fid = rif->fid;
5122         struct mlxsw_sp_vr *vr;
5123
5124         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
5125         vr = &mlxsw_sp->router->vrs[rif->vr_id];
5126
5127         vr->rif_count--;
5128         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
5129         mlxsw_sp_rif_counters_free(rif);
5130         ops->deconfigure(rif);
5131         if (fid)
5132                 /* Loopback RIFs are not associated with a FID. */
5133                 mlxsw_sp_fid_put(fid);
5134         kfree(rif);
5135         mlxsw_sp_vr_put(vr);
5136 }
5137
5138 static void
5139 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
5140                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
5141 {
5142         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
5143
5144         params->vid = mlxsw_sp_port_vlan->vid;
5145         params->lag = mlxsw_sp_port->lagged;
5146         if (params->lag)
5147                 params->lag_id = mlxsw_sp_port->lag_id;
5148         else
5149                 params->system_port = mlxsw_sp_port->local_port;
5150 }
5151
5152 static int
5153 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
5154                                struct net_device *l3_dev)
5155 {
5156         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
5157         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
5158         u16 vid = mlxsw_sp_port_vlan->vid;
5159         struct mlxsw_sp_rif *rif;
5160         struct mlxsw_sp_fid *fid;
5161         int err;
5162
5163         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5164         if (!rif) {
5165                 struct mlxsw_sp_rif_params params = {
5166                         .dev = l3_dev,
5167                 };
5168
5169                 mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
5170                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
5171                 if (IS_ERR(rif))
5172                         return PTR_ERR(rif);
5173         }
5174
5175         /* FID was already created, just take a reference */
5176         fid = rif->ops->fid_get(rif);
5177         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
5178         if (err)
5179                 goto err_fid_port_vid_map;
5180
5181         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
5182         if (err)
5183                 goto err_port_vid_learning_set;
5184
5185         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
5186                                         BR_STATE_FORWARDING);
5187         if (err)
5188                 goto err_port_vid_stp_set;
5189
5190         mlxsw_sp_port_vlan->fid = fid;
5191
5192         return 0;
5193
5194 err_port_vid_stp_set:
5195         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
5196 err_port_vid_learning_set:
5197         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
5198 err_fid_port_vid_map:
5199         mlxsw_sp_fid_put(fid);
5200         return err;
5201 }
5202
5203 void
5204 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
5205 {
5206         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
5207         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
5208         u16 vid = mlxsw_sp_port_vlan->vid;
5209
5210         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
5211                 return;
5212
5213         mlxsw_sp_port_vlan->fid = NULL;
5214         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
5215         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
5216         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
5217         /* If router port holds the last reference on the rFID, then the
5218          * associated Sub-port RIF will be destroyed.
5219          */
5220         mlxsw_sp_fid_put(fid);
5221 }
5222
5223 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
5224                                              struct net_device *port_dev,
5225                                              unsigned long event, u16 vid)
5226 {
5227         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
5228         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
5229
5230         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
5231         if (WARN_ON(!mlxsw_sp_port_vlan))
5232                 return -EINVAL;
5233
5234         switch (event) {
5235         case NETDEV_UP:
5236                 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
5237                                                       l3_dev);
5238         case NETDEV_DOWN:
5239                 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
5240                 break;
5241         }
5242
5243         return 0;
5244 }
5245
5246 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
5247                                         unsigned long event)
5248 {
5249         if (netif_is_bridge_port(port_dev) ||
5250             netif_is_lag_port(port_dev) ||
5251             netif_is_ovs_port(port_dev))
5252                 return 0;
5253
5254         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
5255 }
5256
5257 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
5258                                          struct net_device *lag_dev,
5259                                          unsigned long event, u16 vid)
5260 {
5261         struct net_device *port_dev;
5262         struct list_head *iter;
5263         int err;
5264
5265         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
5266                 if (mlxsw_sp_port_dev_check(port_dev)) {
5267                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
5268                                                                 port_dev,
5269                                                                 event, vid);
5270                         if (err)
5271                                 return err;
5272                 }
5273         }
5274
5275         return 0;
5276 }
5277
5278 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
5279                                        unsigned long event)
5280 {
5281         if (netif_is_bridge_port(lag_dev))
5282                 return 0;
5283
5284         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
5285 }
5286
5287 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
5288                                           unsigned long event)
5289 {
5290         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
5291         struct mlxsw_sp_rif_params params = {
5292                 .dev = l3_dev,
5293         };
5294         struct mlxsw_sp_rif *rif;
5295
5296         switch (event) {
5297         case NETDEV_UP:
5298                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
5299                 if (IS_ERR(rif))
5300                         return PTR_ERR(rif);
5301                 break;
5302         case NETDEV_DOWN:
5303                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5304                 mlxsw_sp_rif_destroy(rif);
5305                 break;
5306         }
5307
5308         return 0;
5309 }
5310
5311 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
5312                                         unsigned long event)
5313 {
5314         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
5315         u16 vid = vlan_dev_vlan_id(vlan_dev);
5316
5317         if (netif_is_bridge_port(vlan_dev))
5318                 return 0;
5319
5320         if (mlxsw_sp_port_dev_check(real_dev))
5321                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
5322                                                          event, vid);
5323         else if (netif_is_lag_master(real_dev))
5324                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
5325                                                      vid);
5326         else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
5327                 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
5328
5329         return 0;
5330 }
5331
5332 static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
5333                                      unsigned long event)
5334 {
5335         if (mlxsw_sp_port_dev_check(dev))
5336                 return mlxsw_sp_inetaddr_port_event(dev, event);
5337         else if (netif_is_lag_master(dev))
5338                 return mlxsw_sp_inetaddr_lag_event(dev, event);
5339         else if (netif_is_bridge_master(dev))
5340                 return mlxsw_sp_inetaddr_bridge_event(dev, event);
5341         else if (is_vlan_dev(dev))
5342                 return mlxsw_sp_inetaddr_vlan_event(dev, event);
5343         else
5344                 return 0;
5345 }
5346
5347 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
5348                             unsigned long event, void *ptr)
5349 {
5350         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
5351         struct net_device *dev = ifa->ifa_dev->dev;
5352         struct mlxsw_sp *mlxsw_sp;
5353         struct mlxsw_sp_rif *rif;
5354         int err = 0;
5355
5356         mlxsw_sp = mlxsw_sp_lower_get(dev);
5357         if (!mlxsw_sp)
5358                 goto out;
5359
5360         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5361         if (!mlxsw_sp_rif_should_config(rif, dev, event))
5362                 goto out;
5363
5364         err = __mlxsw_sp_inetaddr_event(dev, event);
5365 out:
5366         return notifier_from_errno(err);
5367 }
5368
5369 struct mlxsw_sp_inet6addr_event_work {
5370         struct work_struct work;
5371         struct net_device *dev;
5372         unsigned long event;
5373 };
5374
5375 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
5376 {
5377         struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
5378                 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
5379         struct net_device *dev = inet6addr_work->dev;
5380         unsigned long event = inet6addr_work->event;
5381         struct mlxsw_sp *mlxsw_sp;
5382         struct mlxsw_sp_rif *rif;
5383
5384         rtnl_lock();
5385         mlxsw_sp = mlxsw_sp_lower_get(dev);
5386         if (!mlxsw_sp)
5387                 goto out;
5388
5389         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5390         if (!mlxsw_sp_rif_should_config(rif, dev, event))
5391                 goto out;
5392
5393         __mlxsw_sp_inetaddr_event(dev, event);
5394 out:
5395         rtnl_unlock();
5396         dev_put(dev);
5397         kfree(inet6addr_work);
5398 }
5399
5400 /* Called with rcu_read_lock() */
5401 int mlxsw_sp_inet6addr_event(struct notifier_block *unused,
5402                              unsigned long event, void *ptr)
5403 {
5404         struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
5405         struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
5406         struct net_device *dev = if6->idev->dev;
5407
5408         if (!mlxsw_sp_port_dev_lower_find_rcu(dev))
5409                 return NOTIFY_DONE;
5410
5411         inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
5412         if (!inet6addr_work)
5413                 return NOTIFY_BAD;
5414
5415         INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
5416         inet6addr_work->dev = dev;
5417         inet6addr_work->event = event;
5418         dev_hold(dev);
5419         mlxsw_core_schedule_work(&inet6addr_work->work);
5420
5421         return NOTIFY_DONE;
5422 }
5423
5424 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
5425                              const char *mac, int mtu)
5426 {
5427         char ritr_pl[MLXSW_REG_RITR_LEN];
5428         int err;
5429
5430         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
5431         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5432         if (err)
5433                 return err;
5434
5435         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
5436         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
5437         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
5438         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5439 }
5440
5441 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
5442 {
5443         struct mlxsw_sp *mlxsw_sp;
5444         struct mlxsw_sp_rif *rif;
5445         u16 fid_index;
5446         int err;
5447
5448         mlxsw_sp = mlxsw_sp_lower_get(dev);
5449         if (!mlxsw_sp)
5450                 return 0;
5451
5452         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5453         if (!rif)
5454                 return 0;
5455         fid_index = mlxsw_sp_fid_index(rif->fid);
5456
5457         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
5458         if (err)
5459                 return err;
5460
5461         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
5462                                 dev->mtu);
5463         if (err)
5464                 goto err_rif_edit;
5465
5466         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
5467         if (err)
5468                 goto err_rif_fdb_op;
5469
5470         ether_addr_copy(rif->addr, dev->dev_addr);
5471         rif->mtu = dev->mtu;
5472
5473         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
5474
5475         return 0;
5476
5477 err_rif_fdb_op:
5478         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
5479 err_rif_edit:
5480         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
5481         return err;
5482 }
5483
5484 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
5485                                   struct net_device *l3_dev)
5486 {
5487         struct mlxsw_sp_rif *rif;
5488
5489         /* If netdev is already associated with a RIF, then we need to
5490          * destroy it and create a new one with the new virtual router ID.
5491          */
5492         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5493         if (rif)
5494                 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
5495
5496         return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
5497 }
5498
5499 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
5500                                     struct net_device *l3_dev)
5501 {
5502         struct mlxsw_sp_rif *rif;
5503
5504         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5505         if (!rif)
5506                 return;
5507         __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
5508 }
5509
5510 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
5511                                  struct netdev_notifier_changeupper_info *info)
5512 {
5513         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
5514         int err = 0;
5515
5516         if (!mlxsw_sp)
5517                 return 0;
5518
5519         switch (event) {
5520         case NETDEV_PRECHANGEUPPER:
5521                 return 0;
5522         case NETDEV_CHANGEUPPER:
5523                 if (info->linking)
5524                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
5525                 else
5526                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
5527                 break;
5528         }
5529
5530         return err;
5531 }
5532
5533 static struct mlxsw_sp_rif_subport *
5534 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
5535 {
5536         return container_of(rif, struct mlxsw_sp_rif_subport, common);
5537 }
5538
5539 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
5540                                        const struct mlxsw_sp_rif_params *params)
5541 {
5542         struct mlxsw_sp_rif_subport *rif_subport;
5543
5544         rif_subport = mlxsw_sp_rif_subport_rif(rif);
5545         rif_subport->vid = params->vid;
5546         rif_subport->lag = params->lag;
5547         if (params->lag)
5548                 rif_subport->lag_id = params->lag_id;
5549         else
5550                 rif_subport->system_port = params->system_port;
5551 }
5552
5553 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
5554 {
5555         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5556         struct mlxsw_sp_rif_subport *rif_subport;
5557         char ritr_pl[MLXSW_REG_RITR_LEN];
5558
5559         rif_subport = mlxsw_sp_rif_subport_rif(rif);
5560         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
5561                             rif->rif_index, rif->vr_id, rif->dev->mtu);
5562         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
5563         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
5564                                   rif_subport->lag ? rif_subport->lag_id :
5565                                                      rif_subport->system_port,
5566                                   rif_subport->vid);
5567
5568         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5569 }
5570
5571 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
5572 {
5573         int err;
5574
5575         err = mlxsw_sp_rif_subport_op(rif, true);
5576         if (err)
5577                 return err;
5578
5579         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5580                                   mlxsw_sp_fid_index(rif->fid), true);
5581         if (err)
5582                 goto err_rif_fdb_op;
5583
5584         mlxsw_sp_fid_rif_set(rif->fid, rif);
5585         return 0;
5586
5587 err_rif_fdb_op:
5588         mlxsw_sp_rif_subport_op(rif, false);
5589         return err;
5590 }
5591
5592 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
5593 {
5594         struct mlxsw_sp_fid *fid = rif->fid;
5595
5596         mlxsw_sp_fid_rif_set(fid, NULL);
5597         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5598                             mlxsw_sp_fid_index(fid), false);
5599         mlxsw_sp_rif_subport_op(rif, false);
5600 }
5601
5602 static struct mlxsw_sp_fid *
5603 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
5604 {
5605         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
5606 }
5607
5608 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
5609         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
5610         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
5611         .setup                  = mlxsw_sp_rif_subport_setup,
5612         .configure              = mlxsw_sp_rif_subport_configure,
5613         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
5614         .fid_get                = mlxsw_sp_rif_subport_fid_get,
5615 };
5616
5617 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
5618                                     enum mlxsw_reg_ritr_if_type type,
5619                                     u16 vid_fid, bool enable)
5620 {
5621         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5622         char ritr_pl[MLXSW_REG_RITR_LEN];
5623
5624         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
5625                             rif->dev->mtu);
5626         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
5627         mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
5628
5629         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5630 }
5631
5632 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
5633 {
5634         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
5635 }
5636
5637 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
5638 {
5639         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5640         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5641         int err;
5642
5643         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
5644         if (err)
5645                 return err;
5646
5647         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5648                                      mlxsw_sp_router_port(mlxsw_sp), true);
5649         if (err)
5650                 goto err_fid_mc_flood_set;
5651
5652         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5653                                      mlxsw_sp_router_port(mlxsw_sp), true);
5654         if (err)
5655                 goto err_fid_bc_flood_set;
5656
5657         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5658                                   mlxsw_sp_fid_index(rif->fid), true);
5659         if (err)
5660                 goto err_rif_fdb_op;
5661
5662         mlxsw_sp_fid_rif_set(rif->fid, rif);
5663         return 0;
5664
5665 err_rif_fdb_op:
5666         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5667                                mlxsw_sp_router_port(mlxsw_sp), false);
5668 err_fid_bc_flood_set:
5669         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5670                                mlxsw_sp_router_port(mlxsw_sp), false);
5671 err_fid_mc_flood_set:
5672         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5673         return err;
5674 }
5675
5676 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
5677 {
5678         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5679         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5680         struct mlxsw_sp_fid *fid = rif->fid;
5681
5682         mlxsw_sp_fid_rif_set(fid, NULL);
5683         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5684                             mlxsw_sp_fid_index(fid), false);
5685         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5686                                mlxsw_sp_router_port(mlxsw_sp), false);
5687         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5688                                mlxsw_sp_router_port(mlxsw_sp), false);
5689         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5690 }
5691
5692 static struct mlxsw_sp_fid *
5693 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
5694 {
5695         u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
5696
5697         return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
5698 }
5699
5700 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
5701         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
5702         .rif_size               = sizeof(struct mlxsw_sp_rif),
5703         .configure              = mlxsw_sp_rif_vlan_configure,
5704         .deconfigure            = mlxsw_sp_rif_vlan_deconfigure,
5705         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
5706 };
5707
5708 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
5709 {
5710         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5711         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5712         int err;
5713
5714         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
5715                                        true);
5716         if (err)
5717                 return err;
5718
5719         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5720                                      mlxsw_sp_router_port(mlxsw_sp), true);
5721         if (err)
5722                 goto err_fid_mc_flood_set;
5723
5724         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5725                                      mlxsw_sp_router_port(mlxsw_sp), true);
5726         if (err)
5727                 goto err_fid_bc_flood_set;
5728
5729         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5730                                   mlxsw_sp_fid_index(rif->fid), true);
5731         if (err)
5732                 goto err_rif_fdb_op;
5733
5734         mlxsw_sp_fid_rif_set(rif->fid, rif);
5735         return 0;
5736
5737 err_rif_fdb_op:
5738         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5739                                mlxsw_sp_router_port(mlxsw_sp), false);
5740 err_fid_bc_flood_set:
5741         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5742                                mlxsw_sp_router_port(mlxsw_sp), false);
5743 err_fid_mc_flood_set:
5744         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5745         return err;
5746 }
5747
5748 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
5749 {
5750         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5751         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5752         struct mlxsw_sp_fid *fid = rif->fid;
5753
5754         mlxsw_sp_fid_rif_set(fid, NULL);
5755         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5756                             mlxsw_sp_fid_index(fid), false);
5757         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5758                                mlxsw_sp_router_port(mlxsw_sp), false);
5759         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5760                                mlxsw_sp_router_port(mlxsw_sp), false);
5761         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5762 }
5763
5764 static struct mlxsw_sp_fid *
5765 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
5766 {
5767         return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
5768 }
5769
5770 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
5771         .type                   = MLXSW_SP_RIF_TYPE_FID,
5772         .rif_size               = sizeof(struct mlxsw_sp_rif),
5773         .configure              = mlxsw_sp_rif_fid_configure,
5774         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
5775         .fid_get                = mlxsw_sp_rif_fid_fid_get,
5776 };
5777
5778 static struct mlxsw_sp_rif_ipip_lb *
5779 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
5780 {
5781         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
5782 }
5783
5784 static void
5785 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
5786                            const struct mlxsw_sp_rif_params *params)
5787 {
5788         struct mlxsw_sp_rif_params_ipip_lb *params_lb;
5789         struct mlxsw_sp_rif_ipip_lb *rif_lb;
5790
5791         params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
5792                                  common);
5793         rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
5794         rif_lb->lb_config = params_lb->lb_config;
5795 }
5796
5797 static int
5798 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif,
5799                         struct mlxsw_sp_vr *ul_vr, bool enable)
5800 {
5801         struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
5802         struct mlxsw_sp_rif *rif = &lb_rif->common;
5803         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5804         char ritr_pl[MLXSW_REG_RITR_LEN];
5805         u32 saddr4;
5806
5807         switch (lb_cf.ul_protocol) {
5808         case MLXSW_SP_L3_PROTO_IPV4:
5809                 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
5810                 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
5811                                     rif->rif_index, rif->vr_id, rif->dev->mtu);
5812                 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
5813                             MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
5814                             ul_vr->id, saddr4, lb_cf.okey);
5815                 break;
5816
5817         case MLXSW_SP_L3_PROTO_IPV6:
5818                 return -EAFNOSUPPORT;
5819         }
5820
5821         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5822 }
5823
5824 static int
5825 mlxsw_sp_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
5826 {
5827         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5828         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
5829         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5830         struct mlxsw_sp_vr *ul_vr;
5831         int err;
5832
5833         ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id);
5834         if (IS_ERR(ul_vr))
5835                 return PTR_ERR(ul_vr);
5836
5837         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, true);
5838         if (err)
5839                 goto err_loopback_op;
5840
5841         lb_rif->ul_vr_id = ul_vr->id;
5842         ++ul_vr->rif_count;
5843         return 0;
5844
5845 err_loopback_op:
5846         mlxsw_sp_vr_put(ul_vr);
5847         return err;
5848 }
5849
5850 static void mlxsw_sp_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
5851 {
5852         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5853         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5854         struct mlxsw_sp_vr *ul_vr;
5855
5856         ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
5857         mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, false);
5858
5859         --ul_vr->rif_count;
5860         mlxsw_sp_vr_put(ul_vr);
5861 }
5862
5863 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_ipip_lb_ops = {
5864         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
5865         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
5866         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
5867         .configure              = mlxsw_sp_rif_ipip_lb_configure,
5868         .deconfigure            = mlxsw_sp_rif_ipip_lb_deconfigure,
5869 };
5870
5871 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
5872         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
5873         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_ops,
5874         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
5875         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp_rif_ipip_lb_ops,
5876 };
5877
5878 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
5879 {
5880         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5881
5882         mlxsw_sp->router->rifs = kcalloc(max_rifs,
5883                                          sizeof(struct mlxsw_sp_rif *),
5884                                          GFP_KERNEL);
5885         if (!mlxsw_sp->router->rifs)
5886                 return -ENOMEM;
5887
5888         mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
5889
5890         return 0;
5891 }
5892
5893 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
5894 {
5895         int i;
5896
5897         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
5898                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
5899
5900         kfree(mlxsw_sp->router->rifs);
5901 }
5902
5903 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
5904 {
5905         mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
5906         INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
5907         return 0;
5908 }
5909
5910 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
5911 {
5912         WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
5913 }
5914
5915 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
5916 {
5917         struct mlxsw_sp_router *router;
5918
5919         /* Flush pending FIB notifications and then flush the device's
5920          * table before requesting another dump. The FIB notification
5921          * block is unregistered, so no need to take RTNL.
5922          */
5923         mlxsw_core_flush_owq();
5924         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
5925         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
5926 }
5927
5928 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5929 {
5930         char rgcr_pl[MLXSW_REG_RGCR_LEN];
5931         u64 max_rifs;
5932         int err;
5933
5934         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
5935                 return -EIO;
5936         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5937
5938         mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
5939         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
5940         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
5941         if (err)
5942                 return err;
5943         return 0;
5944 }
5945
5946 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
5947 {
5948         char rgcr_pl[MLXSW_REG_RGCR_LEN];
5949
5950         mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
5951         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
5952 }
5953
5954 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5955 {
5956         struct mlxsw_sp_router *router;
5957         int err;
5958
5959         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
5960         if (!router)
5961                 return -ENOMEM;
5962         mlxsw_sp->router = router;
5963         router->mlxsw_sp = mlxsw_sp;
5964
5965         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
5966         err = __mlxsw_sp_router_init(mlxsw_sp);
5967         if (err)
5968                 goto err_router_init;
5969
5970         err = mlxsw_sp_rifs_init(mlxsw_sp);
5971         if (err)
5972                 goto err_rifs_init;
5973
5974         err = mlxsw_sp_ipips_init(mlxsw_sp);
5975         if (err)
5976                 goto err_ipips_init;
5977
5978         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
5979                               &mlxsw_sp_nexthop_ht_params);
5980         if (err)
5981                 goto err_nexthop_ht_init;
5982
5983         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
5984                               &mlxsw_sp_nexthop_group_ht_params);
5985         if (err)
5986                 goto err_nexthop_group_ht_init;
5987
5988         err = mlxsw_sp_lpm_init(mlxsw_sp);
5989         if (err)
5990                 goto err_lpm_init;
5991
5992         err = mlxsw_sp_vrs_init(mlxsw_sp);
5993         if (err)
5994                 goto err_vrs_init;
5995
5996         err = mlxsw_sp_neigh_init(mlxsw_sp);
5997         if (err)
5998                 goto err_neigh_init;
5999
6000         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
6001         err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
6002                                     mlxsw_sp_router_fib_dump_flush);
6003         if (err)
6004                 goto err_register_fib_notifier;
6005
6006         return 0;
6007
6008 err_register_fib_notifier:
6009         mlxsw_sp_neigh_fini(mlxsw_sp);
6010 err_neigh_init:
6011         mlxsw_sp_vrs_fini(mlxsw_sp);
6012 err_vrs_init:
6013         mlxsw_sp_lpm_fini(mlxsw_sp);
6014 err_lpm_init:
6015         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
6016 err_nexthop_group_ht_init:
6017         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
6018 err_nexthop_ht_init:
6019         mlxsw_sp_ipips_fini(mlxsw_sp);
6020 err_ipips_init:
6021         mlxsw_sp_rifs_fini(mlxsw_sp);
6022 err_rifs_init:
6023         __mlxsw_sp_router_fini(mlxsw_sp);
6024 err_router_init:
6025         kfree(mlxsw_sp->router);
6026         return err;
6027 }
6028
6029 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
6030 {
6031         unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
6032         mlxsw_sp_neigh_fini(mlxsw_sp);
6033         mlxsw_sp_vrs_fini(mlxsw_sp);
6034         mlxsw_sp_lpm_fini(mlxsw_sp);
6035         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
6036         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
6037         mlxsw_sp_ipips_fini(mlxsw_sp);
6038         mlxsw_sp_rifs_fini(mlxsw_sp);
6039         __mlxsw_sp_router_fini(mlxsw_sp);
6040         kfree(mlxsw_sp->router);
6041 }