drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/types.h>
   6 #include <linux/rhashtable.h>
   7 #include <linux/bitops.h>
   8 #include <linux/in6.h>
   9 #include <linux/notifier.h>
  10 #include <linux/inetdevice.h>
  11 #include <linux/netdevice.h>
  12 #include <linux/if_bridge.h>
  13 #include <linux/socket.h>
  14 #include <linux/route.h>
  15 #include <linux/gcd.h>
  16 #include <linux/if_macvlan.h>
  17 #include <linux/refcount.h>
  18 #include <linux/jhash.h>
  19 #include <linux/net_namespace.h>
  20 #include <linux/mutex.h>
  21 #include <net/netevent.h>
  22 #include <net/neighbour.h>
  23 #include <net/arp.h>
  24 #include <net/ip_fib.h>
  25 #include <net/ip6_fib.h>
  26 #include <net/nexthop.h>
  27 #include <net/fib_rules.h>
  28 #include <net/ip_tunnels.h>
  29 #include <net/l3mdev.h>
  30 #include <net/addrconf.h>
  31 #include <net/ndisc.h>
  32 #include <net/ipv6.h>
  33 #include <net/fib_notifier.h>
  34 #include <net/switchdev.h>
  35
  36 #include "spectrum.h"
  37 #include "core.h"
  38 #include "reg.h"
  39 #include "spectrum_cnt.h"
  40 #include "spectrum_dpipe.h"
  41 #include "spectrum_ipip.h"
  42 #include "spectrum_mr.h"
  43 #include "spectrum_mr_tcam.h"
  44 #include "spectrum_router.h"
  45 #include "spectrum_span.h"
  46
  47 struct mlxsw_sp_fib;
  48 struct mlxsw_sp_vr;
  49 struct mlxsw_sp_lpm_tree;
  50 struct mlxsw_sp_rif_ops;
  51
  52 struct mlxsw_sp_rif {
  53         struct list_head nexthop_list;
  54         struct list_head neigh_list;
  55         struct net_device *dev; /* NULL for underlay RIF */
  56         struct mlxsw_sp_fid *fid;
  57         unsigned char addr[ETH_ALEN];
  58         int mtu;
  59         u16 rif_index;
  60         u16 vr_id;
  61         const struct mlxsw_sp_rif_ops *ops;
  62         struct mlxsw_sp *mlxsw_sp;
  63
  64         unsigned int counter_ingress;
  65         bool counter_ingress_valid;
  66         unsigned int counter_egress;
  67         bool counter_egress_valid;
  68 };
  69
  70 struct mlxsw_sp_rif_params {
  71         struct net_device *dev;
  72         union {
  73                 u16 system_port;
  74                 u16 lag_id;
  75         };
  76         u16 vid;
  77         bool lag;
  78 };
  79
  80 struct mlxsw_sp_rif_subport {
  81         struct mlxsw_sp_rif common;
  82         refcount_t ref_count;
  83         union {
  84                 u16 system_port;
  85                 u16 lag_id;
  86         };
  87         u16 vid;
  88         bool lag;
  89 };
  90
  91 struct mlxsw_sp_rif_ipip_lb {
  92         struct mlxsw_sp_rif common;
  93         struct mlxsw_sp_rif_ipip_lb_config lb_config;
  94         u16 ul_vr_id; /* Reserved for Spectrum-2. */
  95         u16 ul_rif_id; /* Reserved for Spectrum. */
  96 };
  97
  98 struct mlxsw_sp_rif_params_ipip_lb {
  99         struct mlxsw_sp_rif_params common;
 100         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 101 };
 102
 103 struct mlxsw_sp_rif_ops {
 104         enum mlxsw_sp_rif_type type;
 105         size_t rif_size;
 106
 107         void (*setup)(struct mlxsw_sp_rif *rif,
 108                       const struct mlxsw_sp_rif_params *params);
 109         int (*configure)(struct mlxsw_sp_rif *rif);
 110         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 111         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif,
 112                                          struct netlink_ext_ack *extack);
 113         void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac);
 114 };
 115
 116 static struct mlxsw_sp_rif *
 117 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 118                          const struct net_device *dev);
 119 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif);
 120 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree);
 121 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 122                                   struct mlxsw_sp_lpm_tree *lpm_tree);
 123 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 124                                      const struct mlxsw_sp_fib *fib,
 125                                      u8 tree_id);
 126 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 127                                        const struct mlxsw_sp_fib *fib);
 128
 129 static unsigned int *
 130 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 131                            enum mlxsw_sp_rif_counter_dir dir)
 132 {
 133         switch (dir) {
 134         case MLXSW_SP_RIF_COUNTER_EGRESS:
 135                 return &rif->counter_egress;
 136         case MLXSW_SP_RIF_COUNTER_INGRESS:
 137                 return &rif->counter_ingress;
 138         }
 139         return NULL;
 140 }
 141
 142 static bool
 143 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 144                                enum mlxsw_sp_rif_counter_dir dir)
 145 {
 146         switch (dir) {
 147         case MLXSW_SP_RIF_COUNTER_EGRESS:
 148                 return rif->counter_egress_valid;
 149         case MLXSW_SP_RIF_COUNTER_INGRESS:
 150                 return rif->counter_ingress_valid;
 151         }
 152         return false;
 153 }
 154
 155 static void
 156 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 157                                enum mlxsw_sp_rif_counter_dir dir,
 158                                bool valid)
 159 {
 160         switch (dir) {
 161         case MLXSW_SP_RIF_COUNTER_EGRESS:
 162                 rif->counter_egress_valid = valid;
 163                 break;
 164         case MLXSW_SP_RIF_COUNTER_INGRESS:
 165                 rif->counter_ingress_valid = valid;
 166                 break;
 167         }
 168 }
 169
 170 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 171                                      unsigned int counter_index, bool enable,
 172                                      enum mlxsw_sp_rif_counter_dir dir)
 173 {
 174         char ritr_pl[MLXSW_REG_RITR_LEN];
 175         bool is_egress = false;
 176         int err;
 177
 178         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 179                 is_egress = true;
 180         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 181         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 182         if (err)
 183                 return err;
 184
 185         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 186                                     is_egress);
 187         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 188 }
 189
 190 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 191                                    struct mlxsw_sp_rif *rif,
 192                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 193 {
 194         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 195         unsigned int *p_counter_index;
 196         bool valid;
 197         int err;
 198
 199         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 200         if (!valid)
 201                 return -EINVAL;
 202
 203         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 204         if (!p_counter_index)
 205                 return -EINVAL;
 206         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 207                              MLXSW_REG_RICNT_OPCODE_NOP);
 208         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 209         if (err)
 210                 return err;
 211         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 212         return 0;
 213 }
 214
 215 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 216                                       unsigned int counter_index)
 217 {
 218         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 219
 220         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 221                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 222         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 223 }
 224
 225 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 226                                struct mlxsw_sp_rif *rif,
 227                                enum mlxsw_sp_rif_counter_dir dir)
 228 {
 229         unsigned int *p_counter_index;
 230         int err;
 231
 232         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 233         if (!p_counter_index)
 234                 return -EINVAL;
 235         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 236                                      p_counter_index);
 237         if (err)
 238                 return err;
 239
 240         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 241         if (err)
 242                 goto err_counter_clear;
 243
 244         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 245                                         *p_counter_index, true, dir);
 246         if (err)
 247                 goto err_counter_edit;
 248         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 249         return 0;
 250
 251 err_counter_edit:
 252 err_counter_clear:
 253         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 254                               *p_counter_index);
 255         return err;
 256 }
 257
 258 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 259                                struct mlxsw_sp_rif *rif,
 260                                enum mlxsw_sp_rif_counter_dir dir)
 261 {
 262         unsigned int *p_counter_index;
 263
 264         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 265                 return;
 266
 267         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 268         if (WARN_ON(!p_counter_index))
 269                 return;
 270         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 271                                   *p_counter_index, false, dir);
 272         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 273                               *p_counter_index);
 274         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 275 }
 276
 277 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 278 {
 279         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 280         struct devlink *devlink;
 281
 282         devlink = priv_to_devlink(mlxsw_sp->core);
 283         if (!devlink_dpipe_table_counter_enabled(devlink,
 284                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 285                 return;
 286         mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 287 }
 288
 289 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 290 {
 291         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 292
 293         mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 294 }
 295
 296 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
 297
 298 struct mlxsw_sp_prefix_usage {
 299         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 300 };
 301
 302 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 303         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 304
 305 static bool
 306 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 307                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 308 {
 309         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 310 }
 311
 312 static void
 313 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 314                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 315 {
 316         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 317 }
 318
 319 static void
 320 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 321                           unsigned char prefix_len)
 322 {
 323         set_bit(prefix_len, prefix_usage->b);
 324 }
 325
 326 static void
 327 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 328                             unsigned char prefix_len)
 329 {
 330         clear_bit(prefix_len, prefix_usage->b);
 331 }
 332
 333 struct mlxsw_sp_fib_key {
 334         unsigned char addr[sizeof(struct in6_addr)];
 335         unsigned char prefix_len;
 336 };
 337
 338 enum mlxsw_sp_fib_entry_type {
 339         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 340         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 341         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 342         MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE,
 343         MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE,
 344
 345         /* This is a special case of local delivery, where a packet should be
 346          * decapsulated on reception. Note that there is no corresponding ENCAP,
 347          * because that's a type of next hop, not of FIB entry. (There can be
 348          * several next hops in a REMOTE entry, and some of them may be
 349          * encapsulating entries.)
 350          */
 351         MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
 352         MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP,
 353 };
 354
 355 struct mlxsw_sp_nexthop_group_info;
 356 struct mlxsw_sp_nexthop_group;
 357 struct mlxsw_sp_fib_entry;
 358
 359 struct mlxsw_sp_fib_node {
 360         struct mlxsw_sp_fib_entry *fib_entry;
 361         struct list_head list;
 362         struct rhash_head ht_node;
 363         struct mlxsw_sp_fib *fib;
 364         struct mlxsw_sp_fib_key key;
 365 };
 366
 367 struct mlxsw_sp_fib_entry_decap {
 368         struct mlxsw_sp_ipip_entry *ipip_entry;
 369         u32 tunnel_index;
 370 };
 371
 372 static struct mlxsw_sp_fib_entry_priv *
 373 mlxsw_sp_fib_entry_priv_create(const struct mlxsw_sp_router_ll_ops *ll_ops)
 374 {
 375         struct mlxsw_sp_fib_entry_priv *priv;
 376
 377         if (!ll_ops->fib_entry_priv_size)
 378                 /* No need to have priv */
 379                 return NULL;
 380
 381         priv = kzalloc(sizeof(*priv) + ll_ops->fib_entry_priv_size, GFP_KERNEL);
 382         if (!priv)
 383                 return ERR_PTR(-ENOMEM);
 384         refcount_set(&priv->refcnt, 1);
 385         return priv;
 386 }
 387
 388 static void
 389 mlxsw_sp_fib_entry_priv_destroy(struct mlxsw_sp_fib_entry_priv *priv)
 390 {
 391         kfree(priv);
 392 }
 393
 394 static void mlxsw_sp_fib_entry_priv_hold(struct mlxsw_sp_fib_entry_priv *priv)
 395 {
 396         refcount_inc(&priv->refcnt);
 397 }
 398
 399 static void mlxsw_sp_fib_entry_priv_put(struct mlxsw_sp_fib_entry_priv *priv)
 400 {
 401         if (!priv || !refcount_dec_and_test(&priv->refcnt))
 402                 return;
 403         mlxsw_sp_fib_entry_priv_destroy(priv);
 404 }
 405
 406 static void mlxsw_sp_fib_entry_op_ctx_priv_hold(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
 407                                                 struct mlxsw_sp_fib_entry_priv *priv)
 408 {
 409         if (!priv)
 410                 return;
 411         mlxsw_sp_fib_entry_priv_hold(priv);
 412         list_add(&priv->list, &op_ctx->fib_entry_priv_list);
 413 }
 414
 415 static void mlxsw_sp_fib_entry_op_ctx_priv_put_all(struct mlxsw_sp_fib_entry_op_ctx *op_ctx)
 416 {
 417         struct mlxsw_sp_fib_entry_priv *priv, *tmp;
 418
 419         list_for_each_entry_safe(priv, tmp, &op_ctx->fib_entry_priv_list, list)
 420                 mlxsw_sp_fib_entry_priv_put(priv);
 421         INIT_LIST_HEAD(&op_ctx->fib_entry_priv_list);
 422 }
 423
 424 struct mlxsw_sp_fib_entry {
 425         struct mlxsw_sp_fib_node *fib_node;
 426         enum mlxsw_sp_fib_entry_type type;
 427         struct list_head nexthop_group_node;
 428         struct mlxsw_sp_nexthop_group *nh_group;
 429         struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
 430         struct mlxsw_sp_fib_entry_priv *priv;
 431 };
 432
 433 struct mlxsw_sp_fib4_entry {
 434         struct mlxsw_sp_fib_entry common;
 435         struct fib_info *fi;
 436         u32 tb_id;
 437         u8 tos;
 438         u8 type;
 439 };
 440
 441 struct mlxsw_sp_fib6_entry {
 442         struct mlxsw_sp_fib_entry common;
 443         struct list_head rt6_list;
 444         unsigned int nrt6;
 445 };
 446
 447 struct mlxsw_sp_rt6 {
 448         struct list_head list;
 449         struct fib6_info *rt;
 450 };
 451
 452 struct mlxsw_sp_lpm_tree {
 453         u8 id; /* tree ID */
 454         unsigned int ref_count;
 455         enum mlxsw_sp_l3proto proto;
 456         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 457         struct mlxsw_sp_prefix_usage prefix_usage;
 458 };
 459
 460 struct mlxsw_sp_fib {
 461         struct rhashtable ht;
 462         struct list_head node_list;
 463         struct mlxsw_sp_vr *vr;
 464         struct mlxsw_sp_lpm_tree *lpm_tree;
 465         enum mlxsw_sp_l3proto proto;
 466         const struct mlxsw_sp_router_ll_ops *ll_ops;
 467 };
 468
 469 struct mlxsw_sp_vr {
 470         u16 id; /* virtual router ID */
 471         u32 tb_id; /* kernel fib table id */
 472         unsigned int rif_count;
 473         struct mlxsw_sp_fib *fib4;
 474         struct mlxsw_sp_fib *fib6;
 475         struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX];
 476         struct mlxsw_sp_rif *ul_rif;
 477         refcount_t ul_rif_refcnt;
 478 };
 479
 480 static int mlxsw_sp_router_ll_basic_ralta_write(struct mlxsw_sp *mlxsw_sp, char *xralta_pl)
 481 {
 482         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta),
 483                                xralta_pl + MLXSW_REG_XRALTA_RALTA_OFFSET);
 484 }
 485
 486 static int mlxsw_sp_router_ll_basic_ralst_write(struct mlxsw_sp *mlxsw_sp, char *xralst_pl)
 487 {
 488         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst),
 489                                xralst_pl + MLXSW_REG_XRALST_RALST_OFFSET);
 490 }
 491
 492 static int mlxsw_sp_router_ll_basic_raltb_write(struct mlxsw_sp *mlxsw_sp, char *xraltb_pl)
 493 {
 494         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
 495                                xraltb_pl + MLXSW_REG_XRALTB_RALTB_OFFSET);
 496 }
 497
 498 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 499
 500 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp,
 501                                                 struct mlxsw_sp_vr *vr,
 502                                                 enum mlxsw_sp_l3proto proto)
 503 {
 504         const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
 505         struct mlxsw_sp_lpm_tree *lpm_tree;
 506         struct mlxsw_sp_fib *fib;
 507         int err;
 508
 509         lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 510         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 511         if (!fib)
 512                 return ERR_PTR(-ENOMEM);
 513         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 514         if (err)
 515                 goto err_rhashtable_init;
 516         INIT_LIST_HEAD(&fib->node_list);
 517         fib->proto = proto;
 518         fib->vr = vr;
 519         fib->lpm_tree = lpm_tree;
 520         fib->ll_ops = ll_ops;
 521         mlxsw_sp_lpm_tree_hold(lpm_tree);
 522         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id);
 523         if (err)
 524                 goto err_lpm_tree_bind;
 525         return fib;
 526
 527 err_lpm_tree_bind:
 528         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 529 err_rhashtable_init:
 530         kfree(fib);
 531         return ERR_PTR(err);
 532 }
 533
 534 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp,
 535                                  struct mlxsw_sp_fib *fib)
 536 {
 537         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
 538         mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
 539         WARN_ON(!list_empty(&fib->node_list));
 540         rhashtable_destroy(&fib->ht);
 541         kfree(fib);
 542 }
 543
 544 static struct mlxsw_sp_lpm_tree *
 545 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 546 {
 547         static struct mlxsw_sp_lpm_tree *lpm_tree;
 548         int i;
 549
 550         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 551                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 552                 if (lpm_tree->ref_count == 0)
 553                         return lpm_tree;
 554         }
 555         return NULL;
 556 }
 557
 558 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 559                                    const struct mlxsw_sp_router_ll_ops *ll_ops,
 560                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 561 {
 562         char xralta_pl[MLXSW_REG_XRALTA_LEN];
 563
 564         mlxsw_reg_xralta_pack(xralta_pl, true,
 565                               (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 566                               lpm_tree->id);
 567         return ll_ops->ralta_write(mlxsw_sp, xralta_pl);
 568 }
 569
 570 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 571                                    const struct mlxsw_sp_router_ll_ops *ll_ops,
 572                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 573 {
 574         char xralta_pl[MLXSW_REG_XRALTA_LEN];
 575
 576         mlxsw_reg_xralta_pack(xralta_pl, false,
 577                               (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 578                               lpm_tree->id);
 579         ll_ops->ralta_write(mlxsw_sp, xralta_pl);
 580 }
 581
 582 static int
 583 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 584                                   const struct mlxsw_sp_router_ll_ops *ll_ops,
 585                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 586                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 587 {
 588         char xralst_pl[MLXSW_REG_XRALST_LEN];
 589         u8 root_bin = 0;
 590         u8 prefix;
 591         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 592
 593         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 594                 root_bin = prefix;
 595
 596         mlxsw_reg_xralst_pack(xralst_pl, root_bin, lpm_tree->id);
 597         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 598                 if (prefix == 0)
 599                         continue;
 600                 mlxsw_reg_xralst_bin_pack(xralst_pl, prefix, last_prefix,
 601                                           MLXSW_REG_RALST_BIN_NO_CHILD);
 602                 last_prefix = prefix;
 603         }
 604         return ll_ops->ralst_write(mlxsw_sp, xralst_pl);
 605 }
 606
 607 static struct mlxsw_sp_lpm_tree *
 608 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 609                          const struct mlxsw_sp_router_ll_ops *ll_ops,
 610                          struct mlxsw_sp_prefix_usage *prefix_usage,
 611                          enum mlxsw_sp_l3proto proto)
 612 {
 613         struct mlxsw_sp_lpm_tree *lpm_tree;
 614         int err;
 615
 616         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 617         if (!lpm_tree)
 618                 return ERR_PTR(-EBUSY);
 619         lpm_tree->proto = proto;
 620         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, ll_ops, lpm_tree);
 621         if (err)
 622                 return ERR_PTR(err);
 623
 624         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, ll_ops, prefix_usage, lpm_tree);
 625         if (err)
 626                 goto err_left_struct_set;
 627         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 628                sizeof(lpm_tree->prefix_usage));
 629         memset(&lpm_tree->prefix_ref_count, 0,
 630                sizeof(lpm_tree->prefix_ref_count));
 631         lpm_tree->ref_count = 1;
 632         return lpm_tree;
 633
 634 err_left_struct_set:
 635         mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree);
 636         return ERR_PTR(err);
 637 }
 638
 639 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 640                                       const struct mlxsw_sp_router_ll_ops *ll_ops,
 641                                       struct mlxsw_sp_lpm_tree *lpm_tree)
 642 {
 643         mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree);
 644 }
 645
 646 static struct mlxsw_sp_lpm_tree *
 647 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 648                       struct mlxsw_sp_prefix_usage *prefix_usage,
 649                       enum mlxsw_sp_l3proto proto)
 650 {
 651         const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
 652         struct mlxsw_sp_lpm_tree *lpm_tree;
 653         int i;
 654
 655         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 656                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 657                 if (lpm_tree->ref_count != 0 &&
 658                     lpm_tree->proto == proto &&
 659                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 660                                              prefix_usage)) {
 661                         mlxsw_sp_lpm_tree_hold(lpm_tree);
 662                         return lpm_tree;
 663                 }
 664         }
 665         return mlxsw_sp_lpm_tree_create(mlxsw_sp, ll_ops, prefix_usage, proto);
 666 }
 667
 668 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
 669 {
 670         lpm_tree->ref_count++;
 671 }
 672
 673 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 674                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 675 {
 676         const struct mlxsw_sp_router_ll_ops *ll_ops =
 677                                 mlxsw_sp->router->proto_ll_ops[lpm_tree->proto];
 678
 679         if (--lpm_tree->ref_count == 0)
 680                 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, ll_ops, lpm_tree);
 681 }
 682
 683 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 684
 685 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 686 {
 687         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
 688         struct mlxsw_sp_lpm_tree *lpm_tree;
 689         u64 max_trees;
 690         int err, i;
 691
 692         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 693                 return -EIO;
 694
 695         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 696         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 697         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 698                                              sizeof(struct mlxsw_sp_lpm_tree),
 699                                              GFP_KERNEL);
 700         if (!mlxsw_sp->router->lpm.trees)
 701                 return -ENOMEM;
 702
 703         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 704                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 705                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 706         }
 707
 708         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 709                                          MLXSW_SP_L3_PROTO_IPV4);
 710         if (IS_ERR(lpm_tree)) {
 711                 err = PTR_ERR(lpm_tree);
 712                 goto err_ipv4_tree_get;
 713         }
 714         mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree;
 715
 716         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 717                                          MLXSW_SP_L3_PROTO_IPV6);
 718         if (IS_ERR(lpm_tree)) {
 719                 err = PTR_ERR(lpm_tree);
 720                 goto err_ipv6_tree_get;
 721         }
 722         mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree;
 723
 724         return 0;
 725
 726 err_ipv6_tree_get:
 727         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
 728         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 729 err_ipv4_tree_get:
 730         kfree(mlxsw_sp->router->lpm.trees);
 731         return err;
 732 }
 733
 734 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 735 {
 736         struct mlxsw_sp_lpm_tree *lpm_tree;
 737
 738         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6];
 739         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 740
 741         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
 742         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 743
 744         kfree(mlxsw_sp->router->lpm.trees);
 745 }
 746
 747 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 748 {
 749         return !!vr->fib4 || !!vr->fib6 ||
 750                !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] ||
 751                !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
 752 }
 753
 754 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 755 {
 756         struct mlxsw_sp_vr *vr;
 757         int i;
 758
 759         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 760                 vr = &mlxsw_sp->router->vrs[i];
 761                 if (!mlxsw_sp_vr_is_used(vr))
 762                         return vr;
 763         }
 764         return NULL;
 765 }
 766
 767 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 768                                      const struct mlxsw_sp_fib *fib, u8 tree_id)
 769 {
 770         char xraltb_pl[MLXSW_REG_XRALTB_LEN];
 771
 772         mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id,
 773                               (enum mlxsw_reg_ralxx_protocol) fib->proto,
 774                               tree_id);
 775         return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
 776 }
 777
 778 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 779                                        const struct mlxsw_sp_fib *fib)
 780 {
 781         char xraltb_pl[MLXSW_REG_XRALTB_LEN];
 782
 783         /* Bind to tree 0 which is default */
 784         mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id,
 785                               (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 786         return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
 787 }
 788
 789 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 790 {
 791         /* For our purpose, squash main, default and local tables into one */
 792         if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT)
 793                 tb_id = RT_TABLE_MAIN;
 794         return tb_id;
 795 }
 796
 797 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 798                                             u32 tb_id)
 799 {
 800         struct mlxsw_sp_vr *vr;
 801         int i;
 802
 803         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 804
 805         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 806                 vr = &mlxsw_sp->router->vrs[i];
 807                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 808                         return vr;
 809         }
 810         return NULL;
 811 }
 812
 813 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
 814                                 u16 *vr_id)
 815 {
 816         struct mlxsw_sp_vr *vr;
 817         int err = 0;
 818
 819         mutex_lock(&mlxsw_sp->router->lock);
 820         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 821         if (!vr) {
 822                 err = -ESRCH;
 823                 goto out;
 824         }
 825         *vr_id = vr->id;
 826 out:
 827         mutex_unlock(&mlxsw_sp->router->lock);
 828         return err;
 829 }
 830
 831 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 832                                             enum mlxsw_sp_l3proto proto)
 833 {
 834         switch (proto) {
 835         case MLXSW_SP_L3_PROTO_IPV4:
 836                 return vr->fib4;
 837         case MLXSW_SP_L3_PROTO_IPV6:
 838                 return vr->fib6;
 839         }
 840         return NULL;
 841 }
 842
 843 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 844                                               u32 tb_id,
 845                                               struct netlink_ext_ack *extack)
 846 {
 847         struct mlxsw_sp_mr_table *mr4_table, *mr6_table;
 848         struct mlxsw_sp_fib *fib4;
 849         struct mlxsw_sp_fib *fib6;
 850         struct mlxsw_sp_vr *vr;
 851         int err;
 852
 853         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 854         if (!vr) {
 855                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers");
 856                 return ERR_PTR(-EBUSY);
 857         }
 858         fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
 859         if (IS_ERR(fib4))
 860                 return ERR_CAST(fib4);
 861         fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
 862         if (IS_ERR(fib6)) {
 863                 err = PTR_ERR(fib6);
 864                 goto err_fib6_create;
 865         }
 866         mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
 867                                              MLXSW_SP_L3_PROTO_IPV4);
 868         if (IS_ERR(mr4_table)) {
 869                 err = PTR_ERR(mr4_table);
 870                 goto err_mr4_table_create;
 871         }
 872         mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
 873                                              MLXSW_SP_L3_PROTO_IPV6);
 874         if (IS_ERR(mr6_table)) {
 875                 err = PTR_ERR(mr6_table);
 876                 goto err_mr6_table_create;
 877         }
 878
 879         vr->fib4 = fib4;
 880         vr->fib6 = fib6;
 881         vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table;
 882         vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table;
 883         vr->tb_id = tb_id;
 884         return vr;
 885
 886 err_mr6_table_create:
 887         mlxsw_sp_mr_table_destroy(mr4_table);
 888 err_mr4_table_create:
 889         mlxsw_sp_fib_destroy(mlxsw_sp, fib6);
 890 err_fib6_create:
 891         mlxsw_sp_fib_destroy(mlxsw_sp, fib4);
 892         return ERR_PTR(err);
 893 }
 894
 895 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
 896                                 struct mlxsw_sp_vr *vr)
 897 {
 898         mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]);
 899         vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL;
 900         mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]);
 901         vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL;
 902         mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6);
 903         vr->fib6 = NULL;
 904         mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4);
 905         vr->fib4 = NULL;
 906 }
 907
 908 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
 909                                            struct netlink_ext_ack *extack)
 910 {
 911         struct mlxsw_sp_vr *vr;
 912
 913         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 914         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 915         if (!vr)
 916                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack);
 917         return vr;
 918 }
 919
 920 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
 921 {
 922         if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
 923             list_empty(&vr->fib6->node_list) &&
 924             mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) &&
 925             mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]))
 926                 mlxsw_sp_vr_destroy(mlxsw_sp, vr);
 927 }
 928
 929 static bool
 930 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
 931                                     enum mlxsw_sp_l3proto proto, u8 tree_id)
 932 {
 933         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 934
 935         if (!mlxsw_sp_vr_is_used(vr))
 936                 return false;
 937         if (fib->lpm_tree->id == tree_id)
 938                 return true;
 939         return false;
 940 }
 941
 942 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 943                                         struct mlxsw_sp_fib *fib,
 944                                         struct mlxsw_sp_lpm_tree *new_tree)
 945 {
 946         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 947         int err;
 948
 949         fib->lpm_tree = new_tree;
 950         mlxsw_sp_lpm_tree_hold(new_tree);
 951         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 952         if (err)
 953                 goto err_tree_bind;
 954         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 955         return 0;
 956
 957 err_tree_bind:
 958         mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 959         fib->lpm_tree = old_tree;
 960         return err;
 961 }
 962
 963 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 964                                          struct mlxsw_sp_fib *fib,
 965                                          struct mlxsw_sp_lpm_tree *new_tree)
 966 {
 967         enum mlxsw_sp_l3proto proto = fib->proto;
 968         struct mlxsw_sp_lpm_tree *old_tree;
 969         u8 old_id, new_id = new_tree->id;
 970         struct mlxsw_sp_vr *vr;
 971         int i, err;
 972
 973         old_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 974         old_id = old_tree->id;
 975
 976         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 977                 vr = &mlxsw_sp->router->vrs[i];
 978                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
 979                         continue;
 980                 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 981                                                    mlxsw_sp_vr_fib(vr, proto),
 982                                                    new_tree);
 983                 if (err)
 984                         goto err_tree_replace;
 985         }
 986
 987         memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count,
 988                sizeof(new_tree->prefix_ref_count));
 989         mlxsw_sp->router->lpm.proto_trees[proto] = new_tree;
 990         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 991
 992         return 0;
 993
 994 err_tree_replace:
 995         for (i--; i >= 0; i--) {
 996                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
 997                         continue;
 998                 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 999                                              mlxsw_sp_vr_fib(vr, proto),
1000                                              old_tree);
1001         }
1002         return err;
1003 }
1004
1005 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
1006 {
1007         struct mlxsw_sp_vr *vr;
1008         u64 max_vrs;
1009         int i;
1010
1011         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
1012                 return -EIO;
1013
1014         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
1015         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
1016                                         GFP_KERNEL);
1017         if (!mlxsw_sp->router->vrs)
1018                 return -ENOMEM;
1019
1020         for (i = 0; i < max_vrs; i++) {
1021                 vr = &mlxsw_sp->router->vrs[i];
1022                 vr->id = i;
1023         }
1024
1025         return 0;
1026 }
1027
1028 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
1029
1030 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
1031 {
1032         /* At this stage we're guaranteed not to have new incoming
1033          * FIB notifications and the work queue is free from FIBs
1034          * sitting on top of mlxsw netdevs. However, we can still
1035          * have other FIBs queued. Flush the queue before flushing
1036          * the device's tables. No need for locks, as we're the only
1037          * writer.
1038          */
1039         mlxsw_core_flush_owq();
1040         mlxsw_sp_router_fib_flush(mlxsw_sp);
1041         kfree(mlxsw_sp->router->vrs);
1042 }
1043
1044 static struct net_device *
1045 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
1046 {
1047         struct ip_tunnel *tun = netdev_priv(ol_dev);
1048         struct net *net = dev_net(ol_dev);
1049
1050         return dev_get_by_index_rcu(net, tun->parms.link);
1051 }
1052
1053 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
1054 {
1055         struct net_device *d;
1056         u32 tb_id;
1057
1058         rcu_read_lock();
1059         d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1060         if (d)
1061                 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
1062         else
1063                 tb_id = RT_TABLE_MAIN;
1064         rcu_read_unlock();
1065
1066         return tb_id;
1067 }
1068
1069 static struct mlxsw_sp_rif *
1070 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
1071                     const struct mlxsw_sp_rif_params *params,
1072                     struct netlink_ext_ack *extack);
1073
1074 static struct mlxsw_sp_rif_ipip_lb *
1075 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
1076                                 enum mlxsw_sp_ipip_type ipipt,
1077                                 struct net_device *ol_dev,
1078                                 struct netlink_ext_ack *extack)
1079 {
1080         struct mlxsw_sp_rif_params_ipip_lb lb_params;
1081         const struct mlxsw_sp_ipip_ops *ipip_ops;
1082         struct mlxsw_sp_rif *rif;
1083
1084         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1085         lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
1086                 .common.dev = ol_dev,
1087                 .common.lag = false,
1088                 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
1089         };
1090
1091         rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack);
1092         if (IS_ERR(rif))
1093                 return ERR_CAST(rif);
1094         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
1095 }
1096
1097 static struct mlxsw_sp_ipip_entry *
1098 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
1099                           enum mlxsw_sp_ipip_type ipipt,
1100                           struct net_device *ol_dev)
1101 {
1102         const struct mlxsw_sp_ipip_ops *ipip_ops;
1103         struct mlxsw_sp_ipip_entry *ipip_entry;
1104         struct mlxsw_sp_ipip_entry *ret = NULL;
1105
1106         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1107         ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
1108         if (!ipip_entry)
1109                 return ERR_PTR(-ENOMEM);
1110
1111         ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
1112                                                             ol_dev, NULL);
1113         if (IS_ERR(ipip_entry->ol_lb)) {
1114                 ret = ERR_CAST(ipip_entry->ol_lb);
1115                 goto err_ol_ipip_lb_create;
1116         }
1117
1118         ipip_entry->ipipt = ipipt;
1119         ipip_entry->ol_dev = ol_dev;
1120
1121         switch (ipip_ops->ul_proto) {
1122         case MLXSW_SP_L3_PROTO_IPV4:
1123                 ipip_entry->parms4 = mlxsw_sp_ipip_netdev_parms4(ol_dev);
1124                 break;
1125         case MLXSW_SP_L3_PROTO_IPV6:
1126                 WARN_ON(1);
1127                 break;
1128         }
1129
1130         return ipip_entry;
1131
1132 err_ol_ipip_lb_create:
1133         kfree(ipip_entry);
1134         return ret;
1135 }
1136
1137 static void
1138 mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry)
1139 {
1140         mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
1141         kfree(ipip_entry);
1142 }
1143
1144 static bool
1145 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1146                                   const enum mlxsw_sp_l3proto ul_proto,
1147                                   union mlxsw_sp_l3addr saddr,
1148                                   u32 ul_tb_id,
1149                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1150 {
1151         u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1152         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1153         union mlxsw_sp_l3addr tun_saddr;
1154
1155         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1156                 return false;
1157
1158         tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1159         return tun_ul_tb_id == ul_tb_id &&
1160                mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1161 }
1162
1163 static int
1164 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1165                               struct mlxsw_sp_fib_entry *fib_entry,
1166                               struct mlxsw_sp_ipip_entry *ipip_entry)
1167 {
1168         u32 tunnel_index;
1169         int err;
1170
1171         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1172                                   1, &tunnel_index);
1173         if (err)
1174                 return err;
1175
1176         ipip_entry->decap_fib_entry = fib_entry;
1177         fib_entry->decap.ipip_entry = ipip_entry;
1178         fib_entry->decap.tunnel_index = tunnel_index;
1179         return 0;
1180 }
1181
1182 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1183                                           struct mlxsw_sp_fib_entry *fib_entry)
1184 {
1185         /* Unlink this node from the IPIP entry that it's the decap entry of. */
1186         fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1187         fib_entry->decap.ipip_entry = NULL;
1188         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1189                            1, fib_entry->decap.tunnel_index);
1190 }
1191
1192 static struct mlxsw_sp_fib_node *
1193 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1194                          size_t addr_len, unsigned char prefix_len);
1195 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1196                                      struct mlxsw_sp_fib_entry *fib_entry);
1197
1198 static void
1199 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1200                                  struct mlxsw_sp_ipip_entry *ipip_entry)
1201 {
1202         struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1203
1204         mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1205         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1206
1207         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1208 }
1209
1210 static void
1211 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1212                                   struct mlxsw_sp_ipip_entry *ipip_entry,
1213                                   struct mlxsw_sp_fib_entry *decap_fib_entry)
1214 {
1215         if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1216                                           ipip_entry))
1217                 return;
1218         decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1219
1220         if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1221                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1222 }
1223
1224 static struct mlxsw_sp_fib_entry *
1225 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
1226                                      enum mlxsw_sp_l3proto proto,
1227                                      const union mlxsw_sp_l3addr *addr,
1228                                      enum mlxsw_sp_fib_entry_type type)
1229 {
1230         struct mlxsw_sp_fib_node *fib_node;
1231         unsigned char addr_prefix_len;
1232         struct mlxsw_sp_fib *fib;
1233         struct mlxsw_sp_vr *vr;
1234         const void *addrp;
1235         size_t addr_len;
1236         u32 addr4;
1237
1238         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
1239         if (!vr)
1240                 return NULL;
1241         fib = mlxsw_sp_vr_fib(vr, proto);
1242
1243         switch (proto) {
1244         case MLXSW_SP_L3_PROTO_IPV4:
1245                 addr4 = be32_to_cpu(addr->addr4);
1246                 addrp = &addr4;
1247                 addr_len = 4;
1248                 addr_prefix_len = 32;
1249                 break;
1250         case MLXSW_SP_L3_PROTO_IPV6:
1251         default:
1252                 WARN_ON(1);
1253                 return NULL;
1254         }
1255
1256         fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len,
1257                                             addr_prefix_len);
1258         if (!fib_node || fib_node->fib_entry->type != type)
1259                 return NULL;
1260
1261         return fib_node->fib_entry;
1262 }
1263
1264 /* Given an IPIP entry, find the corresponding decap route. */
1265 static struct mlxsw_sp_fib_entry *
1266 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1267                                struct mlxsw_sp_ipip_entry *ipip_entry)
1268 {
1269         static struct mlxsw_sp_fib_node *fib_node;
1270         const struct mlxsw_sp_ipip_ops *ipip_ops;
1271         unsigned char saddr_prefix_len;
1272         union mlxsw_sp_l3addr saddr;
1273         struct mlxsw_sp_fib *ul_fib;
1274         struct mlxsw_sp_vr *ul_vr;
1275         const void *saddrp;
1276         size_t saddr_len;
1277         u32 ul_tb_id;
1278         u32 saddr4;
1279
1280         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1281
1282         ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1283         ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1284         if (!ul_vr)
1285                 return NULL;
1286
1287         ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1288         saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1289                                            ipip_entry->ol_dev);
1290
1291         switch (ipip_ops->ul_proto) {
1292         case MLXSW_SP_L3_PROTO_IPV4:
1293                 saddr4 = be32_to_cpu(saddr.addr4);
1294                 saddrp = &saddr4;
1295                 saddr_len = 4;
1296                 saddr_prefix_len = 32;
1297                 break;
1298         default:
1299                 WARN_ON(1);
1300                 return NULL;
1301         }
1302
1303         fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1304                                             saddr_prefix_len);
1305         if (!fib_node ||
1306             fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1307                 return NULL;
1308
1309         return fib_node->fib_entry;
1310 }
1311
1312 static struct mlxsw_sp_ipip_entry *
1313 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp,
1314                            enum mlxsw_sp_ipip_type ipipt,
1315                            struct net_device *ol_dev)
1316 {
1317         struct mlxsw_sp_ipip_entry *ipip_entry;
1318
1319         ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1320         if (IS_ERR(ipip_entry))
1321                 return ipip_entry;
1322
1323         list_add_tail(&ipip_entry->ipip_list_node,
1324                       &mlxsw_sp->router->ipip_list);
1325
1326         return ipip_entry;
1327 }
1328
1329 static void
1330 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1331                             struct mlxsw_sp_ipip_entry *ipip_entry)
1332 {
1333         list_del(&ipip_entry->ipip_list_node);
1334         mlxsw_sp_ipip_entry_dealloc(ipip_entry);
1335 }
1336
1337 static bool
1338 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1339                                   const struct net_device *ul_dev,
1340                                   enum mlxsw_sp_l3proto ul_proto,
1341                                   union mlxsw_sp_l3addr ul_dip,
1342                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1343 {
1344         u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1345         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1346
1347         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1348                 return false;
1349
1350         return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1351                                                  ul_tb_id, ipip_entry);
1352 }
1353
1354 /* Given decap parameters, find the corresponding IPIP entry. */
1355 static struct mlxsw_sp_ipip_entry *
1356 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex,
1357                                   enum mlxsw_sp_l3proto ul_proto,
1358                                   union mlxsw_sp_l3addr ul_dip)
1359 {
1360         struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1361         struct net_device *ul_dev;
1362
1363         rcu_read_lock();
1364
1365         ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex);
1366         if (!ul_dev)
1367                 goto out_unlock;
1368
1369         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1370                             ipip_list_node)
1371                 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1372                                                       ul_proto, ul_dip,
1373                                                       ipip_entry))
1374                         goto out_unlock;
1375
1376         rcu_read_unlock();
1377
1378         return NULL;
1379
1380 out_unlock:
1381         rcu_read_unlock();
1382         return ipip_entry;
1383 }
1384
1385 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
1386                                       const struct net_device *dev,
1387                                       enum mlxsw_sp_ipip_type *p_type)
1388 {
1389         struct mlxsw_sp_router *router = mlxsw_sp->router;
1390         const struct mlxsw_sp_ipip_ops *ipip_ops;
1391         enum mlxsw_sp_ipip_type ipipt;
1392
1393         for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
1394                 ipip_ops = router->ipip_ops_arr[ipipt];
1395                 if (dev->type == ipip_ops->dev_type) {
1396                         if (p_type)
1397                                 *p_type = ipipt;
1398                         return true;
1399                 }
1400         }
1401         return false;
1402 }
1403
1404 bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp,
1405                                 const struct net_device *dev)
1406 {
1407         return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
1408 }
1409
1410 static struct mlxsw_sp_ipip_entry *
1411 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp,
1412                                    const struct net_device *ol_dev)
1413 {
1414         struct mlxsw_sp_ipip_entry *ipip_entry;
1415
1416         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1417                             ipip_list_node)
1418                 if (ipip_entry->ol_dev == ol_dev)
1419                         return ipip_entry;
1420
1421         return NULL;
1422 }
1423
1424 static struct mlxsw_sp_ipip_entry *
1425 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp,
1426                                    const struct net_device *ul_dev,
1427                                    struct mlxsw_sp_ipip_entry *start)
1428 {
1429         struct mlxsw_sp_ipip_entry *ipip_entry;
1430
1431         ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list,
1432                                         ipip_list_node);
1433         list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list,
1434                                      ipip_list_node) {
1435                 struct net_device *ol_dev = ipip_entry->ol_dev;
1436                 struct net_device *ipip_ul_dev;
1437
1438                 rcu_read_lock();
1439                 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1440                 rcu_read_unlock();
1441
1442                 if (ipip_ul_dev == ul_dev)
1443                         return ipip_entry;
1444         }
1445
1446         return NULL;
1447 }
1448
1449 bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp,
1450                                 const struct net_device *dev)
1451 {
1452         bool is_ipip_ul;
1453
1454         mutex_lock(&mlxsw_sp->router->lock);
1455         is_ipip_ul = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL);
1456         mutex_unlock(&mlxsw_sp->router->lock);
1457
1458         return is_ipip_ul;
1459 }
1460
1461 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp,
1462                                                 const struct net_device *ol_dev,
1463                                                 enum mlxsw_sp_ipip_type ipipt)
1464 {
1465         const struct mlxsw_sp_ipip_ops *ops
1466                 = mlxsw_sp->router->ipip_ops_arr[ipipt];
1467
1468         return ops->can_offload(mlxsw_sp, ol_dev);
1469 }
1470
1471 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp,
1472                                                 struct net_device *ol_dev)
1473 {
1474         enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX;
1475         struct mlxsw_sp_ipip_entry *ipip_entry;
1476         enum mlxsw_sp_l3proto ul_proto;
1477         union mlxsw_sp_l3addr saddr;
1478         u32 ul_tb_id;
1479
1480         mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt);
1481         if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) {
1482                 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1483                 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto;
1484                 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1485                 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1486                                                           saddr, ul_tb_id,
1487                                                           NULL)) {
1488                         ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt,
1489                                                                 ol_dev);
1490                         if (IS_ERR(ipip_entry))
1491                                 return PTR_ERR(ipip_entry);
1492                 }
1493         }
1494
1495         return 0;
1496 }
1497
1498 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp,
1499                                                    struct net_device *ol_dev)
1500 {
1501         struct mlxsw_sp_ipip_entry *ipip_entry;
1502
1503         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1504         if (ipip_entry)
1505                 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1506 }
1507
1508 static void
1509 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1510                                 struct mlxsw_sp_ipip_entry *ipip_entry)
1511 {
1512         struct mlxsw_sp_fib_entry *decap_fib_entry;
1513
1514         decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1515         if (decap_fib_entry)
1516                 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1517                                                   decap_fib_entry);
1518 }
1519
1520 static int
1521 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id,
1522                         u16 ul_rif_id, bool enable)
1523 {
1524         struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
1525         struct mlxsw_sp_rif *rif = &lb_rif->common;
1526         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
1527         char ritr_pl[MLXSW_REG_RITR_LEN];
1528         u32 saddr4;
1529
1530         switch (lb_cf.ul_protocol) {
1531         case MLXSW_SP_L3_PROTO_IPV4:
1532                 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
1533                 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
1534                                     rif->rif_index, rif->vr_id, rif->dev->mtu);
1535                 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
1536                             MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
1537                             ul_vr_id, ul_rif_id, saddr4, lb_cf.okey);
1538                 break;
1539
1540         case MLXSW_SP_L3_PROTO_IPV6:
1541                 return -EAFNOSUPPORT;
1542         }
1543
1544         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
1545 }
1546
1547 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp,
1548                                                  struct net_device *ol_dev)
1549 {
1550         struct mlxsw_sp_ipip_entry *ipip_entry;
1551         struct mlxsw_sp_rif_ipip_lb *lb_rif;
1552         int err = 0;
1553
1554         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1555         if (ipip_entry) {
1556                 lb_rif = ipip_entry->ol_lb;
1557                 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id,
1558                                               lb_rif->ul_rif_id, true);
1559                 if (err)
1560                         goto out;
1561                 lb_rif->common.mtu = ol_dev->mtu;
1562         }
1563
1564 out:
1565         return err;
1566 }
1567
1568 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1569                                                 struct net_device *ol_dev)
1570 {
1571         struct mlxsw_sp_ipip_entry *ipip_entry;
1572
1573         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1574         if (ipip_entry)
1575                 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1576 }
1577
1578 static void
1579 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1580                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1581 {
1582         if (ipip_entry->decap_fib_entry)
1583                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1584 }
1585
1586 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1587                                                   struct net_device *ol_dev)
1588 {
1589         struct mlxsw_sp_ipip_entry *ipip_entry;
1590
1591         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1592         if (ipip_entry)
1593                 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1594 }
1595
1596 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
1597                                          struct mlxsw_sp_rif *old_rif,
1598                                          struct mlxsw_sp_rif *new_rif);
1599 static int
1600 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp,
1601                                  struct mlxsw_sp_ipip_entry *ipip_entry,
1602                                  bool keep_encap,
1603                                  struct netlink_ext_ack *extack)
1604 {
1605         struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb;
1606         struct mlxsw_sp_rif_ipip_lb *new_lb_rif;
1607
1608         new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp,
1609                                                      ipip_entry->ipipt,
1610                                                      ipip_entry->ol_dev,
1611                                                      extack);
1612         if (IS_ERR(new_lb_rif))
1613                 return PTR_ERR(new_lb_rif);
1614         ipip_entry->ol_lb = new_lb_rif;
1615
1616         if (keep_encap)
1617                 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common,
1618                                              &new_lb_rif->common);
1619
1620         mlxsw_sp_rif_destroy(&old_lb_rif->common);
1621
1622         return 0;
1623 }
1624
1625 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
1626                                         struct mlxsw_sp_rif *rif);
1627
1628 /**
1629  * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry.
1630  * @mlxsw_sp: mlxsw_sp.
1631  * @ipip_entry: IPIP entry.
1632  * @recreate_loopback: Recreates the associated loopback RIF.
1633  * @keep_encap: Updates next hops that use the tunnel netdevice. This is only
1634  *              relevant when recreate_loopback is true.
1635  * @update_nexthops: Updates next hops, keeping the current loopback RIF. This
1636  *                   is only relevant when recreate_loopback is false.
1637  * @extack: extack.
1638  *
1639  * Return: Non-zero value on failure.
1640  */
1641 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp,
1642                                         struct mlxsw_sp_ipip_entry *ipip_entry,
1643                                         bool recreate_loopback,
1644                                         bool keep_encap,
1645                                         bool update_nexthops,
1646                                         struct netlink_ext_ack *extack)
1647 {
1648         int err;
1649
1650         /* RIFs can't be edited, so to update loopback, we need to destroy and
1651          * recreate it. That creates a window of opportunity where RALUE and
1652          * RATR registers end up referencing a RIF that's already gone. RATRs
1653          * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care
1654          * of RALUE, demote the decap route back.
1655          */
1656         if (ipip_entry->decap_fib_entry)
1657                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1658
1659         if (recreate_loopback) {
1660                 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry,
1661                                                        keep_encap, extack);
1662                 if (err)
1663                         return err;
1664         } else if (update_nexthops) {
1665                 mlxsw_sp_nexthop_rif_update(mlxsw_sp,
1666                                             &ipip_entry->ol_lb->common);
1667         }
1668
1669         if (ipip_entry->ol_dev->flags & IFF_UP)
1670                 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1671
1672         return 0;
1673 }
1674
1675 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp,
1676                                                 struct net_device *ol_dev,
1677                                                 struct netlink_ext_ack *extack)
1678 {
1679         struct mlxsw_sp_ipip_entry *ipip_entry =
1680                 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1681
1682         if (!ipip_entry)
1683                 return 0;
1684
1685         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1686                                                    true, false, false, extack);
1687 }
1688
1689 static int
1690 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp,
1691                                      struct mlxsw_sp_ipip_entry *ipip_entry,
1692                                      struct net_device *ul_dev,
1693                                      bool *demote_this,
1694                                      struct netlink_ext_ack *extack)
1695 {
1696         u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1697         enum mlxsw_sp_l3proto ul_proto;
1698         union mlxsw_sp_l3addr saddr;
1699
1700         /* Moving underlay to a different VRF might cause local address
1701          * conflict, and the conflicting tunnels need to be demoted.
1702          */
1703         ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1704         saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1705         if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1706                                                  saddr, ul_tb_id,
1707                                                  ipip_entry)) {
1708                 *demote_this = true;
1709                 return 0;
1710         }
1711
1712         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1713                                                    true, true, false, extack);
1714 }
1715
1716 static int
1717 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp,
1718                                     struct mlxsw_sp_ipip_entry *ipip_entry,
1719                                     struct net_device *ul_dev)
1720 {
1721         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1722                                                    false, false, true, NULL);
1723 }
1724
1725 static int
1726 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp,
1727                                       struct mlxsw_sp_ipip_entry *ipip_entry,
1728                                       struct net_device *ul_dev)
1729 {
1730         /* A down underlay device causes encapsulated packets to not be
1731          * forwarded, but decap still works. So refresh next hops without
1732          * touching anything else.
1733          */
1734         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1735                                                    false, false, true, NULL);
1736 }
1737
1738 static int
1739 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp,
1740                                         struct net_device *ol_dev,
1741                                         struct netlink_ext_ack *extack)
1742 {
1743         const struct mlxsw_sp_ipip_ops *ipip_ops;
1744         struct mlxsw_sp_ipip_entry *ipip_entry;
1745         int err;
1746
1747         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1748         if (!ipip_entry)
1749                 /* A change might make a tunnel eligible for offloading, but
1750                  * that is currently not implemented. What falls to slow path
1751                  * stays there.
1752                  */
1753                 return 0;
1754
1755         /* A change might make a tunnel not eligible for offloading. */
1756         if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev,
1757                                                  ipip_entry->ipipt)) {
1758                 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1759                 return 0;
1760         }
1761
1762         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1763         err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack);
1764         return err;
1765 }
1766
1767 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp,
1768                                        struct mlxsw_sp_ipip_entry *ipip_entry)
1769 {
1770         struct net_device *ol_dev = ipip_entry->ol_dev;
1771
1772         if (ol_dev->flags & IFF_UP)
1773                 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1774         mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1775 }
1776
1777 /* The configuration where several tunnels have the same local address in the
1778  * same underlay table needs special treatment in the HW. That is currently not
1779  * implemented in the driver. This function finds and demotes the first tunnel
1780  * with a given source address, except the one passed in in the argument
1781  * `except'.
1782  */
1783 bool
1784 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp,
1785                                      enum mlxsw_sp_l3proto ul_proto,
1786                                      union mlxsw_sp_l3addr saddr,
1787                                      u32 ul_tb_id,
1788                                      const struct mlxsw_sp_ipip_entry *except)
1789 {
1790         struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1791
1792         list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1793                                  ipip_list_node) {
1794                 if (ipip_entry != except &&
1795                     mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1796                                                       ul_tb_id, ipip_entry)) {
1797                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1798                         return true;
1799                 }
1800         }
1801
1802         return false;
1803 }
1804
1805 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp,
1806                                                      struct net_device *ul_dev)
1807 {
1808         struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1809
1810         list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1811                                  ipip_list_node) {
1812                 struct net_device *ol_dev = ipip_entry->ol_dev;
1813                 struct net_device *ipip_ul_dev;
1814
1815                 rcu_read_lock();
1816                 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1817                 rcu_read_unlock();
1818                 if (ipip_ul_dev == ul_dev)
1819                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1820         }
1821 }
1822
1823 int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp,
1824                                      struct net_device *ol_dev,
1825                                      unsigned long event,
1826                                      struct netdev_notifier_info *info)
1827 {
1828         struct netdev_notifier_changeupper_info *chup;
1829         struct netlink_ext_ack *extack;
1830         int err = 0;
1831
1832         mutex_lock(&mlxsw_sp->router->lock);
1833         switch (event) {
1834         case NETDEV_REGISTER:
1835                 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev);
1836                 break;
1837         case NETDEV_UNREGISTER:
1838                 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev);
1839                 break;
1840         case NETDEV_UP:
1841                 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev);
1842                 break;
1843         case NETDEV_DOWN:
1844                 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev);
1845                 break;
1846         case NETDEV_CHANGEUPPER:
1847                 chup = container_of(info, typeof(*chup), info);
1848                 extack = info->extack;
1849                 if (netif_is_l3_master(chup->upper_dev))
1850                         err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp,
1851                                                                    ol_dev,
1852                                                                    extack);
1853                 break;
1854         case NETDEV_CHANGE:
1855                 extack = info->extack;
1856                 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp,
1857                                                               ol_dev, extack);
1858                 break;
1859         case NETDEV_CHANGEMTU:
1860                 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev);
1861                 break;
1862         }
1863         mutex_unlock(&mlxsw_sp->router->lock);
1864         return err;
1865 }
1866
1867 static int
1868 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1869                                    struct mlxsw_sp_ipip_entry *ipip_entry,
1870                                    struct net_device *ul_dev,
1871                                    bool *demote_this,
1872                                    unsigned long event,
1873                                    struct netdev_notifier_info *info)
1874 {
1875         struct netdev_notifier_changeupper_info *chup;
1876         struct netlink_ext_ack *extack;
1877
1878         switch (event) {
1879         case NETDEV_CHANGEUPPER:
1880                 chup = container_of(info, typeof(*chup), info);
1881                 extack = info->extack;
1882                 if (netif_is_l3_master(chup->upper_dev))
1883                         return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp,
1884                                                                     ipip_entry,
1885                                                                     ul_dev,
1886                                                                     demote_this,
1887                                                                     extack);
1888                 break;
1889
1890         case NETDEV_UP:
1891                 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry,
1892                                                            ul_dev);
1893         case NETDEV_DOWN:
1894                 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp,
1895                                                              ipip_entry,
1896                                                              ul_dev);
1897         }
1898         return 0;
1899 }
1900
1901 int
1902 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1903                                  struct net_device *ul_dev,
1904                                  unsigned long event,
1905                                  struct netdev_notifier_info *info)
1906 {
1907         struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1908         int err = 0;
1909
1910         mutex_lock(&mlxsw_sp->router->lock);
1911         while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp,
1912                                                                 ul_dev,
1913                                                                 ipip_entry))) {
1914                 struct mlxsw_sp_ipip_entry *prev;
1915                 bool demote_this = false;
1916
1917                 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry,
1918                                                          ul_dev, &demote_this,
1919                                                          event, info);
1920                 if (err) {
1921                         mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp,
1922                                                                  ul_dev);
1923                         break;
1924                 }
1925
1926                 if (demote_this) {
1927                         if (list_is_first(&ipip_entry->ipip_list_node,
1928                                           &mlxsw_sp->router->ipip_list))
1929                                 prev = NULL;
1930                         else
1931                                 /* This can't be cached from previous iteration,
1932                                  * because that entry could be gone now.
1933                                  */
1934                                 prev = list_prev_entry(ipip_entry,
1935                                                        ipip_list_node);
1936                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1937                         ipip_entry = prev;
1938                 }
1939         }
1940         mutex_unlock(&mlxsw_sp->router->lock);
1941
1942         return err;
1943 }
1944
1945 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
1946                                       enum mlxsw_sp_l3proto ul_proto,
1947                                       const union mlxsw_sp_l3addr *ul_sip,
1948                                       u32 tunnel_index)
1949 {
1950         enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1951         struct mlxsw_sp_router *router = mlxsw_sp->router;
1952         struct mlxsw_sp_fib_entry *fib_entry;
1953         int err = 0;
1954
1955         mutex_lock(&mlxsw_sp->router->lock);
1956
1957         if (WARN_ON_ONCE(router->nve_decap_config.valid)) {
1958                 err = -EINVAL;
1959                 goto out;
1960         }
1961
1962         router->nve_decap_config.ul_tb_id = ul_tb_id;
1963         router->nve_decap_config.tunnel_index = tunnel_index;
1964         router->nve_decap_config.ul_proto = ul_proto;
1965         router->nve_decap_config.ul_sip = *ul_sip;
1966         router->nve_decap_config.valid = true;
1967
1968         /* It is valid to create a tunnel with a local IP and only later
1969          * assign this IP address to a local interface
1970          */
1971         fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
1972                                                          ul_proto, ul_sip,
1973                                                          type);
1974         if (!fib_entry)
1975                 goto out;
1976
1977         fib_entry->decap.tunnel_index = tunnel_index;
1978         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
1979
1980         err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1981         if (err)
1982                 goto err_fib_entry_update;
1983
1984         goto out;
1985
1986 err_fib_entry_update:
1987         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1988         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1989 out:
1990         mutex_unlock(&mlxsw_sp->router->lock);
1991         return err;
1992 }
1993
1994 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
1995                                       enum mlxsw_sp_l3proto ul_proto,
1996                                       const union mlxsw_sp_l3addr *ul_sip)
1997 {
1998         enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
1999         struct mlxsw_sp_router *router = mlxsw_sp->router;
2000         struct mlxsw_sp_fib_entry *fib_entry;
2001
2002         mutex_lock(&mlxsw_sp->router->lock);
2003
2004         if (WARN_ON_ONCE(!router->nve_decap_config.valid))
2005                 goto out;
2006
2007         router->nve_decap_config.valid = false;
2008
2009         fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
2010                                                          ul_proto, ul_sip,
2011                                                          type);
2012         if (!fib_entry)
2013                 goto out;
2014
2015         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2016         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2017 out:
2018         mutex_unlock(&mlxsw_sp->router->lock);
2019 }
2020
2021 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp,
2022                                          u32 ul_tb_id,
2023                                          enum mlxsw_sp_l3proto ul_proto,
2024                                          const union mlxsw_sp_l3addr *ul_sip)
2025 {
2026         struct mlxsw_sp_router *router = mlxsw_sp->router;
2027
2028         return router->nve_decap_config.valid &&
2029                router->nve_decap_config.ul_tb_id == ul_tb_id &&
2030                router->nve_decap_config.ul_proto == ul_proto &&
2031                !memcmp(&router->nve_decap_config.ul_sip, ul_sip,
2032                        sizeof(*ul_sip));
2033 }
2034
2035 struct mlxsw_sp_neigh_key {
2036         struct neighbour *n;
2037 };
2038
2039 struct mlxsw_sp_neigh_entry {
2040         struct list_head rif_list_node;
2041         struct rhash_head ht_node;
2042         struct mlxsw_sp_neigh_key key;
2043         u16 rif;
2044         bool connected;
2045         unsigned char ha[ETH_ALEN];
2046         struct list_head nexthop_list; /* list of nexthops using
2047                                         * this neigh entry
2048                                         */
2049         struct list_head nexthop_neighs_list_node;
2050         unsigned int counter_index;
2051         bool counter_valid;
2052 };
2053
2054 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
2055         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
2056         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
2057         .key_len = sizeof(struct mlxsw_sp_neigh_key),
2058 };
2059
2060 struct mlxsw_sp_neigh_entry *
2061 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
2062                         struct mlxsw_sp_neigh_entry *neigh_entry)
2063 {
2064         if (!neigh_entry) {
2065                 if (list_empty(&rif->neigh_list))
2066                         return NULL;
2067                 else
2068                         return list_first_entry(&rif->neigh_list,
2069                                                 typeof(*neigh_entry),
2070                                                 rif_list_node);
2071         }
2072         if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list))
2073                 return NULL;
2074         return list_next_entry(neigh_entry, rif_list_node);
2075 }
2076
2077 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
2078 {
2079         return neigh_entry->key.n->tbl->family;
2080 }
2081
2082 unsigned char *
2083 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
2084 {
2085         return neigh_entry->ha;
2086 }
2087
2088 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2089 {
2090         struct neighbour *n;
2091
2092         n = neigh_entry->key.n;
2093         return ntohl(*((__be32 *) n->primary_key));
2094 }
2095
2096 struct in6_addr *
2097 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2098 {
2099         struct neighbour *n;
2100
2101         n = neigh_entry->key.n;
2102         return (struct in6_addr *) &n->primary_key;
2103 }
2104
2105 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
2106                                struct mlxsw_sp_neigh_entry *neigh_entry,
2107                                u64 *p_counter)
2108 {
2109         if (!neigh_entry->counter_valid)
2110                 return -EINVAL;
2111
2112         return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
2113                                          p_counter, NULL);
2114 }
2115
2116 static struct mlxsw_sp_neigh_entry *
2117 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
2118                            u16 rif)
2119 {
2120         struct mlxsw_sp_neigh_entry *neigh_entry;
2121
2122         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
2123         if (!neigh_entry)
2124                 return NULL;
2125
2126         neigh_entry->key.n = n;
2127         neigh_entry->rif = rif;
2128         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
2129
2130         return neigh_entry;
2131 }
2132
2133 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
2134 {
2135         kfree(neigh_entry);
2136 }
2137
2138 static int
2139 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
2140                             struct mlxsw_sp_neigh_entry *neigh_entry)
2141 {
2142         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
2143                                       &neigh_entry->ht_node,
2144                                       mlxsw_sp_neigh_ht_params);
2145 }
2146
2147 static void
2148 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
2149                             struct mlxsw_sp_neigh_entry *neigh_entry)
2150 {
2151         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
2152                                &neigh_entry->ht_node,
2153                                mlxsw_sp_neigh_ht_params);
2154 }
2155
2156 static bool
2157 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
2158                                     struct mlxsw_sp_neigh_entry *neigh_entry)
2159 {
2160         struct devlink *devlink;
2161         const char *table_name;
2162
2163         switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
2164         case AF_INET:
2165                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
2166                 break;
2167         case AF_INET6:
2168                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
2169                 break;
2170         default:
2171                 WARN_ON(1);
2172                 return false;
2173         }
2174
2175         devlink = priv_to_devlink(mlxsw_sp->core);
2176         return devlink_dpipe_table_counter_enabled(devlink, table_name);
2177 }
2178
2179 static void
2180 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2181                              struct mlxsw_sp_neigh_entry *neigh_entry)
2182 {
2183         if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
2184                 return;
2185
2186         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
2187                 return;
2188
2189         neigh_entry->counter_valid = true;
2190 }
2191
2192 static void
2193 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
2194                             struct mlxsw_sp_neigh_entry *neigh_entry)
2195 {
2196         if (!neigh_entry->counter_valid)
2197                 return;
2198         mlxsw_sp_flow_counter_free(mlxsw_sp,
2199                                    neigh_entry->counter_index);
2200         neigh_entry->counter_valid = false;
2201 }
2202
2203 static struct mlxsw_sp_neigh_entry *
2204 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2205 {
2206         struct mlxsw_sp_neigh_entry *neigh_entry;
2207         struct mlxsw_sp_rif *rif;
2208         int err;
2209
2210         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
2211         if (!rif)
2212                 return ERR_PTR(-EINVAL);
2213
2214         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
2215         if (!neigh_entry)
2216                 return ERR_PTR(-ENOMEM);
2217
2218         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
2219         if (err)
2220                 goto err_neigh_entry_insert;
2221
2222         mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2223         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
2224
2225         return neigh_entry;
2226
2227 err_neigh_entry_insert:
2228         mlxsw_sp_neigh_entry_free(neigh_entry);
2229         return ERR_PTR(err);
2230 }
2231
2232 static void
2233 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2234                              struct mlxsw_sp_neigh_entry *neigh_entry)
2235 {
2236         list_del(&neigh_entry->rif_list_node);
2237         mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2238         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
2239         mlxsw_sp_neigh_entry_free(neigh_entry);
2240 }
2241
2242 static struct mlxsw_sp_neigh_entry *
2243 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2244 {
2245         struct mlxsw_sp_neigh_key key;
2246
2247         key.n = n;
2248         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
2249                                       &key, mlxsw_sp_neigh_ht_params);
2250 }
2251
2252 static void
2253 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
2254 {
2255         unsigned long interval;
2256
2257 #if IS_ENABLED(CONFIG_IPV6)
2258         interval = min_t(unsigned long,
2259                          NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
2260                          NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
2261 #else
2262         interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
2263 #endif
2264         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
2265 }
2266
2267 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2268                                                    char *rauhtd_pl,
2269                                                    int ent_index)
2270 {
2271         struct net_device *dev;
2272         struct neighbour *n;
2273         __be32 dipn;
2274         u32 dip;
2275         u16 rif;
2276
2277         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
2278
2279         if (!mlxsw_sp->router->rifs[rif]) {
2280                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2281                 return;
2282         }
2283
2284         dipn = htonl(dip);
2285         dev = mlxsw_sp->router->rifs[rif]->dev;
2286         n = neigh_lookup(&arp_tbl, &dipn, dev);
2287         if (!n)
2288                 return;
2289
2290         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
2291         neigh_event_send(n, NULL);
2292         neigh_release(n);
2293 }
2294
2295 #if IS_ENABLED(CONFIG_IPV6)
2296 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2297                                                    char *rauhtd_pl,
2298                                                    int rec_index)
2299 {
2300         struct net_device *dev;
2301         struct neighbour *n;
2302         struct in6_addr dip;
2303         u16 rif;
2304
2305         mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
2306                                          (char *) &dip);
2307
2308         if (!mlxsw_sp->router->rifs[rif]) {
2309                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2310                 return;
2311         }
2312
2313         dev = mlxsw_sp->router->rifs[rif]->dev;
2314         n = neigh_lookup(&nd_tbl, &dip, dev);
2315         if (!n)
2316                 return;
2317
2318         netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
2319         neigh_event_send(n, NULL);
2320         neigh_release(n);
2321 }
2322 #else
2323 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2324                                                    char *rauhtd_pl,
2325                                                    int rec_index)
2326 {
2327 }
2328 #endif
2329
2330 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2331                                                    char *rauhtd_pl,
2332                                                    int rec_index)
2333 {
2334         u8 num_entries;
2335         int i;
2336
2337         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2338                                                                 rec_index);
2339         /* Hardware starts counting at 0, so add 1. */
2340         num_entries++;
2341
2342         /* Each record consists of several neighbour entries. */
2343         for (i = 0; i < num_entries; i++) {
2344                 int ent_index;
2345
2346                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
2347                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
2348                                                        ent_index);
2349         }
2350
2351 }
2352
2353 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2354                                                    char *rauhtd_pl,
2355                                                    int rec_index)
2356 {
2357         /* One record contains one entry. */
2358         mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
2359                                                rec_index);
2360 }
2361
2362 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
2363                                               char *rauhtd_pl, int rec_index)
2364 {
2365         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
2366         case MLXSW_REG_RAUHTD_TYPE_IPV4:
2367                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
2368                                                        rec_index);
2369                 break;
2370         case MLXSW_REG_RAUHTD_TYPE_IPV6:
2371                 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
2372                                                        rec_index);
2373                 break;
2374         }
2375 }
2376
2377 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
2378 {
2379         u8 num_rec, last_rec_index, num_entries;
2380
2381         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2382         last_rec_index = num_rec - 1;
2383
2384         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
2385                 return false;
2386         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
2387             MLXSW_REG_RAUHTD_TYPE_IPV6)
2388                 return true;
2389
2390         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2391                                                                 last_rec_index);
2392         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
2393                 return true;
2394         return false;
2395 }
2396
2397 static int
2398 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
2399                                        char *rauhtd_pl,
2400                                        enum mlxsw_reg_rauhtd_type type)
2401 {
2402         int i, num_rec;
2403         int err;
2404
2405         /* Ensure the RIF we read from the device does not change mid-dump. */
2406         mutex_lock(&mlxsw_sp->router->lock);
2407         do {
2408                 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
2409                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
2410                                       rauhtd_pl);
2411                 if (err) {
2412                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n");
2413                         break;
2414                 }
2415                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2416                 for (i = 0; i < num_rec; i++)
2417                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
2418                                                           i);
2419         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
2420         mutex_unlock(&mlxsw_sp->router->lock);
2421
2422         return err;
2423 }
2424
2425 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
2426 {
2427         enum mlxsw_reg_rauhtd_type type;
2428         char *rauhtd_pl;
2429         int err;
2430
2431         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
2432         if (!rauhtd_pl)
2433                 return -ENOMEM;
2434
2435         type = MLXSW_REG_RAUHTD_TYPE_IPV4;
2436         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2437         if (err)
2438                 goto out;
2439
2440         type = MLXSW_REG_RAUHTD_TYPE_IPV6;
2441         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2442 out:
2443         kfree(rauhtd_pl);
2444         return err;
2445 }
2446
2447 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
2448 {
2449         struct mlxsw_sp_neigh_entry *neigh_entry;
2450
2451         mutex_lock(&mlxsw_sp->router->lock);
2452         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
2453                             nexthop_neighs_list_node)
2454                 /* If this neigh have nexthops, make the kernel think this neigh
2455                  * is active regardless of the traffic.
2456                  */
2457                 neigh_event_send(neigh_entry->key.n, NULL);
2458         mutex_unlock(&mlxsw_sp->router->lock);
2459 }
2460
2461 static void
2462 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
2463 {
2464         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
2465
2466         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
2467                                msecs_to_jiffies(interval));
2468 }
2469
2470 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
2471 {
2472         struct mlxsw_sp_router *router;
2473         int err;
2474
2475         router = container_of(work, struct mlxsw_sp_router,
2476                               neighs_update.dw.work);
2477         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
2478         if (err)
2479                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
2480
2481         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
2482
2483         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
2484 }
2485
2486 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
2487 {
2488         struct mlxsw_sp_neigh_entry *neigh_entry;
2489         struct mlxsw_sp_router *router;
2490
2491         router = container_of(work, struct mlxsw_sp_router,
2492                               nexthop_probe_dw.work);
2493         /* Iterate over nexthop neighbours, find those who are unresolved and
2494          * send arp on them. This solves the chicken-egg problem when
2495          * the nexthop wouldn't get offloaded until the neighbor is resolved
2496          * but it wouldn't get resolved ever in case traffic is flowing in HW
2497          * using different nexthop.
2498          */
2499         mutex_lock(&router->lock);
2500         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
2501                             nexthop_neighs_list_node)
2502                 if (!neigh_entry->connected)
2503                         neigh_event_send(neigh_entry->key.n, NULL);
2504         mutex_unlock(&router->lock);
2505
2506         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
2507                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
2508 }
2509
2510 static void
2511 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2512                               struct mlxsw_sp_neigh_entry *neigh_entry,
2513                               bool removing, bool dead);
2514
2515 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
2516 {
2517         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
2518                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
2519 }
2520
2521 static int
2522 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
2523                                 struct mlxsw_sp_neigh_entry *neigh_entry,
2524                                 enum mlxsw_reg_rauht_op op)
2525 {
2526         struct neighbour *n = neigh_entry->key.n;
2527         u32 dip = ntohl(*((__be32 *) n->primary_key));
2528         char rauht_pl[MLXSW_REG_RAUHT_LEN];
2529
2530         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2531                               dip);
2532         if (neigh_entry->counter_valid)
2533                 mlxsw_reg_rauht_pack_counter(rauht_pl,
2534                                              neigh_entry->counter_index);
2535         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2536 }
2537
2538 static int
2539 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
2540                                 struct mlxsw_sp_neigh_entry *neigh_entry,
2541                                 enum mlxsw_reg_rauht_op op)
2542 {
2543         struct neighbour *n = neigh_entry->key.n;
2544         char rauht_pl[MLXSW_REG_RAUHT_LEN];
2545         const char *dip = n->primary_key;
2546
2547         mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2548                               dip);
2549         if (neigh_entry->counter_valid)
2550                 mlxsw_reg_rauht_pack_counter(rauht_pl,
2551                                              neigh_entry->counter_index);
2552         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2553 }
2554
2555 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
2556 {
2557         struct neighbour *n = neigh_entry->key.n;
2558
2559         /* Packets with a link-local destination address are trapped
2560          * after LPM lookup and never reach the neighbour table, so
2561          * there is no need to program such neighbours to the device.
2562          */
2563         if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
2564             IPV6_ADDR_LINKLOCAL)
2565                 return true;
2566         return false;
2567 }
2568
2569 static void
2570 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
2571                             struct mlxsw_sp_neigh_entry *neigh_entry,
2572                             bool adding)
2573 {
2574         enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding);
2575         int err;
2576
2577         if (!adding && !neigh_entry->connected)
2578                 return;
2579         neigh_entry->connected = adding;
2580         if (neigh_entry->key.n->tbl->family == AF_INET) {
2581                 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
2582                                                       op);
2583                 if (err)
2584                         return;
2585         } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
2586                 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
2587                         return;
2588                 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
2589                                                       op);
2590                 if (err)
2591                         return;
2592         } else {
2593                 WARN_ON_ONCE(1);
2594                 return;
2595         }
2596
2597         if (adding)
2598                 neigh_entry->key.n->flags |= NTF_OFFLOADED;
2599         else
2600                 neigh_entry->key.n->flags &= ~NTF_OFFLOADED;
2601 }
2602
2603 void
2604 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
2605                                     struct mlxsw_sp_neigh_entry *neigh_entry,
2606                                     bool adding)
2607 {
2608         if (adding)
2609                 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2610         else
2611                 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2612         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
2613 }
2614
2615 struct mlxsw_sp_netevent_work {
2616         struct work_struct work;
2617         struct mlxsw_sp *mlxsw_sp;
2618         struct neighbour *n;
2619 };
2620
2621 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
2622 {
2623         struct mlxsw_sp_netevent_work *net_work =
2624                 container_of(work, struct mlxsw_sp_netevent_work, work);
2625         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2626         struct mlxsw_sp_neigh_entry *neigh_entry;
2627         struct neighbour *n = net_work->n;
2628         unsigned char ha[ETH_ALEN];
2629         bool entry_connected;
2630         u8 nud_state, dead;
2631
2632         /* If these parameters are changed after we release the lock,
2633          * then we are guaranteed to receive another event letting us
2634          * know about it.
2635          */
2636         read_lock_bh(&n->lock);
2637         memcpy(ha, n->ha, ETH_ALEN);
2638         nud_state = n->nud_state;
2639         dead = n->dead;
2640         read_unlock_bh(&n->lock);
2641
2642         mutex_lock(&mlxsw_sp->router->lock);
2643         mlxsw_sp_span_respin(mlxsw_sp);
2644
2645         entry_connected = nud_state & NUD_VALID && !dead;
2646         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2647         if (!entry_connected && !neigh_entry)
2648                 goto out;
2649         if (!neigh_entry) {
2650                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2651                 if (IS_ERR(neigh_entry))
2652                         goto out;
2653         }
2654
2655         memcpy(neigh_entry->ha, ha, ETH_ALEN);
2656         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
2657         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected,
2658                                       dead);
2659
2660         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2661                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2662
2663 out:
2664         mutex_unlock(&mlxsw_sp->router->lock);
2665         neigh_release(n);
2666         kfree(net_work);
2667 }
2668
2669 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp);
2670
2671 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work)
2672 {
2673         struct mlxsw_sp_netevent_work *net_work =
2674                 container_of(work, struct mlxsw_sp_netevent_work, work);
2675         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2676
2677         mlxsw_sp_mp_hash_init(mlxsw_sp);
2678         kfree(net_work);
2679 }
2680
2681 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp);
2682
2683 static void mlxsw_sp_router_update_priority_work(struct work_struct *work)
2684 {
2685         struct mlxsw_sp_netevent_work *net_work =
2686                 container_of(work, struct mlxsw_sp_netevent_work, work);
2687         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2688
2689         __mlxsw_sp_router_init(mlxsw_sp);
2690         kfree(net_work);
2691 }
2692
2693 static int mlxsw_sp_router_schedule_work(struct net *net,
2694                                          struct notifier_block *nb,
2695                                          void (*cb)(struct work_struct *))
2696 {
2697         struct mlxsw_sp_netevent_work *net_work;
2698         struct mlxsw_sp_router *router;
2699
2700         router = container_of(nb, struct mlxsw_sp_router, netevent_nb);
2701         if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp)))
2702                 return NOTIFY_DONE;
2703
2704         net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2705         if (!net_work)
2706                 return NOTIFY_BAD;
2707
2708         INIT_WORK(&net_work->work, cb);
2709         net_work->mlxsw_sp = router->mlxsw_sp;
2710         mlxsw_core_schedule_work(&net_work->work);
2711         return NOTIFY_DONE;
2712 }
2713
2714 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb,
2715                                           unsigned long event, void *ptr)
2716 {
2717         struct mlxsw_sp_netevent_work *net_work;
2718         struct mlxsw_sp_port *mlxsw_sp_port;
2719         struct mlxsw_sp *mlxsw_sp;
2720         unsigned long interval;
2721         struct neigh_parms *p;
2722         struct neighbour *n;
2723
2724         switch (event) {
2725         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
2726                 p = ptr;
2727
2728                 /* We don't care about changes in the default table. */
2729                 if (!p->dev || (p->tbl->family != AF_INET &&
2730                                 p->tbl->family != AF_INET6))
2731                         return NOTIFY_DONE;
2732
2733                 /* We are in atomic context and can't take RTNL mutex,
2734                  * so use RCU variant to walk the device chain.
2735                  */
2736                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
2737                 if (!mlxsw_sp_port)
2738                         return NOTIFY_DONE;
2739
2740                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2741                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
2742                 mlxsw_sp->router->neighs_update.interval = interval;
2743
2744                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2745                 break;
2746         case NETEVENT_NEIGH_UPDATE:
2747                 n = ptr;
2748
2749                 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
2750                         return NOTIFY_DONE;
2751
2752                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
2753                 if (!mlxsw_sp_port)
2754                         return NOTIFY_DONE;
2755
2756                 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2757                 if (!net_work) {
2758                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
2759                         return NOTIFY_BAD;
2760                 }
2761
2762                 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work);
2763                 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2764                 net_work->n = n;
2765
2766                 /* Take a reference to ensure the neighbour won't be
2767                  * destructed until we drop the reference in delayed
2768                  * work.
2769                  */
2770                 neigh_clone(n);
2771                 mlxsw_core_schedule_work(&net_work->work);
2772                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2773                 break;
2774         case NETEVENT_IPV4_MPATH_HASH_UPDATE:
2775         case NETEVENT_IPV6_MPATH_HASH_UPDATE:
2776                 return mlxsw_sp_router_schedule_work(ptr, nb,
2777                                 mlxsw_sp_router_mp_hash_event_work);
2778
2779         case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE:
2780                 return mlxsw_sp_router_schedule_work(ptr, nb,
2781                                 mlxsw_sp_router_update_priority_work);
2782         }
2783
2784         return NOTIFY_DONE;
2785 }
2786
2787 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
2788 {
2789         int err;
2790
2791         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
2792                               &mlxsw_sp_neigh_ht_params);
2793         if (err)
2794                 return err;
2795
2796         /* Initialize the polling interval according to the default
2797          * table.
2798          */
2799         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
2800
2801         /* Create the delayed works for the activity_update */
2802         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
2803                           mlxsw_sp_router_neighs_update_work);
2804         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
2805                           mlxsw_sp_router_probe_unresolved_nexthops);
2806         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
2807         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
2808         return 0;
2809 }
2810
2811 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2812 {
2813         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
2814         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
2815         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2816 }
2817
2818 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2819                                          struct mlxsw_sp_rif *rif)
2820 {
2821         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2822
2823         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2824                                  rif_list_node) {
2825                 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2826                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2827         }
2828 }
2829
2830 enum mlxsw_sp_nexthop_type {
2831         MLXSW_SP_NEXTHOP_TYPE_ETH,
2832         MLXSW_SP_NEXTHOP_TYPE_IPIP,
2833 };
2834
2835 struct mlxsw_sp_nexthop_key {
2836         struct fib_nh *fib_nh;
2837 };
2838
2839 struct mlxsw_sp_nexthop {
2840         struct list_head neigh_list_node; /* member of neigh entry list */
2841         struct list_head rif_list_node;
2842         struct list_head router_list_node;
2843         struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group
2844                                                    * this nexthop belongs to
2845                                                    */
2846         struct rhash_head ht_node;
2847         struct neigh_table *neigh_tbl;
2848         struct mlxsw_sp_nexthop_key key;
2849         unsigned char gw_addr[sizeof(struct in6_addr)];
2850         int ifindex;
2851         int nh_weight;
2852         int norm_nh_weight;
2853         int num_adj_entries;
2854         struct mlxsw_sp_rif *rif;
2855         u8 should_offload:1, /* set indicates this neigh is connected and
2856                               * should be put to KVD linear area of this group.
2857                               */
2858            offloaded:1, /* set in case the neigh is actually put into
2859                          * KVD linear area of this group.
2860                          */
2861            update:1; /* set indicates that MAC of this neigh should be
2862                       * updated in HW
2863                       */
2864         enum mlxsw_sp_nexthop_type type;
2865         union {
2866                 struct mlxsw_sp_neigh_entry *neigh_entry;
2867                 struct mlxsw_sp_ipip_entry *ipip_entry;
2868         };
2869         unsigned int counter_index;
2870         bool counter_valid;
2871 };
2872
2873 enum mlxsw_sp_nexthop_group_type {
2874         MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4,
2875         MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6,
2876 };
2877
2878 struct mlxsw_sp_nexthop_group_info {
2879         struct mlxsw_sp_nexthop_group *nh_grp;
2880         u32 adj_index;
2881         u16 ecmp_size;
2882         u16 count;
2883         int sum_norm_weight;
2884         u8 adj_index_valid:1,
2885            gateway:1; /* routes using the group use a gateway */
2886         struct mlxsw_sp_nexthop nexthops[0];
2887 #define nh_rif  nexthops[0].rif
2888 };
2889
2890 struct mlxsw_sp_nexthop_group {
2891         struct rhash_head ht_node;
2892         struct list_head fib_list; /* list of fib entries that use this group */
2893         union {
2894                 struct {
2895                         struct fib_info *fi;
2896                 } ipv4;
2897         };
2898         struct mlxsw_sp_nexthop_group_info *nhgi;
2899         enum mlxsw_sp_nexthop_group_type type;
2900         bool can_destroy;
2901 };
2902
2903 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2904                                     struct mlxsw_sp_nexthop *nh)
2905 {
2906         struct devlink *devlink;
2907
2908         devlink = priv_to_devlink(mlxsw_sp->core);
2909         if (!devlink_dpipe_table_counter_enabled(devlink,
2910                                                  MLXSW_SP_DPIPE_TABLE_NAME_ADJ))
2911                 return;
2912
2913         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index))
2914                 return;
2915
2916         nh->counter_valid = true;
2917 }
2918
2919 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp,
2920                                    struct mlxsw_sp_nexthop *nh)
2921 {
2922         if (!nh->counter_valid)
2923                 return;
2924         mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index);
2925         nh->counter_valid = false;
2926 }
2927
2928 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp,
2929                                  struct mlxsw_sp_nexthop *nh, u64 *p_counter)
2930 {
2931         if (!nh->counter_valid)
2932                 return -EINVAL;
2933
2934         return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index,
2935                                          p_counter, NULL);
2936 }
2937
2938 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router,
2939                                                struct mlxsw_sp_nexthop *nh)
2940 {
2941         if (!nh) {
2942                 if (list_empty(&router->nexthop_list))
2943                         return NULL;
2944                 else
2945                         return list_first_entry(&router->nexthop_list,
2946                                                 typeof(*nh), router_list_node);
2947         }
2948         if (list_is_last(&nh->router_list_node, &router->nexthop_list))
2949                 return NULL;
2950         return list_next_entry(nh, router_list_node);
2951 }
2952
2953 bool mlxsw_sp_nexthop_offload(struct mlxsw_sp_nexthop *nh)
2954 {
2955         return nh->offloaded;
2956 }
2957
2958 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh)
2959 {
2960         if (!nh->offloaded)
2961                 return NULL;
2962         return nh->neigh_entry->ha;
2963 }
2964
2965 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index,
2966                              u32 *p_adj_size, u32 *p_adj_hash_index)
2967 {
2968         struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
2969         u32 adj_hash_index = 0;
2970         int i;
2971
2972         if (!nh->offloaded || !nhgi->adj_index_valid)
2973                 return -EINVAL;
2974
2975         *p_adj_index = nhgi->adj_index;
2976         *p_adj_size = nhgi->ecmp_size;
2977
2978         for (i = 0; i < nhgi->count; i++) {
2979                 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
2980
2981                 if (nh_iter == nh)
2982                         break;
2983                 if (nh_iter->offloaded)
2984                         adj_hash_index += nh_iter->num_adj_entries;
2985         }
2986
2987         *p_adj_hash_index = adj_hash_index;
2988         return 0;
2989 }
2990
2991 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh)
2992 {
2993         return nh->rif;
2994 }
2995
2996 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh)
2997 {
2998         struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
2999         int i;
3000
3001         for (i = 0; i < nhgi->count; i++) {
3002                 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
3003
3004                 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP)
3005                         return true;
3006         }
3007         return false;
3008 }
3009
3010 struct mlxsw_sp_nexthop_group_cmp_arg {
3011         enum mlxsw_sp_nexthop_group_type type;
3012         union {
3013                 struct fib_info *fi;
3014                 struct mlxsw_sp_fib6_entry *fib6_entry;
3015         };
3016 };
3017
3018 static bool
3019 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
3020                                     const struct in6_addr *gw, int ifindex,
3021                                     int weight)
3022 {
3023         int i;
3024
3025         for (i = 0; i < nh_grp->nhgi->count; i++) {
3026                 const struct mlxsw_sp_nexthop *nh;
3027
3028                 nh = &nh_grp->nhgi->nexthops[i];
3029                 if (nh->ifindex == ifindex && nh->nh_weight == weight &&
3030                     ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
3031                         return true;
3032         }
3033
3034         return false;
3035 }
3036
3037 static bool
3038 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
3039                             const struct mlxsw_sp_fib6_entry *fib6_entry)
3040 {
3041         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3042
3043         if (nh_grp->nhgi->count != fib6_entry->nrt6)
3044                 return false;
3045
3046         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3047                 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3048                 struct in6_addr *gw;
3049                 int ifindex, weight;
3050
3051                 ifindex = fib6_nh->fib_nh_dev->ifindex;
3052                 weight = fib6_nh->fib_nh_weight;
3053                 gw = &fib6_nh->fib_nh_gw6;
3054                 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex,
3055                                                          weight))
3056                         return false;
3057         }
3058
3059         return true;
3060 }
3061
3062 static int
3063 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
3064 {
3065         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
3066         const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
3067
3068         if (nh_grp->type != cmp_arg->type)
3069                 return 1;
3070
3071         switch (cmp_arg->type) {
3072         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3073                 return cmp_arg->fi != nh_grp->ipv4.fi;
3074         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3075                 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
3076                                                     cmp_arg->fib6_entry);
3077         default:
3078                 WARN_ON(1);
3079                 return 1;
3080         }
3081 }
3082
3083 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
3084 {
3085         const struct mlxsw_sp_nexthop_group *nh_grp = data;
3086         const struct mlxsw_sp_nexthop *nh;
3087         struct fib_info *fi;
3088         unsigned int val;
3089         int i;
3090
3091         switch (nh_grp->type) {
3092         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3093                 fi = nh_grp->ipv4.fi;
3094                 return jhash(&fi, sizeof(fi), seed);
3095         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3096                 val = nh_grp->nhgi->count;
3097                 for (i = 0; i < nh_grp->nhgi->count; i++) {
3098                         nh = &nh_grp->nhgi->nexthops[i];
3099                         val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed);
3100                         val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed);
3101                 }
3102                 return jhash(&val, sizeof(val), seed);
3103         default:
3104                 WARN_ON(1);
3105                 return 0;
3106         }
3107 }
3108
3109 static u32
3110 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
3111 {
3112         unsigned int val = fib6_entry->nrt6;
3113         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3114
3115         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3116                 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3117                 struct net_device *dev = fib6_nh->fib_nh_dev;
3118                 struct in6_addr *gw = &fib6_nh->fib_nh_gw6;
3119
3120                 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed);
3121                 val ^= jhash(gw, sizeof(*gw), seed);
3122         }
3123
3124         return jhash(&val, sizeof(val), seed);
3125 }
3126
3127 static u32
3128 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
3129 {
3130         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
3131
3132         switch (cmp_arg->type) {
3133         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3134                 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
3135         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3136                 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
3137         default:
3138                 WARN_ON(1);
3139                 return 0;
3140         }
3141 }
3142
3143 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
3144         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
3145         .hashfn      = mlxsw_sp_nexthop_group_hash,
3146         .obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
3147         .obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
3148 };
3149
3150 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
3151                                          struct mlxsw_sp_nexthop_group *nh_grp)
3152 {
3153         if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3154             !nh_grp->nhgi->gateway)
3155                 return 0;
3156
3157         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
3158                                       &nh_grp->ht_node,
3159                                       mlxsw_sp_nexthop_group_ht_params);
3160 }
3161
3162 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
3163                                           struct mlxsw_sp_nexthop_group *nh_grp)
3164 {
3165         if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3166             !nh_grp->nhgi->gateway)
3167                 return;
3168
3169         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
3170                                &nh_grp->ht_node,
3171                                mlxsw_sp_nexthop_group_ht_params);
3172 }
3173
3174 static struct mlxsw_sp_nexthop_group *
3175 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
3176                                struct fib_info *fi)
3177 {
3178         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3179
3180         cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
3181         cmp_arg.fi = fi;
3182         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3183                                       &cmp_arg,
3184                                       mlxsw_sp_nexthop_group_ht_params);
3185 }
3186
3187 static struct mlxsw_sp_nexthop_group *
3188 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
3189                                struct mlxsw_sp_fib6_entry *fib6_entry)
3190 {
3191         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3192
3193         cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
3194         cmp_arg.fib6_entry = fib6_entry;
3195         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3196                                       &cmp_arg,
3197                                       mlxsw_sp_nexthop_group_ht_params);
3198 }
3199
3200 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
3201         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
3202         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
3203         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
3204 };
3205
3206 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
3207                                    struct mlxsw_sp_nexthop *nh)
3208 {
3209         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
3210                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
3211 }
3212
3213 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
3214                                     struct mlxsw_sp_nexthop *nh)
3215 {
3216         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
3217                                mlxsw_sp_nexthop_ht_params);
3218 }
3219
3220 static struct mlxsw_sp_nexthop *
3221 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
3222                         struct mlxsw_sp_nexthop_key key)
3223 {
3224         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
3225                                       mlxsw_sp_nexthop_ht_params);
3226 }
3227
3228 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
3229                                              const struct mlxsw_sp_fib *fib,
3230                                              u32 adj_index, u16 ecmp_size,
3231                                              u32 new_adj_index,
3232                                              u16 new_ecmp_size)
3233 {
3234         char raleu_pl[MLXSW_REG_RALEU_LEN];
3235
3236         mlxsw_reg_raleu_pack(raleu_pl,
3237                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
3238                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
3239                              new_ecmp_size);
3240         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
3241 }
3242
3243 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
3244                                           struct mlxsw_sp_nexthop_group *nh_grp,
3245                                           u32 old_adj_index, u16 old_ecmp_size)
3246 {
3247         struct mlxsw_sp_fib_entry *fib_entry;
3248         struct mlxsw_sp_fib *fib = NULL;
3249         int err;
3250
3251         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3252                 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
3253
3254                 if (fib == fib_entry->fib_node->fib)
3255                         continue;
3256                 fib = fib_entry->fib_node->fib;
3257                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
3258                                                         old_adj_index,
3259                                                         old_ecmp_size,
3260                                                         nhgi->adj_index,
3261                                                         nhgi->ecmp_size);
3262                 if (err)
3263                         return err;
3264         }
3265         return 0;
3266 }
3267
3268 static int __mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3269                                      struct mlxsw_sp_nexthop *nh)
3270 {
3271         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3272         char ratr_pl[MLXSW_REG_RATR_LEN];
3273
3274         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
3275                             true, MLXSW_REG_RATR_TYPE_ETHERNET,
3276                             adj_index, neigh_entry->rif);
3277         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
3278         if (nh->counter_valid)
3279                 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true);
3280         else
3281                 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false);
3282
3283         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
3284 }
3285
3286 int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3287                             struct mlxsw_sp_nexthop *nh)
3288 {
3289         int i;
3290
3291         for (i = 0; i < nh->num_adj_entries; i++) {
3292                 int err;
3293
3294                 err = __mlxsw_sp_nexthop_update(mlxsw_sp, adj_index + i, nh);
3295                 if (err)
3296                         return err;
3297         }
3298
3299         return 0;
3300 }
3301
3302 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3303                                           u32 adj_index,
3304                                           struct mlxsw_sp_nexthop *nh)
3305 {
3306         const struct mlxsw_sp_ipip_ops *ipip_ops;
3307
3308         ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
3309         return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry);
3310 }
3311
3312 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3313                                         u32 adj_index,
3314                                         struct mlxsw_sp_nexthop *nh)
3315 {
3316         int i;
3317
3318         for (i = 0; i < nh->num_adj_entries; i++) {
3319                 int err;
3320
3321                 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i,
3322                                                      nh);
3323                 if (err)
3324                         return err;
3325         }
3326
3327         return 0;
3328 }
3329
3330 static int
3331 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
3332                               struct mlxsw_sp_nexthop_group_info *nhgi,
3333                               bool reallocate)
3334 {
3335         u32 adj_index = nhgi->adj_index; /* base */
3336         struct mlxsw_sp_nexthop *nh;
3337         int i;
3338
3339         for (i = 0; i < nhgi->count; i++) {
3340                 nh = &nhgi->nexthops[i];
3341
3342                 if (!nh->should_offload) {
3343                         nh->offloaded = 0;
3344                         continue;
3345                 }
3346
3347                 if (nh->update || reallocate) {
3348                         int err = 0;
3349
3350                         switch (nh->type) {
3351                         case MLXSW_SP_NEXTHOP_TYPE_ETH:
3352                                 err = mlxsw_sp_nexthop_update
3353                                             (mlxsw_sp, adj_index, nh);
3354                                 break;
3355                         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
3356                                 err = mlxsw_sp_nexthop_ipip_update
3357                                             (mlxsw_sp, adj_index, nh);
3358                                 break;
3359                         }
3360                         if (err)
3361                                 return err;
3362                         nh->update = 0;
3363                         nh->offloaded = 1;
3364                 }
3365                 adj_index += nh->num_adj_entries;
3366         }
3367         return 0;
3368 }
3369
3370 static int
3371 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
3372                                     struct mlxsw_sp_nexthop_group *nh_grp)
3373 {
3374         struct mlxsw_sp_fib_entry *fib_entry;
3375         int err;
3376
3377         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3378                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3379                 if (err)
3380                         return err;
3381         }
3382         return 0;
3383 }
3384
3385 static void mlxsw_sp_adj_grp_size_round_up(u16 *p_adj_grp_size)
3386 {
3387         /* Valid sizes for an adjacency group are:
3388          * 1-64, 512, 1024, 2048 and 4096.
3389          */
3390         if (*p_adj_grp_size <= 64)
3391                 return;
3392         else if (*p_adj_grp_size <= 512)
3393                 *p_adj_grp_size = 512;
3394         else if (*p_adj_grp_size <= 1024)
3395                 *p_adj_grp_size = 1024;
3396         else if (*p_adj_grp_size <= 2048)
3397                 *p_adj_grp_size = 2048;
3398         else
3399                 *p_adj_grp_size = 4096;
3400 }
3401
3402 static void mlxsw_sp_adj_grp_size_round_down(u16 *p_adj_grp_size,
3403                                              unsigned int alloc_size)
3404 {
3405         if (alloc_size >= 4096)
3406                 *p_adj_grp_size = 4096;
3407         else if (alloc_size >= 2048)
3408                 *p_adj_grp_size = 2048;
3409         else if (alloc_size >= 1024)
3410                 *p_adj_grp_size = 1024;
3411         else if (alloc_size >= 512)
3412                 *p_adj_grp_size = 512;
3413 }
3414
3415 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp,
3416                                      u16 *p_adj_grp_size)
3417 {
3418         unsigned int alloc_size;
3419         int err;
3420
3421         /* Round up the requested group size to the next size supported
3422          * by the device and make sure the request can be satisfied.
3423          */
3424         mlxsw_sp_adj_grp_size_round_up(p_adj_grp_size);
3425         err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
3426                                               MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3427                                               *p_adj_grp_size, &alloc_size);
3428         if (err)
3429                 return err;
3430         /* It is possible the allocation results in more allocated
3431          * entries than requested. Try to use as much of them as
3432          * possible.
3433          */
3434         mlxsw_sp_adj_grp_size_round_down(p_adj_grp_size, alloc_size);
3435
3436         return 0;
3437 }
3438
3439 static void
3440 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi)
3441 {
3442         int i, g = 0, sum_norm_weight = 0;
3443         struct mlxsw_sp_nexthop *nh;
3444
3445         for (i = 0; i < nhgi->count; i++) {
3446                 nh = &nhgi->nexthops[i];
3447
3448                 if (!nh->should_offload)
3449                         continue;
3450                 if (g > 0)
3451                         g = gcd(nh->nh_weight, g);
3452                 else
3453                         g = nh->nh_weight;
3454         }
3455
3456         for (i = 0; i < nhgi->count; i++) {
3457                 nh = &nhgi->nexthops[i];
3458
3459                 if (!nh->should_offload)
3460                         continue;
3461                 nh->norm_nh_weight = nh->nh_weight / g;
3462                 sum_norm_weight += nh->norm_nh_weight;
3463         }
3464
3465         nhgi->sum_norm_weight = sum_norm_weight;
3466 }
3467
3468 static void
3469 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi)
3470 {
3471         int i, weight = 0, lower_bound = 0;
3472         int total = nhgi->sum_norm_weight;
3473         u16 ecmp_size = nhgi->ecmp_size;
3474
3475         for (i = 0; i < nhgi->count; i++) {
3476                 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
3477                 int upper_bound;
3478
3479                 if (!nh->should_offload)
3480                         continue;
3481                 weight += nh->norm_nh_weight;
3482                 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total);
3483                 nh->num_adj_entries = upper_bound - lower_bound;
3484                 lower_bound = upper_bound;
3485         }
3486 }
3487
3488 static struct mlxsw_sp_nexthop *
3489 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
3490                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6);
3491
3492 static void
3493 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3494                                         struct mlxsw_sp_nexthop_group *nh_grp)
3495 {
3496         int i;
3497
3498         for (i = 0; i < nh_grp->nhgi->count; i++) {
3499                 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
3500
3501                 if (nh->offloaded)
3502                         nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3503                 else
3504                         nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3505         }
3506 }
3507
3508 static void
3509 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp,
3510                                           struct mlxsw_sp_fib6_entry *fib6_entry)
3511 {
3512         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3513
3514         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3515                 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3516                 struct mlxsw_sp_nexthop *nh;
3517
3518                 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3519                 if (nh && nh->offloaded)
3520                         fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3521                 else
3522                         fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3523         }
3524 }
3525
3526 static void
3527 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3528                                         struct mlxsw_sp_nexthop_group *nh_grp)
3529 {
3530         struct mlxsw_sp_fib6_entry *fib6_entry;
3531
3532         /* Unfortunately, in IPv6 the route and the nexthop are described by
3533          * the same struct, so we need to iterate over all the routes using the
3534          * nexthop group and set / clear the offload indication for them.
3535          */
3536         list_for_each_entry(fib6_entry, &nh_grp->fib_list,
3537                             common.nexthop_group_node)
3538                 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
3539 }
3540
3541 static void
3542 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3543                                        struct mlxsw_sp_nexthop_group *nh_grp)
3544 {
3545         switch (nh_grp->type) {
3546         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3547                 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp);
3548                 break;
3549         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3550                 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp);
3551                 break;
3552         }
3553 }
3554
3555 static int
3556 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
3557                                struct mlxsw_sp_nexthop_group *nh_grp)
3558 {
3559         struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
3560         u16 ecmp_size, old_ecmp_size;
3561         struct mlxsw_sp_nexthop *nh;
3562         bool offload_change = false;
3563         u32 adj_index;
3564         bool old_adj_index_valid;
3565         int i, err2, err = 0;
3566         u32 old_adj_index;
3567
3568         if (!nhgi->gateway) {
3569                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3570                 return 0;
3571         }
3572
3573         for (i = 0; i < nhgi->count; i++) {
3574                 nh = &nhgi->nexthops[i];
3575
3576                 if (nh->should_offload != nh->offloaded) {
3577                         offload_change = true;
3578                         if (nh->should_offload)
3579                                 nh->update = 1;
3580                 }
3581         }
3582         if (!offload_change) {
3583                 /* Nothing was added or removed, so no need to reallocate. Just
3584                  * update MAC on existing adjacency indexes.
3585                  */
3586                 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false);
3587                 if (err) {
3588                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3589                         goto set_trap;
3590                 }
3591                 return 0;
3592         }
3593         mlxsw_sp_nexthop_group_normalize(nhgi);
3594         if (!nhgi->sum_norm_weight)
3595                 /* No neigh of this group is connected so we just set
3596                  * the trap and let everthing flow through kernel.
3597                  */
3598                 goto set_trap;
3599
3600         ecmp_size = nhgi->sum_norm_weight;
3601         err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size);
3602         if (err)
3603                 /* No valid allocation size available. */
3604                 goto set_trap;
3605
3606         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3607                                   ecmp_size, &adj_index);
3608         if (err) {
3609                 /* We ran out of KVD linear space, just set the
3610                  * trap and let everything flow through kernel.
3611                  */
3612                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
3613                 goto set_trap;
3614         }
3615         old_adj_index_valid = nhgi->adj_index_valid;
3616         old_adj_index = nhgi->adj_index;
3617         old_ecmp_size = nhgi->ecmp_size;
3618         nhgi->adj_index_valid = 1;
3619         nhgi->adj_index = adj_index;
3620         nhgi->ecmp_size = ecmp_size;
3621         mlxsw_sp_nexthop_group_rebalance(nhgi);
3622         err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true);
3623         if (err) {
3624                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3625                 goto set_trap;
3626         }
3627
3628         mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3629
3630         if (!old_adj_index_valid) {
3631                 /* The trap was set for fib entries, so we have to call
3632                  * fib entry update to unset it and use adjacency index.
3633                  */
3634                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3635                 if (err) {
3636                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
3637                         goto set_trap;
3638                 }
3639                 return 0;
3640         }
3641
3642         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
3643                                              old_adj_index, old_ecmp_size);
3644         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3645                            old_ecmp_size, old_adj_index);
3646         if (err) {
3647                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
3648                 goto set_trap;
3649         }
3650
3651         return 0;
3652
3653 set_trap:
3654         old_adj_index_valid = nhgi->adj_index_valid;
3655         nhgi->adj_index_valid = 0;
3656         for (i = 0; i < nhgi->count; i++) {
3657                 nh = &nhgi->nexthops[i];
3658                 nh->offloaded = 0;
3659         }
3660         err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3661         if (err2)
3662                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
3663         mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3664         if (old_adj_index_valid)
3665                 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3666                                    nhgi->ecmp_size, nhgi->adj_index);
3667         return err;
3668 }
3669
3670 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
3671                                             bool removing)
3672 {
3673         if (!removing)
3674                 nh->should_offload = 1;
3675         else
3676                 nh->should_offload = 0;
3677         nh->update = 1;
3678 }
3679
3680 static int
3681 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp,
3682                                     struct mlxsw_sp_neigh_entry *neigh_entry)
3683 {
3684         struct neighbour *n, *old_n = neigh_entry->key.n;
3685         struct mlxsw_sp_nexthop *nh;
3686         bool entry_connected;
3687         u8 nud_state, dead;
3688         int err;
3689
3690         nh = list_first_entry(&neigh_entry->nexthop_list,
3691                               struct mlxsw_sp_nexthop, neigh_list_node);
3692
3693         n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3694         if (!n) {
3695                 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3696                 if (IS_ERR(n))
3697                         return PTR_ERR(n);
3698                 neigh_event_send(n, NULL);
3699         }
3700
3701         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
3702         neigh_entry->key.n = n;
3703         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
3704         if (err)
3705                 goto err_neigh_entry_insert;
3706
3707         read_lock_bh(&n->lock);
3708         nud_state = n->nud_state;
3709         dead = n->dead;
3710         read_unlock_bh(&n->lock);
3711         entry_connected = nud_state & NUD_VALID && !dead;
3712
3713         list_for_each_entry(nh, &neigh_entry->nexthop_list,
3714                             neigh_list_node) {
3715                 neigh_release(old_n);
3716                 neigh_clone(n);
3717                 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected);
3718                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
3719         }
3720
3721         neigh_release(n);
3722
3723         return 0;
3724
3725 err_neigh_entry_insert:
3726         neigh_entry->key.n = old_n;
3727         mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
3728         neigh_release(n);
3729         return err;
3730 }
3731
3732 static void
3733 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
3734                               struct mlxsw_sp_neigh_entry *neigh_entry,
3735                               bool removing, bool dead)
3736 {
3737         struct mlxsw_sp_nexthop *nh;
3738
3739         if (list_empty(&neigh_entry->nexthop_list))
3740                 return;
3741
3742         if (dead) {
3743                 int err;
3744
3745                 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp,
3746                                                           neigh_entry);
3747                 if (err)
3748                         dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n");
3749                 return;
3750         }
3751
3752         list_for_each_entry(nh, &neigh_entry->nexthop_list,
3753                             neigh_list_node) {
3754                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
3755                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
3756         }
3757 }
3758
3759 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
3760                                       struct mlxsw_sp_rif *rif)
3761 {
3762         if (nh->rif)
3763                 return;
3764
3765         nh->rif = rif;
3766         list_add(&nh->rif_list_node, &rif->nexthop_list);
3767 }
3768
3769 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
3770 {
3771         if (!nh->rif)
3772                 return;
3773
3774         list_del(&nh->rif_list_node);
3775         nh->rif = NULL;
3776 }
3777
3778 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
3779                                        struct mlxsw_sp_nexthop *nh)
3780 {
3781         struct mlxsw_sp_neigh_entry *neigh_entry;
3782         struct neighbour *n;
3783         u8 nud_state, dead;
3784         int err;
3785
3786         if (!nh->nhgi->gateway || nh->neigh_entry)
3787                 return 0;
3788
3789         /* Take a reference of neigh here ensuring that neigh would
3790          * not be destructed before the nexthop entry is finished.
3791          * The reference is taken either in neigh_lookup() or
3792          * in neigh_create() in case n is not found.
3793          */
3794         n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3795         if (!n) {
3796                 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3797                 if (IS_ERR(n))
3798                         return PTR_ERR(n);
3799                 neigh_event_send(n, NULL);
3800         }
3801         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
3802         if (!neigh_entry) {
3803                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
3804                 if (IS_ERR(neigh_entry)) {
3805                         err = -EINVAL;
3806                         goto err_neigh_entry_create;
3807                 }
3808         }
3809
3810         /* If that is the first nexthop connected to that neigh, add to
3811          * nexthop_neighs_list
3812          */
3813         if (list_empty(&neigh_entry->nexthop_list))
3814                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
3815                               &mlxsw_sp->router->nexthop_neighs_list);
3816
3817         nh->neigh_entry = neigh_entry;
3818         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
3819         read_lock_bh(&n->lock);
3820         nud_state = n->nud_state;
3821         dead = n->dead;
3822         read_unlock_bh(&n->lock);
3823         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
3824
3825         return 0;
3826
3827 err_neigh_entry_create:
3828         neigh_release(n);
3829         return err;
3830 }
3831
3832 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
3833                                         struct mlxsw_sp_nexthop *nh)
3834 {
3835         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3836         struct neighbour *n;
3837
3838         if (!neigh_entry)
3839                 return;
3840         n = neigh_entry->key.n;
3841
3842         __mlxsw_sp_nexthop_neigh_update(nh, true);
3843         list_del(&nh->neigh_list_node);
3844         nh->neigh_entry = NULL;
3845
3846         /* If that is the last nexthop connected to that neigh, remove from
3847          * nexthop_neighs_list
3848          */
3849         if (list_empty(&neigh_entry->nexthop_list))
3850                 list_del(&neigh_entry->nexthop_neighs_list_node);
3851
3852         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
3853                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
3854
3855         neigh_release(n);
3856 }
3857
3858 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev)
3859 {
3860         struct net_device *ul_dev;
3861         bool is_up;
3862
3863         rcu_read_lock();
3864         ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
3865         is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true;
3866         rcu_read_unlock();
3867
3868         return is_up;
3869 }
3870
3871 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
3872                                        struct mlxsw_sp_nexthop *nh,
3873                                        struct mlxsw_sp_ipip_entry *ipip_entry)
3874 {
3875         bool removing;
3876
3877         if (!nh->nhgi->gateway || nh->ipip_entry)
3878                 return;
3879
3880         nh->ipip_entry = ipip_entry;
3881         removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev);
3882         __mlxsw_sp_nexthop_neigh_update(nh, removing);
3883         mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common);
3884 }
3885
3886 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
3887                                        struct mlxsw_sp_nexthop *nh)
3888 {
3889         struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
3890
3891         if (!ipip_entry)
3892                 return;
3893
3894         __mlxsw_sp_nexthop_neigh_update(nh, true);
3895         nh->ipip_entry = NULL;
3896 }
3897
3898 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
3899                                         const struct fib_nh *fib_nh,
3900                                         enum mlxsw_sp_ipip_type *p_ipipt)
3901 {
3902         struct net_device *dev = fib_nh->fib_nh_dev;
3903
3904         return dev &&
3905                fib_nh->nh_parent->fib_type == RTN_UNICAST &&
3906                mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
3907 }
3908
3909 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp,
3910                                       struct mlxsw_sp_nexthop *nh,
3911                                       const struct net_device *dev)
3912 {
3913         const struct mlxsw_sp_ipip_ops *ipip_ops;
3914         struct mlxsw_sp_ipip_entry *ipip_entry;
3915         struct mlxsw_sp_rif *rif;
3916         int err;
3917
3918         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev);
3919         if (ipip_entry) {
3920                 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
3921                 if (ipip_ops->can_offload(mlxsw_sp, dev)) {
3922                         nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
3923                         mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
3924                         return 0;
3925                 }
3926         }
3927
3928         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
3929         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3930         if (!rif)
3931                 return 0;
3932
3933         mlxsw_sp_nexthop_rif_init(nh, rif);
3934         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
3935         if (err)
3936                 goto err_neigh_init;
3937
3938         return 0;
3939
3940 err_neigh_init:
3941         mlxsw_sp_nexthop_rif_fini(nh);
3942         return err;
3943 }
3944
3945 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
3946                                        struct mlxsw_sp_nexthop *nh)
3947 {
3948         switch (nh->type) {
3949         case MLXSW_SP_NEXTHOP_TYPE_ETH:
3950                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
3951                 mlxsw_sp_nexthop_rif_fini(nh);
3952                 break;
3953         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
3954                 mlxsw_sp_nexthop_rif_fini(nh);
3955                 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
3956                 break;
3957         }
3958 }
3959
3960 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
3961                                   struct mlxsw_sp_nexthop_group *nh_grp,
3962                                   struct mlxsw_sp_nexthop *nh,
3963                                   struct fib_nh *fib_nh)
3964 {
3965         struct net_device *dev = fib_nh->fib_nh_dev;
3966         struct in_device *in_dev;
3967         int err;
3968
3969         nh->nhgi = nh_grp->nhgi;
3970         nh->key.fib_nh = fib_nh;
3971 #ifdef CONFIG_IP_ROUTE_MULTIPATH
3972         nh->nh_weight = fib_nh->fib_nh_weight;
3973 #else
3974         nh->nh_weight = 1;
3975 #endif
3976         memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4));
3977         nh->neigh_tbl = &arp_tbl;
3978         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
3979         if (err)
3980                 return err;
3981
3982         mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
3983         list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
3984
3985         if (!dev)
3986                 return 0;
3987         nh->ifindex = dev->ifindex;
3988
3989         rcu_read_lock();
3990         in_dev = __in_dev_get_rcu(dev);
3991         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
3992             fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) {
3993                 rcu_read_unlock();
3994                 return 0;
3995         }
3996         rcu_read_unlock();
3997
3998         err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
3999         if (err)
4000                 goto err_nexthop_neigh_init;
4001
4002         return 0;
4003
4004 err_nexthop_neigh_init:
4005         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4006         return err;
4007 }
4008
4009 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
4010                                    struct mlxsw_sp_nexthop *nh)
4011 {
4012         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4013         list_del(&nh->router_list_node);
4014         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4015         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4016 }
4017
4018 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
4019                                     unsigned long event, struct fib_nh *fib_nh)
4020 {
4021         struct mlxsw_sp_nexthop_key key;
4022         struct mlxsw_sp_nexthop *nh;
4023
4024         if (mlxsw_sp->router->aborted)
4025                 return;
4026
4027         key.fib_nh = fib_nh;
4028         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
4029         if (!nh)
4030                 return;
4031
4032         switch (event) {
4033         case FIB_EVENT_NH_ADD:
4034                 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev);
4035                 break;
4036         case FIB_EVENT_NH_DEL:
4037                 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4038                 break;
4039         }
4040
4041         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4042 }
4043
4044 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
4045                                         struct mlxsw_sp_rif *rif)
4046 {
4047         struct mlxsw_sp_nexthop *nh;
4048         bool removing;
4049
4050         list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) {
4051                 switch (nh->type) {
4052                 case MLXSW_SP_NEXTHOP_TYPE_ETH:
4053                         removing = false;
4054                         break;
4055                 case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4056                         removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev);
4057                         break;
4058                 default:
4059                         WARN_ON(1);
4060                         continue;
4061                 }
4062
4063                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
4064                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4065         }
4066 }
4067
4068 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
4069                                          struct mlxsw_sp_rif *old_rif,
4070                                          struct mlxsw_sp_rif *new_rif)
4071 {
4072         struct mlxsw_sp_nexthop *nh;
4073
4074         list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list);
4075         list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node)
4076                 nh->rif = new_rif;
4077         mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif);
4078 }
4079
4080 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4081                                            struct mlxsw_sp_rif *rif)
4082 {
4083         struct mlxsw_sp_nexthop *nh, *tmp;
4084
4085         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
4086                 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4087                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4088         }
4089 }
4090
4091 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
4092                                    struct fib_info *fi)
4093 {
4094         const struct fib_nh *nh = fib_info_nh(fi, 0);
4095
4096         return nh->fib_nh_scope == RT_SCOPE_LINK ||
4097                mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL);
4098 }
4099
4100 static int
4101 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp,
4102                                   struct mlxsw_sp_nexthop_group *nh_grp)
4103 {
4104         unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi);
4105         struct mlxsw_sp_nexthop_group_info *nhgi;
4106         struct mlxsw_sp_nexthop *nh;
4107         int err, i;
4108
4109         nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL);
4110         if (!nhgi)
4111                 return -ENOMEM;
4112         nh_grp->nhgi = nhgi;
4113         nhgi->nh_grp = nh_grp;
4114         nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi);
4115         nhgi->count = nhs;
4116         for (i = 0; i < nhgi->count; i++) {
4117                 struct fib_nh *fib_nh;
4118
4119                 nh = &nhgi->nexthops[i];
4120                 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i);
4121                 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
4122                 if (err)
4123                         goto err_nexthop4_init;
4124         }
4125         err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4126         if (err)
4127                 goto err_group_refresh;
4128
4129         return 0;
4130
4131 err_group_refresh:
4132         i = nhgi->count;
4133 err_nexthop4_init:
4134         for (i--; i >= 0; i--) {
4135                 nh = &nhgi->nexthops[i];
4136                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
4137         }
4138         kfree(nhgi);
4139         return err;
4140 }
4141
4142 static void
4143 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp,
4144                                   struct mlxsw_sp_nexthop_group *nh_grp)
4145 {
4146         struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
4147         int i;
4148
4149         for (i = nhgi->count - 1; i >= 0; i--) {
4150                 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
4151
4152                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
4153         }
4154         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4155         WARN_ON_ONCE(nhgi->adj_index_valid);
4156         kfree(nhgi);
4157 }
4158
4159 static struct mlxsw_sp_nexthop_group *
4160 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
4161 {
4162         struct mlxsw_sp_nexthop_group *nh_grp;
4163         int err;
4164
4165         nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
4166         if (!nh_grp)
4167                 return ERR_PTR(-ENOMEM);
4168         INIT_LIST_HEAD(&nh_grp->fib_list);
4169         nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
4170         nh_grp->ipv4.fi = fi;
4171         fib_info_hold(fi);
4172
4173         err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp);
4174         if (err)
4175                 goto err_nexthop_group_info_init;
4176
4177         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
4178         if (err)
4179                 goto err_nexthop_group_insert;
4180
4181         nh_grp->can_destroy = true;
4182
4183         return nh_grp;
4184
4185 err_nexthop_group_insert:
4186         mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
4187 err_nexthop_group_info_init:
4188         fib_info_put(fi);
4189         kfree(nh_grp);
4190         return ERR_PTR(err);
4191 }
4192
4193 static void
4194 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
4195                                 struct mlxsw_sp_nexthop_group *nh_grp)
4196 {
4197         if (!nh_grp->can_destroy)
4198                 return;
4199         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
4200         mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
4201         fib_info_put(nh_grp->ipv4.fi);
4202         kfree(nh_grp);
4203 }
4204
4205 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
4206                                        struct mlxsw_sp_fib_entry *fib_entry,
4207                                        struct fib_info *fi)
4208 {
4209         struct mlxsw_sp_nexthop_group *nh_grp;
4210
4211         nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
4212         if (!nh_grp) {
4213                 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
4214                 if (IS_ERR(nh_grp))
4215                         return PTR_ERR(nh_grp);
4216         }
4217         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
4218         fib_entry->nh_group = nh_grp;
4219         return 0;
4220 }
4221
4222 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
4223                                         struct mlxsw_sp_fib_entry *fib_entry)
4224 {
4225         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
4226
4227         list_del(&fib_entry->nexthop_group_node);
4228         if (!list_empty(&nh_grp->fib_list))
4229                 return;
4230         mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
4231 }
4232
4233 static bool
4234 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
4235 {
4236         struct mlxsw_sp_fib4_entry *fib4_entry;
4237
4238         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
4239                                   common);
4240         return !fib4_entry->tos;
4241 }
4242
4243 static bool
4244 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
4245 {
4246         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
4247
4248         switch (fib_entry->fib_node->fib->proto) {
4249         case MLXSW_SP_L3_PROTO_IPV4:
4250                 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
4251                         return false;
4252                 break;
4253         case MLXSW_SP_L3_PROTO_IPV6:
4254                 break;
4255         }
4256
4257         switch (fib_entry->type) {
4258         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
4259                 return !!nh_group->nhgi->adj_index_valid;
4260         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
4261                 return !!nh_group->nhgi->nh_rif;
4262         case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
4263         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
4264         case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
4265                 return true;
4266         default:
4267                 return false;
4268         }
4269 }
4270
4271 static struct mlxsw_sp_nexthop *
4272 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
4273                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
4274 {
4275         int i;
4276
4277         for (i = 0; i < nh_grp->nhgi->count; i++) {
4278                 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
4279                 struct fib6_info *rt = mlxsw_sp_rt6->rt;
4280
4281                 if (nh->rif && nh->rif->dev == rt->fib6_nh->fib_nh_dev &&
4282                     ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
4283                                     &rt->fib6_nh->fib_nh_gw6))
4284                         return nh;
4285                 continue;
4286         }
4287
4288         return NULL;
4289 }
4290
4291 static void
4292 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
4293                                  struct mlxsw_sp_fib_entry *fib_entry)
4294 {
4295         u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
4296         int dst_len = fib_entry->fib_node->key.prefix_len;
4297         struct mlxsw_sp_fib4_entry *fib4_entry;
4298         struct fib_rt_info fri;
4299         bool should_offload;
4300
4301         should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
4302         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
4303                                   common);
4304         fri.fi = fib4_entry->fi;
4305         fri.tb_id = fib4_entry->tb_id;
4306         fri.dst = cpu_to_be32(*p_dst);
4307         fri.dst_len = dst_len;
4308         fri.tos = fib4_entry->tos;
4309         fri.type = fib4_entry->type;
4310         fri.offload = should_offload;
4311         fri.trap = !should_offload;
4312         fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
4313 }
4314
4315 static void
4316 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
4317                                    struct mlxsw_sp_fib_entry *fib_entry)
4318 {
4319         u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
4320         int dst_len = fib_entry->fib_node->key.prefix_len;
4321         struct mlxsw_sp_fib4_entry *fib4_entry;
4322         struct fib_rt_info fri;
4323
4324         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
4325                                   common);
4326         fri.fi = fib4_entry->fi;
4327         fri.tb_id = fib4_entry->tb_id;
4328         fri.dst = cpu_to_be32(*p_dst);
4329         fri.dst_len = dst_len;
4330         fri.tos = fib4_entry->tos;
4331         fri.type = fib4_entry->type;
4332         fri.offload = false;
4333         fri.trap = false;
4334         fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
4335 }
4336
4337 static void
4338 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
4339                                  struct mlxsw_sp_fib_entry *fib_entry)
4340 {
4341         struct mlxsw_sp_fib6_entry *fib6_entry;
4342         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4343         bool should_offload;
4344
4345         should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
4346
4347         /* In IPv6 a multipath route is represented using multiple routes, so
4348          * we need to set the flags on all of them.
4349          */
4350         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
4351                                   common);
4352         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
4353                 fib6_info_hw_flags_set(mlxsw_sp_rt6->rt, should_offload,
4354                                        !should_offload);
4355 }
4356
4357 static void
4358 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
4359                                    struct mlxsw_sp_fib_entry *fib_entry)
4360 {
4361         struct mlxsw_sp_fib6_entry *fib6_entry;
4362         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4363
4364         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
4365                                   common);
4366         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
4367                 fib6_info_hw_flags_set(mlxsw_sp_rt6->rt, false, false);
4368 }
4369
4370 static void
4371 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
4372                                 struct mlxsw_sp_fib_entry *fib_entry)
4373 {
4374         switch (fib_entry->fib_node->fib->proto) {
4375         case MLXSW_SP_L3_PROTO_IPV4:
4376                 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry);
4377                 break;
4378         case MLXSW_SP_L3_PROTO_IPV6:
4379                 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry);
4380                 break;
4381         }
4382 }
4383
4384 static void
4385 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
4386                                   struct mlxsw_sp_fib_entry *fib_entry)
4387 {
4388         switch (fib_entry->fib_node->fib->proto) {
4389         case MLXSW_SP_L3_PROTO_IPV4:
4390                 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry);
4391                 break;
4392         case MLXSW_SP_L3_PROTO_IPV6:
4393                 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry);
4394                 break;
4395         }
4396 }
4397
4398 static void
4399 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp,
4400                                     struct mlxsw_sp_fib_entry *fib_entry,
4401                                     enum mlxsw_sp_fib_entry_op op)
4402 {
4403         switch (op) {
4404         case MLXSW_SP_FIB_ENTRY_OP_WRITE:
4405         case MLXSW_SP_FIB_ENTRY_OP_UPDATE:
4406                 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry);
4407                 break;
4408         case MLXSW_SP_FIB_ENTRY_OP_DELETE:
4409                 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry);
4410                 break;
4411         default:
4412                 break;
4413         }
4414 }
4415
4416 struct mlxsw_sp_fib_entry_op_ctx_basic {
4417         char ralue_pl[MLXSW_REG_RALUE_LEN];
4418 };
4419
4420 static void
4421 mlxsw_sp_router_ll_basic_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4422                                         enum mlxsw_sp_l3proto proto,
4423                                         enum mlxsw_sp_fib_entry_op op,
4424                                         u16 virtual_router, u8 prefix_len,
4425                                         unsigned char *addr,
4426                                         struct mlxsw_sp_fib_entry_priv *priv)
4427 {
4428         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4429         enum mlxsw_reg_ralxx_protocol ralxx_proto;
4430         char *ralue_pl = op_ctx_basic->ralue_pl;
4431         enum mlxsw_reg_ralue_op ralue_op;
4432
4433         ralxx_proto = (enum mlxsw_reg_ralxx_protocol) proto;
4434
4435         switch (op) {
4436         case MLXSW_SP_FIB_ENTRY_OP_WRITE:
4437         case MLXSW_SP_FIB_ENTRY_OP_UPDATE:
4438                 ralue_op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
4439                 break;
4440         case MLXSW_SP_FIB_ENTRY_OP_DELETE:
4441                 ralue_op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
4442                 break;
4443         default:
4444                 WARN_ON_ONCE(1);
4445                 return;
4446         }
4447
4448         switch (proto) {
4449         case MLXSW_SP_L3_PROTO_IPV4:
4450                 mlxsw_reg_ralue_pack4(ralue_pl, ralxx_proto, ralue_op,
4451                                       virtual_router, prefix_len, (u32 *) addr);
4452                 break;
4453         case MLXSW_SP_L3_PROTO_IPV6:
4454                 mlxsw_reg_ralue_pack6(ralue_pl, ralxx_proto, ralue_op,
4455                                       virtual_router, prefix_len, addr);
4456                 break;
4457         }
4458 }
4459
4460 static void
4461 mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4462                                                    enum mlxsw_reg_ralue_trap_action trap_action,
4463                                                    u16 trap_id, u32 adjacency_index, u16 ecmp_size)
4464 {
4465         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4466
4467         mlxsw_reg_ralue_act_remote_pack(op_ctx_basic->ralue_pl, trap_action,
4468                                         trap_id, adjacency_index, ecmp_size);
4469 }
4470
4471 static void
4472 mlxsw_sp_router_ll_basic_fib_entry_act_local_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4473                                                   enum mlxsw_reg_ralue_trap_action trap_action,
4474                                                   u16 trap_id, u16 local_erif)
4475 {
4476         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4477
4478         mlxsw_reg_ralue_act_local_pack(op_ctx_basic->ralue_pl, trap_action,
4479                                        trap_id, local_erif);
4480 }
4481
4482 static void
4483 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx)
4484 {
4485         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4486
4487         mlxsw_reg_ralue_act_ip2me_pack(op_ctx_basic->ralue_pl);
4488 }
4489
4490 static void
4491 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4492                                                       u32 tunnel_ptr)
4493 {
4494         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4495
4496         mlxsw_reg_ralue_act_ip2me_tun_pack(op_ctx_basic->ralue_pl, tunnel_ptr);
4497 }
4498
4499 static int
4500 mlxsw_sp_router_ll_basic_fib_entry_commit(struct mlxsw_sp *mlxsw_sp,
4501                                           struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4502                                           bool *postponed_for_bulk)
4503 {
4504         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4505
4506         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
4507                                op_ctx_basic->ralue_pl);
4508 }
4509
4510 static bool
4511 mlxsw_sp_router_ll_basic_fib_entry_is_committed(struct mlxsw_sp_fib_entry_priv *priv)
4512 {
4513         return true;
4514 }
4515
4516 static void mlxsw_sp_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4517                                     struct mlxsw_sp_fib_entry *fib_entry,
4518                                     enum mlxsw_sp_fib_entry_op op)
4519 {
4520         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
4521
4522         mlxsw_sp_fib_entry_op_ctx_priv_hold(op_ctx, fib_entry->priv);
4523         fib->ll_ops->fib_entry_pack(op_ctx, fib->proto, op, fib->vr->id,
4524                                     fib_entry->fib_node->key.prefix_len,
4525                                     fib_entry->fib_node->key.addr,
4526                                     fib_entry->priv);
4527 }
4528
4529 int mlxsw_sp_fib_entry_commit(struct mlxsw_sp *mlxsw_sp,
4530                               struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4531                               const struct mlxsw_sp_router_ll_ops *ll_ops)
4532 {
4533         bool postponed_for_bulk = false;
4534         int err;
4535
4536         err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, &postponed_for_bulk);
4537         if (!postponed_for_bulk)
4538                 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
4539         return err;
4540 }
4541
4542 static int mlxsw_sp_adj_discard_write(struct mlxsw_sp *mlxsw_sp, u16 rif_index)
4543 {
4544         enum mlxsw_reg_ratr_trap_action trap_action;
4545         char ratr_pl[MLXSW_REG_RATR_LEN];
4546         int err;
4547
4548         if (mlxsw_sp->router->adj_discard_index_valid)
4549                 return 0;
4550
4551         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4552                                   &mlxsw_sp->router->adj_discard_index);
4553         if (err)
4554                 return err;
4555
4556         trap_action = MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS;
4557         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true,
4558                             MLXSW_REG_RATR_TYPE_ETHERNET,
4559                             mlxsw_sp->router->adj_discard_index, rif_index);
4560         mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action);
4561         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
4562         if (err)
4563                 goto err_ratr_write;
4564
4565         mlxsw_sp->router->adj_discard_index_valid = true;
4566
4567         return 0;
4568
4569 err_ratr_write:
4570         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4571                            mlxsw_sp->router->adj_discard_index);
4572         return err;
4573 }
4574
4575 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
4576                                         struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4577                                         struct mlxsw_sp_fib_entry *fib_entry,
4578                                         enum mlxsw_sp_fib_entry_op op)
4579 {
4580         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4581         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
4582         struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi;
4583         enum mlxsw_reg_ralue_trap_action trap_action;
4584         u16 trap_id = 0;
4585         u32 adjacency_index = 0;
4586         u16 ecmp_size = 0;
4587         int err;
4588
4589         /* In case the nexthop group adjacency index is valid, use it
4590          * with provided ECMP size. Otherwise, setup trap and pass
4591          * traffic to kernel.
4592          */
4593         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
4594                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4595                 adjacency_index = nhgi->adj_index;
4596                 ecmp_size = nhgi->ecmp_size;
4597         } else if (!nhgi->adj_index_valid && nhgi->count && nhgi->nh_rif) {
4598                 err = mlxsw_sp_adj_discard_write(mlxsw_sp,
4599                                                  nhgi->nh_rif->rif_index);
4600                 if (err)
4601                         return err;
4602                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4603                 adjacency_index = mlxsw_sp->router->adj_discard_index;
4604                 ecmp_size = 1;
4605         } else {
4606                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4607                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
4608         }
4609
4610         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4611         ll_ops->fib_entry_act_remote_pack(op_ctx, trap_action, trap_id,
4612                                           adjacency_index, ecmp_size);
4613         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4614 }
4615
4616 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
4617                                        struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4618                                        struct mlxsw_sp_fib_entry *fib_entry,
4619                                        enum mlxsw_sp_fib_entry_op op)
4620 {
4621         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4622         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nhgi->nh_rif;
4623         enum mlxsw_reg_ralue_trap_action trap_action;
4624         u16 trap_id = 0;
4625         u16 rif_index = 0;
4626
4627         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
4628                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4629                 rif_index = rif->rif_index;
4630         } else {
4631                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4632                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
4633         }
4634
4635         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4636         ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, rif_index);
4637         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4638 }
4639
4640 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
4641                                       struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4642                                       struct mlxsw_sp_fib_entry *fib_entry,
4643                                       enum mlxsw_sp_fib_entry_op op)
4644 {
4645         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4646
4647         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4648         ll_ops->fib_entry_act_ip2me_pack(op_ctx);
4649         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4650 }
4651
4652 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp,
4653                                            struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4654                                            struct mlxsw_sp_fib_entry *fib_entry,
4655                                            enum mlxsw_sp_fib_entry_op op)
4656 {
4657         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4658         enum mlxsw_reg_ralue_trap_action trap_action;
4659
4660         trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR;
4661         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4662         ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, 0, 0);
4663         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4664 }
4665
4666 static int
4667 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp,
4668                                   struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4669                                   struct mlxsw_sp_fib_entry *fib_entry,
4670                                   enum mlxsw_sp_fib_entry_op op)
4671 {
4672         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4673         enum mlxsw_reg_ralue_trap_action trap_action;
4674         u16 trap_id;
4675
4676         trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4677         trap_id = MLXSW_TRAP_ID_RTR_INGRESS1;
4678
4679         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4680         ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, 0);
4681         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4682 }
4683
4684 static int
4685 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
4686                                  struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4687                                  struct mlxsw_sp_fib_entry *fib_entry,
4688                                  enum mlxsw_sp_fib_entry_op op)
4689 {
4690         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4691         struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
4692         const struct mlxsw_sp_ipip_ops *ipip_ops;
4693
4694         if (WARN_ON(!ipip_entry))
4695                 return -EINVAL;
4696
4697         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
4698         return ipip_ops->fib_entry_op(mlxsw_sp, ll_ops, op_ctx, ipip_entry, op,
4699                                       fib_entry->decap.tunnel_index, fib_entry->priv);
4700 }
4701
4702 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp,
4703                                            struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4704                                            struct mlxsw_sp_fib_entry *fib_entry,
4705                                            enum mlxsw_sp_fib_entry_op op)
4706 {
4707         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4708
4709         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4710         ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx,
4711                                              fib_entry->decap.tunnel_index);
4712         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4713 }
4714
4715 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
4716                                    struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4717                                    struct mlxsw_sp_fib_entry *fib_entry,
4718                                    enum mlxsw_sp_fib_entry_op op)
4719 {
4720         switch (fib_entry->type) {
4721         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
4722                 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, op_ctx, fib_entry, op);
4723         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
4724                 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, op_ctx, fib_entry, op);
4725         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
4726                 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, op_ctx, fib_entry, op);
4727         case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
4728                 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, op_ctx, fib_entry, op);
4729         case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE:
4730                 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, op_ctx, fib_entry, op);
4731         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
4732                 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, op_ctx, fib_entry, op);
4733         case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
4734                 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, op_ctx, fib_entry, op);
4735         }
4736         return -EINVAL;
4737 }
4738
4739 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
4740                                  struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4741                                  struct mlxsw_sp_fib_entry *fib_entry,
4742                                  enum mlxsw_sp_fib_entry_op op)
4743 {
4744         int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, op);
4745
4746         if (err)
4747                 return err;
4748
4749         mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op);
4750
4751         return err;
4752 }
4753
4754 static int __mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
4755                                        struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4756                                        struct mlxsw_sp_fib_entry *fib_entry,
4757                                        bool is_new)
4758 {
4759         return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry,
4760                                      is_new ? MLXSW_SP_FIB_ENTRY_OP_WRITE :
4761                                               MLXSW_SP_FIB_ENTRY_OP_UPDATE);
4762 }
4763
4764 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
4765                                      struct mlxsw_sp_fib_entry *fib_entry)
4766 {
4767         struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx;
4768
4769         mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
4770         return __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, false);
4771 }
4772
4773 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
4774                                   struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4775                                   struct mlxsw_sp_fib_entry *fib_entry)
4776 {
4777         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4778
4779         if (!ll_ops->fib_entry_is_committed(fib_entry->priv))
4780                 return 0;
4781         return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry,
4782                                      MLXSW_SP_FIB_ENTRY_OP_DELETE);
4783 }
4784
4785 static int
4786 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
4787                              const struct fib_entry_notifier_info *fen_info,
4788                              struct mlxsw_sp_fib_entry *fib_entry)
4789 {
4790         struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi;
4791         union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
4792         struct mlxsw_sp_router *router = mlxsw_sp->router;
4793         u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id);
4794         int ifindex = nhgi->nexthops[0].ifindex;
4795         struct mlxsw_sp_ipip_entry *ipip_entry;
4796
4797         switch (fen_info->type) {
4798         case RTN_LOCAL:
4799                 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex,
4800                                                                MLXSW_SP_L3_PROTO_IPV4, dip);
4801                 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
4802                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
4803                         return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
4804                                                              fib_entry,
4805                                                              ipip_entry);
4806                 }
4807                 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id,
4808                                                  MLXSW_SP_L3_PROTO_IPV4,
4809                                                  &dip)) {
4810                         u32 tunnel_index;
4811
4812                         tunnel_index = router->nve_decap_config.tunnel_index;
4813                         fib_entry->decap.tunnel_index = tunnel_index;
4814                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
4815                         return 0;
4816                 }
4817                 fallthrough;
4818         case RTN_BROADCAST:
4819                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
4820                 return 0;
4821         case RTN_BLACKHOLE:
4822                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
4823                 return 0;
4824         case RTN_UNREACHABLE:
4825         case RTN_PROHIBIT:
4826                 /* Packets hitting these routes need to be trapped, but
4827                  * can do so with a lower priority than packets directed
4828                  * at the host, so use action type local instead of trap.
4829                  */
4830                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
4831                 return 0;
4832         case RTN_UNICAST:
4833                 if (nhgi->gateway)
4834                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
4835                 else
4836                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4837                 return 0;
4838         default:
4839                 return -EINVAL;
4840         }
4841 }
4842
4843 static void
4844 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
4845                                struct mlxsw_sp_fib_entry *fib_entry)
4846 {
4847         switch (fib_entry->type) {
4848         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
4849                 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
4850                 break;
4851         default:
4852                 break;
4853         }
4854 }
4855
4856 static struct mlxsw_sp_fib4_entry *
4857 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
4858                            struct mlxsw_sp_fib_node *fib_node,
4859                            const struct fib_entry_notifier_info *fen_info)
4860 {
4861         struct mlxsw_sp_fib4_entry *fib4_entry;
4862         struct mlxsw_sp_fib_entry *fib_entry;
4863         int err;
4864
4865         fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
4866         if (!fib4_entry)
4867                 return ERR_PTR(-ENOMEM);
4868         fib_entry = &fib4_entry->common;
4869
4870         fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops);
4871         if (IS_ERR(fib_entry->priv)) {
4872                 err = PTR_ERR(fib_entry->priv);
4873                 goto err_fib_entry_priv_create;
4874         }
4875
4876         err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
4877         if (err)
4878                 goto err_nexthop4_group_get;
4879
4880         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
4881         if (err)
4882                 goto err_fib4_entry_type_set;
4883
4884         fib4_entry->fi = fen_info->fi;
4885         fib_info_hold(fib4_entry->fi);
4886         fib4_entry->tb_id = fen_info->tb_id;
4887         fib4_entry->type = fen_info->type;
4888         fib4_entry->tos = fen_info->tos;
4889
4890         fib_entry->fib_node = fib_node;
4891
4892         return fib4_entry;
4893
4894 err_fib4_entry_type_set:
4895         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
4896 err_nexthop4_group_get:
4897         mlxsw_sp_fib_entry_priv_put(fib_entry->priv);
4898 err_fib_entry_priv_create:
4899         kfree(fib4_entry);
4900         return ERR_PTR(err);
4901 }
4902
4903 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
4904                                         struct mlxsw_sp_fib4_entry *fib4_entry)
4905 {
4906         fib_info_put(fib4_entry->fi);
4907         mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, &fib4_entry->common);
4908         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
4909         mlxsw_sp_fib_entry_priv_put(fib4_entry->common.priv);
4910         kfree(fib4_entry);
4911 }
4912
4913 static struct mlxsw_sp_fib4_entry *
4914 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
4915                            const struct fib_entry_notifier_info *fen_info)
4916 {
4917         struct mlxsw_sp_fib4_entry *fib4_entry;
4918         struct mlxsw_sp_fib_node *fib_node;
4919         struct mlxsw_sp_fib *fib;
4920         struct mlxsw_sp_vr *vr;
4921
4922         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
4923         if (!vr)
4924                 return NULL;
4925         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
4926
4927         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
4928                                             sizeof(fen_info->dst),
4929                                             fen_info->dst_len);
4930         if (!fib_node)
4931                 return NULL;
4932
4933         fib4_entry = container_of(fib_node->fib_entry,
4934                                   struct mlxsw_sp_fib4_entry, common);
4935         if (fib4_entry->tb_id == fen_info->tb_id &&
4936             fib4_entry->tos == fen_info->tos &&
4937             fib4_entry->type == fen_info->type &&
4938             fib4_entry->fi == fen_info->fi)
4939                 return fib4_entry;
4940
4941         return NULL;
4942 }
4943
4944 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
4945         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
4946         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
4947         .key_len = sizeof(struct mlxsw_sp_fib_key),
4948         .automatic_shrinking = true,
4949 };
4950
4951 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
4952                                     struct mlxsw_sp_fib_node *fib_node)
4953 {
4954         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
4955                                       mlxsw_sp_fib_ht_params);
4956 }
4957
4958 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
4959                                      struct mlxsw_sp_fib_node *fib_node)
4960 {
4961         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
4962                                mlxsw_sp_fib_ht_params);
4963 }
4964
4965 static struct mlxsw_sp_fib_node *
4966 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
4967                          size_t addr_len, unsigned char prefix_len)
4968 {
4969         struct mlxsw_sp_fib_key key;
4970
4971         memset(&key, 0, sizeof(key));
4972         memcpy(key.addr, addr, addr_len);
4973         key.prefix_len = prefix_len;
4974         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
4975 }
4976
4977 static struct mlxsw_sp_fib_node *
4978 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
4979                          size_t addr_len, unsigned char prefix_len)
4980 {
4981         struct mlxsw_sp_fib_node *fib_node;
4982
4983         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
4984         if (!fib_node)
4985                 return NULL;
4986
4987         list_add(&fib_node->list, &fib->node_list);
4988         memcpy(fib_node->key.addr, addr, addr_len);
4989         fib_node->key.prefix_len = prefix_len;
4990
4991         return fib_node;
4992 }
4993
4994 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
4995 {
4996         list_del(&fib_node->list);
4997         kfree(fib_node);
4998 }
4999
5000 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
5001                                       struct mlxsw_sp_fib_node *fib_node)
5002 {
5003         struct mlxsw_sp_prefix_usage req_prefix_usage;
5004         struct mlxsw_sp_fib *fib = fib_node->fib;
5005         struct mlxsw_sp_lpm_tree *lpm_tree;
5006         int err;
5007
5008         lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto];
5009         if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
5010                 goto out;
5011
5012         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
5013         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
5014         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
5015                                          fib->proto);
5016         if (IS_ERR(lpm_tree))
5017                 return PTR_ERR(lpm_tree);
5018
5019         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
5020         if (err)
5021                 goto err_lpm_tree_replace;
5022
5023 out:
5024         lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++;
5025         return 0;
5026
5027 err_lpm_tree_replace:
5028         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
5029         return err;
5030 }
5031
5032 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
5033                                          struct mlxsw_sp_fib_node *fib_node)
5034 {
5035         struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
5036         struct mlxsw_sp_prefix_usage req_prefix_usage;
5037         struct mlxsw_sp_fib *fib = fib_node->fib;
5038         int err;
5039
5040         if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
5041                 return;
5042         /* Try to construct a new LPM tree from the current prefix usage
5043          * minus the unused one. If we fail, continue using the old one.
5044          */
5045         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
5046         mlxsw_sp_prefix_usage_clear(&req_prefix_usage,
5047                                     fib_node->key.prefix_len);
5048         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
5049                                          fib->proto);
5050         if (IS_ERR(lpm_tree))
5051                 return;
5052
5053         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
5054         if (err)
5055                 goto err_lpm_tree_replace;
5056
5057         return;
5058
5059 err_lpm_tree_replace:
5060         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
5061 }
5062
5063 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
5064                                   struct mlxsw_sp_fib_node *fib_node,
5065                                   struct mlxsw_sp_fib *fib)
5066 {
5067         int err;
5068
5069         err = mlxsw_sp_fib_node_insert(fib, fib_node);
5070         if (err)
5071                 return err;
5072         fib_node->fib = fib;
5073
5074         err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node);
5075         if (err)
5076                 goto err_fib_lpm_tree_link;
5077
5078         return 0;
5079
5080 err_fib_lpm_tree_link:
5081         fib_node->fib = NULL;
5082         mlxsw_sp_fib_node_remove(fib, fib_node);
5083         return err;
5084 }
5085
5086 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
5087                                    struct mlxsw_sp_fib_node *fib_node)
5088 {
5089         struct mlxsw_sp_fib *fib = fib_node->fib;
5090
5091         mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node);
5092         fib_node->fib = NULL;
5093         mlxsw_sp_fib_node_remove(fib, fib_node);
5094 }
5095
5096 static struct mlxsw_sp_fib_node *
5097 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
5098                       size_t addr_len, unsigned char prefix_len,
5099                       enum mlxsw_sp_l3proto proto)
5100 {
5101         struct mlxsw_sp_fib_node *fib_node;
5102         struct mlxsw_sp_fib *fib;
5103         struct mlxsw_sp_vr *vr;
5104         int err;
5105
5106         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL);
5107         if (IS_ERR(vr))
5108                 return ERR_CAST(vr);
5109         fib = mlxsw_sp_vr_fib(vr, proto);
5110
5111         fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
5112         if (fib_node)
5113                 return fib_node;
5114
5115         fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
5116         if (!fib_node) {
5117                 err = -ENOMEM;
5118                 goto err_fib_node_create;
5119         }
5120
5121         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
5122         if (err)
5123                 goto err_fib_node_init;
5124
5125         return fib_node;
5126
5127 err_fib_node_init:
5128         mlxsw_sp_fib_node_destroy(fib_node);
5129 err_fib_node_create:
5130         mlxsw_sp_vr_put(mlxsw_sp, vr);
5131         return ERR_PTR(err);
5132 }
5133
5134 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
5135                                   struct mlxsw_sp_fib_node *fib_node)
5136 {
5137         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
5138
5139         if (fib_node->fib_entry)
5140                 return;
5141         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
5142         mlxsw_sp_fib_node_destroy(fib_node);
5143         mlxsw_sp_vr_put(mlxsw_sp, vr);
5144 }
5145
5146 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp,
5147                                         struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5148                                         struct mlxsw_sp_fib_entry *fib_entry)
5149 {
5150         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
5151         bool is_new = !fib_node->fib_entry;
5152         int err;
5153
5154         fib_node->fib_entry = fib_entry;
5155
5156         err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, is_new);
5157         if (err)
5158                 goto err_fib_entry_update;
5159
5160         return 0;
5161
5162 err_fib_entry_update:
5163         fib_node->fib_entry = NULL;
5164         return err;
5165 }
5166
5167 static int __mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
5168                                             struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5169                                             struct mlxsw_sp_fib_entry *fib_entry)
5170 {
5171         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
5172         int err;
5173
5174         err = mlxsw_sp_fib_entry_del(mlxsw_sp, op_ctx, fib_entry);
5175         fib_node->fib_entry = NULL;
5176         return err;
5177 }
5178
5179 static void mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
5180                                            struct mlxsw_sp_fib_entry *fib_entry)
5181 {
5182         struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx;
5183
5184         mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
5185         __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, fib_entry);
5186 }
5187
5188 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry)
5189 {
5190         struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
5191         struct mlxsw_sp_fib4_entry *fib4_replaced;
5192
5193         if (!fib_node->fib_entry)
5194                 return true;
5195
5196         fib4_replaced = container_of(fib_node->fib_entry,
5197                                      struct mlxsw_sp_fib4_entry, common);
5198         if (fib4_entry->tb_id == RT_TABLE_MAIN &&
5199             fib4_replaced->tb_id == RT_TABLE_LOCAL)
5200                 return false;
5201
5202         return true;
5203 }
5204
5205 static int
5206 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp,
5207                              struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5208                              const struct fib_entry_notifier_info *fen_info)
5209 {
5210         struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced;
5211         struct mlxsw_sp_fib_entry *replaced;
5212         struct mlxsw_sp_fib_node *fib_node;
5213         int err;
5214
5215         if (mlxsw_sp->router->aborted)
5216                 return 0;
5217
5218         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
5219                                          &fen_info->dst, sizeof(fen_info->dst),
5220                                          fen_info->dst_len,
5221                                          MLXSW_SP_L3_PROTO_IPV4);
5222         if (IS_ERR(fib_node)) {
5223                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
5224                 return PTR_ERR(fib_node);
5225         }
5226
5227         fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
5228         if (IS_ERR(fib4_entry)) {
5229                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
5230                 err = PTR_ERR(fib4_entry);
5231                 goto err_fib4_entry_create;
5232         }
5233
5234         if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) {
5235                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
5236                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5237                 return 0;
5238         }
5239
5240         replaced = fib_node->fib_entry;
5241         err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib4_entry->common);
5242         if (err) {
5243                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
5244                 goto err_fib_node_entry_link;
5245         }
5246
5247         /* Nothing to replace */
5248         if (!replaced)
5249                 return 0;
5250
5251         mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
5252         fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry,
5253                                      common);
5254         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced);
5255
5256         return 0;
5257
5258 err_fib_node_entry_link:
5259         fib_node->fib_entry = replaced;
5260         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
5261 err_fib4_entry_create:
5262         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5263         return err;
5264 }
5265
5266 static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
5267                                     struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5268                                     struct fib_entry_notifier_info *fen_info)
5269 {
5270         struct mlxsw_sp_fib4_entry *fib4_entry;
5271         struct mlxsw_sp_fib_node *fib_node;
5272         int err;
5273
5274         if (mlxsw_sp->router->aborted)
5275                 return 0;
5276
5277         fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
5278         if (!fib4_entry)
5279                 return 0;
5280         fib_node = fib4_entry->common.fib_node;
5281
5282         err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib4_entry->common);
5283         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
5284         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5285         return err;
5286 }
5287
5288 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt)
5289 {
5290         /* Multicast routes aren't supported, so ignore them. Neighbour
5291          * Discovery packets are specifically trapped.
5292          */
5293         if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST)
5294                 return true;
5295
5296         /* Cloned routes are irrelevant in the forwarding path. */
5297         if (rt->fib6_flags & RTF_CACHE)
5298                 return true;
5299
5300         return false;
5301 }
5302
5303 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt)
5304 {
5305         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5306
5307         mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
5308         if (!mlxsw_sp_rt6)
5309                 return ERR_PTR(-ENOMEM);
5310
5311         /* In case of route replace, replaced route is deleted with
5312          * no notification. Take reference to prevent accessing freed
5313          * memory.
5314          */
5315         mlxsw_sp_rt6->rt = rt;
5316         fib6_info_hold(rt);
5317
5318         return mlxsw_sp_rt6;
5319 }
5320
5321 #if IS_ENABLED(CONFIG_IPV6)
5322 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
5323 {
5324         fib6_info_release(rt);
5325 }
5326 #else
5327 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
5328 {
5329 }
5330 #endif
5331
5332 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
5333 {
5334         struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
5335
5336         if (!mlxsw_sp_rt6->rt->nh)
5337                 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
5338         mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
5339         kfree(mlxsw_sp_rt6);
5340 }
5341
5342 static struct fib6_info *
5343 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
5344 {
5345         return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
5346                                 list)->rt;
5347 }
5348
5349 static struct mlxsw_sp_rt6 *
5350 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
5351                             const struct fib6_info *rt)
5352 {
5353         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5354
5355         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
5356                 if (mlxsw_sp_rt6->rt == rt)
5357                         return mlxsw_sp_rt6;
5358         }
5359
5360         return NULL;
5361 }
5362
5363 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
5364                                         const struct fib6_info *rt,
5365                                         enum mlxsw_sp_ipip_type *ret)
5366 {
5367         return rt->fib6_nh->fib_nh_dev &&
5368                mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret);
5369 }
5370
5371 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
5372                                   struct mlxsw_sp_nexthop_group *nh_grp,
5373                                   struct mlxsw_sp_nexthop *nh,
5374                                   const struct fib6_info *rt)
5375 {
5376         struct net_device *dev = rt->fib6_nh->fib_nh_dev;
5377
5378         nh->nhgi = nh_grp->nhgi;
5379         nh->nh_weight = rt->fib6_nh->fib_nh_weight;
5380         memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr));
5381 #if IS_ENABLED(CONFIG_IPV6)
5382         nh->neigh_tbl = &nd_tbl;
5383 #endif
5384         mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
5385
5386         list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
5387
5388         if (!dev)
5389                 return 0;
5390         nh->ifindex = dev->ifindex;
5391
5392         return mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
5393 }
5394
5395 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
5396                                    struct mlxsw_sp_nexthop *nh)
5397 {
5398         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
5399         list_del(&nh->router_list_node);
5400         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
5401 }
5402
5403 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
5404                                     const struct fib6_info *rt)
5405 {
5406         return rt->fib6_nh->fib_nh_gw_family ||
5407                mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
5408 }
5409
5410 static int
5411 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp,
5412                                   struct mlxsw_sp_nexthop_group *nh_grp,
5413                                   struct mlxsw_sp_fib6_entry *fib6_entry)
5414 {
5415         struct mlxsw_sp_nexthop_group_info *nhgi;
5416         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5417         struct mlxsw_sp_nexthop *nh;
5418         int err, i;
5419
5420         nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6),
5421                        GFP_KERNEL);
5422         if (!nhgi)
5423                 return -ENOMEM;
5424         nh_grp->nhgi = nhgi;
5425         nhgi->nh_grp = nh_grp;
5426         mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
5427                                         struct mlxsw_sp_rt6, list);
5428         nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
5429         nhgi->count = fib6_entry->nrt6;
5430         for (i = 0; i < nhgi->count; i++) {
5431                 struct fib6_info *rt = mlxsw_sp_rt6->rt;
5432
5433                 nh = &nhgi->nexthops[i];
5434                 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
5435                 if (err)
5436                         goto err_nexthop6_init;
5437                 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
5438         }
5439         nh_grp->nhgi = nhgi;
5440         err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5441         if (err)
5442                 goto err_group_refresh;
5443
5444         return 0;
5445
5446 err_group_refresh:
5447         i = nhgi->count;
5448 err_nexthop6_init:
5449         for (i--; i >= 0; i--) {
5450                 nh = &nhgi->nexthops[i];
5451                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
5452         }
5453         kfree(nhgi);
5454         return err;
5455 }
5456
5457 static void
5458 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp,
5459                                   struct mlxsw_sp_nexthop_group *nh_grp)
5460 {
5461         struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
5462         int i;
5463
5464         for (i = nhgi->count - 1; i >= 0; i--) {
5465                 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
5466
5467                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
5468         }
5469         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5470         WARN_ON_ONCE(nhgi->adj_index_valid);
5471         kfree(nhgi);
5472 }
5473
5474 static struct mlxsw_sp_nexthop_group *
5475 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
5476                                struct mlxsw_sp_fib6_entry *fib6_entry)
5477 {
5478         struct mlxsw_sp_nexthop_group *nh_grp;
5479         int err;
5480
5481         nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
5482         if (!nh_grp)
5483                 return ERR_PTR(-ENOMEM);
5484         INIT_LIST_HEAD(&nh_grp->fib_list);
5485         nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
5486
5487         err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry);
5488         if (err)
5489                 goto err_nexthop_group_info_init;
5490
5491         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
5492         if (err)
5493                 goto err_nexthop_group_insert;
5494
5495         nh_grp->can_destroy = true;
5496
5497         return nh_grp;
5498
5499 err_nexthop_group_insert:
5500         mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
5501 err_nexthop_group_info_init:
5502         kfree(nh_grp);
5503         return ERR_PTR(err);
5504 }
5505
5506 static void
5507 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
5508                                 struct mlxsw_sp_nexthop_group *nh_grp)
5509 {
5510         if (!nh_grp->can_destroy)
5511                 return;
5512         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
5513         mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
5514         kfree(nh_grp);
5515 }
5516
5517 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
5518                                        struct mlxsw_sp_fib6_entry *fib6_entry)
5519 {
5520         struct mlxsw_sp_nexthop_group *nh_grp;
5521
5522         nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
5523         if (!nh_grp) {
5524                 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
5525                 if (IS_ERR(nh_grp))
5526                         return PTR_ERR(nh_grp);
5527         }
5528
5529         /* The route and the nexthop are described by the same struct, so we
5530          * need to the update the nexthop offload indication for the new route.
5531          */
5532         __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
5533
5534         list_add_tail(&fib6_entry->common.nexthop_group_node,
5535                       &nh_grp->fib_list);
5536         fib6_entry->common.nh_group = nh_grp;
5537
5538         return 0;
5539 }
5540
5541 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
5542                                         struct mlxsw_sp_fib_entry *fib_entry)
5543 {
5544         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
5545
5546         list_del(&fib_entry->nexthop_group_node);
5547         if (!list_empty(&nh_grp->fib_list))
5548                 return;
5549         mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
5550 }
5551
5552 static int mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
5553                                           struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5554                                           struct mlxsw_sp_fib6_entry *fib6_entry)
5555 {
5556         struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
5557         int err;
5558
5559         fib6_entry->common.nh_group = NULL;
5560         list_del(&fib6_entry->common.nexthop_group_node);
5561
5562         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
5563         if (err)
5564                 goto err_nexthop6_group_get;
5565
5566         /* In case this entry is offloaded, then the adjacency index
5567          * currently associated with it in the device's table is that
5568          * of the old group. Start using the new one instead.
5569          */
5570         err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx,
5571                                           &fib6_entry->common, false);
5572         if (err)
5573                 goto err_fib_entry_update;
5574
5575         if (list_empty(&old_nh_grp->fib_list))
5576                 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
5577
5578         return 0;
5579
5580 err_fib_entry_update:
5581         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
5582 err_nexthop6_group_get:
5583         list_add_tail(&fib6_entry->common.nexthop_group_node,
5584                       &old_nh_grp->fib_list);
5585         fib6_entry->common.nh_group = old_nh_grp;
5586         return err;
5587 }
5588
5589 static int
5590 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
5591                                 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5592                                 struct mlxsw_sp_fib6_entry *fib6_entry,
5593                                 struct fib6_info **rt_arr, unsigned int nrt6)
5594 {
5595         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5596         int err, i;
5597
5598         for (i = 0; i < nrt6; i++) {
5599                 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
5600                 if (IS_ERR(mlxsw_sp_rt6)) {
5601                         err = PTR_ERR(mlxsw_sp_rt6);
5602                         goto err_rt6_create;
5603                 }
5604
5605                 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
5606                 fib6_entry->nrt6++;
5607         }
5608
5609         err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry);
5610         if (err)
5611                 goto err_nexthop6_group_update;
5612
5613         return 0;
5614
5615 err_nexthop6_group_update:
5616         i = nrt6;
5617 err_rt6_create:
5618         for (i--; i >= 0; i--) {
5619                 fib6_entry->nrt6--;
5620                 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
5621                                                struct mlxsw_sp_rt6, list);
5622                 list_del(&mlxsw_sp_rt6->list);
5623                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5624         }
5625         return err;
5626 }
5627
5628 static void
5629 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
5630                                 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5631                                 struct mlxsw_sp_fib6_entry *fib6_entry,
5632                                 struct fib6_info **rt_arr, unsigned int nrt6)
5633 {
5634         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5635         int i;
5636
5637         for (i = 0; i < nrt6; i++) {
5638                 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry,
5639                                                            rt_arr[i]);
5640                 if (WARN_ON_ONCE(!mlxsw_sp_rt6))
5641                         continue;
5642
5643                 fib6_entry->nrt6--;
5644                 list_del(&mlxsw_sp_rt6->list);
5645                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5646         }
5647
5648         mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry);
5649 }
5650
5651 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
5652                                          struct mlxsw_sp_fib_entry *fib_entry,
5653                                          const struct fib6_info *rt)
5654 {
5655         if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))
5656                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
5657         else if (rt->fib6_type == RTN_BLACKHOLE)
5658                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
5659         else if (rt->fib6_flags & RTF_REJECT)
5660                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
5661         else if (fib_entry->nh_group->nhgi->gateway)
5662                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
5663         else
5664                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
5665 }
5666
5667 static void
5668 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
5669 {
5670         struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
5671
5672         list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
5673                                  list) {
5674                 fib6_entry->nrt6--;
5675                 list_del(&mlxsw_sp_rt6->list);
5676                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5677         }
5678 }
5679
5680 static struct mlxsw_sp_fib6_entry *
5681 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
5682                            struct mlxsw_sp_fib_node *fib_node,
5683                            struct fib6_info **rt_arr, unsigned int nrt6)
5684 {
5685         struct mlxsw_sp_fib6_entry *fib6_entry;
5686         struct mlxsw_sp_fib_entry *fib_entry;
5687         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5688         int err, i;
5689
5690         fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
5691         if (!fib6_entry)
5692                 return ERR_PTR(-ENOMEM);
5693         fib_entry = &fib6_entry->common;
5694
5695         fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops);
5696         if (IS_ERR(fib_entry->priv)) {
5697                 err = PTR_ERR(fib_entry->priv);
5698                 goto err_fib_entry_priv_create;
5699         }
5700
5701         INIT_LIST_HEAD(&fib6_entry->rt6_list);
5702
5703         for (i = 0; i < nrt6; i++) {
5704                 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
5705                 if (IS_ERR(mlxsw_sp_rt6)) {
5706                         err = PTR_ERR(mlxsw_sp_rt6);
5707                         goto err_rt6_create;
5708                 }
5709                 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
5710                 fib6_entry->nrt6++;
5711         }
5712
5713         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
5714         if (err)
5715                 goto err_nexthop6_group_get;
5716
5717         mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]);
5718
5719         fib_entry->fib_node = fib_node;
5720
5721         return fib6_entry;
5722
5723 err_nexthop6_group_get:
5724         i = nrt6;
5725 err_rt6_create:
5726         for (i--; i >= 0; i--) {
5727                 fib6_entry->nrt6--;
5728                 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
5729                                                struct mlxsw_sp_rt6, list);
5730                 list_del(&mlxsw_sp_rt6->list);
5731                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5732         }
5733         mlxsw_sp_fib_entry_priv_put(fib_entry->priv);
5734 err_fib_entry_priv_create:
5735         kfree(fib6_entry);
5736         return ERR_PTR(err);
5737 }
5738
5739 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
5740                                         struct mlxsw_sp_fib6_entry *fib6_entry)
5741 {
5742         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
5743         mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
5744         WARN_ON(fib6_entry->nrt6);
5745         mlxsw_sp_fib_entry_priv_put(fib6_entry->common.priv);
5746         kfree(fib6_entry);
5747 }
5748
5749 static struct mlxsw_sp_fib6_entry *
5750 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
5751                            const struct fib6_info *rt)
5752 {
5753         struct mlxsw_sp_fib6_entry *fib6_entry;
5754         struct mlxsw_sp_fib_node *fib_node;
5755         struct mlxsw_sp_fib *fib;
5756         struct fib6_info *cmp_rt;
5757         struct mlxsw_sp_vr *vr;
5758
5759         vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id);
5760         if (!vr)
5761                 return NULL;
5762         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
5763
5764         fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr,
5765                                             sizeof(rt->fib6_dst.addr),
5766                                             rt->fib6_dst.plen);
5767         if (!fib_node)
5768                 return NULL;
5769
5770         fib6_entry = container_of(fib_node->fib_entry,
5771                                   struct mlxsw_sp_fib6_entry, common);
5772         cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
5773         if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id &&
5774             rt->fib6_metric == cmp_rt->fib6_metric &&
5775             mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
5776                 return fib6_entry;
5777
5778         return NULL;
5779 }
5780
5781 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry)
5782 {
5783         struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
5784         struct mlxsw_sp_fib6_entry *fib6_replaced;
5785         struct fib6_info *rt, *rt_replaced;
5786
5787         if (!fib_node->fib_entry)
5788                 return true;
5789
5790         fib6_replaced = container_of(fib_node->fib_entry,
5791                                      struct mlxsw_sp_fib6_entry,
5792                                      common);
5793         rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
5794         rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced);
5795         if (rt->fib6_table->tb6_id == RT_TABLE_MAIN &&
5796             rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL)
5797                 return false;
5798
5799         return true;
5800 }
5801
5802 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp,
5803                                         struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5804                                         struct fib6_info **rt_arr, unsigned int nrt6)
5805 {
5806         struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced;
5807         struct mlxsw_sp_fib_entry *replaced;
5808         struct mlxsw_sp_fib_node *fib_node;
5809         struct fib6_info *rt = rt_arr[0];
5810         int err;
5811
5812         if (mlxsw_sp->router->aborted)
5813                 return 0;
5814
5815         if (rt->fib6_src.plen)
5816                 return -EINVAL;
5817
5818         if (mlxsw_sp_fib6_rt_should_ignore(rt))
5819                 return 0;
5820
5821         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
5822                                          &rt->fib6_dst.addr,
5823                                          sizeof(rt->fib6_dst.addr),
5824                                          rt->fib6_dst.plen,
5825                                          MLXSW_SP_L3_PROTO_IPV6);
5826         if (IS_ERR(fib_node))
5827                 return PTR_ERR(fib_node);
5828
5829         fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr,
5830                                                 nrt6);
5831         if (IS_ERR(fib6_entry)) {
5832                 err = PTR_ERR(fib6_entry);
5833                 goto err_fib6_entry_create;
5834         }
5835
5836         if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) {
5837                 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5838                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5839                 return 0;
5840         }
5841
5842         replaced = fib_node->fib_entry;
5843         err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib6_entry->common);
5844         if (err)
5845                 goto err_fib_node_entry_link;
5846
5847         /* Nothing to replace */
5848         if (!replaced)
5849                 return 0;
5850
5851         mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
5852         fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry,
5853                                      common);
5854         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced);
5855
5856         return 0;
5857
5858 err_fib_node_entry_link:
5859         fib_node->fib_entry = replaced;
5860         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5861 err_fib6_entry_create:
5862         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5863         return err;
5864 }
5865
5866 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp,
5867                                        struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5868                                        struct fib6_info **rt_arr, unsigned int nrt6)
5869 {
5870         struct mlxsw_sp_fib6_entry *fib6_entry;
5871         struct mlxsw_sp_fib_node *fib_node;
5872         struct fib6_info *rt = rt_arr[0];
5873         int err;
5874
5875         if (mlxsw_sp->router->aborted)
5876                 return 0;
5877
5878         if (rt->fib6_src.plen)
5879                 return -EINVAL;
5880
5881         if (mlxsw_sp_fib6_rt_should_ignore(rt))
5882                 return 0;
5883
5884         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
5885                                          &rt->fib6_dst.addr,
5886                                          sizeof(rt->fib6_dst.addr),
5887                                          rt->fib6_dst.plen,
5888                                          MLXSW_SP_L3_PROTO_IPV6);
5889         if (IS_ERR(fib_node))
5890                 return PTR_ERR(fib_node);
5891
5892         if (WARN_ON_ONCE(!fib_node->fib_entry)) {
5893                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5894                 return -EINVAL;
5895         }
5896
5897         fib6_entry = container_of(fib_node->fib_entry,
5898                                   struct mlxsw_sp_fib6_entry, common);
5899         err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6);
5900         if (err)
5901                 goto err_fib6_entry_nexthop_add;
5902
5903         return 0;
5904
5905 err_fib6_entry_nexthop_add:
5906         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5907         return err;
5908 }
5909
5910 static int mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
5911                                     struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5912                                     struct fib6_info **rt_arr, unsigned int nrt6)
5913 {
5914         struct mlxsw_sp_fib6_entry *fib6_entry;
5915         struct mlxsw_sp_fib_node *fib_node;
5916         struct fib6_info *rt = rt_arr[0];
5917         int err;
5918
5919         if (mlxsw_sp->router->aborted)
5920                 return 0;
5921
5922         if (mlxsw_sp_fib6_rt_should_ignore(rt))
5923                 return 0;
5924
5925         /* Multipath routes are first added to the FIB trie and only then
5926          * notified. If we vetoed the addition, we will get a delete
5927          * notification for a route we do not have. Therefore, do not warn if
5928          * route was not found.
5929          */
5930         fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
5931         if (!fib6_entry)
5932                 return 0;
5933
5934         /* If not all the nexthops are deleted, then only reduce the nexthop
5935          * group.
5936          */
5937         if (nrt6 != fib6_entry->nrt6) {
5938                 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6);
5939                 return 0;
5940         }
5941
5942         fib_node = fib6_entry->common.fib_node;
5943
5944         err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib6_entry->common);
5945         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5946         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5947         return err;
5948 }
5949
5950 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
5951                                             enum mlxsw_sp_l3proto proto,
5952                                             u8 tree_id)
5953 {
5954         const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
5955         enum mlxsw_reg_ralxx_protocol ralxx_proto =
5956                                 (enum mlxsw_reg_ralxx_protocol) proto;
5957         struct mlxsw_sp_fib_entry_priv *priv;
5958         char xralta_pl[MLXSW_REG_XRALTA_LEN];
5959         char xralst_pl[MLXSW_REG_XRALST_LEN];
5960         int i, err;
5961
5962         mlxsw_reg_xralta_pack(xralta_pl, true, ralxx_proto, tree_id);
5963         err = ll_ops->ralta_write(mlxsw_sp, xralta_pl);
5964         if (err)
5965                 return err;
5966
5967         mlxsw_reg_xralst_pack(xralst_pl, 0xff, tree_id);
5968         err = ll_ops->ralst_write(mlxsw_sp, xralst_pl);
5969         if (err)
5970                 return err;
5971
5972         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
5973                 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx;
5974                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
5975                 char xraltb_pl[MLXSW_REG_XRALTB_LEN];
5976
5977                 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
5978                 mlxsw_reg_xraltb_pack(xraltb_pl, vr->id, ralxx_proto, tree_id);
5979                 err = ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
5980                 if (err)
5981                         return err;
5982
5983                 priv = mlxsw_sp_fib_entry_priv_create(ll_ops);
5984                 if (IS_ERR(priv))
5985                         return PTR_ERR(priv);
5986
5987                 ll_ops->fib_entry_pack(op_ctx, proto, MLXSW_SP_FIB_ENTRY_OP_WRITE,
5988                                        vr->id, 0, NULL, priv);
5989                 ll_ops->fib_entry_act_ip2me_pack(op_ctx);
5990                 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, NULL);
5991                 mlxsw_sp_fib_entry_priv_put(priv);
5992                 if (err)
5993                         return err;
5994         }
5995
5996         return 0;
5997 }
5998
5999 static struct mlxsw_sp_mr_table *
6000 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family)
6001 {
6002         if (family == RTNL_FAMILY_IPMR)
6003                 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4];
6004         else
6005                 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
6006 }
6007
6008 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp,
6009                                      struct mfc_entry_notifier_info *men_info,
6010                                      bool replace)
6011 {
6012         struct mlxsw_sp_mr_table *mrt;
6013         struct mlxsw_sp_vr *vr;
6014
6015         if (mlxsw_sp->router->aborted)
6016                 return 0;
6017
6018         vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL);
6019         if (IS_ERR(vr))
6020                 return PTR_ERR(vr);
6021
6022         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
6023         return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace);
6024 }
6025
6026 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp,
6027                                       struct mfc_entry_notifier_info *men_info)
6028 {
6029         struct mlxsw_sp_mr_table *mrt;
6030         struct mlxsw_sp_vr *vr;
6031
6032         if (mlxsw_sp->router->aborted)
6033                 return;
6034
6035         vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id);
6036         if (WARN_ON(!vr))
6037                 return;
6038
6039         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
6040         mlxsw_sp_mr_route_del(mrt, men_info->mfc);
6041         mlxsw_sp_vr_put(mlxsw_sp, vr);
6042 }
6043
6044 static int
6045 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp,
6046                               struct vif_entry_notifier_info *ven_info)
6047 {
6048         struct mlxsw_sp_mr_table *mrt;
6049         struct mlxsw_sp_rif *rif;
6050         struct mlxsw_sp_vr *vr;
6051
6052         if (mlxsw_sp->router->aborted)
6053                 return 0;
6054
6055         vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL);
6056         if (IS_ERR(vr))
6057                 return PTR_ERR(vr);
6058
6059         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
6060         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev);
6061         return mlxsw_sp_mr_vif_add(mrt, ven_info->dev,
6062                                    ven_info->vif_index,
6063                                    ven_info->vif_flags, rif);
6064 }
6065
6066 static void
6067 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp,
6068                               struct vif_entry_notifier_info *ven_info)
6069 {
6070         struct mlxsw_sp_mr_table *mrt;
6071         struct mlxsw_sp_vr *vr;
6072
6073         if (mlxsw_sp->router->aborted)
6074                 return;
6075
6076         vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id);
6077         if (WARN_ON(!vr))
6078                 return;
6079
6080         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
6081         mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index);
6082         mlxsw_sp_vr_put(mlxsw_sp, vr);
6083 }
6084
6085 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
6086 {
6087         enum mlxsw_sp_l3proto proto = MLXSW_SP_L3_PROTO_IPV4;
6088         int err;
6089
6090         err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
6091                                                MLXSW_SP_LPM_TREE_MIN);
6092         if (err)
6093                 return err;
6094
6095         /* The multicast router code does not need an abort trap as by default,
6096          * packets that don't match any routes are trapped to the CPU.
6097          */
6098
6099         proto = MLXSW_SP_L3_PROTO_IPV6;
6100         return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
6101                                                 MLXSW_SP_LPM_TREE_MIN + 1);
6102 }
6103
6104 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
6105                                      struct mlxsw_sp_fib_node *fib_node)
6106 {
6107         struct mlxsw_sp_fib4_entry *fib4_entry;
6108
6109         fib4_entry = container_of(fib_node->fib_entry,
6110                                   struct mlxsw_sp_fib4_entry, common);
6111         mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
6112         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6113         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6114 }
6115
6116 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
6117                                      struct mlxsw_sp_fib_node *fib_node)
6118 {
6119         struct mlxsw_sp_fib6_entry *fib6_entry;
6120
6121         fib6_entry = container_of(fib_node->fib_entry,
6122                                   struct mlxsw_sp_fib6_entry, common);
6123         mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
6124         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
6125         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6126 }
6127
6128 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
6129                                     struct mlxsw_sp_fib_node *fib_node)
6130 {
6131         switch (fib_node->fib->proto) {
6132         case MLXSW_SP_L3_PROTO_IPV4:
6133                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
6134                 break;
6135         case MLXSW_SP_L3_PROTO_IPV6:
6136                 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
6137                 break;
6138         }
6139 }
6140
6141 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
6142                                   struct mlxsw_sp_vr *vr,
6143                                   enum mlxsw_sp_l3proto proto)
6144 {
6145         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
6146         struct mlxsw_sp_fib_node *fib_node, *tmp;
6147
6148         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
6149                 bool do_break = &tmp->list == &fib->node_list;
6150
6151                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
6152                 if (do_break)
6153                         break;
6154         }
6155 }
6156
6157 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
6158 {
6159         int i, j;
6160
6161         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
6162                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
6163
6164                 if (!mlxsw_sp_vr_is_used(vr))
6165                         continue;
6166
6167                 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++)
6168                         mlxsw_sp_mr_table_flush(vr->mr_table[j]);
6169                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
6170
6171                 /* If virtual router was only used for IPv4, then it's no
6172                  * longer used.
6173                  */
6174                 if (!mlxsw_sp_vr_is_used(vr))
6175                         continue;
6176                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
6177         }
6178
6179         /* After flushing all the routes, it is not possible anyone is still
6180          * using the adjacency index that is discarding packets, so free it in
6181          * case it was allocated.
6182          */
6183         if (!mlxsw_sp->router->adj_discard_index_valid)
6184                 return;
6185         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
6186                            mlxsw_sp->router->adj_discard_index);
6187         mlxsw_sp->router->adj_discard_index_valid = false;
6188 }
6189
6190 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
6191 {
6192         int err;
6193
6194         if (mlxsw_sp->router->aborted)
6195                 return;
6196         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
6197         mlxsw_sp_router_fib_flush(mlxsw_sp);
6198         mlxsw_sp->router->aborted = true;
6199         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
6200         if (err)
6201                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
6202 }
6203
6204 struct mlxsw_sp_fib6_event {
6205         struct fib6_info **rt_arr;
6206         unsigned int nrt6;
6207 };
6208
6209 struct mlxsw_sp_fib_event {
6210         struct list_head list; /* node in fib queue */
6211         union {
6212                 struct mlxsw_sp_fib6_event fib6_event;
6213                 struct fib_entry_notifier_info fen_info;
6214                 struct fib_rule_notifier_info fr_info;
6215                 struct fib_nh_notifier_info fnh_info;
6216                 struct mfc_entry_notifier_info men_info;
6217                 struct vif_entry_notifier_info ven_info;
6218         };
6219         struct mlxsw_sp *mlxsw_sp;
6220         unsigned long event;
6221         int family;
6222 };
6223
6224 static int
6225 mlxsw_sp_router_fib6_event_init(struct mlxsw_sp_fib6_event *fib6_event,
6226                                 struct fib6_entry_notifier_info *fen6_info)
6227 {
6228         struct fib6_info *rt = fen6_info->rt;
6229         struct fib6_info **rt_arr;
6230         struct fib6_info *iter;
6231         unsigned int nrt6;
6232         int i = 0;
6233
6234         nrt6 = fen6_info->nsiblings + 1;
6235
6236         rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC);
6237         if (!rt_arr)
6238                 return -ENOMEM;
6239
6240         fib6_event->rt_arr = rt_arr;
6241         fib6_event->nrt6 = nrt6;
6242
6243         rt_arr[0] = rt;
6244         fib6_info_hold(rt);
6245
6246         if (!fen6_info->nsiblings)
6247                 return 0;
6248
6249         list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
6250                 if (i == fen6_info->nsiblings)
6251                         break;
6252
6253                 rt_arr[i + 1] = iter;
6254                 fib6_info_hold(iter);
6255                 i++;
6256         }
6257         WARN_ON_ONCE(i != fen6_info->nsiblings);
6258
6259         return 0;
6260 }
6261
6262 static void
6263 mlxsw_sp_router_fib6_event_fini(struct mlxsw_sp_fib6_event *fib6_event)
6264 {
6265         int i;
6266
6267         for (i = 0; i < fib6_event->nrt6; i++)
6268                 mlxsw_sp_rt6_release(fib6_event->rt_arr[i]);
6269         kfree(fib6_event->rt_arr);
6270 }
6271
6272 static void mlxsw_sp_router_fib4_event_process(struct mlxsw_sp *mlxsw_sp,
6273                                                struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
6274                                                struct mlxsw_sp_fib_event *fib_event)
6275 {
6276         int err;
6277
6278         mlxsw_sp_span_respin(mlxsw_sp);
6279
6280         switch (fib_event->event) {
6281         case FIB_EVENT_ENTRY_REPLACE:
6282                 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, op_ctx, &fib_event->fen_info);
6283                 if (err) {
6284                         mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
6285                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6286                 }
6287                 fib_info_put(fib_event->fen_info.fi);
6288                 break;
6289         case FIB_EVENT_ENTRY_DEL:
6290                 err = mlxsw_sp_router_fib4_del(mlxsw_sp, op_ctx, &fib_event->fen_info);
6291                 if (err)
6292                         mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
6293                 fib_info_put(fib_event->fen_info.fi);
6294                 break;
6295         case FIB_EVENT_NH_ADD:
6296         case FIB_EVENT_NH_DEL:
6297                 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_event->event, fib_event->fnh_info.fib_nh);
6298                 fib_info_put(fib_event->fnh_info.fib_nh->nh_parent);
6299                 break;
6300         }
6301 }
6302
6303 static void mlxsw_sp_router_fib6_event_process(struct mlxsw_sp *mlxsw_sp,
6304                                                struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
6305                                                struct mlxsw_sp_fib_event *fib_event)
6306 {
6307         int err;
6308
6309         mlxsw_sp_span_respin(mlxsw_sp);
6310
6311         switch (fib_event->event) {
6312         case FIB_EVENT_ENTRY_REPLACE:
6313                 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
6314                                                    fib_event->fib6_event.nrt6);
6315                 if (err) {
6316                         mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
6317                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6318                 }
6319                 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
6320                 break;
6321         case FIB_EVENT_ENTRY_APPEND:
6322                 err = mlxsw_sp_router_fib6_append(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
6323                                                   fib_event->fib6_event.nrt6);
6324                 if (err) {
6325                         mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
6326                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6327                 }
6328                 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
6329                 break;
6330         case FIB_EVENT_ENTRY_DEL:
6331                 err = mlxsw_sp_router_fib6_del(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
6332                                                fib_event->fib6_event.nrt6);
6333                 if (err)
6334                         mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
6335                 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
6336                 break;
6337         }
6338 }
6339
6340 static void mlxsw_sp_router_fibmr_event_process(struct mlxsw_sp *mlxsw_sp,
6341                                                 struct mlxsw_sp_fib_event *fib_event)
6342 {
6343         bool replace;
6344         int err;
6345
6346         rtnl_lock();
6347         mutex_lock(&mlxsw_sp->router->lock);
6348         switch (fib_event->event) {
6349         case FIB_EVENT_ENTRY_REPLACE:
6350         case FIB_EVENT_ENTRY_ADD:
6351                 replace = fib_event->event == FIB_EVENT_ENTRY_REPLACE;
6352
6353                 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_event->men_info, replace);
6354                 if (err)
6355                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6356                 mr_cache_put(fib_event->men_info.mfc);
6357                 break;
6358         case FIB_EVENT_ENTRY_DEL:
6359                 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_event->men_info);
6360                 mr_cache_put(fib_event->men_info.mfc);
6361                 break;
6362         case FIB_EVENT_VIF_ADD:
6363                 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp,
6364                                                     &fib_event->ven_info);
6365                 if (err)
6366                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6367                 dev_put(fib_event->ven_info.dev);
6368                 break;
6369         case FIB_EVENT_VIF_DEL:
6370                 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, &fib_event->ven_info);
6371                 dev_put(fib_event->ven_info.dev);
6372                 break;
6373         }
6374         mutex_unlock(&mlxsw_sp->router->lock);
6375         rtnl_unlock();
6376 }
6377
6378 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
6379 {
6380         struct mlxsw_sp_router *router = container_of(work, struct mlxsw_sp_router, fib_event_work);
6381         struct mlxsw_sp_fib_entry_op_ctx *op_ctx = router->ll_op_ctx;
6382         struct mlxsw_sp *mlxsw_sp = router->mlxsw_sp;
6383         struct mlxsw_sp_fib_event *next_fib_event;
6384         struct mlxsw_sp_fib_event *fib_event;
6385         int last_family = AF_UNSPEC;
6386         LIST_HEAD(fib_event_queue);
6387
6388         spin_lock_bh(&router->fib_event_queue_lock);
6389         list_splice_init(&router->fib_event_queue, &fib_event_queue);
6390         spin_unlock_bh(&router->fib_event_queue_lock);
6391
6392         /* Router lock is held here to make sure per-instance
6393          * operation context is not used in between FIB4/6 events
6394          * processing.
6395          */
6396         mutex_lock(&router->lock);
6397         mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
6398         list_for_each_entry_safe(fib_event, next_fib_event,
6399                                  &fib_event_queue, list) {
6400                 /* Check if the next entry in the queue exists and it is
6401                  * of the same type (family and event) as the currect one.
6402                  * In that case it is permitted to do the bulking
6403                  * of multiple FIB entries to a single register write.
6404                  */
6405                 op_ctx->bulk_ok = !list_is_last(&fib_event->list, &fib_event_queue) &&
6406                                   fib_event->family == next_fib_event->family &&
6407                                   fib_event->event == next_fib_event->event;
6408
6409                 /* In case family of this and the previous entry are different, context
6410                  * reinitialization is going to be needed now, indicate that.
6411                  * Note that since last_family is initialized to AF_UNSPEC, this is always
6412                  * going to happen for the first entry processed in the work.
6413                  */
6414                 if (fib_event->family != last_family)
6415                         op_ctx->initialized = false;
6416
6417                 switch (fib_event->family) {
6418                 case AF_INET:
6419                         mlxsw_sp_router_fib4_event_process(mlxsw_sp, op_ctx,
6420                                                            fib_event);
6421                         break;
6422                 case AF_INET6:
6423                         mlxsw_sp_router_fib6_event_process(mlxsw_sp, op_ctx,
6424                                                            fib_event);
6425                         break;
6426                 case RTNL_FAMILY_IP6MR:
6427                 case RTNL_FAMILY_IPMR:
6428                         /* Unlock here as inside FIBMR the lock is taken again
6429                          * under RTNL. The per-instance operation context
6430                          * is not used by FIBMR.
6431                          */
6432                         mutex_unlock(&router->lock);
6433                         mlxsw_sp_router_fibmr_event_process(mlxsw_sp,
6434                                                             fib_event);
6435                         mutex_lock(&router->lock);
6436                         break;
6437                 default:
6438                         WARN_ON_ONCE(1);
6439                 }
6440                 last_family = fib_event->family;
6441                 kfree(fib_event);
6442                 cond_resched();
6443         }
6444         WARN_ON_ONCE(!list_empty(&router->ll_op_ctx->fib_entry_priv_list));
6445         mutex_unlock(&router->lock);
6446 }
6447
6448 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event *fib_event,
6449                                        struct fib_notifier_info *info)
6450 {
6451         struct fib_entry_notifier_info *fen_info;
6452         struct fib_nh_notifier_info *fnh_info;
6453
6454         switch (fib_event->event) {
6455         case FIB_EVENT_ENTRY_REPLACE:
6456         case FIB_EVENT_ENTRY_DEL:
6457                 fen_info = container_of(info, struct fib_entry_notifier_info,
6458                                         info);
6459                 fib_event->fen_info = *fen_info;
6460                 /* Take reference on fib_info to prevent it from being
6461                  * freed while event is queued. Release it afterwards.
6462                  */
6463                 fib_info_hold(fib_event->fen_info.fi);
6464                 break;
6465         case FIB_EVENT_NH_ADD:
6466         case FIB_EVENT_NH_DEL:
6467                 fnh_info = container_of(info, struct fib_nh_notifier_info,
6468                                         info);
6469                 fib_event->fnh_info = *fnh_info;
6470                 fib_info_hold(fib_event->fnh_info.fib_nh->nh_parent);
6471                 break;
6472         }
6473 }
6474
6475 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event *fib_event,
6476                                       struct fib_notifier_info *info)
6477 {
6478         struct fib6_entry_notifier_info *fen6_info;
6479         int err;
6480
6481         switch (fib_event->event) {
6482         case FIB_EVENT_ENTRY_REPLACE:
6483         case FIB_EVENT_ENTRY_APPEND:
6484         case FIB_EVENT_ENTRY_DEL:
6485                 fen6_info = container_of(info, struct fib6_entry_notifier_info,
6486                                          info);
6487                 err = mlxsw_sp_router_fib6_event_init(&fib_event->fib6_event,
6488                                                       fen6_info);
6489                 if (err)
6490                         return err;
6491                 break;
6492         }
6493
6494         return 0;
6495 }
6496
6497 static void
6498 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event *fib_event,
6499                             struct fib_notifier_info *info)
6500 {
6501         switch (fib_event->event) {
6502         case FIB_EVENT_ENTRY_REPLACE:
6503         case FIB_EVENT_ENTRY_ADD:
6504         case FIB_EVENT_ENTRY_DEL:
6505                 memcpy(&fib_event->men_info, info, sizeof(fib_event->men_info));
6506                 mr_cache_hold(fib_event->men_info.mfc);
6507                 break;
6508         case FIB_EVENT_VIF_ADD:
6509         case FIB_EVENT_VIF_DEL:
6510                 memcpy(&fib_event->ven_info, info, sizeof(fib_event->ven_info));
6511                 dev_hold(fib_event->ven_info.dev);
6512                 break;
6513         }
6514 }
6515
6516 static int mlxsw_sp_router_fib_rule_event(unsigned long event,
6517                                           struct fib_notifier_info *info,
6518                                           struct mlxsw_sp *mlxsw_sp)
6519 {
6520         struct netlink_ext_ack *extack = info->extack;
6521         struct fib_rule_notifier_info *fr_info;
6522         struct fib_rule *rule;
6523         int err = 0;
6524
6525         /* nothing to do at the moment */
6526         if (event == FIB_EVENT_RULE_DEL)
6527                 return 0;
6528
6529         if (mlxsw_sp->router->aborted)
6530                 return 0;
6531
6532         fr_info = container_of(info, struct fib_rule_notifier_info, info);
6533         rule = fr_info->rule;
6534
6535         /* Rule only affects locally generated traffic */
6536         if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex)
6537                 return 0;
6538
6539         switch (info->family) {
6540         case AF_INET:
6541                 if (!fib4_rule_default(rule) && !rule->l3mdev)
6542                         err = -EOPNOTSUPP;
6543                 break;
6544         case AF_INET6:
6545                 if (!fib6_rule_default(rule) && !rule->l3mdev)
6546                         err = -EOPNOTSUPP;
6547                 break;
6548         case RTNL_FAMILY_IPMR:
6549                 if (!ipmr_rule_default(rule) && !rule->l3mdev)
6550                         err = -EOPNOTSUPP;
6551                 break;
6552         case RTNL_FAMILY_IP6MR:
6553                 if (!ip6mr_rule_default(rule) && !rule->l3mdev)
6554                         err = -EOPNOTSUPP;
6555                 break;
6556         }
6557
6558         if (err < 0)
6559                 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported");
6560
6561         return err;
6562 }
6563
6564 /* Called with rcu_read_lock() */
6565 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
6566                                      unsigned long event, void *ptr)
6567 {
6568         struct mlxsw_sp_fib_event *fib_event;
6569         struct fib_notifier_info *info = ptr;
6570         struct mlxsw_sp_router *router;
6571         int err;
6572
6573         if ((info->family != AF_INET && info->family != AF_INET6 &&
6574              info->family != RTNL_FAMILY_IPMR &&
6575              info->family != RTNL_FAMILY_IP6MR))
6576                 return NOTIFY_DONE;
6577
6578         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
6579
6580         switch (event) {
6581         case FIB_EVENT_RULE_ADD:
6582         case FIB_EVENT_RULE_DEL:
6583                 err = mlxsw_sp_router_fib_rule_event(event, info,
6584                                                      router->mlxsw_sp);
6585                 return notifier_from_errno(err);
6586         case FIB_EVENT_ENTRY_ADD:
6587         case FIB_EVENT_ENTRY_REPLACE:
6588         case FIB_EVENT_ENTRY_APPEND:
6589                 if (router->aborted) {
6590                         NL_SET_ERR_MSG_MOD(info->extack, "FIB offload was aborted. Not configuring route");
6591                         return notifier_from_errno(-EINVAL);
6592                 }
6593                 if (info->family == AF_INET) {
6594                         struct fib_entry_notifier_info *fen_info = ptr;
6595
6596                         if (fen_info->fi->fib_nh_is_v6) {
6597                                 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported");
6598                                 return notifier_from_errno(-EINVAL);
6599                         }
6600                         if (fen_info->fi->nh) {
6601                                 NL_SET_ERR_MSG_MOD(info->extack, "IPv4 route with nexthop objects is not supported");
6602                                 return notifier_from_errno(-EINVAL);
6603                         }
6604                 } else if (info->family == AF_INET6) {
6605                         struct fib6_entry_notifier_info *fen6_info;
6606
6607                         fen6_info = container_of(info,
6608                                                  struct fib6_entry_notifier_info,
6609                                                  info);
6610                         if (fen6_info->rt->nh) {
6611                                 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 route with nexthop objects is not supported");
6612                                 return notifier_from_errno(-EINVAL);
6613                         }
6614                 }
6615                 break;
6616         }
6617
6618         fib_event = kzalloc(sizeof(*fib_event), GFP_ATOMIC);
6619         if (!fib_event)
6620                 return NOTIFY_BAD;
6621
6622         fib_event->mlxsw_sp = router->mlxsw_sp;
6623         fib_event->event = event;
6624         fib_event->family = info->family;
6625
6626         switch (info->family) {
6627         case AF_INET:
6628                 mlxsw_sp_router_fib4_event(fib_event, info);
6629                 break;
6630         case AF_INET6:
6631                 err = mlxsw_sp_router_fib6_event(fib_event, info);
6632                 if (err)
6633                         goto err_fib_event;
6634                 break;
6635         case RTNL_FAMILY_IP6MR:
6636         case RTNL_FAMILY_IPMR:
6637                 mlxsw_sp_router_fibmr_event(fib_event, info);
6638                 break;
6639         }
6640
6641         /* Enqueue the event and trigger the work */
6642         spin_lock_bh(&router->fib_event_queue_lock);
6643         list_add_tail(&fib_event->list, &router->fib_event_queue);
6644         spin_unlock_bh(&router->fib_event_queue_lock);
6645         mlxsw_core_schedule_work(&router->fib_event_work);
6646
6647         return NOTIFY_DONE;
6648
6649 err_fib_event:
6650         kfree(fib_event);
6651         return NOTIFY_BAD;
6652 }
6653
6654 static struct mlxsw_sp_rif *
6655 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
6656                          const struct net_device *dev)
6657 {
6658         int i;
6659
6660         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
6661                 if (mlxsw_sp->router->rifs[i] &&
6662                     mlxsw_sp->router->rifs[i]->dev == dev)
6663                         return mlxsw_sp->router->rifs[i];
6664
6665         return NULL;
6666 }
6667
6668 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp,
6669                          const struct net_device *dev)
6670 {
6671         struct mlxsw_sp_rif *rif;
6672
6673         mutex_lock(&mlxsw_sp->router->lock);
6674         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6675         mutex_unlock(&mlxsw_sp->router->lock);
6676
6677         return rif;
6678 }
6679
6680 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev)
6681 {
6682         struct mlxsw_sp_rif *rif;
6683         u16 vid = 0;
6684
6685         mutex_lock(&mlxsw_sp->router->lock);
6686         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6687         if (!rif)
6688                 goto out;
6689
6690         /* We only return the VID for VLAN RIFs. Otherwise we return an
6691          * invalid value (0).
6692          */
6693         if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN)
6694                 goto out;
6695
6696         vid = mlxsw_sp_fid_8021q_vid(rif->fid);
6697
6698 out:
6699         mutex_unlock(&mlxsw_sp->router->lock);
6700         return vid;
6701 }
6702
6703 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
6704 {
6705         char ritr_pl[MLXSW_REG_RITR_LEN];
6706         int err;
6707
6708         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
6709         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6710         if (err)
6711                 return err;
6712
6713         mlxsw_reg_ritr_enable_set(ritr_pl, false);
6714         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6715 }
6716
6717 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
6718                                           struct mlxsw_sp_rif *rif)
6719 {
6720         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
6721         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
6722         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
6723 }
6724
6725 static bool
6726 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
6727                            unsigned long event)
6728 {
6729         struct inet6_dev *inet6_dev;
6730         bool addr_list_empty = true;
6731         struct in_device *idev;
6732
6733         switch (event) {
6734         case NETDEV_UP:
6735                 return rif == NULL;
6736         case NETDEV_DOWN:
6737                 rcu_read_lock();
6738                 idev = __in_dev_get_rcu(dev);
6739                 if (idev && idev->ifa_list)
6740                         addr_list_empty = false;
6741
6742                 inet6_dev = __in6_dev_get(dev);
6743                 if (addr_list_empty && inet6_dev &&
6744                     !list_empty(&inet6_dev->addr_list))
6745                         addr_list_empty = false;
6746                 rcu_read_unlock();
6747
6748                 /* macvlans do not have a RIF, but rather piggy back on the
6749                  * RIF of their lower device.
6750                  */
6751                 if (netif_is_macvlan(dev) && addr_list_empty)
6752                         return true;
6753
6754                 if (rif && addr_list_empty &&
6755                     !netif_is_l3_slave(rif->dev))
6756                         return true;
6757                 /* It is possible we already removed the RIF ourselves
6758                  * if it was assigned to a netdev that is now a bridge
6759                  * or LAG slave.
6760                  */
6761                 return false;
6762         }
6763
6764         return false;
6765 }
6766
6767 static enum mlxsw_sp_rif_type
6768 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
6769                       const struct net_device *dev)
6770 {
6771         enum mlxsw_sp_fid_type type;
6772
6773         if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
6774                 return MLXSW_SP_RIF_TYPE_IPIP_LB;
6775
6776         /* Otherwise RIF type is derived from the type of the underlying FID. */
6777         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
6778                 type = MLXSW_SP_FID_TYPE_8021Q;
6779         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
6780                 type = MLXSW_SP_FID_TYPE_8021Q;
6781         else if (netif_is_bridge_master(dev))
6782                 type = MLXSW_SP_FID_TYPE_8021D;
6783         else
6784                 type = MLXSW_SP_FID_TYPE_RFID;
6785
6786         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
6787 }
6788
6789 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
6790 {
6791         int i;
6792
6793         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
6794                 if (!mlxsw_sp->router->rifs[i]) {
6795                         *p_rif_index = i;
6796                         return 0;
6797                 }
6798         }
6799
6800         return -ENOBUFS;
6801 }
6802
6803 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
6804                                                u16 vr_id,
6805                                                struct net_device *l3_dev)
6806 {
6807         struct mlxsw_sp_rif *rif;
6808
6809         rif = kzalloc(rif_size, GFP_KERNEL);
6810         if (!rif)
6811                 return NULL;
6812
6813         INIT_LIST_HEAD(&rif->nexthop_list);
6814         INIT_LIST_HEAD(&rif->neigh_list);
6815         if (l3_dev) {
6816                 ether_addr_copy(rif->addr, l3_dev->dev_addr);
6817                 rif->mtu = l3_dev->mtu;
6818                 rif->dev = l3_dev;
6819         }
6820         rif->vr_id = vr_id;
6821         rif->rif_index = rif_index;
6822
6823         return rif;
6824 }
6825
6826 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
6827                                            u16 rif_index)
6828 {
6829         return mlxsw_sp->router->rifs[rif_index];
6830 }
6831
6832 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
6833 {
6834         return rif->rif_index;
6835 }
6836
6837 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6838 {
6839         return lb_rif->common.rif_index;
6840 }
6841
6842 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6843 {
6844         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev);
6845         struct mlxsw_sp_vr *ul_vr;
6846
6847         ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL);
6848         if (WARN_ON(IS_ERR(ul_vr)))
6849                 return 0;
6850
6851         return ul_vr->id;
6852 }
6853
6854 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6855 {
6856         return lb_rif->ul_rif_id;
6857 }
6858
6859 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
6860 {
6861         return rif->dev->ifindex;
6862 }
6863
6864 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif)
6865 {
6866         return rif->dev;
6867 }
6868
6869 static struct mlxsw_sp_rif *
6870 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
6871                     const struct mlxsw_sp_rif_params *params,
6872                     struct netlink_ext_ack *extack)
6873 {
6874         u32 tb_id = l3mdev_fib_table(params->dev);
6875         const struct mlxsw_sp_rif_ops *ops;
6876         struct mlxsw_sp_fid *fid = NULL;
6877         enum mlxsw_sp_rif_type type;
6878         struct mlxsw_sp_rif *rif;
6879         struct mlxsw_sp_vr *vr;
6880         u16 rif_index;
6881         int i, err;
6882
6883         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
6884         ops = mlxsw_sp->rif_ops_arr[type];
6885
6886         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack);
6887         if (IS_ERR(vr))
6888                 return ERR_CAST(vr);
6889         vr->rif_count++;
6890
6891         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
6892         if (err) {
6893                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
6894                 goto err_rif_index_alloc;
6895         }
6896
6897         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
6898         if (!rif) {
6899                 err = -ENOMEM;
6900                 goto err_rif_alloc;
6901         }
6902         dev_hold(rif->dev);
6903         mlxsw_sp->router->rifs[rif_index] = rif;
6904         rif->mlxsw_sp = mlxsw_sp;
6905         rif->ops = ops;
6906
6907         if (ops->fid_get) {
6908                 fid = ops->fid_get(rif, extack);
6909                 if (IS_ERR(fid)) {
6910                         err = PTR_ERR(fid);
6911                         goto err_fid_get;
6912                 }
6913                 rif->fid = fid;
6914         }
6915
6916         if (ops->setup)
6917                 ops->setup(rif, params);
6918
6919         err = ops->configure(rif);
6920         if (err)
6921                 goto err_configure;
6922
6923         for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
6924                 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif);
6925                 if (err)
6926                         goto err_mr_rif_add;
6927         }
6928
6929         mlxsw_sp_rif_counters_alloc(rif);
6930
6931         return rif;
6932
6933 err_mr_rif_add:
6934         for (i--; i >= 0; i--)
6935                 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
6936         ops->deconfigure(rif);
6937 err_configure:
6938         if (fid)
6939                 mlxsw_sp_fid_put(fid);
6940 err_fid_get:
6941         mlxsw_sp->router->rifs[rif_index] = NULL;
6942         dev_put(rif->dev);
6943         kfree(rif);
6944 err_rif_alloc:
6945 err_rif_index_alloc:
6946         vr->rif_count--;
6947         mlxsw_sp_vr_put(mlxsw_sp, vr);
6948         return ERR_PTR(err);
6949 }
6950
6951 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
6952 {
6953         const struct mlxsw_sp_rif_ops *ops = rif->ops;
6954         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
6955         struct mlxsw_sp_fid *fid = rif->fid;
6956         struct mlxsw_sp_vr *vr;
6957         int i;
6958
6959         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
6960         vr = &mlxsw_sp->router->vrs[rif->vr_id];
6961
6962         mlxsw_sp_rif_counters_free(rif);
6963         for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
6964                 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
6965         ops->deconfigure(rif);
6966         if (fid)
6967                 /* Loopback RIFs are not associated with a FID. */
6968                 mlxsw_sp_fid_put(fid);
6969         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
6970         dev_put(rif->dev);
6971         kfree(rif);
6972         vr->rif_count--;
6973         mlxsw_sp_vr_put(mlxsw_sp, vr);
6974 }
6975
6976 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp,
6977                                  struct net_device *dev)
6978 {
6979         struct mlxsw_sp_rif *rif;
6980
6981         mutex_lock(&mlxsw_sp->router->lock);
6982         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6983         if (!rif)
6984                 goto out;
6985         mlxsw_sp_rif_destroy(rif);
6986 out:
6987         mutex_unlock(&mlxsw_sp->router->lock);
6988 }
6989
6990 static void
6991 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
6992                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
6993 {
6994         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
6995
6996         params->vid = mlxsw_sp_port_vlan->vid;
6997         params->lag = mlxsw_sp_port->lagged;
6998         if (params->lag)
6999                 params->lag_id = mlxsw_sp_port->lag_id;
7000         else
7001                 params->system_port = mlxsw_sp_port->local_port;
7002 }
7003
7004 static struct mlxsw_sp_rif_subport *
7005 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
7006 {
7007         return container_of(rif, struct mlxsw_sp_rif_subport, common);
7008 }
7009
7010 static struct mlxsw_sp_rif *
7011 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp,
7012                          const struct mlxsw_sp_rif_params *params,
7013                          struct netlink_ext_ack *extack)
7014 {
7015         struct mlxsw_sp_rif_subport *rif_subport;
7016         struct mlxsw_sp_rif *rif;
7017
7018         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev);
7019         if (!rif)
7020                 return mlxsw_sp_rif_create(mlxsw_sp, params, extack);
7021
7022         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7023         refcount_inc(&rif_subport->ref_count);
7024         return rif;
7025 }
7026
7027 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif)
7028 {
7029         struct mlxsw_sp_rif_subport *rif_subport;
7030
7031         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7032         if (!refcount_dec_and_test(&rif_subport->ref_count))
7033                 return;
7034
7035         mlxsw_sp_rif_destroy(rif);
7036 }
7037
7038 static int
7039 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
7040                                struct net_device *l3_dev,
7041                                struct netlink_ext_ack *extack)
7042 {
7043         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
7044         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
7045         struct mlxsw_sp_rif_params params = {
7046                 .dev = l3_dev,
7047         };
7048         u16 vid = mlxsw_sp_port_vlan->vid;
7049         struct mlxsw_sp_rif *rif;
7050         struct mlxsw_sp_fid *fid;
7051         int err;
7052
7053         mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
7054         rif = mlxsw_sp_rif_subport_get(mlxsw_sp, &params, extack);
7055         if (IS_ERR(rif))
7056                 return PTR_ERR(rif);
7057
7058         /* FID was already created, just take a reference */
7059         fid = rif->ops->fid_get(rif, extack);
7060         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
7061         if (err)
7062                 goto err_fid_port_vid_map;
7063
7064         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
7065         if (err)
7066                 goto err_port_vid_learning_set;
7067
7068         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
7069                                         BR_STATE_FORWARDING);
7070         if (err)
7071                 goto err_port_vid_stp_set;
7072
7073         mlxsw_sp_port_vlan->fid = fid;
7074
7075         return 0;
7076
7077 err_port_vid_stp_set:
7078         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
7079 err_port_vid_learning_set:
7080         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
7081 err_fid_port_vid_map:
7082         mlxsw_sp_fid_put(fid);
7083         mlxsw_sp_rif_subport_put(rif);
7084         return err;
7085 }
7086
7087 static void
7088 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
7089 {
7090         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
7091         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
7092         struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid);
7093         u16 vid = mlxsw_sp_port_vlan->vid;
7094
7095         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
7096                 return;
7097
7098         mlxsw_sp_port_vlan->fid = NULL;
7099         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
7100         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
7101         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
7102         mlxsw_sp_fid_put(fid);
7103         mlxsw_sp_rif_subport_put(rif);
7104 }
7105
7106 void
7107 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
7108 {
7109         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
7110
7111         mutex_lock(&mlxsw_sp->router->lock);
7112         __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
7113         mutex_unlock(&mlxsw_sp->router->lock);
7114 }
7115
7116 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
7117                                              struct net_device *port_dev,
7118                                              unsigned long event, u16 vid,
7119                                              struct netlink_ext_ack *extack)
7120 {
7121         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
7122         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
7123
7124         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
7125         if (WARN_ON(!mlxsw_sp_port_vlan))
7126                 return -EINVAL;
7127
7128         switch (event) {
7129         case NETDEV_UP:
7130                 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
7131                                                       l3_dev, extack);
7132         case NETDEV_DOWN:
7133                 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
7134                 break;
7135         }
7136
7137         return 0;
7138 }
7139
7140 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
7141                                         unsigned long event,
7142                                         struct netlink_ext_ack *extack)
7143 {
7144         if (netif_is_bridge_port(port_dev) ||
7145             netif_is_lag_port(port_dev) ||
7146             netif_is_ovs_port(port_dev))
7147                 return 0;
7148
7149         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event,
7150                                                  MLXSW_SP_DEFAULT_VID, extack);
7151 }
7152
7153 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
7154                                          struct net_device *lag_dev,
7155                                          unsigned long event, u16 vid,
7156                                          struct netlink_ext_ack *extack)
7157 {
7158         struct net_device *port_dev;
7159         struct list_head *iter;
7160         int err;
7161
7162         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
7163                 if (mlxsw_sp_port_dev_check(port_dev)) {
7164                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
7165                                                                 port_dev,
7166                                                                 event, vid,
7167                                                                 extack);
7168                         if (err)
7169                                 return err;
7170                 }
7171         }
7172
7173         return 0;
7174 }
7175
7176 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
7177                                        unsigned long event,
7178                                        struct netlink_ext_ack *extack)
7179 {
7180         if (netif_is_bridge_port(lag_dev))
7181                 return 0;
7182
7183         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event,
7184                                              MLXSW_SP_DEFAULT_VID, extack);
7185 }
7186
7187 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp,
7188                                           struct net_device *l3_dev,
7189                                           unsigned long event,
7190                                           struct netlink_ext_ack *extack)
7191 {
7192         struct mlxsw_sp_rif_params params = {
7193                 .dev = l3_dev,
7194         };
7195         struct mlxsw_sp_rif *rif;
7196
7197         switch (event) {
7198         case NETDEV_UP:
7199                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params, extack);
7200                 if (IS_ERR(rif))
7201                         return PTR_ERR(rif);
7202                 break;
7203         case NETDEV_DOWN:
7204                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
7205                 mlxsw_sp_rif_destroy(rif);
7206                 break;
7207         }
7208
7209         return 0;
7210 }
7211
7212 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp,
7213                                         struct net_device *vlan_dev,
7214                                         unsigned long event,
7215                                         struct netlink_ext_ack *extack)
7216 {
7217         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
7218         u16 vid = vlan_dev_vlan_id(vlan_dev);
7219
7220         if (netif_is_bridge_port(vlan_dev))
7221                 return 0;
7222
7223         if (mlxsw_sp_port_dev_check(real_dev))
7224                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
7225                                                          event, vid, extack);
7226         else if (netif_is_lag_master(real_dev))
7227                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
7228                                                      vid, extack);
7229         else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
7230                 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event,
7231                                                       extack);
7232
7233         return 0;
7234 }
7235
7236 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac)
7237 {
7238         u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 };
7239         u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
7240
7241         return ether_addr_equal_masked(mac, vrrp4, mask);
7242 }
7243
7244 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac)
7245 {
7246         u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 };
7247         u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
7248
7249         return ether_addr_equal_masked(mac, vrrp6, mask);
7250 }
7251
7252 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
7253                                 const u8 *mac, bool adding)
7254 {
7255         char ritr_pl[MLXSW_REG_RITR_LEN];
7256         u8 vrrp_id = adding ? mac[5] : 0;
7257         int err;
7258
7259         if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) &&
7260             !mlxsw_sp_rif_macvlan_is_vrrp6(mac))
7261                 return 0;
7262
7263         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
7264         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7265         if (err)
7266                 return err;
7267
7268         if (mlxsw_sp_rif_macvlan_is_vrrp4(mac))
7269                 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id);
7270         else
7271                 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id);
7272
7273         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7274 }
7275
7276 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp,
7277                                     const struct net_device *macvlan_dev,
7278                                     struct netlink_ext_ack *extack)
7279 {
7280         struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
7281         struct mlxsw_sp_rif *rif;
7282         int err;
7283
7284         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
7285         if (!rif) {
7286                 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces");
7287                 return -EOPNOTSUPP;
7288         }
7289
7290         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
7291                                   mlxsw_sp_fid_index(rif->fid), true);
7292         if (err)
7293                 return err;
7294
7295         err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index,
7296                                    macvlan_dev->dev_addr, true);
7297         if (err)
7298                 goto err_rif_vrrp_add;
7299
7300         /* Make sure the bridge driver does not have this MAC pointing at
7301          * some other port.
7302          */
7303         if (rif->ops->fdb_del)
7304                 rif->ops->fdb_del(rif, macvlan_dev->dev_addr);
7305
7306         return 0;
7307
7308 err_rif_vrrp_add:
7309         mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
7310                             mlxsw_sp_fid_index(rif->fid), false);
7311         return err;
7312 }
7313
7314 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
7315                                        const struct net_device *macvlan_dev)
7316 {
7317         struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
7318         struct mlxsw_sp_rif *rif;
7319
7320         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
7321         /* If we do not have a RIF, then we already took care of
7322          * removing the macvlan's MAC during RIF deletion.
7323          */
7324         if (!rif)
7325                 return;
7326         mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr,
7327                              false);
7328         mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
7329                             mlxsw_sp_fid_index(rif->fid), false);
7330 }
7331
7332 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
7333                               const struct net_device *macvlan_dev)
7334 {
7335         mutex_lock(&mlxsw_sp->router->lock);
7336         __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
7337         mutex_unlock(&mlxsw_sp->router->lock);
7338 }
7339
7340 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp,
7341                                            struct net_device *macvlan_dev,
7342                                            unsigned long event,
7343                                            struct netlink_ext_ack *extack)
7344 {
7345         switch (event) {
7346         case NETDEV_UP:
7347                 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack);
7348         case NETDEV_DOWN:
7349                 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
7350                 break;
7351         }
7352
7353         return 0;
7354 }
7355
7356 static int mlxsw_sp_router_port_check_rif_addr(struct mlxsw_sp *mlxsw_sp,
7357                                                struct net_device *dev,
7358                                                const unsigned char *dev_addr,
7359                                                struct netlink_ext_ack *extack)
7360 {
7361         struct mlxsw_sp_rif *rif;
7362         int i;
7363
7364         /* A RIF is not created for macvlan netdevs. Their MAC is used to
7365          * populate the FDB
7366          */
7367         if (netif_is_macvlan(dev) || netif_is_l3_master(dev))
7368                 return 0;
7369
7370         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
7371                 rif = mlxsw_sp->router->rifs[i];
7372                 if (rif && rif->ops &&
7373                     rif->ops->type == MLXSW_SP_RIF_TYPE_IPIP_LB)
7374                         continue;
7375                 if (rif && rif->dev && rif->dev != dev &&
7376                     !ether_addr_equal_masked(rif->dev->dev_addr, dev_addr,
7377                                              mlxsw_sp->mac_mask)) {
7378                         NL_SET_ERR_MSG_MOD(extack, "All router interface MAC addresses must have the same prefix");
7379                         return -EINVAL;
7380                 }
7381         }
7382
7383         return 0;
7384 }
7385
7386 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp,
7387                                      struct net_device *dev,
7388                                      unsigned long event,
7389                                      struct netlink_ext_ack *extack)
7390 {
7391         if (mlxsw_sp_port_dev_check(dev))
7392                 return mlxsw_sp_inetaddr_port_event(dev, event, extack);
7393         else if (netif_is_lag_master(dev))
7394                 return mlxsw_sp_inetaddr_lag_event(dev, event, extack);
7395         else if (netif_is_bridge_master(dev))
7396                 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event,
7397                                                       extack);
7398         else if (is_vlan_dev(dev))
7399                 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event,
7400                                                     extack);
7401         else if (netif_is_macvlan(dev))
7402                 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event,
7403                                                        extack);
7404         else
7405                 return 0;
7406 }
7407
7408 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb,
7409                                    unsigned long event, void *ptr)
7410 {
7411         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
7412         struct net_device *dev = ifa->ifa_dev->dev;
7413         struct mlxsw_sp_router *router;
7414         struct mlxsw_sp_rif *rif;
7415         int err = 0;
7416
7417         /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */
7418         if (event == NETDEV_UP)
7419                 return NOTIFY_DONE;
7420
7421         router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb);
7422         mutex_lock(&router->lock);
7423         rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev);
7424         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7425                 goto out;
7426
7427         err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL);
7428 out:
7429         mutex_unlock(&router->lock);
7430         return notifier_from_errno(err);
7431 }
7432
7433 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused,
7434                                   unsigned long event, void *ptr)
7435 {
7436         struct in_validator_info *ivi = (struct in_validator_info *) ptr;
7437         struct net_device *dev = ivi->ivi_dev->dev;
7438         struct mlxsw_sp *mlxsw_sp;
7439         struct mlxsw_sp_rif *rif;
7440         int err = 0;
7441
7442         mlxsw_sp = mlxsw_sp_lower_get(dev);
7443         if (!mlxsw_sp)
7444                 return NOTIFY_DONE;
7445
7446         mutex_lock(&mlxsw_sp->router->lock);
7447         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7448         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7449                 goto out;
7450
7451         err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr,
7452                                                   ivi->extack);
7453         if (err)
7454                 goto out;
7455
7456         err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack);
7457 out:
7458         mutex_unlock(&mlxsw_sp->router->lock);
7459         return notifier_from_errno(err);
7460 }
7461
7462 struct mlxsw_sp_inet6addr_event_work {
7463         struct work_struct work;
7464         struct mlxsw_sp *mlxsw_sp;
7465         struct net_device *dev;
7466         unsigned long event;
7467 };
7468
7469 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
7470 {
7471         struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
7472                 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
7473         struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp;
7474         struct net_device *dev = inet6addr_work->dev;
7475         unsigned long event = inet6addr_work->event;
7476         struct mlxsw_sp_rif *rif;
7477
7478         rtnl_lock();
7479         mutex_lock(&mlxsw_sp->router->lock);
7480
7481         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7482         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7483                 goto out;
7484
7485         __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL);
7486 out:
7487         mutex_unlock(&mlxsw_sp->router->lock);
7488         rtnl_unlock();
7489         dev_put(dev);
7490         kfree(inet6addr_work);
7491 }
7492
7493 /* Called with rcu_read_lock() */
7494 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb,
7495                                     unsigned long event, void *ptr)
7496 {
7497         struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
7498         struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
7499         struct net_device *dev = if6->idev->dev;
7500         struct mlxsw_sp_router *router;
7501
7502         /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */
7503         if (event == NETDEV_UP)
7504                 return NOTIFY_DONE;
7505
7506         inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
7507         if (!inet6addr_work)
7508                 return NOTIFY_BAD;
7509
7510         router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb);
7511         INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
7512         inet6addr_work->mlxsw_sp = router->mlxsw_sp;
7513         inet6addr_work->dev = dev;
7514         inet6addr_work->event = event;
7515         dev_hold(dev);
7516         mlxsw_core_schedule_work(&inet6addr_work->work);
7517
7518         return NOTIFY_DONE;
7519 }
7520
7521 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused,
7522                                    unsigned long event, void *ptr)
7523 {
7524         struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr;
7525         struct net_device *dev = i6vi->i6vi_dev->dev;
7526         struct mlxsw_sp *mlxsw_sp;
7527         struct mlxsw_sp_rif *rif;
7528         int err = 0;
7529
7530         mlxsw_sp = mlxsw_sp_lower_get(dev);
7531         if (!mlxsw_sp)
7532                 return NOTIFY_DONE;
7533
7534         mutex_lock(&mlxsw_sp->router->lock);
7535         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7536         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7537                 goto out;
7538
7539         err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr,
7540                                                   i6vi->extack);
7541         if (err)
7542                 goto out;
7543
7544         err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack);
7545 out:
7546         mutex_unlock(&mlxsw_sp->router->lock);
7547         return notifier_from_errno(err);
7548 }
7549
7550 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
7551                              const char *mac, int mtu)
7552 {
7553         char ritr_pl[MLXSW_REG_RITR_LEN];
7554         int err;
7555
7556         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
7557         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7558         if (err)
7559                 return err;
7560
7561         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
7562         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
7563         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
7564         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7565 }
7566
7567 static int
7568 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp,
7569                                   struct mlxsw_sp_rif *rif)
7570 {
7571         struct net_device *dev = rif->dev;
7572         u16 fid_index;
7573         int err;
7574
7575         fid_index = mlxsw_sp_fid_index(rif->fid);
7576
7577         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
7578         if (err)
7579                 return err;
7580
7581         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
7582                                 dev->mtu);
7583         if (err)
7584                 goto err_rif_edit;
7585
7586         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
7587         if (err)
7588                 goto err_rif_fdb_op;
7589
7590         if (rif->mtu != dev->mtu) {
7591                 struct mlxsw_sp_vr *vr;
7592                 int i;
7593
7594                 /* The RIF is relevant only to its mr_table instance, as unlike
7595                  * unicast routing, in multicast routing a RIF cannot be shared
7596                  * between several multicast routing tables.
7597                  */
7598                 vr = &mlxsw_sp->router->vrs[rif->vr_id];
7599                 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
7600                         mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i],
7601                                                    rif, dev->mtu);
7602         }
7603
7604         ether_addr_copy(rif->addr, dev->dev_addr);
7605         rif->mtu = dev->mtu;
7606
7607         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
7608
7609         return 0;
7610
7611 err_rif_fdb_op:
7612         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
7613 err_rif_edit:
7614         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
7615         return err;
7616 }
7617
7618 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif,
7619                             struct netdev_notifier_pre_changeaddr_info *info)
7620 {
7621         struct netlink_ext_ack *extack;
7622
7623         extack = netdev_notifier_info_to_extack(&info->info);
7624         return mlxsw_sp_router_port_check_rif_addr(rif->mlxsw_sp, rif->dev,
7625                                                    info->dev_addr, extack);
7626 }
7627
7628 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev,
7629                                          unsigned long event, void *ptr)
7630 {
7631         struct mlxsw_sp *mlxsw_sp;
7632         struct mlxsw_sp_rif *rif;
7633         int err = 0;
7634
7635         mlxsw_sp = mlxsw_sp_lower_get(dev);
7636         if (!mlxsw_sp)
7637                 return 0;
7638
7639         mutex_lock(&mlxsw_sp->router->lock);
7640         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7641         if (!rif)
7642                 goto out;
7643
7644         switch (event) {
7645         case NETDEV_CHANGEMTU:
7646         case NETDEV_CHANGEADDR:
7647                 err = mlxsw_sp_router_port_change_event(mlxsw_sp, rif);
7648                 break;
7649         case NETDEV_PRE_CHANGEADDR:
7650                 err = mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr);
7651                 break;
7652         }
7653
7654 out:
7655         mutex_unlock(&mlxsw_sp->router->lock);
7656         return err;
7657 }
7658
7659 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
7660                                   struct net_device *l3_dev,
7661                                   struct netlink_ext_ack *extack)
7662 {
7663         struct mlxsw_sp_rif *rif;
7664
7665         /* If netdev is already associated with a RIF, then we need to
7666          * destroy it and create a new one with the new virtual router ID.
7667          */
7668         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
7669         if (rif)
7670                 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN,
7671                                           extack);
7672
7673         return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack);
7674 }
7675
7676 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
7677                                     struct net_device *l3_dev)
7678 {
7679         struct mlxsw_sp_rif *rif;
7680
7681         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
7682         if (!rif)
7683                 return;
7684         __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL);
7685 }
7686
7687 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
7688                                  struct netdev_notifier_changeupper_info *info)
7689 {
7690         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
7691         int err = 0;
7692
7693         /* We do not create a RIF for a macvlan, but only use it to
7694          * direct more MAC addresses to the router.
7695          */
7696         if (!mlxsw_sp || netif_is_macvlan(l3_dev))
7697                 return 0;
7698
7699         mutex_lock(&mlxsw_sp->router->lock);
7700         switch (event) {
7701         case NETDEV_PRECHANGEUPPER:
7702                 break;
7703         case NETDEV_CHANGEUPPER:
7704                 if (info->linking) {
7705                         struct netlink_ext_ack *extack;
7706
7707                         extack = netdev_notifier_info_to_extack(&info->info);
7708                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack);
7709                 } else {
7710                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
7711                 }
7712                 break;
7713         }
7714         mutex_unlock(&mlxsw_sp->router->lock);
7715
7716         return err;
7717 }
7718
7719 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev,
7720                                         struct netdev_nested_priv *priv)
7721 {
7722         struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data;
7723
7724         if (!netif_is_macvlan(dev))
7725                 return 0;
7726
7727         return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
7728                                    mlxsw_sp_fid_index(rif->fid), false);
7729 }
7730
7731 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif)
7732 {
7733         struct netdev_nested_priv priv = {
7734                 .data = (void *)rif,
7735         };
7736
7737         if (!netif_is_macvlan_port(rif->dev))
7738                 return 0;
7739
7740         netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n");
7741         return netdev_walk_all_upper_dev_rcu(rif->dev,
7742                                              __mlxsw_sp_rif_macvlan_flush, &priv);
7743 }
7744
7745 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
7746                                        const struct mlxsw_sp_rif_params *params)
7747 {
7748         struct mlxsw_sp_rif_subport *rif_subport;
7749
7750         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7751         refcount_set(&rif_subport->ref_count, 1);
7752         rif_subport->vid = params->vid;
7753         rif_subport->lag = params->lag;
7754         if (params->lag)
7755                 rif_subport->lag_id = params->lag_id;
7756         else
7757                 rif_subport->system_port = params->system_port;
7758 }
7759
7760 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
7761 {
7762         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7763         struct mlxsw_sp_rif_subport *rif_subport;
7764         char ritr_pl[MLXSW_REG_RITR_LEN];
7765
7766         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7767         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
7768                             rif->rif_index, rif->vr_id, rif->dev->mtu);
7769         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
7770         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
7771                                   rif_subport->lag ? rif_subport->lag_id :
7772                                                      rif_subport->system_port,
7773                                   rif_subport->vid);
7774
7775         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7776 }
7777
7778 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
7779 {
7780         int err;
7781
7782         err = mlxsw_sp_rif_subport_op(rif, true);
7783         if (err)
7784                 return err;
7785
7786         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7787                                   mlxsw_sp_fid_index(rif->fid), true);
7788         if (err)
7789                 goto err_rif_fdb_op;
7790
7791         mlxsw_sp_fid_rif_set(rif->fid, rif);
7792         return 0;
7793
7794 err_rif_fdb_op:
7795         mlxsw_sp_rif_subport_op(rif, false);
7796         return err;
7797 }
7798
7799 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
7800 {
7801         struct mlxsw_sp_fid *fid = rif->fid;
7802
7803         mlxsw_sp_fid_rif_set(fid, NULL);
7804         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7805                             mlxsw_sp_fid_index(fid), false);
7806         mlxsw_sp_rif_macvlan_flush(rif);
7807         mlxsw_sp_rif_subport_op(rif, false);
7808 }
7809
7810 static struct mlxsw_sp_fid *
7811 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif,
7812                              struct netlink_ext_ack *extack)
7813 {
7814         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
7815 }
7816
7817 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
7818         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
7819         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
7820         .setup                  = mlxsw_sp_rif_subport_setup,
7821         .configure              = mlxsw_sp_rif_subport_configure,
7822         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
7823         .fid_get                = mlxsw_sp_rif_subport_fid_get,
7824 };
7825
7826 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
7827                                     enum mlxsw_reg_ritr_if_type type,
7828                                     u16 vid_fid, bool enable)
7829 {
7830         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7831         char ritr_pl[MLXSW_REG_RITR_LEN];
7832
7833         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
7834                             rif->dev->mtu);
7835         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
7836         mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
7837
7838         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7839 }
7840
7841 u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
7842 {
7843         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
7844 }
7845
7846 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
7847 {
7848         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7849         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
7850         int err;
7851
7852         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
7853                                        true);
7854         if (err)
7855                 return err;
7856
7857         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7858                                      mlxsw_sp_router_port(mlxsw_sp), true);
7859         if (err)
7860                 goto err_fid_mc_flood_set;
7861
7862         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7863                                      mlxsw_sp_router_port(mlxsw_sp), true);
7864         if (err)
7865                 goto err_fid_bc_flood_set;
7866
7867         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7868                                   mlxsw_sp_fid_index(rif->fid), true);
7869         if (err)
7870                 goto err_rif_fdb_op;
7871
7872         mlxsw_sp_fid_rif_set(rif->fid, rif);
7873         return 0;
7874
7875 err_rif_fdb_op:
7876         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7877                                mlxsw_sp_router_port(mlxsw_sp), false);
7878 err_fid_bc_flood_set:
7879         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7880                                mlxsw_sp_router_port(mlxsw_sp), false);
7881 err_fid_mc_flood_set:
7882         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
7883         return err;
7884 }
7885
7886 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
7887 {
7888         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
7889         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7890         struct mlxsw_sp_fid *fid = rif->fid;
7891
7892         mlxsw_sp_fid_rif_set(fid, NULL);
7893         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7894                             mlxsw_sp_fid_index(fid), false);
7895         mlxsw_sp_rif_macvlan_flush(rif);
7896         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7897                                mlxsw_sp_router_port(mlxsw_sp), false);
7898         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7899                                mlxsw_sp_router_port(mlxsw_sp), false);
7900         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
7901 }
7902
7903 static struct mlxsw_sp_fid *
7904 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif,
7905                          struct netlink_ext_ack *extack)
7906 {
7907         return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
7908 }
7909
7910 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
7911 {
7912         struct switchdev_notifier_fdb_info info;
7913         struct net_device *dev;
7914
7915         dev = br_fdb_find_port(rif->dev, mac, 0);
7916         if (!dev)
7917                 return;
7918
7919         info.addr = mac;
7920         info.vid = 0;
7921         call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
7922                                  NULL);
7923 }
7924
7925 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
7926         .type                   = MLXSW_SP_RIF_TYPE_FID,
7927         .rif_size               = sizeof(struct mlxsw_sp_rif),
7928         .configure              = mlxsw_sp_rif_fid_configure,
7929         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
7930         .fid_get                = mlxsw_sp_rif_fid_fid_get,
7931         .fdb_del                = mlxsw_sp_rif_fid_fdb_del,
7932 };
7933
7934 static struct mlxsw_sp_fid *
7935 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif,
7936                           struct netlink_ext_ack *extack)
7937 {
7938         struct net_device *br_dev;
7939         u16 vid;
7940         int err;
7941
7942         if (is_vlan_dev(rif->dev)) {
7943                 vid = vlan_dev_vlan_id(rif->dev);
7944                 br_dev = vlan_dev_real_dev(rif->dev);
7945                 if (WARN_ON(!netif_is_bridge_master(br_dev)))
7946                         return ERR_PTR(-EINVAL);
7947         } else {
7948                 err = br_vlan_get_pvid(rif->dev, &vid);
7949                 if (err < 0 || !vid) {
7950                         NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID");
7951                         return ERR_PTR(-EINVAL);
7952                 }
7953         }
7954
7955         return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
7956 }
7957
7958 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
7959 {
7960         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
7961         struct switchdev_notifier_fdb_info info;
7962         struct net_device *br_dev;
7963         struct net_device *dev;
7964
7965         br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev;
7966         dev = br_fdb_find_port(br_dev, mac, vid);
7967         if (!dev)
7968                 return;
7969
7970         info.addr = mac;
7971         info.vid = vid;
7972         call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
7973                                  NULL);
7974 }
7975
7976 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = {
7977         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
7978         .rif_size               = sizeof(struct mlxsw_sp_rif),
7979         .configure              = mlxsw_sp_rif_fid_configure,
7980         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
7981         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
7982         .fdb_del                = mlxsw_sp_rif_vlan_fdb_del,
7983 };
7984
7985 static struct mlxsw_sp_rif_ipip_lb *
7986 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
7987 {
7988         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
7989 }
7990
7991 static void
7992 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
7993                            const struct mlxsw_sp_rif_params *params)
7994 {
7995         struct mlxsw_sp_rif_params_ipip_lb *params_lb;
7996         struct mlxsw_sp_rif_ipip_lb *rif_lb;
7997
7998         params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
7999                                  common);
8000         rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
8001         rif_lb->lb_config = params_lb->lb_config;
8002 }
8003
8004 static int
8005 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
8006 {
8007         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
8008         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
8009         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8010         struct mlxsw_sp_vr *ul_vr;
8011         int err;
8012
8013         ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL);
8014         if (IS_ERR(ul_vr))
8015                 return PTR_ERR(ul_vr);
8016
8017         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true);
8018         if (err)
8019                 goto err_loopback_op;
8020
8021         lb_rif->ul_vr_id = ul_vr->id;
8022         lb_rif->ul_rif_id = 0;
8023         ++ul_vr->rif_count;
8024         return 0;
8025
8026 err_loopback_op:
8027         mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
8028         return err;
8029 }
8030
8031 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
8032 {
8033         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
8034         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8035         struct mlxsw_sp_vr *ul_vr;
8036
8037         ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
8038         mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false);
8039
8040         --ul_vr->rif_count;
8041         mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
8042 }
8043
8044 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = {
8045         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
8046         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
8047         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
8048         .configure              = mlxsw_sp1_rif_ipip_lb_configure,
8049         .deconfigure            = mlxsw_sp1_rif_ipip_lb_deconfigure,
8050 };
8051
8052 const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = {
8053         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
8054         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_emu_ops,
8055         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
8056         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp1_rif_ipip_lb_ops,
8057 };
8058
8059 static int
8060 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable)
8061 {
8062         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
8063         char ritr_pl[MLXSW_REG_RITR_LEN];
8064
8065         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
8066                             ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU);
8067         mlxsw_reg_ritr_loopback_protocol_set(ritr_pl,
8068                                              MLXSW_REG_RITR_LOOPBACK_GENERIC);
8069
8070         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
8071 }
8072
8073 static struct mlxsw_sp_rif *
8074 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
8075                        struct netlink_ext_ack *extack)
8076 {
8077         struct mlxsw_sp_rif *ul_rif;
8078         u16 rif_index;
8079         int err;
8080
8081         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
8082         if (err) {
8083                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
8084                 return ERR_PTR(err);
8085         }
8086
8087         ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL);
8088         if (!ul_rif)
8089                 return ERR_PTR(-ENOMEM);
8090
8091         mlxsw_sp->router->rifs[rif_index] = ul_rif;
8092         ul_rif->mlxsw_sp = mlxsw_sp;
8093         err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true);
8094         if (err)
8095                 goto ul_rif_op_err;
8096
8097         return ul_rif;
8098
8099 ul_rif_op_err:
8100         mlxsw_sp->router->rifs[rif_index] = NULL;
8101         kfree(ul_rif);
8102         return ERR_PTR(err);
8103 }
8104
8105 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif)
8106 {
8107         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
8108
8109         mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false);
8110         mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL;
8111         kfree(ul_rif);
8112 }
8113
8114 static struct mlxsw_sp_rif *
8115 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
8116                     struct netlink_ext_ack *extack)
8117 {
8118         struct mlxsw_sp_vr *vr;
8119         int err;
8120
8121         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack);
8122         if (IS_ERR(vr))
8123                 return ERR_CAST(vr);
8124
8125         if (refcount_inc_not_zero(&vr->ul_rif_refcnt))
8126                 return vr->ul_rif;
8127
8128         vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack);
8129         if (IS_ERR(vr->ul_rif)) {
8130                 err = PTR_ERR(vr->ul_rif);
8131                 goto err_ul_rif_create;
8132         }
8133
8134         vr->rif_count++;
8135         refcount_set(&vr->ul_rif_refcnt, 1);
8136
8137         return vr->ul_rif;
8138
8139 err_ul_rif_create:
8140         mlxsw_sp_vr_put(mlxsw_sp, vr);
8141         return ERR_PTR(err);
8142 }
8143
8144 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif)
8145 {
8146         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
8147         struct mlxsw_sp_vr *vr;
8148
8149         vr = &mlxsw_sp->router->vrs[ul_rif->vr_id];
8150
8151         if (!refcount_dec_and_test(&vr->ul_rif_refcnt))
8152                 return;
8153
8154         vr->rif_count--;
8155         mlxsw_sp_ul_rif_destroy(ul_rif);
8156         mlxsw_sp_vr_put(mlxsw_sp, vr);
8157 }
8158
8159 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
8160                                u16 *ul_rif_index)
8161 {
8162         struct mlxsw_sp_rif *ul_rif;
8163         int err = 0;
8164
8165         mutex_lock(&mlxsw_sp->router->lock);
8166         ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
8167         if (IS_ERR(ul_rif)) {
8168                 err = PTR_ERR(ul_rif);
8169                 goto out;
8170         }
8171         *ul_rif_index = ul_rif->rif_index;
8172 out:
8173         mutex_unlock(&mlxsw_sp->router->lock);
8174         return err;
8175 }
8176
8177 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index)
8178 {
8179         struct mlxsw_sp_rif *ul_rif;
8180
8181         mutex_lock(&mlxsw_sp->router->lock);
8182         ul_rif = mlxsw_sp->router->rifs[ul_rif_index];
8183         if (WARN_ON(!ul_rif))
8184                 goto out;
8185
8186         mlxsw_sp_ul_rif_put(ul_rif);
8187 out:
8188         mutex_unlock(&mlxsw_sp->router->lock);
8189 }
8190
8191 static int
8192 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
8193 {
8194         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
8195         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
8196         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8197         struct mlxsw_sp_rif *ul_rif;
8198         int err;
8199
8200         ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
8201         if (IS_ERR(ul_rif))
8202                 return PTR_ERR(ul_rif);
8203
8204         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true);
8205         if (err)
8206                 goto err_loopback_op;
8207
8208         lb_rif->ul_vr_id = 0;
8209         lb_rif->ul_rif_id = ul_rif->rif_index;
8210
8211         return 0;
8212
8213 err_loopback_op:
8214         mlxsw_sp_ul_rif_put(ul_rif);
8215         return err;
8216 }
8217
8218 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
8219 {
8220         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
8221         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8222         struct mlxsw_sp_rif *ul_rif;
8223
8224         ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id);
8225         mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false);
8226         mlxsw_sp_ul_rif_put(ul_rif);
8227 }
8228
8229 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = {
8230         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
8231         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
8232         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
8233         .configure              = mlxsw_sp2_rif_ipip_lb_configure,
8234         .deconfigure            = mlxsw_sp2_rif_ipip_lb_deconfigure,
8235 };
8236
8237 const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = {
8238         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
8239         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_emu_ops,
8240         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
8241         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp2_rif_ipip_lb_ops,
8242 };
8243
8244 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
8245 {
8246         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
8247
8248         mlxsw_sp->router->rifs = kcalloc(max_rifs,
8249                                          sizeof(struct mlxsw_sp_rif *),
8250                                          GFP_KERNEL);
8251         if (!mlxsw_sp->router->rifs)
8252                 return -ENOMEM;
8253
8254         return 0;
8255 }
8256
8257 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
8258 {
8259         int i;
8260
8261         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
8262                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
8263
8264         kfree(mlxsw_sp->router->rifs);
8265 }
8266
8267 static int
8268 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp)
8269 {
8270         char tigcr_pl[MLXSW_REG_TIGCR_LEN];
8271
8272         mlxsw_reg_tigcr_pack(tigcr_pl, true, 0);
8273         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl);
8274 }
8275
8276 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
8277 {
8278         int err;
8279
8280         mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
8281         INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
8282
8283         err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp);
8284         if (err)
8285                 return err;
8286         err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp);
8287         if (err)
8288                 return err;
8289
8290         return mlxsw_sp_ipip_config_tigcr(mlxsw_sp);
8291 }
8292
8293 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
8294 {
8295         WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
8296 }
8297
8298 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
8299 {
8300         struct mlxsw_sp_router *router;
8301
8302         /* Flush pending FIB notifications and then flush the device's
8303          * table before requesting another dump. The FIB notification
8304          * block is unregistered, so no need to take RTNL.
8305          */
8306         mlxsw_core_flush_owq();
8307         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
8308         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
8309 }
8310
8311 #ifdef CONFIG_IP_ROUTE_MULTIPATH
8312 static void mlxsw_sp_mp_hash_header_set(char *recr2_pl, int header)
8313 {
8314         mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, header, true);
8315 }
8316
8317 static void mlxsw_sp_mp_hash_field_set(char *recr2_pl, int field)
8318 {
8319         mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, field, true);
8320 }
8321
8322 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl)
8323 {
8324         struct net *net = mlxsw_sp_net(mlxsw_sp);
8325         bool only_l3 = !net->ipv4.sysctl_fib_multipath_hash_policy;
8326
8327         mlxsw_sp_mp_hash_header_set(recr2_pl,
8328                                     MLXSW_REG_RECR2_IPV4_EN_NOT_TCP_NOT_UDP);
8329         mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV4_EN_TCP_UDP);
8330         mlxsw_reg_recr2_ipv4_sip_enable(recr2_pl);
8331         mlxsw_reg_recr2_ipv4_dip_enable(recr2_pl);
8332         if (only_l3)
8333                 return;
8334         mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_EN_IPV4);
8335         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV4_PROTOCOL);
8336         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_SPORT);
8337         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_DPORT);
8338 }
8339
8340 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl)
8341 {
8342         bool only_l3 = !ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp));
8343
8344         mlxsw_sp_mp_hash_header_set(recr2_pl,
8345                                     MLXSW_REG_RECR2_IPV6_EN_NOT_TCP_NOT_UDP);
8346         mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV6_EN_TCP_UDP);
8347         mlxsw_reg_recr2_ipv6_sip_enable(recr2_pl);
8348         mlxsw_reg_recr2_ipv6_dip_enable(recr2_pl);
8349         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_NEXT_HEADER);
8350         if (only_l3) {
8351                 mlxsw_sp_mp_hash_field_set(recr2_pl,
8352                                            MLXSW_REG_RECR2_IPV6_FLOW_LABEL);
8353         } else {
8354                 mlxsw_sp_mp_hash_header_set(recr2_pl,
8355                                             MLXSW_REG_RECR2_TCP_UDP_EN_IPV6);
8356                 mlxsw_sp_mp_hash_field_set(recr2_pl,
8357                                            MLXSW_REG_RECR2_TCP_UDP_SPORT);
8358                 mlxsw_sp_mp_hash_field_set(recr2_pl,
8359                                            MLXSW_REG_RECR2_TCP_UDP_DPORT);
8360         }
8361 }
8362
8363 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
8364 {
8365         char recr2_pl[MLXSW_REG_RECR2_LEN];
8366         u32 seed;
8367
8368         seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0);
8369         mlxsw_reg_recr2_pack(recr2_pl, seed);
8370         mlxsw_sp_mp4_hash_init(mlxsw_sp, recr2_pl);
8371         mlxsw_sp_mp6_hash_init(mlxsw_sp, recr2_pl);
8372
8373         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl);
8374 }
8375 #else
8376 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
8377 {
8378         return 0;
8379 }
8380 #endif
8381
8382 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp)
8383 {
8384         char rdpm_pl[MLXSW_REG_RDPM_LEN];
8385         unsigned int i;
8386
8387         MLXSW_REG_ZERO(rdpm, rdpm_pl);
8388
8389         /* HW is determining switch priority based on DSCP-bits, but the
8390          * kernel is still doing that based on the ToS. Since there's a
8391          * mismatch in bits we need to make sure to translate the right
8392          * value ToS would observe, skipping the 2 least-significant ECN bits.
8393          */
8394         for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++)
8395                 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2));
8396
8397         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl);
8398 }
8399
8400 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
8401 {
8402         struct net *net = mlxsw_sp_net(mlxsw_sp);
8403         bool usp = net->ipv4.sysctl_ip_fwd_update_priority;
8404         char rgcr_pl[MLXSW_REG_RGCR_LEN];
8405         u64 max_rifs;
8406
8407         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
8408                 return -EIO;
8409         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
8410
8411         mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
8412         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
8413         mlxsw_reg_rgcr_usp_set(rgcr_pl, usp);
8414         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
8415 }
8416
8417 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
8418 {
8419         char rgcr_pl[MLXSW_REG_RGCR_LEN];
8420
8421         mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
8422         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
8423 }
8424
8425 static const struct mlxsw_sp_router_ll_ops mlxsw_sp_router_ll_basic_ops = {
8426         .ralta_write = mlxsw_sp_router_ll_basic_ralta_write,
8427         .ralst_write = mlxsw_sp_router_ll_basic_ralst_write,
8428         .raltb_write = mlxsw_sp_router_ll_basic_raltb_write,
8429         .fib_entry_op_ctx_size = sizeof(struct mlxsw_sp_fib_entry_op_ctx_basic),
8430         .fib_entry_pack = mlxsw_sp_router_ll_basic_fib_entry_pack,
8431         .fib_entry_act_remote_pack = mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack,
8432         .fib_entry_act_local_pack = mlxsw_sp_router_ll_basic_fib_entry_act_local_pack,
8433         .fib_entry_act_ip2me_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack,
8434         .fib_entry_act_ip2me_tun_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack,
8435         .fib_entry_commit = mlxsw_sp_router_ll_basic_fib_entry_commit,
8436         .fib_entry_is_committed = mlxsw_sp_router_ll_basic_fib_entry_is_committed,
8437 };
8438
8439 static int mlxsw_sp_router_ll_op_ctx_init(struct mlxsw_sp_router *router)
8440 {
8441         size_t max_size = 0;
8442         int i;
8443
8444         for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
8445                 size_t size = router->proto_ll_ops[i]->fib_entry_op_ctx_size;
8446
8447                 if (size > max_size)
8448                         max_size = size;
8449         }
8450         router->ll_op_ctx = kzalloc(sizeof(*router->ll_op_ctx) + max_size,
8451                                     GFP_KERNEL);
8452         if (!router->ll_op_ctx)
8453                 return -ENOMEM;
8454         INIT_LIST_HEAD(&router->ll_op_ctx->fib_entry_priv_list);
8455         return 0;
8456 }
8457
8458 static void mlxsw_sp_router_ll_op_ctx_fini(struct mlxsw_sp_router *router)
8459 {
8460         WARN_ON(!list_empty(&router->ll_op_ctx->fib_entry_priv_list));
8461         kfree(router->ll_op_ctx);
8462 }
8463
8464 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp,
8465                          struct netlink_ext_ack *extack)
8466 {
8467         struct mlxsw_sp_router *router;
8468         int err;
8469
8470         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
8471         if (!router)
8472                 return -ENOMEM;
8473         mutex_init(&router->lock);
8474         mlxsw_sp->router = router;
8475         router->mlxsw_sp = mlxsw_sp;
8476
8477         router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV4] = &mlxsw_sp_router_ll_basic_ops;
8478         router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_router_ll_basic_ops;
8479
8480         err = mlxsw_sp_router_ll_op_ctx_init(router);
8481         if (err)
8482                 goto err_ll_op_ctx_init;
8483
8484         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
8485         err = __mlxsw_sp_router_init(mlxsw_sp);
8486         if (err)
8487                 goto err_router_init;
8488
8489         err = mlxsw_sp_rifs_init(mlxsw_sp);
8490         if (err)
8491                 goto err_rifs_init;
8492
8493         err = mlxsw_sp_ipips_init(mlxsw_sp);
8494         if (err)
8495                 goto err_ipips_init;
8496
8497         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
8498                               &mlxsw_sp_nexthop_ht_params);
8499         if (err)
8500                 goto err_nexthop_ht_init;
8501
8502         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
8503                               &mlxsw_sp_nexthop_group_ht_params);
8504         if (err)
8505                 goto err_nexthop_group_ht_init;
8506
8507         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list);
8508         err = mlxsw_sp_lpm_init(mlxsw_sp);
8509         if (err)
8510                 goto err_lpm_init;
8511
8512         err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops);
8513         if (err)
8514                 goto err_mr_init;
8515
8516         err = mlxsw_sp_vrs_init(mlxsw_sp);
8517         if (err)
8518                 goto err_vrs_init;
8519
8520         err = mlxsw_sp_neigh_init(mlxsw_sp);
8521         if (err)
8522                 goto err_neigh_init;
8523
8524         err = mlxsw_sp_mp_hash_init(mlxsw_sp);
8525         if (err)
8526                 goto err_mp_hash_init;
8527
8528         err = mlxsw_sp_dscp_init(mlxsw_sp);
8529         if (err)
8530                 goto err_dscp_init;
8531
8532         INIT_WORK(&router->fib_event_work, mlxsw_sp_router_fib_event_work);
8533         INIT_LIST_HEAD(&router->fib_event_queue);
8534         spin_lock_init(&router->fib_event_queue_lock);
8535
8536         router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event;
8537         err = register_inetaddr_notifier(&router->inetaddr_nb);
8538         if (err)
8539                 goto err_register_inetaddr_notifier;
8540
8541         router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event;
8542         err = register_inet6addr_notifier(&router->inet6addr_nb);
8543         if (err)
8544                 goto err_register_inet6addr_notifier;
8545
8546         mlxsw_sp->router->netevent_nb.notifier_call =
8547                 mlxsw_sp_router_netevent_event;
8548         err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb);
8549         if (err)
8550                 goto err_register_netevent_notifier;
8551
8552         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
8553         err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp),
8554                                     &mlxsw_sp->router->fib_nb,
8555                                     mlxsw_sp_router_fib_dump_flush, extack);
8556         if (err)
8557                 goto err_register_fib_notifier;
8558
8559         return 0;
8560
8561 err_register_fib_notifier:
8562         unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
8563 err_register_netevent_notifier:
8564         unregister_inet6addr_notifier(&router->inet6addr_nb);
8565 err_register_inet6addr_notifier:
8566         unregister_inetaddr_notifier(&router->inetaddr_nb);
8567 err_register_inetaddr_notifier:
8568         mlxsw_core_flush_owq();
8569         WARN_ON(!list_empty(&router->fib_event_queue));
8570 err_dscp_init:
8571 err_mp_hash_init:
8572         mlxsw_sp_neigh_fini(mlxsw_sp);
8573 err_neigh_init:
8574         mlxsw_sp_vrs_fini(mlxsw_sp);
8575 err_vrs_init:
8576         mlxsw_sp_mr_fini(mlxsw_sp);
8577 err_mr_init:
8578         mlxsw_sp_lpm_fini(mlxsw_sp);
8579 err_lpm_init:
8580         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
8581 err_nexthop_group_ht_init:
8582         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
8583 err_nexthop_ht_init:
8584         mlxsw_sp_ipips_fini(mlxsw_sp);
8585 err_ipips_init:
8586         mlxsw_sp_rifs_fini(mlxsw_sp);
8587 err_rifs_init:
8588         __mlxsw_sp_router_fini(mlxsw_sp);
8589 err_router_init:
8590         mlxsw_sp_router_ll_op_ctx_fini(router);
8591 err_ll_op_ctx_init:
8592         mutex_destroy(&mlxsw_sp->router->lock);
8593         kfree(mlxsw_sp->router);
8594         return err;
8595 }
8596
8597 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
8598 {
8599         unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp),
8600                                 &mlxsw_sp->router->fib_nb);
8601         unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
8602         unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb);
8603         unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb);
8604         mlxsw_core_flush_owq();
8605         WARN_ON(!list_empty(&mlxsw_sp->router->fib_event_queue));
8606         mlxsw_sp_neigh_fini(mlxsw_sp);
8607         mlxsw_sp_vrs_fini(mlxsw_sp);
8608         mlxsw_sp_mr_fini(mlxsw_sp);
8609         mlxsw_sp_lpm_fini(mlxsw_sp);
8610         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
8611         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
8612         mlxsw_sp_ipips_fini(mlxsw_sp);
8613         mlxsw_sp_rifs_fini(mlxsw_sp);
8614         __mlxsw_sp_router_fini(mlxsw_sp);
8615         mlxsw_sp_router_ll_op_ctx_fini(mlxsw_sp->router);
8616         mutex_destroy(&mlxsw_sp->router->lock);
8617         kfree(mlxsw_sp->router);
8618 }