drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/types.h>
   6 #include <linux/rhashtable.h>
   7 #include <linux/bitops.h>
   8 #include <linux/in6.h>
   9 #include <linux/notifier.h>
  10 #include <linux/inetdevice.h>
  11 #include <linux/netdevice.h>
  12 #include <linux/if_bridge.h>
  13 #include <linux/socket.h>
  14 #include <linux/route.h>
  15 #include <linux/gcd.h>
  16 #include <linux/if_macvlan.h>
  17 #include <linux/refcount.h>
  18 #include <linux/jhash.h>
  19 #include <linux/net_namespace.h>
  20 #include <linux/mutex.h>
  21 #include <net/netevent.h>
  22 #include <net/neighbour.h>
  23 #include <net/arp.h>
  24 #include <net/ip_fib.h>
  25 #include <net/ip6_fib.h>
  26 #include <net/nexthop.h>
  27 #include <net/fib_rules.h>
  28 #include <net/ip_tunnels.h>
  29 #include <net/l3mdev.h>
  30 #include <net/addrconf.h>
  31 #include <net/ndisc.h>
  32 #include <net/ipv6.h>
  33 #include <net/fib_notifier.h>
  34 #include <net/switchdev.h>
  35
  36 #include "spectrum.h"
  37 #include "core.h"
  38 #include "reg.h"
  39 #include "spectrum_cnt.h"
  40 #include "spectrum_dpipe.h"
  41 #include "spectrum_ipip.h"
  42 #include "spectrum_mr.h"
  43 #include "spectrum_mr_tcam.h"
  44 #include "spectrum_router.h"
  45 #include "spectrum_span.h"
  46
  47 struct mlxsw_sp_fib;
  48 struct mlxsw_sp_vr;
  49 struct mlxsw_sp_lpm_tree;
  50 struct mlxsw_sp_rif_ops;
  51
  52 struct mlxsw_sp_rif {
  53         struct list_head nexthop_list;
  54         struct list_head neigh_list;
  55         struct net_device *dev; /* NULL for underlay RIF */
  56         struct mlxsw_sp_fid *fid;
  57         unsigned char addr[ETH_ALEN];
  58         int mtu;
  59         u16 rif_index;
  60         u16 vr_id;
  61         const struct mlxsw_sp_rif_ops *ops;
  62         struct mlxsw_sp *mlxsw_sp;
  63
  64         unsigned int counter_ingress;
  65         bool counter_ingress_valid;
  66         unsigned int counter_egress;
  67         bool counter_egress_valid;
  68 };
  69
  70 struct mlxsw_sp_rif_params {
  71         struct net_device *dev;
  72         union {
  73                 u16 system_port;
  74                 u16 lag_id;
  75         };
  76         u16 vid;
  77         bool lag;
  78 };
  79
  80 struct mlxsw_sp_rif_subport {
  81         struct mlxsw_sp_rif common;
  82         refcount_t ref_count;
  83         union {
  84                 u16 system_port;
  85                 u16 lag_id;
  86         };
  87         u16 vid;
  88         bool lag;
  89 };
  90
  91 struct mlxsw_sp_rif_ipip_lb {
  92         struct mlxsw_sp_rif common;
  93         struct mlxsw_sp_rif_ipip_lb_config lb_config;
  94         u16 ul_vr_id; /* Reserved for Spectrum-2. */
  95         u16 ul_rif_id; /* Reserved for Spectrum. */
  96 };
  97
  98 struct mlxsw_sp_rif_params_ipip_lb {
  99         struct mlxsw_sp_rif_params common;
 100         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 101 };
 102
 103 struct mlxsw_sp_rif_ops {
 104         enum mlxsw_sp_rif_type type;
 105         size_t rif_size;
 106
 107         void (*setup)(struct mlxsw_sp_rif *rif,
 108                       const struct mlxsw_sp_rif_params *params);
 109         int (*configure)(struct mlxsw_sp_rif *rif);
 110         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 111         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif,
 112                                          struct netlink_ext_ack *extack);
 113         void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac);
 114 };
 115
 116 static struct mlxsw_sp_rif *
 117 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 118                          const struct net_device *dev);
 119 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif);
 120 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree);
 121 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 122                                   struct mlxsw_sp_lpm_tree *lpm_tree);
 123 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 124                                      const struct mlxsw_sp_fib *fib,
 125                                      u8 tree_id);
 126 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 127                                        const struct mlxsw_sp_fib *fib);
 128
 129 static unsigned int *
 130 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 131                            enum mlxsw_sp_rif_counter_dir dir)
 132 {
 133         switch (dir) {
 134         case MLXSW_SP_RIF_COUNTER_EGRESS:
 135                 return &rif->counter_egress;
 136         case MLXSW_SP_RIF_COUNTER_INGRESS:
 137                 return &rif->counter_ingress;
 138         }
 139         return NULL;
 140 }
 141
 142 static bool
 143 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 144                                enum mlxsw_sp_rif_counter_dir dir)
 145 {
 146         switch (dir) {
 147         case MLXSW_SP_RIF_COUNTER_EGRESS:
 148                 return rif->counter_egress_valid;
 149         case MLXSW_SP_RIF_COUNTER_INGRESS:
 150                 return rif->counter_ingress_valid;
 151         }
 152         return false;
 153 }
 154
 155 static void
 156 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 157                                enum mlxsw_sp_rif_counter_dir dir,
 158                                bool valid)
 159 {
 160         switch (dir) {
 161         case MLXSW_SP_RIF_COUNTER_EGRESS:
 162                 rif->counter_egress_valid = valid;
 163                 break;
 164         case MLXSW_SP_RIF_COUNTER_INGRESS:
 165                 rif->counter_ingress_valid = valid;
 166                 break;
 167         }
 168 }
 169
 170 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 171                                      unsigned int counter_index, bool enable,
 172                                      enum mlxsw_sp_rif_counter_dir dir)
 173 {
 174         char ritr_pl[MLXSW_REG_RITR_LEN];
 175         bool is_egress = false;
 176         int err;
 177
 178         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 179                 is_egress = true;
 180         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 181         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 182         if (err)
 183                 return err;
 184
 185         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 186                                     is_egress);
 187         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 188 }
 189
 190 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 191                                    struct mlxsw_sp_rif *rif,
 192                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 193 {
 194         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 195         unsigned int *p_counter_index;
 196         bool valid;
 197         int err;
 198
 199         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 200         if (!valid)
 201                 return -EINVAL;
 202
 203         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 204         if (!p_counter_index)
 205                 return -EINVAL;
 206         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 207                              MLXSW_REG_RICNT_OPCODE_NOP);
 208         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 209         if (err)
 210                 return err;
 211         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 212         return 0;
 213 }
 214
 215 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 216                                       unsigned int counter_index)
 217 {
 218         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 219
 220         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 221                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 222         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 223 }
 224
 225 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 226                                struct mlxsw_sp_rif *rif,
 227                                enum mlxsw_sp_rif_counter_dir dir)
 228 {
 229         unsigned int *p_counter_index;
 230         int err;
 231
 232         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 233         if (!p_counter_index)
 234                 return -EINVAL;
 235         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 236                                      p_counter_index);
 237         if (err)
 238                 return err;
 239
 240         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 241         if (err)
 242                 goto err_counter_clear;
 243
 244         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 245                                         *p_counter_index, true, dir);
 246         if (err)
 247                 goto err_counter_edit;
 248         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 249         return 0;
 250
 251 err_counter_edit:
 252 err_counter_clear:
 253         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 254                               *p_counter_index);
 255         return err;
 256 }
 257
 258 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 259                                struct mlxsw_sp_rif *rif,
 260                                enum mlxsw_sp_rif_counter_dir dir)
 261 {
 262         unsigned int *p_counter_index;
 263
 264         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 265                 return;
 266
 267         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 268         if (WARN_ON(!p_counter_index))
 269                 return;
 270         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 271                                   *p_counter_index, false, dir);
 272         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 273                               *p_counter_index);
 274         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 275 }
 276
 277 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 278 {
 279         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 280         struct devlink *devlink;
 281
 282         devlink = priv_to_devlink(mlxsw_sp->core);
 283         if (!devlink_dpipe_table_counter_enabled(devlink,
 284                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 285                 return;
 286         mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 287 }
 288
 289 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 290 {
 291         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 292
 293         mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 294 }
 295
 296 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
 297
 298 struct mlxsw_sp_prefix_usage {
 299         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 300 };
 301
 302 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 303         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 304
 305 static bool
 306 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 307                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 308 {
 309         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 310 }
 311
 312 static void
 313 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 314                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 315 {
 316         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 317 }
 318
 319 static void
 320 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 321                           unsigned char prefix_len)
 322 {
 323         set_bit(prefix_len, prefix_usage->b);
 324 }
 325
 326 static void
 327 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 328                             unsigned char prefix_len)
 329 {
 330         clear_bit(prefix_len, prefix_usage->b);
 331 }
 332
 333 struct mlxsw_sp_fib_key {
 334         unsigned char addr[sizeof(struct in6_addr)];
 335         unsigned char prefix_len;
 336 };
 337
 338 enum mlxsw_sp_fib_entry_type {
 339         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 340         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 341         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 342         MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE,
 343         MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE,
 344
 345         /* This is a special case of local delivery, where a packet should be
 346          * decapsulated on reception. Note that there is no corresponding ENCAP,
 347          * because that's a type of next hop, not of FIB entry. (There can be
 348          * several next hops in a REMOTE entry, and some of them may be
 349          * encapsulating entries.)
 350          */
 351         MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
 352         MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP,
 353 };
 354
 355 struct mlxsw_sp_nexthop_group_info;
 356 struct mlxsw_sp_nexthop_group;
 357 struct mlxsw_sp_fib_entry;
 358
 359 struct mlxsw_sp_fib_node {
 360         struct mlxsw_sp_fib_entry *fib_entry;
 361         struct list_head list;
 362         struct rhash_head ht_node;
 363         struct mlxsw_sp_fib *fib;
 364         struct mlxsw_sp_fib_key key;
 365 };
 366
 367 struct mlxsw_sp_fib_entry_decap {
 368         struct mlxsw_sp_ipip_entry *ipip_entry;
 369         u32 tunnel_index;
 370 };
 371
 372 static struct mlxsw_sp_fib_entry_priv *
 373 mlxsw_sp_fib_entry_priv_create(const struct mlxsw_sp_router_ll_ops *ll_ops)
 374 {
 375         struct mlxsw_sp_fib_entry_priv *priv;
 376
 377         if (!ll_ops->fib_entry_priv_size)
 378                 /* No need to have priv */
 379                 return NULL;
 380
 381         priv = kzalloc(sizeof(*priv) + ll_ops->fib_entry_priv_size, GFP_KERNEL);
 382         if (!priv)
 383                 return ERR_PTR(-ENOMEM);
 384         refcount_set(&priv->refcnt, 1);
 385         return priv;
 386 }
 387
 388 static void
 389 mlxsw_sp_fib_entry_priv_destroy(struct mlxsw_sp_fib_entry_priv *priv)
 390 {
 391         kfree(priv);
 392 }
 393
 394 static void mlxsw_sp_fib_entry_priv_hold(struct mlxsw_sp_fib_entry_priv *priv)
 395 {
 396         refcount_inc(&priv->refcnt);
 397 }
 398
 399 static void mlxsw_sp_fib_entry_priv_put(struct mlxsw_sp_fib_entry_priv *priv)
 400 {
 401         if (!priv || !refcount_dec_and_test(&priv->refcnt))
 402                 return;
 403         mlxsw_sp_fib_entry_priv_destroy(priv);
 404 }
 405
 406 static void mlxsw_sp_fib_entry_op_ctx_priv_hold(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
 407                                                 struct mlxsw_sp_fib_entry_priv *priv)
 408 {
 409         if (!priv)
 410                 return;
 411         mlxsw_sp_fib_entry_priv_hold(priv);
 412         list_add(&priv->list, &op_ctx->fib_entry_priv_list);
 413 }
 414
 415 static void mlxsw_sp_fib_entry_op_ctx_priv_put_all(struct mlxsw_sp_fib_entry_op_ctx *op_ctx)
 416 {
 417         struct mlxsw_sp_fib_entry_priv *priv, *tmp;
 418
 419         list_for_each_entry_safe(priv, tmp, &op_ctx->fib_entry_priv_list, list)
 420                 mlxsw_sp_fib_entry_priv_put(priv);
 421         INIT_LIST_HEAD(&op_ctx->fib_entry_priv_list);
 422 }
 423
 424 struct mlxsw_sp_fib_entry {
 425         struct mlxsw_sp_fib_node *fib_node;
 426         enum mlxsw_sp_fib_entry_type type;
 427         struct list_head nexthop_group_node;
 428         struct mlxsw_sp_nexthop_group *nh_group;
 429         struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
 430         struct mlxsw_sp_fib_entry_priv *priv;
 431 };
 432
 433 struct mlxsw_sp_fib4_entry {
 434         struct mlxsw_sp_fib_entry common;
 435         struct fib_info *fi;
 436         u32 tb_id;
 437         u8 tos;
 438         u8 type;
 439 };
 440
 441 struct mlxsw_sp_fib6_entry {
 442         struct mlxsw_sp_fib_entry common;
 443         struct list_head rt6_list;
 444         unsigned int nrt6;
 445 };
 446
 447 struct mlxsw_sp_rt6 {
 448         struct list_head list;
 449         struct fib6_info *rt;
 450 };
 451
 452 struct mlxsw_sp_lpm_tree {
 453         u8 id; /* tree ID */
 454         unsigned int ref_count;
 455         enum mlxsw_sp_l3proto proto;
 456         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 457         struct mlxsw_sp_prefix_usage prefix_usage;
 458 };
 459
 460 struct mlxsw_sp_fib {
 461         struct rhashtable ht;
 462         struct list_head node_list;
 463         struct mlxsw_sp_vr *vr;
 464         struct mlxsw_sp_lpm_tree *lpm_tree;
 465         enum mlxsw_sp_l3proto proto;
 466         const struct mlxsw_sp_router_ll_ops *ll_ops;
 467 };
 468
 469 struct mlxsw_sp_vr {
 470         u16 id; /* virtual router ID */
 471         u32 tb_id; /* kernel fib table id */
 472         unsigned int rif_count;
 473         struct mlxsw_sp_fib *fib4;
 474         struct mlxsw_sp_fib *fib6;
 475         struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX];
 476         struct mlxsw_sp_rif *ul_rif;
 477         refcount_t ul_rif_refcnt;
 478 };
 479
 480 static int mlxsw_sp_router_ll_basic_ralta_write(struct mlxsw_sp *mlxsw_sp, char *xralta_pl)
 481 {
 482         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta),
 483                                xralta_pl + MLXSW_REG_XRALTA_RALTA_OFFSET);
 484 }
 485
 486 static int mlxsw_sp_router_ll_basic_ralst_write(struct mlxsw_sp *mlxsw_sp, char *xralst_pl)
 487 {
 488         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst),
 489                                xralst_pl + MLXSW_REG_XRALST_RALST_OFFSET);
 490 }
 491
 492 static int mlxsw_sp_router_ll_basic_raltb_write(struct mlxsw_sp *mlxsw_sp, char *xraltb_pl)
 493 {
 494         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
 495                                xraltb_pl + MLXSW_REG_XRALTB_RALTB_OFFSET);
 496 }
 497
 498 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 499
 500 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp,
 501                                                 struct mlxsw_sp_vr *vr,
 502                                                 enum mlxsw_sp_l3proto proto)
 503 {
 504         const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
 505         struct mlxsw_sp_lpm_tree *lpm_tree;
 506         struct mlxsw_sp_fib *fib;
 507         int err;
 508
 509         lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 510         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 511         if (!fib)
 512                 return ERR_PTR(-ENOMEM);
 513         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 514         if (err)
 515                 goto err_rhashtable_init;
 516         INIT_LIST_HEAD(&fib->node_list);
 517         fib->proto = proto;
 518         fib->vr = vr;
 519         fib->lpm_tree = lpm_tree;
 520         fib->ll_ops = ll_ops;
 521         mlxsw_sp_lpm_tree_hold(lpm_tree);
 522         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id);
 523         if (err)
 524                 goto err_lpm_tree_bind;
 525         return fib;
 526
 527 err_lpm_tree_bind:
 528         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 529 err_rhashtable_init:
 530         kfree(fib);
 531         return ERR_PTR(err);
 532 }
 533
 534 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp,
 535                                  struct mlxsw_sp_fib *fib)
 536 {
 537         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
 538         mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
 539         WARN_ON(!list_empty(&fib->node_list));
 540         rhashtable_destroy(&fib->ht);
 541         kfree(fib);
 542 }
 543
 544 static struct mlxsw_sp_lpm_tree *
 545 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 546 {
 547         static struct mlxsw_sp_lpm_tree *lpm_tree;
 548         int i;
 549
 550         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 551                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 552                 if (lpm_tree->ref_count == 0)
 553                         return lpm_tree;
 554         }
 555         return NULL;
 556 }
 557
 558 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 559                                    const struct mlxsw_sp_router_ll_ops *ll_ops,
 560                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 561 {
 562         char xralta_pl[MLXSW_REG_XRALTA_LEN];
 563
 564         mlxsw_reg_xralta_pack(xralta_pl, true,
 565                               (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 566                               lpm_tree->id);
 567         return ll_ops->ralta_write(mlxsw_sp, xralta_pl);
 568 }
 569
 570 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 571                                    const struct mlxsw_sp_router_ll_ops *ll_ops,
 572                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 573 {
 574         char xralta_pl[MLXSW_REG_XRALTA_LEN];
 575
 576         mlxsw_reg_xralta_pack(xralta_pl, false,
 577                               (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 578                               lpm_tree->id);
 579         ll_ops->ralta_write(mlxsw_sp, xralta_pl);
 580 }
 581
 582 static int
 583 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 584                                   const struct mlxsw_sp_router_ll_ops *ll_ops,
 585                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 586                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 587 {
 588         char xralst_pl[MLXSW_REG_XRALST_LEN];
 589         u8 root_bin = 0;
 590         u8 prefix;
 591         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 592
 593         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 594                 root_bin = prefix;
 595
 596         mlxsw_reg_xralst_pack(xralst_pl, root_bin, lpm_tree->id);
 597         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 598                 if (prefix == 0)
 599                         continue;
 600                 mlxsw_reg_xralst_bin_pack(xralst_pl, prefix, last_prefix,
 601                                           MLXSW_REG_RALST_BIN_NO_CHILD);
 602                 last_prefix = prefix;
 603         }
 604         return ll_ops->ralst_write(mlxsw_sp, xralst_pl);
 605 }
 606
 607 static struct mlxsw_sp_lpm_tree *
 608 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 609                          const struct mlxsw_sp_router_ll_ops *ll_ops,
 610                          struct mlxsw_sp_prefix_usage *prefix_usage,
 611                          enum mlxsw_sp_l3proto proto)
 612 {
 613         struct mlxsw_sp_lpm_tree *lpm_tree;
 614         int err;
 615
 616         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 617         if (!lpm_tree)
 618                 return ERR_PTR(-EBUSY);
 619         lpm_tree->proto = proto;
 620         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, ll_ops, lpm_tree);
 621         if (err)
 622                 return ERR_PTR(err);
 623
 624         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, ll_ops, prefix_usage, lpm_tree);
 625         if (err)
 626                 goto err_left_struct_set;
 627         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 628                sizeof(lpm_tree->prefix_usage));
 629         memset(&lpm_tree->prefix_ref_count, 0,
 630                sizeof(lpm_tree->prefix_ref_count));
 631         lpm_tree->ref_count = 1;
 632         return lpm_tree;
 633
 634 err_left_struct_set:
 635         mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree);
 636         return ERR_PTR(err);
 637 }
 638
 639 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 640                                       const struct mlxsw_sp_router_ll_ops *ll_ops,
 641                                       struct mlxsw_sp_lpm_tree *lpm_tree)
 642 {
 643         mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree);
 644 }
 645
 646 static struct mlxsw_sp_lpm_tree *
 647 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 648                       struct mlxsw_sp_prefix_usage *prefix_usage,
 649                       enum mlxsw_sp_l3proto proto)
 650 {
 651         const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
 652         struct mlxsw_sp_lpm_tree *lpm_tree;
 653         int i;
 654
 655         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 656                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 657                 if (lpm_tree->ref_count != 0 &&
 658                     lpm_tree->proto == proto &&
 659                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 660                                              prefix_usage)) {
 661                         mlxsw_sp_lpm_tree_hold(lpm_tree);
 662                         return lpm_tree;
 663                 }
 664         }
 665         return mlxsw_sp_lpm_tree_create(mlxsw_sp, ll_ops, prefix_usage, proto);
 666 }
 667
 668 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
 669 {
 670         lpm_tree->ref_count++;
 671 }
 672
 673 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 674                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 675 {
 676         const struct mlxsw_sp_router_ll_ops *ll_ops =
 677                                 mlxsw_sp->router->proto_ll_ops[lpm_tree->proto];
 678
 679         if (--lpm_tree->ref_count == 0)
 680                 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, ll_ops, lpm_tree);
 681 }
 682
 683 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 684
 685 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 686 {
 687         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
 688         struct mlxsw_sp_lpm_tree *lpm_tree;
 689         u64 max_trees;
 690         int err, i;
 691
 692         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 693                 return -EIO;
 694
 695         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 696         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 697         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 698                                              sizeof(struct mlxsw_sp_lpm_tree),
 699                                              GFP_KERNEL);
 700         if (!mlxsw_sp->router->lpm.trees)
 701                 return -ENOMEM;
 702
 703         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 704                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 705                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 706         }
 707
 708         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 709                                          MLXSW_SP_L3_PROTO_IPV4);
 710         if (IS_ERR(lpm_tree)) {
 711                 err = PTR_ERR(lpm_tree);
 712                 goto err_ipv4_tree_get;
 713         }
 714         mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree;
 715
 716         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 717                                          MLXSW_SP_L3_PROTO_IPV6);
 718         if (IS_ERR(lpm_tree)) {
 719                 err = PTR_ERR(lpm_tree);
 720                 goto err_ipv6_tree_get;
 721         }
 722         mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree;
 723
 724         return 0;
 725
 726 err_ipv6_tree_get:
 727         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
 728         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 729 err_ipv4_tree_get:
 730         kfree(mlxsw_sp->router->lpm.trees);
 731         return err;
 732 }
 733
 734 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 735 {
 736         struct mlxsw_sp_lpm_tree *lpm_tree;
 737
 738         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6];
 739         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 740
 741         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
 742         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 743
 744         kfree(mlxsw_sp->router->lpm.trees);
 745 }
 746
 747 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 748 {
 749         return !!vr->fib4 || !!vr->fib6 ||
 750                !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] ||
 751                !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
 752 }
 753
 754 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 755 {
 756         struct mlxsw_sp_vr *vr;
 757         int i;
 758
 759         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 760                 vr = &mlxsw_sp->router->vrs[i];
 761                 if (!mlxsw_sp_vr_is_used(vr))
 762                         return vr;
 763         }
 764         return NULL;
 765 }
 766
 767 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 768                                      const struct mlxsw_sp_fib *fib, u8 tree_id)
 769 {
 770         char xraltb_pl[MLXSW_REG_XRALTB_LEN];
 771
 772         mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id,
 773                               (enum mlxsw_reg_ralxx_protocol) fib->proto,
 774                               tree_id);
 775         return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
 776 }
 777
 778 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 779                                        const struct mlxsw_sp_fib *fib)
 780 {
 781         char xraltb_pl[MLXSW_REG_XRALTB_LEN];
 782
 783         /* Bind to tree 0 which is default */
 784         mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id,
 785                               (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 786         return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
 787 }
 788
 789 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 790 {
 791         /* For our purpose, squash main, default and local tables into one */
 792         if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT)
 793                 tb_id = RT_TABLE_MAIN;
 794         return tb_id;
 795 }
 796
 797 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 798                                             u32 tb_id)
 799 {
 800         struct mlxsw_sp_vr *vr;
 801         int i;
 802
 803         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 804
 805         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 806                 vr = &mlxsw_sp->router->vrs[i];
 807                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 808                         return vr;
 809         }
 810         return NULL;
 811 }
 812
 813 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
 814                                 u16 *vr_id)
 815 {
 816         struct mlxsw_sp_vr *vr;
 817         int err = 0;
 818
 819         mutex_lock(&mlxsw_sp->router->lock);
 820         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 821         if (!vr) {
 822                 err = -ESRCH;
 823                 goto out;
 824         }
 825         *vr_id = vr->id;
 826 out:
 827         mutex_unlock(&mlxsw_sp->router->lock);
 828         return err;
 829 }
 830
 831 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 832                                             enum mlxsw_sp_l3proto proto)
 833 {
 834         switch (proto) {
 835         case MLXSW_SP_L3_PROTO_IPV4:
 836                 return vr->fib4;
 837         case MLXSW_SP_L3_PROTO_IPV6:
 838                 return vr->fib6;
 839         }
 840         return NULL;
 841 }
 842
 843 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 844                                               u32 tb_id,
 845                                               struct netlink_ext_ack *extack)
 846 {
 847         struct mlxsw_sp_mr_table *mr4_table, *mr6_table;
 848         struct mlxsw_sp_fib *fib4;
 849         struct mlxsw_sp_fib *fib6;
 850         struct mlxsw_sp_vr *vr;
 851         int err;
 852
 853         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 854         if (!vr) {
 855                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers");
 856                 return ERR_PTR(-EBUSY);
 857         }
 858         fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
 859         if (IS_ERR(fib4))
 860                 return ERR_CAST(fib4);
 861         fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
 862         if (IS_ERR(fib6)) {
 863                 err = PTR_ERR(fib6);
 864                 goto err_fib6_create;
 865         }
 866         mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
 867                                              MLXSW_SP_L3_PROTO_IPV4);
 868         if (IS_ERR(mr4_table)) {
 869                 err = PTR_ERR(mr4_table);
 870                 goto err_mr4_table_create;
 871         }
 872         mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
 873                                              MLXSW_SP_L3_PROTO_IPV6);
 874         if (IS_ERR(mr6_table)) {
 875                 err = PTR_ERR(mr6_table);
 876                 goto err_mr6_table_create;
 877         }
 878
 879         vr->fib4 = fib4;
 880         vr->fib6 = fib6;
 881         vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table;
 882         vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table;
 883         vr->tb_id = tb_id;
 884         return vr;
 885
 886 err_mr6_table_create:
 887         mlxsw_sp_mr_table_destroy(mr4_table);
 888 err_mr4_table_create:
 889         mlxsw_sp_fib_destroy(mlxsw_sp, fib6);
 890 err_fib6_create:
 891         mlxsw_sp_fib_destroy(mlxsw_sp, fib4);
 892         return ERR_PTR(err);
 893 }
 894
 895 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
 896                                 struct mlxsw_sp_vr *vr)
 897 {
 898         mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]);
 899         vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL;
 900         mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]);
 901         vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL;
 902         mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6);
 903         vr->fib6 = NULL;
 904         mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4);
 905         vr->fib4 = NULL;
 906 }
 907
 908 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
 909                                            struct netlink_ext_ack *extack)
 910 {
 911         struct mlxsw_sp_vr *vr;
 912
 913         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 914         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 915         if (!vr)
 916                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack);
 917         return vr;
 918 }
 919
 920 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
 921 {
 922         if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
 923             list_empty(&vr->fib6->node_list) &&
 924             mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) &&
 925             mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]))
 926                 mlxsw_sp_vr_destroy(mlxsw_sp, vr);
 927 }
 928
 929 static bool
 930 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
 931                                     enum mlxsw_sp_l3proto proto, u8 tree_id)
 932 {
 933         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 934
 935         if (!mlxsw_sp_vr_is_used(vr))
 936                 return false;
 937         if (fib->lpm_tree->id == tree_id)
 938                 return true;
 939         return false;
 940 }
 941
 942 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 943                                         struct mlxsw_sp_fib *fib,
 944                                         struct mlxsw_sp_lpm_tree *new_tree)
 945 {
 946         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 947         int err;
 948
 949         fib->lpm_tree = new_tree;
 950         mlxsw_sp_lpm_tree_hold(new_tree);
 951         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 952         if (err)
 953                 goto err_tree_bind;
 954         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 955         return 0;
 956
 957 err_tree_bind:
 958         mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 959         fib->lpm_tree = old_tree;
 960         return err;
 961 }
 962
 963 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 964                                          struct mlxsw_sp_fib *fib,
 965                                          struct mlxsw_sp_lpm_tree *new_tree)
 966 {
 967         enum mlxsw_sp_l3proto proto = fib->proto;
 968         struct mlxsw_sp_lpm_tree *old_tree;
 969         u8 old_id, new_id = new_tree->id;
 970         struct mlxsw_sp_vr *vr;
 971         int i, err;
 972
 973         old_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 974         old_id = old_tree->id;
 975
 976         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 977                 vr = &mlxsw_sp->router->vrs[i];
 978                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
 979                         continue;
 980                 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 981                                                    mlxsw_sp_vr_fib(vr, proto),
 982                                                    new_tree);
 983                 if (err)
 984                         goto err_tree_replace;
 985         }
 986
 987         memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count,
 988                sizeof(new_tree->prefix_ref_count));
 989         mlxsw_sp->router->lpm.proto_trees[proto] = new_tree;
 990         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 991
 992         return 0;
 993
 994 err_tree_replace:
 995         for (i--; i >= 0; i--) {
 996                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
 997                         continue;
 998                 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 999                                              mlxsw_sp_vr_fib(vr, proto),
1000                                              old_tree);
1001         }
1002         return err;
1003 }
1004
1005 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
1006 {
1007         struct mlxsw_sp_vr *vr;
1008         u64 max_vrs;
1009         int i;
1010
1011         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
1012                 return -EIO;
1013
1014         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
1015         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
1016                                         GFP_KERNEL);
1017         if (!mlxsw_sp->router->vrs)
1018                 return -ENOMEM;
1019
1020         for (i = 0; i < max_vrs; i++) {
1021                 vr = &mlxsw_sp->router->vrs[i];
1022                 vr->id = i;
1023         }
1024
1025         return 0;
1026 }
1027
1028 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
1029
1030 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
1031 {
1032         /* At this stage we're guaranteed not to have new incoming
1033          * FIB notifications and the work queue is free from FIBs
1034          * sitting on top of mlxsw netdevs. However, we can still
1035          * have other FIBs queued. Flush the queue before flushing
1036          * the device's tables. No need for locks, as we're the only
1037          * writer.
1038          */
1039         mlxsw_core_flush_owq();
1040         mlxsw_sp_router_fib_flush(mlxsw_sp);
1041         kfree(mlxsw_sp->router->vrs);
1042 }
1043
1044 static struct net_device *
1045 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
1046 {
1047         struct ip_tunnel *tun = netdev_priv(ol_dev);
1048         struct net *net = dev_net(ol_dev);
1049
1050         return dev_get_by_index_rcu(net, tun->parms.link);
1051 }
1052
1053 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
1054 {
1055         struct net_device *d;
1056         u32 tb_id;
1057
1058         rcu_read_lock();
1059         d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1060         if (d)
1061                 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
1062         else
1063                 tb_id = RT_TABLE_MAIN;
1064         rcu_read_unlock();
1065
1066         return tb_id;
1067 }
1068
1069 static struct mlxsw_sp_rif *
1070 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
1071                     const struct mlxsw_sp_rif_params *params,
1072                     struct netlink_ext_ack *extack);
1073
1074 static struct mlxsw_sp_rif_ipip_lb *
1075 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
1076                                 enum mlxsw_sp_ipip_type ipipt,
1077                                 struct net_device *ol_dev,
1078                                 struct netlink_ext_ack *extack)
1079 {
1080         struct mlxsw_sp_rif_params_ipip_lb lb_params;
1081         const struct mlxsw_sp_ipip_ops *ipip_ops;
1082         struct mlxsw_sp_rif *rif;
1083
1084         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1085         lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
1086                 .common.dev = ol_dev,
1087                 .common.lag = false,
1088                 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
1089         };
1090
1091         rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack);
1092         if (IS_ERR(rif))
1093                 return ERR_CAST(rif);
1094         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
1095 }
1096
1097 static struct mlxsw_sp_ipip_entry *
1098 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
1099                           enum mlxsw_sp_ipip_type ipipt,
1100                           struct net_device *ol_dev)
1101 {
1102         const struct mlxsw_sp_ipip_ops *ipip_ops;
1103         struct mlxsw_sp_ipip_entry *ipip_entry;
1104         struct mlxsw_sp_ipip_entry *ret = NULL;
1105
1106         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1107         ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
1108         if (!ipip_entry)
1109                 return ERR_PTR(-ENOMEM);
1110
1111         ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
1112                                                             ol_dev, NULL);
1113         if (IS_ERR(ipip_entry->ol_lb)) {
1114                 ret = ERR_CAST(ipip_entry->ol_lb);
1115                 goto err_ol_ipip_lb_create;
1116         }
1117
1118         ipip_entry->ipipt = ipipt;
1119         ipip_entry->ol_dev = ol_dev;
1120
1121         switch (ipip_ops->ul_proto) {
1122         case MLXSW_SP_L3_PROTO_IPV4:
1123                 ipip_entry->parms4 = mlxsw_sp_ipip_netdev_parms4(ol_dev);
1124                 break;
1125         case MLXSW_SP_L3_PROTO_IPV6:
1126                 WARN_ON(1);
1127                 break;
1128         }
1129
1130         return ipip_entry;
1131
1132 err_ol_ipip_lb_create:
1133         kfree(ipip_entry);
1134         return ret;
1135 }
1136
1137 static void
1138 mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry)
1139 {
1140         mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
1141         kfree(ipip_entry);
1142 }
1143
1144 static bool
1145 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1146                                   const enum mlxsw_sp_l3proto ul_proto,
1147                                   union mlxsw_sp_l3addr saddr,
1148                                   u32 ul_tb_id,
1149                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1150 {
1151         u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1152         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1153         union mlxsw_sp_l3addr tun_saddr;
1154
1155         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1156                 return false;
1157
1158         tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1159         return tun_ul_tb_id == ul_tb_id &&
1160                mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1161 }
1162
1163 static int
1164 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1165                               struct mlxsw_sp_fib_entry *fib_entry,
1166                               struct mlxsw_sp_ipip_entry *ipip_entry)
1167 {
1168         u32 tunnel_index;
1169         int err;
1170
1171         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1172                                   1, &tunnel_index);
1173         if (err)
1174                 return err;
1175
1176         ipip_entry->decap_fib_entry = fib_entry;
1177         fib_entry->decap.ipip_entry = ipip_entry;
1178         fib_entry->decap.tunnel_index = tunnel_index;
1179         return 0;
1180 }
1181
1182 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1183                                           struct mlxsw_sp_fib_entry *fib_entry)
1184 {
1185         /* Unlink this node from the IPIP entry that it's the decap entry of. */
1186         fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1187         fib_entry->decap.ipip_entry = NULL;
1188         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1189                            1, fib_entry->decap.tunnel_index);
1190 }
1191
1192 static struct mlxsw_sp_fib_node *
1193 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1194                          size_t addr_len, unsigned char prefix_len);
1195 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1196                                      struct mlxsw_sp_fib_entry *fib_entry);
1197
1198 static void
1199 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1200                                  struct mlxsw_sp_ipip_entry *ipip_entry)
1201 {
1202         struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1203
1204         mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1205         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1206
1207         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1208 }
1209
1210 static void
1211 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1212                                   struct mlxsw_sp_ipip_entry *ipip_entry,
1213                                   struct mlxsw_sp_fib_entry *decap_fib_entry)
1214 {
1215         if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1216                                           ipip_entry))
1217                 return;
1218         decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1219
1220         if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1221                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1222 }
1223
1224 static struct mlxsw_sp_fib_entry *
1225 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
1226                                      enum mlxsw_sp_l3proto proto,
1227                                      const union mlxsw_sp_l3addr *addr,
1228                                      enum mlxsw_sp_fib_entry_type type)
1229 {
1230         struct mlxsw_sp_fib_node *fib_node;
1231         unsigned char addr_prefix_len;
1232         struct mlxsw_sp_fib *fib;
1233         struct mlxsw_sp_vr *vr;
1234         const void *addrp;
1235         size_t addr_len;
1236         u32 addr4;
1237
1238         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
1239         if (!vr)
1240                 return NULL;
1241         fib = mlxsw_sp_vr_fib(vr, proto);
1242
1243         switch (proto) {
1244         case MLXSW_SP_L3_PROTO_IPV4:
1245                 addr4 = be32_to_cpu(addr->addr4);
1246                 addrp = &addr4;
1247                 addr_len = 4;
1248                 addr_prefix_len = 32;
1249                 break;
1250         case MLXSW_SP_L3_PROTO_IPV6:
1251         default:
1252                 WARN_ON(1);
1253                 return NULL;
1254         }
1255
1256         fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len,
1257                                             addr_prefix_len);
1258         if (!fib_node || fib_node->fib_entry->type != type)
1259                 return NULL;
1260
1261         return fib_node->fib_entry;
1262 }
1263
1264 /* Given an IPIP entry, find the corresponding decap route. */
1265 static struct mlxsw_sp_fib_entry *
1266 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1267                                struct mlxsw_sp_ipip_entry *ipip_entry)
1268 {
1269         static struct mlxsw_sp_fib_node *fib_node;
1270         const struct mlxsw_sp_ipip_ops *ipip_ops;
1271         unsigned char saddr_prefix_len;
1272         union mlxsw_sp_l3addr saddr;
1273         struct mlxsw_sp_fib *ul_fib;
1274         struct mlxsw_sp_vr *ul_vr;
1275         const void *saddrp;
1276         size_t saddr_len;
1277         u32 ul_tb_id;
1278         u32 saddr4;
1279
1280         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1281
1282         ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1283         ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1284         if (!ul_vr)
1285                 return NULL;
1286
1287         ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1288         saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1289                                            ipip_entry->ol_dev);
1290
1291         switch (ipip_ops->ul_proto) {
1292         case MLXSW_SP_L3_PROTO_IPV4:
1293                 saddr4 = be32_to_cpu(saddr.addr4);
1294                 saddrp = &saddr4;
1295                 saddr_len = 4;
1296                 saddr_prefix_len = 32;
1297                 break;
1298         default:
1299                 WARN_ON(1);
1300                 return NULL;
1301         }
1302
1303         fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1304                                             saddr_prefix_len);
1305         if (!fib_node ||
1306             fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1307                 return NULL;
1308
1309         return fib_node->fib_entry;
1310 }
1311
1312 static struct mlxsw_sp_ipip_entry *
1313 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp,
1314                            enum mlxsw_sp_ipip_type ipipt,
1315                            struct net_device *ol_dev)
1316 {
1317         struct mlxsw_sp_ipip_entry *ipip_entry;
1318
1319         ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1320         if (IS_ERR(ipip_entry))
1321                 return ipip_entry;
1322
1323         list_add_tail(&ipip_entry->ipip_list_node,
1324                       &mlxsw_sp->router->ipip_list);
1325
1326         return ipip_entry;
1327 }
1328
1329 static void
1330 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1331                             struct mlxsw_sp_ipip_entry *ipip_entry)
1332 {
1333         list_del(&ipip_entry->ipip_list_node);
1334         mlxsw_sp_ipip_entry_dealloc(ipip_entry);
1335 }
1336
1337 static bool
1338 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1339                                   const struct net_device *ul_dev,
1340                                   enum mlxsw_sp_l3proto ul_proto,
1341                                   union mlxsw_sp_l3addr ul_dip,
1342                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1343 {
1344         u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1345         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1346
1347         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1348                 return false;
1349
1350         return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1351                                                  ul_tb_id, ipip_entry);
1352 }
1353
1354 /* Given decap parameters, find the corresponding IPIP entry. */
1355 static struct mlxsw_sp_ipip_entry *
1356 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex,
1357                                   enum mlxsw_sp_l3proto ul_proto,
1358                                   union mlxsw_sp_l3addr ul_dip)
1359 {
1360         struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1361         struct net_device *ul_dev;
1362
1363         rcu_read_lock();
1364
1365         ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex);
1366         if (!ul_dev)
1367                 goto out_unlock;
1368
1369         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1370                             ipip_list_node)
1371                 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1372                                                       ul_proto, ul_dip,
1373                                                       ipip_entry))
1374                         goto out_unlock;
1375
1376         rcu_read_unlock();
1377
1378         return NULL;
1379
1380 out_unlock:
1381         rcu_read_unlock();
1382         return ipip_entry;
1383 }
1384
1385 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
1386                                       const struct net_device *dev,
1387                                       enum mlxsw_sp_ipip_type *p_type)
1388 {
1389         struct mlxsw_sp_router *router = mlxsw_sp->router;
1390         const struct mlxsw_sp_ipip_ops *ipip_ops;
1391         enum mlxsw_sp_ipip_type ipipt;
1392
1393         for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
1394                 ipip_ops = router->ipip_ops_arr[ipipt];
1395                 if (dev->type == ipip_ops->dev_type) {
1396                         if (p_type)
1397                                 *p_type = ipipt;
1398                         return true;
1399                 }
1400         }
1401         return false;
1402 }
1403
1404 bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp,
1405                                 const struct net_device *dev)
1406 {
1407         return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
1408 }
1409
1410 static struct mlxsw_sp_ipip_entry *
1411 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp,
1412                                    const struct net_device *ol_dev)
1413 {
1414         struct mlxsw_sp_ipip_entry *ipip_entry;
1415
1416         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1417                             ipip_list_node)
1418                 if (ipip_entry->ol_dev == ol_dev)
1419                         return ipip_entry;
1420
1421         return NULL;
1422 }
1423
1424 static struct mlxsw_sp_ipip_entry *
1425 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp,
1426                                    const struct net_device *ul_dev,
1427                                    struct mlxsw_sp_ipip_entry *start)
1428 {
1429         struct mlxsw_sp_ipip_entry *ipip_entry;
1430
1431         ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list,
1432                                         ipip_list_node);
1433         list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list,
1434                                      ipip_list_node) {
1435                 struct net_device *ol_dev = ipip_entry->ol_dev;
1436                 struct net_device *ipip_ul_dev;
1437
1438                 rcu_read_lock();
1439                 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1440                 rcu_read_unlock();
1441
1442                 if (ipip_ul_dev == ul_dev)
1443                         return ipip_entry;
1444         }
1445
1446         return NULL;
1447 }
1448
1449 bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp,
1450                                 const struct net_device *dev)
1451 {
1452         bool is_ipip_ul;
1453
1454         mutex_lock(&mlxsw_sp->router->lock);
1455         is_ipip_ul = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL);
1456         mutex_unlock(&mlxsw_sp->router->lock);
1457
1458         return is_ipip_ul;
1459 }
1460
1461 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp,
1462                                                 const struct net_device *ol_dev,
1463                                                 enum mlxsw_sp_ipip_type ipipt)
1464 {
1465         const struct mlxsw_sp_ipip_ops *ops
1466                 = mlxsw_sp->router->ipip_ops_arr[ipipt];
1467
1468         return ops->can_offload(mlxsw_sp, ol_dev);
1469 }
1470
1471 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp,
1472                                                 struct net_device *ol_dev)
1473 {
1474         enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX;
1475         struct mlxsw_sp_ipip_entry *ipip_entry;
1476         enum mlxsw_sp_l3proto ul_proto;
1477         union mlxsw_sp_l3addr saddr;
1478         u32 ul_tb_id;
1479
1480         mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt);
1481         if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) {
1482                 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1483                 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto;
1484                 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1485                 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1486                                                           saddr, ul_tb_id,
1487                                                           NULL)) {
1488                         ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt,
1489                                                                 ol_dev);
1490                         if (IS_ERR(ipip_entry))
1491                                 return PTR_ERR(ipip_entry);
1492                 }
1493         }
1494
1495         return 0;
1496 }
1497
1498 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp,
1499                                                    struct net_device *ol_dev)
1500 {
1501         struct mlxsw_sp_ipip_entry *ipip_entry;
1502
1503         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1504         if (ipip_entry)
1505                 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1506 }
1507
1508 static void
1509 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1510                                 struct mlxsw_sp_ipip_entry *ipip_entry)
1511 {
1512         struct mlxsw_sp_fib_entry *decap_fib_entry;
1513
1514         decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1515         if (decap_fib_entry)
1516                 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1517                                                   decap_fib_entry);
1518 }
1519
1520 static int
1521 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id,
1522                         u16 ul_rif_id, bool enable)
1523 {
1524         struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
1525         struct mlxsw_sp_rif *rif = &lb_rif->common;
1526         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
1527         char ritr_pl[MLXSW_REG_RITR_LEN];
1528         u32 saddr4;
1529
1530         switch (lb_cf.ul_protocol) {
1531         case MLXSW_SP_L3_PROTO_IPV4:
1532                 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
1533                 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
1534                                     rif->rif_index, rif->vr_id, rif->dev->mtu);
1535                 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
1536                             MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
1537                             ul_vr_id, ul_rif_id, saddr4, lb_cf.okey);
1538                 break;
1539
1540         case MLXSW_SP_L3_PROTO_IPV6:
1541                 return -EAFNOSUPPORT;
1542         }
1543
1544         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
1545 }
1546
1547 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp,
1548                                                  struct net_device *ol_dev)
1549 {
1550         struct mlxsw_sp_ipip_entry *ipip_entry;
1551         struct mlxsw_sp_rif_ipip_lb *lb_rif;
1552         int err = 0;
1553
1554         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1555         if (ipip_entry) {
1556                 lb_rif = ipip_entry->ol_lb;
1557                 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id,
1558                                               lb_rif->ul_rif_id, true);
1559                 if (err)
1560                         goto out;
1561                 lb_rif->common.mtu = ol_dev->mtu;
1562         }
1563
1564 out:
1565         return err;
1566 }
1567
1568 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1569                                                 struct net_device *ol_dev)
1570 {
1571         struct mlxsw_sp_ipip_entry *ipip_entry;
1572
1573         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1574         if (ipip_entry)
1575                 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1576 }
1577
1578 static void
1579 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1580                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1581 {
1582         if (ipip_entry->decap_fib_entry)
1583                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1584 }
1585
1586 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1587                                                   struct net_device *ol_dev)
1588 {
1589         struct mlxsw_sp_ipip_entry *ipip_entry;
1590
1591         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1592         if (ipip_entry)
1593                 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1594 }
1595
1596 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
1597                                          struct mlxsw_sp_rif *old_rif,
1598                                          struct mlxsw_sp_rif *new_rif);
1599 static int
1600 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp,
1601                                  struct mlxsw_sp_ipip_entry *ipip_entry,
1602                                  bool keep_encap,
1603                                  struct netlink_ext_ack *extack)
1604 {
1605         struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb;
1606         struct mlxsw_sp_rif_ipip_lb *new_lb_rif;
1607
1608         new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp,
1609                                                      ipip_entry->ipipt,
1610                                                      ipip_entry->ol_dev,
1611                                                      extack);
1612         if (IS_ERR(new_lb_rif))
1613                 return PTR_ERR(new_lb_rif);
1614         ipip_entry->ol_lb = new_lb_rif;
1615
1616         if (keep_encap)
1617                 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common,
1618                                              &new_lb_rif->common);
1619
1620         mlxsw_sp_rif_destroy(&old_lb_rif->common);
1621
1622         return 0;
1623 }
1624
1625 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
1626                                         struct mlxsw_sp_rif *rif);
1627
1628 /**
1629  * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry.
1630  * @mlxsw_sp: mlxsw_sp.
1631  * @ipip_entry: IPIP entry.
1632  * @recreate_loopback: Recreates the associated loopback RIF.
1633  * @keep_encap: Updates next hops that use the tunnel netdevice. This is only
1634  *              relevant when recreate_loopback is true.
1635  * @update_nexthops: Updates next hops, keeping the current loopback RIF. This
1636  *                   is only relevant when recreate_loopback is false.
1637  * @extack: extack.
1638  *
1639  * Return: Non-zero value on failure.
1640  */
1641 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp,
1642                                         struct mlxsw_sp_ipip_entry *ipip_entry,
1643                                         bool recreate_loopback,
1644                                         bool keep_encap,
1645                                         bool update_nexthops,
1646                                         struct netlink_ext_ack *extack)
1647 {
1648         int err;
1649
1650         /* RIFs can't be edited, so to update loopback, we need to destroy and
1651          * recreate it. That creates a window of opportunity where RALUE and
1652          * RATR registers end up referencing a RIF that's already gone. RATRs
1653          * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care
1654          * of RALUE, demote the decap route back.
1655          */
1656         if (ipip_entry->decap_fib_entry)
1657                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1658
1659         if (recreate_loopback) {
1660                 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry,
1661                                                        keep_encap, extack);
1662                 if (err)
1663                         return err;
1664         } else if (update_nexthops) {
1665                 mlxsw_sp_nexthop_rif_update(mlxsw_sp,
1666                                             &ipip_entry->ol_lb->common);
1667         }
1668
1669         if (ipip_entry->ol_dev->flags & IFF_UP)
1670                 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1671
1672         return 0;
1673 }
1674
1675 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp,
1676                                                 struct net_device *ol_dev,
1677                                                 struct netlink_ext_ack *extack)
1678 {
1679         struct mlxsw_sp_ipip_entry *ipip_entry =
1680                 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1681
1682         if (!ipip_entry)
1683                 return 0;
1684
1685         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1686                                                    true, false, false, extack);
1687 }
1688
1689 static int
1690 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp,
1691                                      struct mlxsw_sp_ipip_entry *ipip_entry,
1692                                      struct net_device *ul_dev,
1693                                      bool *demote_this,
1694                                      struct netlink_ext_ack *extack)
1695 {
1696         u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1697         enum mlxsw_sp_l3proto ul_proto;
1698         union mlxsw_sp_l3addr saddr;
1699
1700         /* Moving underlay to a different VRF might cause local address
1701          * conflict, and the conflicting tunnels need to be demoted.
1702          */
1703         ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1704         saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1705         if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1706                                                  saddr, ul_tb_id,
1707                                                  ipip_entry)) {
1708                 *demote_this = true;
1709                 return 0;
1710         }
1711
1712         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1713                                                    true, true, false, extack);
1714 }
1715
1716 static int
1717 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp,
1718                                     struct mlxsw_sp_ipip_entry *ipip_entry,
1719                                     struct net_device *ul_dev)
1720 {
1721         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1722                                                    false, false, true, NULL);
1723 }
1724
1725 static int
1726 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp,
1727                                       struct mlxsw_sp_ipip_entry *ipip_entry,
1728                                       struct net_device *ul_dev)
1729 {
1730         /* A down underlay device causes encapsulated packets to not be
1731          * forwarded, but decap still works. So refresh next hops without
1732          * touching anything else.
1733          */
1734         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1735                                                    false, false, true, NULL);
1736 }
1737
1738 static int
1739 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp,
1740                                         struct net_device *ol_dev,
1741                                         struct netlink_ext_ack *extack)
1742 {
1743         const struct mlxsw_sp_ipip_ops *ipip_ops;
1744         struct mlxsw_sp_ipip_entry *ipip_entry;
1745         int err;
1746
1747         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1748         if (!ipip_entry)
1749                 /* A change might make a tunnel eligible for offloading, but
1750                  * that is currently not implemented. What falls to slow path
1751                  * stays there.
1752                  */
1753                 return 0;
1754
1755         /* A change might make a tunnel not eligible for offloading. */
1756         if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev,
1757                                                  ipip_entry->ipipt)) {
1758                 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1759                 return 0;
1760         }
1761
1762         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1763         err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack);
1764         return err;
1765 }
1766
1767 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp,
1768                                        struct mlxsw_sp_ipip_entry *ipip_entry)
1769 {
1770         struct net_device *ol_dev = ipip_entry->ol_dev;
1771
1772         if (ol_dev->flags & IFF_UP)
1773                 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1774         mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1775 }
1776
1777 /* The configuration where several tunnels have the same local address in the
1778  * same underlay table needs special treatment in the HW. That is currently not
1779  * implemented in the driver. This function finds and demotes the first tunnel
1780  * with a given source address, except the one passed in in the argument
1781  * `except'.
1782  */
1783 bool
1784 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp,
1785                                      enum mlxsw_sp_l3proto ul_proto,
1786                                      union mlxsw_sp_l3addr saddr,
1787                                      u32 ul_tb_id,
1788                                      const struct mlxsw_sp_ipip_entry *except)
1789 {
1790         struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1791
1792         list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1793                                  ipip_list_node) {
1794                 if (ipip_entry != except &&
1795                     mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1796                                                       ul_tb_id, ipip_entry)) {
1797                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1798                         return true;
1799                 }
1800         }
1801
1802         return false;
1803 }
1804
1805 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp,
1806                                                      struct net_device *ul_dev)
1807 {
1808         struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1809
1810         list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1811                                  ipip_list_node) {
1812                 struct net_device *ol_dev = ipip_entry->ol_dev;
1813                 struct net_device *ipip_ul_dev;
1814
1815                 rcu_read_lock();
1816                 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1817                 rcu_read_unlock();
1818                 if (ipip_ul_dev == ul_dev)
1819                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1820         }
1821 }
1822
1823 int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp,
1824                                      struct net_device *ol_dev,
1825                                      unsigned long event,
1826                                      struct netdev_notifier_info *info)
1827 {
1828         struct netdev_notifier_changeupper_info *chup;
1829         struct netlink_ext_ack *extack;
1830         int err = 0;
1831
1832         mutex_lock(&mlxsw_sp->router->lock);
1833         switch (event) {
1834         case NETDEV_REGISTER:
1835                 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev);
1836                 break;
1837         case NETDEV_UNREGISTER:
1838                 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev);
1839                 break;
1840         case NETDEV_UP:
1841                 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev);
1842                 break;
1843         case NETDEV_DOWN:
1844                 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev);
1845                 break;
1846         case NETDEV_CHANGEUPPER:
1847                 chup = container_of(info, typeof(*chup), info);
1848                 extack = info->extack;
1849                 if (netif_is_l3_master(chup->upper_dev))
1850                         err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp,
1851                                                                    ol_dev,
1852                                                                    extack);
1853                 break;
1854         case NETDEV_CHANGE:
1855                 extack = info->extack;
1856                 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp,
1857                                                               ol_dev, extack);
1858                 break;
1859         case NETDEV_CHANGEMTU:
1860                 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev);
1861                 break;
1862         }
1863         mutex_unlock(&mlxsw_sp->router->lock);
1864         return err;
1865 }
1866
1867 static int
1868 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1869                                    struct mlxsw_sp_ipip_entry *ipip_entry,
1870                                    struct net_device *ul_dev,
1871                                    bool *demote_this,
1872                                    unsigned long event,
1873                                    struct netdev_notifier_info *info)
1874 {
1875         struct netdev_notifier_changeupper_info *chup;
1876         struct netlink_ext_ack *extack;
1877
1878         switch (event) {
1879         case NETDEV_CHANGEUPPER:
1880                 chup = container_of(info, typeof(*chup), info);
1881                 extack = info->extack;
1882                 if (netif_is_l3_master(chup->upper_dev))
1883                         return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp,
1884                                                                     ipip_entry,
1885                                                                     ul_dev,
1886                                                                     demote_this,
1887                                                                     extack);
1888                 break;
1889
1890         case NETDEV_UP:
1891                 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry,
1892                                                            ul_dev);
1893         case NETDEV_DOWN:
1894                 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp,
1895                                                              ipip_entry,
1896                                                              ul_dev);
1897         }
1898         return 0;
1899 }
1900
1901 int
1902 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1903                                  struct net_device *ul_dev,
1904                                  unsigned long event,
1905                                  struct netdev_notifier_info *info)
1906 {
1907         struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1908         int err = 0;
1909
1910         mutex_lock(&mlxsw_sp->router->lock);
1911         while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp,
1912                                                                 ul_dev,
1913                                                                 ipip_entry))) {
1914                 struct mlxsw_sp_ipip_entry *prev;
1915                 bool demote_this = false;
1916
1917                 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry,
1918                                                          ul_dev, &demote_this,
1919                                                          event, info);
1920                 if (err) {
1921                         mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp,
1922                                                                  ul_dev);
1923                         break;
1924                 }
1925
1926                 if (demote_this) {
1927                         if (list_is_first(&ipip_entry->ipip_list_node,
1928                                           &mlxsw_sp->router->ipip_list))
1929                                 prev = NULL;
1930                         else
1931                                 /* This can't be cached from previous iteration,
1932                                  * because that entry could be gone now.
1933                                  */
1934                                 prev = list_prev_entry(ipip_entry,
1935                                                        ipip_list_node);
1936                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1937                         ipip_entry = prev;
1938                 }
1939         }
1940         mutex_unlock(&mlxsw_sp->router->lock);
1941
1942         return err;
1943 }
1944
1945 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
1946                                       enum mlxsw_sp_l3proto ul_proto,
1947                                       const union mlxsw_sp_l3addr *ul_sip,
1948                                       u32 tunnel_index)
1949 {
1950         enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1951         struct mlxsw_sp_router *router = mlxsw_sp->router;
1952         struct mlxsw_sp_fib_entry *fib_entry;
1953         int err = 0;
1954
1955         mutex_lock(&mlxsw_sp->router->lock);
1956
1957         if (WARN_ON_ONCE(router->nve_decap_config.valid)) {
1958                 err = -EINVAL;
1959                 goto out;
1960         }
1961
1962         router->nve_decap_config.ul_tb_id = ul_tb_id;
1963         router->nve_decap_config.tunnel_index = tunnel_index;
1964         router->nve_decap_config.ul_proto = ul_proto;
1965         router->nve_decap_config.ul_sip = *ul_sip;
1966         router->nve_decap_config.valid = true;
1967
1968         /* It is valid to create a tunnel with a local IP and only later
1969          * assign this IP address to a local interface
1970          */
1971         fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
1972                                                          ul_proto, ul_sip,
1973                                                          type);
1974         if (!fib_entry)
1975                 goto out;
1976
1977         fib_entry->decap.tunnel_index = tunnel_index;
1978         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
1979
1980         err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1981         if (err)
1982                 goto err_fib_entry_update;
1983
1984         goto out;
1985
1986 err_fib_entry_update:
1987         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1988         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1989 out:
1990         mutex_unlock(&mlxsw_sp->router->lock);
1991         return err;
1992 }
1993
1994 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
1995                                       enum mlxsw_sp_l3proto ul_proto,
1996                                       const union mlxsw_sp_l3addr *ul_sip)
1997 {
1998         enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
1999         struct mlxsw_sp_router *router = mlxsw_sp->router;
2000         struct mlxsw_sp_fib_entry *fib_entry;
2001
2002         mutex_lock(&mlxsw_sp->router->lock);
2003
2004         if (WARN_ON_ONCE(!router->nve_decap_config.valid))
2005                 goto out;
2006
2007         router->nve_decap_config.valid = false;
2008
2009         fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
2010                                                          ul_proto, ul_sip,
2011                                                          type);
2012         if (!fib_entry)
2013                 goto out;
2014
2015         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2016         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2017 out:
2018         mutex_unlock(&mlxsw_sp->router->lock);
2019 }
2020
2021 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp,
2022                                          u32 ul_tb_id,
2023                                          enum mlxsw_sp_l3proto ul_proto,
2024                                          const union mlxsw_sp_l3addr *ul_sip)
2025 {
2026         struct mlxsw_sp_router *router = mlxsw_sp->router;
2027
2028         return router->nve_decap_config.valid &&
2029                router->nve_decap_config.ul_tb_id == ul_tb_id &&
2030                router->nve_decap_config.ul_proto == ul_proto &&
2031                !memcmp(&router->nve_decap_config.ul_sip, ul_sip,
2032                        sizeof(*ul_sip));
2033 }
2034
2035 struct mlxsw_sp_neigh_key {
2036         struct neighbour *n;
2037 };
2038
2039 struct mlxsw_sp_neigh_entry {
2040         struct list_head rif_list_node;
2041         struct rhash_head ht_node;
2042         struct mlxsw_sp_neigh_key key;
2043         u16 rif;
2044         bool connected;
2045         unsigned char ha[ETH_ALEN];
2046         struct list_head nexthop_list; /* list of nexthops using
2047                                         * this neigh entry
2048                                         */
2049         struct list_head nexthop_neighs_list_node;
2050         unsigned int counter_index;
2051         bool counter_valid;
2052 };
2053
2054 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
2055         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
2056         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
2057         .key_len = sizeof(struct mlxsw_sp_neigh_key),
2058 };
2059
2060 struct mlxsw_sp_neigh_entry *
2061 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
2062                         struct mlxsw_sp_neigh_entry *neigh_entry)
2063 {
2064         if (!neigh_entry) {
2065                 if (list_empty(&rif->neigh_list))
2066                         return NULL;
2067                 else
2068                         return list_first_entry(&rif->neigh_list,
2069                                                 typeof(*neigh_entry),
2070                                                 rif_list_node);
2071         }
2072         if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list))
2073                 return NULL;
2074         return list_next_entry(neigh_entry, rif_list_node);
2075 }
2076
2077 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
2078 {
2079         return neigh_entry->key.n->tbl->family;
2080 }
2081
2082 unsigned char *
2083 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
2084 {
2085         return neigh_entry->ha;
2086 }
2087
2088 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2089 {
2090         struct neighbour *n;
2091
2092         n = neigh_entry->key.n;
2093         return ntohl(*((__be32 *) n->primary_key));
2094 }
2095
2096 struct in6_addr *
2097 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2098 {
2099         struct neighbour *n;
2100
2101         n = neigh_entry->key.n;
2102         return (struct in6_addr *) &n->primary_key;
2103 }
2104
2105 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
2106                                struct mlxsw_sp_neigh_entry *neigh_entry,
2107                                u64 *p_counter)
2108 {
2109         if (!neigh_entry->counter_valid)
2110                 return -EINVAL;
2111
2112         return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
2113                                          p_counter, NULL);
2114 }
2115
2116 static struct mlxsw_sp_neigh_entry *
2117 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
2118                            u16 rif)
2119 {
2120         struct mlxsw_sp_neigh_entry *neigh_entry;
2121
2122         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
2123         if (!neigh_entry)
2124                 return NULL;
2125
2126         neigh_entry->key.n = n;
2127         neigh_entry->rif = rif;
2128         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
2129
2130         return neigh_entry;
2131 }
2132
2133 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
2134 {
2135         kfree(neigh_entry);
2136 }
2137
2138 static int
2139 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
2140                             struct mlxsw_sp_neigh_entry *neigh_entry)
2141 {
2142         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
2143                                       &neigh_entry->ht_node,
2144                                       mlxsw_sp_neigh_ht_params);
2145 }
2146
2147 static void
2148 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
2149                             struct mlxsw_sp_neigh_entry *neigh_entry)
2150 {
2151         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
2152                                &neigh_entry->ht_node,
2153                                mlxsw_sp_neigh_ht_params);
2154 }
2155
2156 static bool
2157 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
2158                                     struct mlxsw_sp_neigh_entry *neigh_entry)
2159 {
2160         struct devlink *devlink;
2161         const char *table_name;
2162
2163         switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
2164         case AF_INET:
2165                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
2166                 break;
2167         case AF_INET6:
2168                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
2169                 break;
2170         default:
2171                 WARN_ON(1);
2172                 return false;
2173         }
2174
2175         devlink = priv_to_devlink(mlxsw_sp->core);
2176         return devlink_dpipe_table_counter_enabled(devlink, table_name);
2177 }
2178
2179 static void
2180 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2181                              struct mlxsw_sp_neigh_entry *neigh_entry)
2182 {
2183         if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
2184                 return;
2185
2186         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
2187                 return;
2188
2189         neigh_entry->counter_valid = true;
2190 }
2191
2192 static void
2193 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
2194                             struct mlxsw_sp_neigh_entry *neigh_entry)
2195 {
2196         if (!neigh_entry->counter_valid)
2197                 return;
2198         mlxsw_sp_flow_counter_free(mlxsw_sp,
2199                                    neigh_entry->counter_index);
2200         neigh_entry->counter_valid = false;
2201 }
2202
2203 static struct mlxsw_sp_neigh_entry *
2204 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2205 {
2206         struct mlxsw_sp_neigh_entry *neigh_entry;
2207         struct mlxsw_sp_rif *rif;
2208         int err;
2209
2210         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
2211         if (!rif)
2212                 return ERR_PTR(-EINVAL);
2213
2214         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
2215         if (!neigh_entry)
2216                 return ERR_PTR(-ENOMEM);
2217
2218         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
2219         if (err)
2220                 goto err_neigh_entry_insert;
2221
2222         mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2223         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
2224
2225         return neigh_entry;
2226
2227 err_neigh_entry_insert:
2228         mlxsw_sp_neigh_entry_free(neigh_entry);
2229         return ERR_PTR(err);
2230 }
2231
2232 static void
2233 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2234                              struct mlxsw_sp_neigh_entry *neigh_entry)
2235 {
2236         list_del(&neigh_entry->rif_list_node);
2237         mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2238         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
2239         mlxsw_sp_neigh_entry_free(neigh_entry);
2240 }
2241
2242 static struct mlxsw_sp_neigh_entry *
2243 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2244 {
2245         struct mlxsw_sp_neigh_key key;
2246
2247         key.n = n;
2248         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
2249                                       &key, mlxsw_sp_neigh_ht_params);
2250 }
2251
2252 static void
2253 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
2254 {
2255         unsigned long interval;
2256
2257 #if IS_ENABLED(CONFIG_IPV6)
2258         interval = min_t(unsigned long,
2259                          NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
2260                          NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
2261 #else
2262         interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
2263 #endif
2264         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
2265 }
2266
2267 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2268                                                    char *rauhtd_pl,
2269                                                    int ent_index)
2270 {
2271         struct net_device *dev;
2272         struct neighbour *n;
2273         __be32 dipn;
2274         u32 dip;
2275         u16 rif;
2276
2277         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
2278
2279         if (!mlxsw_sp->router->rifs[rif]) {
2280                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2281                 return;
2282         }
2283
2284         dipn = htonl(dip);
2285         dev = mlxsw_sp->router->rifs[rif]->dev;
2286         n = neigh_lookup(&arp_tbl, &dipn, dev);
2287         if (!n)
2288                 return;
2289
2290         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
2291         neigh_event_send(n, NULL);
2292         neigh_release(n);
2293 }
2294
2295 #if IS_ENABLED(CONFIG_IPV6)
2296 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2297                                                    char *rauhtd_pl,
2298                                                    int rec_index)
2299 {
2300         struct net_device *dev;
2301         struct neighbour *n;
2302         struct in6_addr dip;
2303         u16 rif;
2304
2305         mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
2306                                          (char *) &dip);
2307
2308         if (!mlxsw_sp->router->rifs[rif]) {
2309                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2310                 return;
2311         }
2312
2313         dev = mlxsw_sp->router->rifs[rif]->dev;
2314         n = neigh_lookup(&nd_tbl, &dip, dev);
2315         if (!n)
2316                 return;
2317
2318         netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
2319         neigh_event_send(n, NULL);
2320         neigh_release(n);
2321 }
2322 #else
2323 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2324                                                    char *rauhtd_pl,
2325                                                    int rec_index)
2326 {
2327 }
2328 #endif
2329
2330 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2331                                                    char *rauhtd_pl,
2332                                                    int rec_index)
2333 {
2334         u8 num_entries;
2335         int i;
2336
2337         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2338                                                                 rec_index);
2339         /* Hardware starts counting at 0, so add 1. */
2340         num_entries++;
2341
2342         /* Each record consists of several neighbour entries. */
2343         for (i = 0; i < num_entries; i++) {
2344                 int ent_index;
2345
2346                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
2347                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
2348                                                        ent_index);
2349         }
2350
2351 }
2352
2353 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2354                                                    char *rauhtd_pl,
2355                                                    int rec_index)
2356 {
2357         /* One record contains one entry. */
2358         mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
2359                                                rec_index);
2360 }
2361
2362 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
2363                                               char *rauhtd_pl, int rec_index)
2364 {
2365         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
2366         case MLXSW_REG_RAUHTD_TYPE_IPV4:
2367                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
2368                                                        rec_index);
2369                 break;
2370         case MLXSW_REG_RAUHTD_TYPE_IPV6:
2371                 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
2372                                                        rec_index);
2373                 break;
2374         }
2375 }
2376
2377 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
2378 {
2379         u8 num_rec, last_rec_index, num_entries;
2380
2381         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2382         last_rec_index = num_rec - 1;
2383
2384         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
2385                 return false;
2386         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
2387             MLXSW_REG_RAUHTD_TYPE_IPV6)
2388                 return true;
2389
2390         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2391                                                                 last_rec_index);
2392         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
2393                 return true;
2394         return false;
2395 }
2396
2397 static int
2398 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
2399                                        char *rauhtd_pl,
2400                                        enum mlxsw_reg_rauhtd_type type)
2401 {
2402         int i, num_rec;
2403         int err;
2404
2405         /* Ensure the RIF we read from the device does not change mid-dump. */
2406         mutex_lock(&mlxsw_sp->router->lock);
2407         do {
2408                 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
2409                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
2410                                       rauhtd_pl);
2411                 if (err) {
2412                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n");
2413                         break;
2414                 }
2415                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2416                 for (i = 0; i < num_rec; i++)
2417                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
2418                                                           i);
2419         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
2420         mutex_unlock(&mlxsw_sp->router->lock);
2421
2422         return err;
2423 }
2424
2425 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
2426 {
2427         enum mlxsw_reg_rauhtd_type type;
2428         char *rauhtd_pl;
2429         int err;
2430
2431         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
2432         if (!rauhtd_pl)
2433                 return -ENOMEM;
2434
2435         type = MLXSW_REG_RAUHTD_TYPE_IPV4;
2436         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2437         if (err)
2438                 goto out;
2439
2440         type = MLXSW_REG_RAUHTD_TYPE_IPV6;
2441         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2442 out:
2443         kfree(rauhtd_pl);
2444         return err;
2445 }
2446
2447 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
2448 {
2449         struct mlxsw_sp_neigh_entry *neigh_entry;
2450
2451         mutex_lock(&mlxsw_sp->router->lock);
2452         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
2453                             nexthop_neighs_list_node)
2454                 /* If this neigh have nexthops, make the kernel think this neigh
2455                  * is active regardless of the traffic.
2456                  */
2457                 neigh_event_send(neigh_entry->key.n, NULL);
2458         mutex_unlock(&mlxsw_sp->router->lock);
2459 }
2460
2461 static void
2462 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
2463 {
2464         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
2465
2466         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
2467                                msecs_to_jiffies(interval));
2468 }
2469
2470 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
2471 {
2472         struct mlxsw_sp_router *router;
2473         int err;
2474
2475         router = container_of(work, struct mlxsw_sp_router,
2476                               neighs_update.dw.work);
2477         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
2478         if (err)
2479                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
2480
2481         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
2482
2483         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
2484 }
2485
2486 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
2487 {
2488         struct mlxsw_sp_neigh_entry *neigh_entry;
2489         struct mlxsw_sp_router *router;
2490
2491         router = container_of(work, struct mlxsw_sp_router,
2492                               nexthop_probe_dw.work);
2493         /* Iterate over nexthop neighbours, find those who are unresolved and
2494          * send arp on them. This solves the chicken-egg problem when
2495          * the nexthop wouldn't get offloaded until the neighbor is resolved
2496          * but it wouldn't get resolved ever in case traffic is flowing in HW
2497          * using different nexthop.
2498          */
2499         mutex_lock(&router->lock);
2500         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
2501                             nexthop_neighs_list_node)
2502                 if (!neigh_entry->connected)
2503                         neigh_event_send(neigh_entry->key.n, NULL);
2504         mutex_unlock(&router->lock);
2505
2506         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
2507                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
2508 }
2509
2510 static void
2511 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2512                               struct mlxsw_sp_neigh_entry *neigh_entry,
2513                               bool removing, bool dead);
2514
2515 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
2516 {
2517         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
2518                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
2519 }
2520
2521 static int
2522 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
2523                                 struct mlxsw_sp_neigh_entry *neigh_entry,
2524                                 enum mlxsw_reg_rauht_op op)
2525 {
2526         struct neighbour *n = neigh_entry->key.n;
2527         u32 dip = ntohl(*((__be32 *) n->primary_key));
2528         char rauht_pl[MLXSW_REG_RAUHT_LEN];
2529
2530         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2531                               dip);
2532         if (neigh_entry->counter_valid)
2533                 mlxsw_reg_rauht_pack_counter(rauht_pl,
2534                                              neigh_entry->counter_index);
2535         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2536 }
2537
2538 static int
2539 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
2540                                 struct mlxsw_sp_neigh_entry *neigh_entry,
2541                                 enum mlxsw_reg_rauht_op op)
2542 {
2543         struct neighbour *n = neigh_entry->key.n;
2544         char rauht_pl[MLXSW_REG_RAUHT_LEN];
2545         const char *dip = n->primary_key;
2546
2547         mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2548                               dip);
2549         if (neigh_entry->counter_valid)
2550                 mlxsw_reg_rauht_pack_counter(rauht_pl,
2551                                              neigh_entry->counter_index);
2552         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2553 }
2554
2555 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
2556 {
2557         struct neighbour *n = neigh_entry->key.n;
2558
2559         /* Packets with a link-local destination address are trapped
2560          * after LPM lookup and never reach the neighbour table, so
2561          * there is no need to program such neighbours to the device.
2562          */
2563         if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
2564             IPV6_ADDR_LINKLOCAL)
2565                 return true;
2566         return false;
2567 }
2568
2569 static void
2570 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
2571                             struct mlxsw_sp_neigh_entry *neigh_entry,
2572                             bool adding)
2573 {
2574         enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding);
2575         int err;
2576
2577         if (!adding && !neigh_entry->connected)
2578                 return;
2579         neigh_entry->connected = adding;
2580         if (neigh_entry->key.n->tbl->family == AF_INET) {
2581                 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
2582                                                       op);
2583                 if (err)
2584                         return;
2585         } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
2586                 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
2587                         return;
2588                 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
2589                                                       op);
2590                 if (err)
2591                         return;
2592         } else {
2593                 WARN_ON_ONCE(1);
2594                 return;
2595         }
2596
2597         if (adding)
2598                 neigh_entry->key.n->flags |= NTF_OFFLOADED;
2599         else
2600                 neigh_entry->key.n->flags &= ~NTF_OFFLOADED;
2601 }
2602
2603 void
2604 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
2605                                     struct mlxsw_sp_neigh_entry *neigh_entry,
2606                                     bool adding)
2607 {
2608         if (adding)
2609                 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2610         else
2611                 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2612         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
2613 }
2614
2615 struct mlxsw_sp_netevent_work {
2616         struct work_struct work;
2617         struct mlxsw_sp *mlxsw_sp;
2618         struct neighbour *n;
2619 };
2620
2621 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
2622 {
2623         struct mlxsw_sp_netevent_work *net_work =
2624                 container_of(work, struct mlxsw_sp_netevent_work, work);
2625         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2626         struct mlxsw_sp_neigh_entry *neigh_entry;
2627         struct neighbour *n = net_work->n;
2628         unsigned char ha[ETH_ALEN];
2629         bool entry_connected;
2630         u8 nud_state, dead;
2631
2632         /* If these parameters are changed after we release the lock,
2633          * then we are guaranteed to receive another event letting us
2634          * know about it.
2635          */
2636         read_lock_bh(&n->lock);
2637         memcpy(ha, n->ha, ETH_ALEN);
2638         nud_state = n->nud_state;
2639         dead = n->dead;
2640         read_unlock_bh(&n->lock);
2641
2642         mutex_lock(&mlxsw_sp->router->lock);
2643         mlxsw_sp_span_respin(mlxsw_sp);
2644
2645         entry_connected = nud_state & NUD_VALID && !dead;
2646         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2647         if (!entry_connected && !neigh_entry)
2648                 goto out;
2649         if (!neigh_entry) {
2650                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2651                 if (IS_ERR(neigh_entry))
2652                         goto out;
2653         }
2654
2655         memcpy(neigh_entry->ha, ha, ETH_ALEN);
2656         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
2657         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected,
2658                                       dead);
2659
2660         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2661                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2662
2663 out:
2664         mutex_unlock(&mlxsw_sp->router->lock);
2665         neigh_release(n);
2666         kfree(net_work);
2667 }
2668
2669 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp);
2670
2671 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work)
2672 {
2673         struct mlxsw_sp_netevent_work *net_work =
2674                 container_of(work, struct mlxsw_sp_netevent_work, work);
2675         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2676
2677         mlxsw_sp_mp_hash_init(mlxsw_sp);
2678         kfree(net_work);
2679 }
2680
2681 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp);
2682
2683 static void mlxsw_sp_router_update_priority_work(struct work_struct *work)
2684 {
2685         struct mlxsw_sp_netevent_work *net_work =
2686                 container_of(work, struct mlxsw_sp_netevent_work, work);
2687         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2688
2689         __mlxsw_sp_router_init(mlxsw_sp);
2690         kfree(net_work);
2691 }
2692
2693 static int mlxsw_sp_router_schedule_work(struct net *net,
2694                                          struct notifier_block *nb,
2695                                          void (*cb)(struct work_struct *))
2696 {
2697         struct mlxsw_sp_netevent_work *net_work;
2698         struct mlxsw_sp_router *router;
2699
2700         router = container_of(nb, struct mlxsw_sp_router, netevent_nb);
2701         if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp)))
2702                 return NOTIFY_DONE;
2703
2704         net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2705         if (!net_work)
2706                 return NOTIFY_BAD;
2707
2708         INIT_WORK(&net_work->work, cb);
2709         net_work->mlxsw_sp = router->mlxsw_sp;
2710         mlxsw_core_schedule_work(&net_work->work);
2711         return NOTIFY_DONE;
2712 }
2713
2714 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb,
2715                                           unsigned long event, void *ptr)
2716 {
2717         struct mlxsw_sp_netevent_work *net_work;
2718         struct mlxsw_sp_port *mlxsw_sp_port;
2719         struct mlxsw_sp *mlxsw_sp;
2720         unsigned long interval;
2721         struct neigh_parms *p;
2722         struct neighbour *n;
2723
2724         switch (event) {
2725         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
2726                 p = ptr;
2727
2728                 /* We don't care about changes in the default table. */
2729                 if (!p->dev || (p->tbl->family != AF_INET &&
2730                                 p->tbl->family != AF_INET6))
2731                         return NOTIFY_DONE;
2732
2733                 /* We are in atomic context and can't take RTNL mutex,
2734                  * so use RCU variant to walk the device chain.
2735                  */
2736                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
2737                 if (!mlxsw_sp_port)
2738                         return NOTIFY_DONE;
2739
2740                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2741                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
2742                 mlxsw_sp->router->neighs_update.interval = interval;
2743
2744                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2745                 break;
2746         case NETEVENT_NEIGH_UPDATE:
2747                 n = ptr;
2748
2749                 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
2750                         return NOTIFY_DONE;
2751
2752                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
2753                 if (!mlxsw_sp_port)
2754                         return NOTIFY_DONE;
2755
2756                 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2757                 if (!net_work) {
2758                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
2759                         return NOTIFY_BAD;
2760                 }
2761
2762                 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work);
2763                 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2764                 net_work->n = n;
2765
2766                 /* Take a reference to ensure the neighbour won't be
2767                  * destructed until we drop the reference in delayed
2768                  * work.
2769                  */
2770                 neigh_clone(n);
2771                 mlxsw_core_schedule_work(&net_work->work);
2772                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2773                 break;
2774         case NETEVENT_IPV4_MPATH_HASH_UPDATE:
2775         case NETEVENT_IPV6_MPATH_HASH_UPDATE:
2776                 return mlxsw_sp_router_schedule_work(ptr, nb,
2777                                 mlxsw_sp_router_mp_hash_event_work);
2778
2779         case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE:
2780                 return mlxsw_sp_router_schedule_work(ptr, nb,
2781                                 mlxsw_sp_router_update_priority_work);
2782         }
2783
2784         return NOTIFY_DONE;
2785 }
2786
2787 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
2788 {
2789         int err;
2790
2791         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
2792                               &mlxsw_sp_neigh_ht_params);
2793         if (err)
2794                 return err;
2795
2796         /* Initialize the polling interval according to the default
2797          * table.
2798          */
2799         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
2800
2801         /* Create the delayed works for the activity_update */
2802         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
2803                           mlxsw_sp_router_neighs_update_work);
2804         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
2805                           mlxsw_sp_router_probe_unresolved_nexthops);
2806         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
2807         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
2808         return 0;
2809 }
2810
2811 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2812 {
2813         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
2814         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
2815         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2816 }
2817
2818 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2819                                          struct mlxsw_sp_rif *rif)
2820 {
2821         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2822
2823         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2824                                  rif_list_node) {
2825                 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2826                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2827         }
2828 }
2829
2830 enum mlxsw_sp_nexthop_type {
2831         MLXSW_SP_NEXTHOP_TYPE_ETH,
2832         MLXSW_SP_NEXTHOP_TYPE_IPIP,
2833 };
2834
2835 struct mlxsw_sp_nexthop_key {
2836         struct fib_nh *fib_nh;
2837 };
2838
2839 struct mlxsw_sp_nexthop {
2840         struct list_head neigh_list_node; /* member of neigh entry list */
2841         struct list_head rif_list_node;
2842         struct list_head router_list_node;
2843         struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group
2844                                                    * this nexthop belongs to
2845                                                    */
2846         struct rhash_head ht_node;
2847         struct neigh_table *neigh_tbl;
2848         struct mlxsw_sp_nexthop_key key;
2849         unsigned char gw_addr[sizeof(struct in6_addr)];
2850         int ifindex;
2851         int nh_weight;
2852         int norm_nh_weight;
2853         int num_adj_entries;
2854         struct mlxsw_sp_rif *rif;
2855         u8 should_offload:1, /* set indicates this neigh is connected and
2856                               * should be put to KVD linear area of this group.
2857                               */
2858            offloaded:1, /* set in case the neigh is actually put into
2859                          * KVD linear area of this group.
2860                          */
2861            update:1; /* set indicates that MAC of this neigh should be
2862                       * updated in HW
2863                       */
2864         enum mlxsw_sp_nexthop_type type;
2865         union {
2866                 struct mlxsw_sp_neigh_entry *neigh_entry;
2867                 struct mlxsw_sp_ipip_entry *ipip_entry;
2868         };
2869         unsigned int counter_index;
2870         bool counter_valid;
2871 };
2872
2873 enum mlxsw_sp_nexthop_group_type {
2874         MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4,
2875         MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6,
2876 };
2877
2878 struct mlxsw_sp_nexthop_group_info {
2879         struct mlxsw_sp_nexthop_group *nh_grp;
2880         u32 adj_index;
2881         u16 ecmp_size;
2882         u16 count;
2883         int sum_norm_weight;
2884         u8 adj_index_valid:1,
2885            gateway:1; /* routes using the group use a gateway */
2886         struct mlxsw_sp_nexthop nexthops[0];
2887 #define nh_rif  nexthops[0].rif
2888 };
2889
2890 struct mlxsw_sp_nexthop_group {
2891         struct rhash_head ht_node;
2892         struct list_head fib_list; /* list of fib entries that use this group */
2893         union {
2894                 struct {
2895                         struct fib_info *fi;
2896                 } ipv4;
2897         };
2898         struct mlxsw_sp_nexthop_group_info *nhgi;
2899         enum mlxsw_sp_nexthop_group_type type;
2900 };
2901
2902 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2903                                     struct mlxsw_sp_nexthop *nh)
2904 {
2905         struct devlink *devlink;
2906
2907         devlink = priv_to_devlink(mlxsw_sp->core);
2908         if (!devlink_dpipe_table_counter_enabled(devlink,
2909                                                  MLXSW_SP_DPIPE_TABLE_NAME_ADJ))
2910                 return;
2911
2912         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index))
2913                 return;
2914
2915         nh->counter_valid = true;
2916 }
2917
2918 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp,
2919                                    struct mlxsw_sp_nexthop *nh)
2920 {
2921         if (!nh->counter_valid)
2922                 return;
2923         mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index);
2924         nh->counter_valid = false;
2925 }
2926
2927 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp,
2928                                  struct mlxsw_sp_nexthop *nh, u64 *p_counter)
2929 {
2930         if (!nh->counter_valid)
2931                 return -EINVAL;
2932
2933         return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index,
2934                                          p_counter, NULL);
2935 }
2936
2937 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router,
2938                                                struct mlxsw_sp_nexthop *nh)
2939 {
2940         if (!nh) {
2941                 if (list_empty(&router->nexthop_list))
2942                         return NULL;
2943                 else
2944                         return list_first_entry(&router->nexthop_list,
2945                                                 typeof(*nh), router_list_node);
2946         }
2947         if (list_is_last(&nh->router_list_node, &router->nexthop_list))
2948                 return NULL;
2949         return list_next_entry(nh, router_list_node);
2950 }
2951
2952 bool mlxsw_sp_nexthop_offload(struct mlxsw_sp_nexthop *nh)
2953 {
2954         return nh->offloaded;
2955 }
2956
2957 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh)
2958 {
2959         if (!nh->offloaded)
2960                 return NULL;
2961         return nh->neigh_entry->ha;
2962 }
2963
2964 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index,
2965                              u32 *p_adj_size, u32 *p_adj_hash_index)
2966 {
2967         struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
2968         u32 adj_hash_index = 0;
2969         int i;
2970
2971         if (!nh->offloaded || !nhgi->adj_index_valid)
2972                 return -EINVAL;
2973
2974         *p_adj_index = nhgi->adj_index;
2975         *p_adj_size = nhgi->ecmp_size;
2976
2977         for (i = 0; i < nhgi->count; i++) {
2978                 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
2979
2980                 if (nh_iter == nh)
2981                         break;
2982                 if (nh_iter->offloaded)
2983                         adj_hash_index += nh_iter->num_adj_entries;
2984         }
2985
2986         *p_adj_hash_index = adj_hash_index;
2987         return 0;
2988 }
2989
2990 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh)
2991 {
2992         return nh->rif;
2993 }
2994
2995 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh)
2996 {
2997         struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
2998         int i;
2999
3000         for (i = 0; i < nhgi->count; i++) {
3001                 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
3002
3003                 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP)
3004                         return true;
3005         }
3006         return false;
3007 }
3008
3009 struct mlxsw_sp_nexthop_group_cmp_arg {
3010         enum mlxsw_sp_nexthop_group_type type;
3011         union {
3012                 struct fib_info *fi;
3013                 struct mlxsw_sp_fib6_entry *fib6_entry;
3014         };
3015 };
3016
3017 static bool
3018 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
3019                                     const struct in6_addr *gw, int ifindex,
3020                                     int weight)
3021 {
3022         int i;
3023
3024         for (i = 0; i < nh_grp->nhgi->count; i++) {
3025                 const struct mlxsw_sp_nexthop *nh;
3026
3027                 nh = &nh_grp->nhgi->nexthops[i];
3028                 if (nh->ifindex == ifindex && nh->nh_weight == weight &&
3029                     ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
3030                         return true;
3031         }
3032
3033         return false;
3034 }
3035
3036 static bool
3037 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
3038                             const struct mlxsw_sp_fib6_entry *fib6_entry)
3039 {
3040         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3041
3042         if (nh_grp->nhgi->count != fib6_entry->nrt6)
3043                 return false;
3044
3045         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3046                 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3047                 struct in6_addr *gw;
3048                 int ifindex, weight;
3049
3050                 ifindex = fib6_nh->fib_nh_dev->ifindex;
3051                 weight = fib6_nh->fib_nh_weight;
3052                 gw = &fib6_nh->fib_nh_gw6;
3053                 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex,
3054                                                          weight))
3055                         return false;
3056         }
3057
3058         return true;
3059 }
3060
3061 static int
3062 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
3063 {
3064         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
3065         const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
3066
3067         if (nh_grp->type != cmp_arg->type)
3068                 return 1;
3069
3070         switch (cmp_arg->type) {
3071         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3072                 return cmp_arg->fi != nh_grp->ipv4.fi;
3073         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3074                 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
3075                                                     cmp_arg->fib6_entry);
3076         default:
3077                 WARN_ON(1);
3078                 return 1;
3079         }
3080 }
3081
3082 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
3083 {
3084         const struct mlxsw_sp_nexthop_group *nh_grp = data;
3085         const struct mlxsw_sp_nexthop *nh;
3086         struct fib_info *fi;
3087         unsigned int val;
3088         int i;
3089
3090         switch (nh_grp->type) {
3091         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3092                 fi = nh_grp->ipv4.fi;
3093                 return jhash(&fi, sizeof(fi), seed);
3094         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3095                 val = nh_grp->nhgi->count;
3096                 for (i = 0; i < nh_grp->nhgi->count; i++) {
3097                         nh = &nh_grp->nhgi->nexthops[i];
3098                         val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed);
3099                         val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed);
3100                 }
3101                 return jhash(&val, sizeof(val), seed);
3102         default:
3103                 WARN_ON(1);
3104                 return 0;
3105         }
3106 }
3107
3108 static u32
3109 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
3110 {
3111         unsigned int val = fib6_entry->nrt6;
3112         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3113
3114         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3115                 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3116                 struct net_device *dev = fib6_nh->fib_nh_dev;
3117                 struct in6_addr *gw = &fib6_nh->fib_nh_gw6;
3118
3119                 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed);
3120                 val ^= jhash(gw, sizeof(*gw), seed);
3121         }
3122
3123         return jhash(&val, sizeof(val), seed);
3124 }
3125
3126 static u32
3127 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
3128 {
3129         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
3130
3131         switch (cmp_arg->type) {
3132         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3133                 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
3134         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3135                 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
3136         default:
3137                 WARN_ON(1);
3138                 return 0;
3139         }
3140 }
3141
3142 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
3143         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
3144         .hashfn      = mlxsw_sp_nexthop_group_hash,
3145         .obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
3146         .obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
3147 };
3148
3149 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
3150                                          struct mlxsw_sp_nexthop_group *nh_grp)
3151 {
3152         if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3153             !nh_grp->nhgi->gateway)
3154                 return 0;
3155
3156         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
3157                                       &nh_grp->ht_node,
3158                                       mlxsw_sp_nexthop_group_ht_params);
3159 }
3160
3161 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
3162                                           struct mlxsw_sp_nexthop_group *nh_grp)
3163 {
3164         if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3165             !nh_grp->nhgi->gateway)
3166                 return;
3167
3168         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
3169                                &nh_grp->ht_node,
3170                                mlxsw_sp_nexthop_group_ht_params);
3171 }
3172
3173 static struct mlxsw_sp_nexthop_group *
3174 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
3175                                struct fib_info *fi)
3176 {
3177         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3178
3179         cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
3180         cmp_arg.fi = fi;
3181         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3182                                       &cmp_arg,
3183                                       mlxsw_sp_nexthop_group_ht_params);
3184 }
3185
3186 static struct mlxsw_sp_nexthop_group *
3187 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
3188                                struct mlxsw_sp_fib6_entry *fib6_entry)
3189 {
3190         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3191
3192         cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
3193         cmp_arg.fib6_entry = fib6_entry;
3194         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3195                                       &cmp_arg,
3196                                       mlxsw_sp_nexthop_group_ht_params);
3197 }
3198
3199 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
3200         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
3201         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
3202         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
3203 };
3204
3205 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
3206                                    struct mlxsw_sp_nexthop *nh)
3207 {
3208         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
3209                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
3210 }
3211
3212 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
3213                                     struct mlxsw_sp_nexthop *nh)
3214 {
3215         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
3216                                mlxsw_sp_nexthop_ht_params);
3217 }
3218
3219 static struct mlxsw_sp_nexthop *
3220 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
3221                         struct mlxsw_sp_nexthop_key key)
3222 {
3223         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
3224                                       mlxsw_sp_nexthop_ht_params);
3225 }
3226
3227 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
3228                                              const struct mlxsw_sp_fib *fib,
3229                                              u32 adj_index, u16 ecmp_size,
3230                                              u32 new_adj_index,
3231                                              u16 new_ecmp_size)
3232 {
3233         char raleu_pl[MLXSW_REG_RALEU_LEN];
3234
3235         mlxsw_reg_raleu_pack(raleu_pl,
3236                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
3237                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
3238                              new_ecmp_size);
3239         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
3240 }
3241
3242 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
3243                                           struct mlxsw_sp_nexthop_group *nh_grp,
3244                                           u32 old_adj_index, u16 old_ecmp_size)
3245 {
3246         struct mlxsw_sp_fib_entry *fib_entry;
3247         struct mlxsw_sp_fib *fib = NULL;
3248         int err;
3249
3250         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3251                 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
3252
3253                 if (fib == fib_entry->fib_node->fib)
3254                         continue;
3255                 fib = fib_entry->fib_node->fib;
3256                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
3257                                                         old_adj_index,
3258                                                         old_ecmp_size,
3259                                                         nhgi->adj_index,
3260                                                         nhgi->ecmp_size);
3261                 if (err)
3262                         return err;
3263         }
3264         return 0;
3265 }
3266
3267 static int __mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3268                                      struct mlxsw_sp_nexthop *nh)
3269 {
3270         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3271         char ratr_pl[MLXSW_REG_RATR_LEN];
3272
3273         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
3274                             true, MLXSW_REG_RATR_TYPE_ETHERNET,
3275                             adj_index, neigh_entry->rif);
3276         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
3277         if (nh->counter_valid)
3278                 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true);
3279         else
3280                 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false);
3281
3282         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
3283 }
3284
3285 int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3286                             struct mlxsw_sp_nexthop *nh)
3287 {
3288         int i;
3289
3290         for (i = 0; i < nh->num_adj_entries; i++) {
3291                 int err;
3292
3293                 err = __mlxsw_sp_nexthop_update(mlxsw_sp, adj_index + i, nh);
3294                 if (err)
3295                         return err;
3296         }
3297
3298         return 0;
3299 }
3300
3301 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3302                                           u32 adj_index,
3303                                           struct mlxsw_sp_nexthop *nh)
3304 {
3305         const struct mlxsw_sp_ipip_ops *ipip_ops;
3306
3307         ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
3308         return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry);
3309 }
3310
3311 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3312                                         u32 adj_index,
3313                                         struct mlxsw_sp_nexthop *nh)
3314 {
3315         int i;
3316
3317         for (i = 0; i < nh->num_adj_entries; i++) {
3318                 int err;
3319
3320                 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i,
3321                                                      nh);
3322                 if (err)
3323                         return err;
3324         }
3325
3326         return 0;
3327 }
3328
3329 static int
3330 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
3331                               struct mlxsw_sp_nexthop_group_info *nhgi,
3332                               bool reallocate)
3333 {
3334         u32 adj_index = nhgi->adj_index; /* base */
3335         struct mlxsw_sp_nexthop *nh;
3336         int i;
3337
3338         for (i = 0; i < nhgi->count; i++) {
3339                 nh = &nhgi->nexthops[i];
3340
3341                 if (!nh->should_offload) {
3342                         nh->offloaded = 0;
3343                         continue;
3344                 }
3345
3346                 if (nh->update || reallocate) {
3347                         int err = 0;
3348
3349                         switch (nh->type) {
3350                         case MLXSW_SP_NEXTHOP_TYPE_ETH:
3351                                 err = mlxsw_sp_nexthop_update
3352                                             (mlxsw_sp, adj_index, nh);
3353                                 break;
3354                         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
3355                                 err = mlxsw_sp_nexthop_ipip_update
3356                                             (mlxsw_sp, adj_index, nh);
3357                                 break;
3358                         }
3359                         if (err)
3360                                 return err;
3361                         nh->update = 0;
3362                         nh->offloaded = 1;
3363                 }
3364                 adj_index += nh->num_adj_entries;
3365         }
3366         return 0;
3367 }
3368
3369 static int
3370 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
3371                                     struct mlxsw_sp_nexthop_group *nh_grp)
3372 {
3373         struct mlxsw_sp_fib_entry *fib_entry;
3374         int err;
3375
3376         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3377                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3378                 if (err)
3379                         return err;
3380         }
3381         return 0;
3382 }
3383
3384 static void mlxsw_sp_adj_grp_size_round_up(u16 *p_adj_grp_size)
3385 {
3386         /* Valid sizes for an adjacency group are:
3387          * 1-64, 512, 1024, 2048 and 4096.
3388          */
3389         if (*p_adj_grp_size <= 64)
3390                 return;
3391         else if (*p_adj_grp_size <= 512)
3392                 *p_adj_grp_size = 512;
3393         else if (*p_adj_grp_size <= 1024)
3394                 *p_adj_grp_size = 1024;
3395         else if (*p_adj_grp_size <= 2048)
3396                 *p_adj_grp_size = 2048;
3397         else
3398                 *p_adj_grp_size = 4096;
3399 }
3400
3401 static void mlxsw_sp_adj_grp_size_round_down(u16 *p_adj_grp_size,
3402                                              unsigned int alloc_size)
3403 {
3404         if (alloc_size >= 4096)
3405                 *p_adj_grp_size = 4096;
3406         else if (alloc_size >= 2048)
3407                 *p_adj_grp_size = 2048;
3408         else if (alloc_size >= 1024)
3409                 *p_adj_grp_size = 1024;
3410         else if (alloc_size >= 512)
3411                 *p_adj_grp_size = 512;
3412 }
3413
3414 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp,
3415                                      u16 *p_adj_grp_size)
3416 {
3417         unsigned int alloc_size;
3418         int err;
3419
3420         /* Round up the requested group size to the next size supported
3421          * by the device and make sure the request can be satisfied.
3422          */
3423         mlxsw_sp_adj_grp_size_round_up(p_adj_grp_size);
3424         err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
3425                                               MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3426                                               *p_adj_grp_size, &alloc_size);
3427         if (err)
3428                 return err;
3429         /* It is possible the allocation results in more allocated
3430          * entries than requested. Try to use as much of them as
3431          * possible.
3432          */
3433         mlxsw_sp_adj_grp_size_round_down(p_adj_grp_size, alloc_size);
3434
3435         return 0;
3436 }
3437
3438 static void
3439 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi)
3440 {
3441         int i, g = 0, sum_norm_weight = 0;
3442         struct mlxsw_sp_nexthop *nh;
3443
3444         for (i = 0; i < nhgi->count; i++) {
3445                 nh = &nhgi->nexthops[i];
3446
3447                 if (!nh->should_offload)
3448                         continue;
3449                 if (g > 0)
3450                         g = gcd(nh->nh_weight, g);
3451                 else
3452                         g = nh->nh_weight;
3453         }
3454
3455         for (i = 0; i < nhgi->count; i++) {
3456                 nh = &nhgi->nexthops[i];
3457
3458                 if (!nh->should_offload)
3459                         continue;
3460                 nh->norm_nh_weight = nh->nh_weight / g;
3461                 sum_norm_weight += nh->norm_nh_weight;
3462         }
3463
3464         nhgi->sum_norm_weight = sum_norm_weight;
3465 }
3466
3467 static void
3468 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi)
3469 {
3470         int i, weight = 0, lower_bound = 0;
3471         int total = nhgi->sum_norm_weight;
3472         u16 ecmp_size = nhgi->ecmp_size;
3473
3474         for (i = 0; i < nhgi->count; i++) {
3475                 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
3476                 int upper_bound;
3477
3478                 if (!nh->should_offload)
3479                         continue;
3480                 weight += nh->norm_nh_weight;
3481                 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total);
3482                 nh->num_adj_entries = upper_bound - lower_bound;
3483                 lower_bound = upper_bound;
3484         }
3485 }
3486
3487 static struct mlxsw_sp_nexthop *
3488 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
3489                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6);
3490
3491 static void
3492 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3493                                         struct mlxsw_sp_nexthop_group *nh_grp)
3494 {
3495         int i;
3496
3497         for (i = 0; i < nh_grp->nhgi->count; i++) {
3498                 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
3499
3500                 if (nh->offloaded)
3501                         nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3502                 else
3503                         nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3504         }
3505 }
3506
3507 static void
3508 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp,
3509                                           struct mlxsw_sp_fib6_entry *fib6_entry)
3510 {
3511         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3512
3513         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3514                 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3515                 struct mlxsw_sp_nexthop *nh;
3516
3517                 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3518                 if (nh && nh->offloaded)
3519                         fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3520                 else
3521                         fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3522         }
3523 }
3524
3525 static void
3526 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3527                                         struct mlxsw_sp_nexthop_group *nh_grp)
3528 {
3529         struct mlxsw_sp_fib6_entry *fib6_entry;
3530
3531         /* Unfortunately, in IPv6 the route and the nexthop are described by
3532          * the same struct, so we need to iterate over all the routes using the
3533          * nexthop group and set / clear the offload indication for them.
3534          */
3535         list_for_each_entry(fib6_entry, &nh_grp->fib_list,
3536                             common.nexthop_group_node)
3537                 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
3538 }
3539
3540 static void
3541 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3542                                        struct mlxsw_sp_nexthop_group *nh_grp)
3543 {
3544         switch (nh_grp->type) {
3545         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3546                 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp);
3547                 break;
3548         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3549                 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp);
3550                 break;
3551         }
3552 }
3553
3554 static void
3555 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
3556                                struct mlxsw_sp_nexthop_group *nh_grp)
3557 {
3558         struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
3559         u16 ecmp_size, old_ecmp_size;
3560         struct mlxsw_sp_nexthop *nh;
3561         bool offload_change = false;
3562         u32 adj_index;
3563         bool old_adj_index_valid;
3564         u32 old_adj_index;
3565         int i;
3566         int err;
3567
3568         if (!nhgi->gateway) {
3569                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3570                 return;
3571         }
3572
3573         for (i = 0; i < nhgi->count; i++) {
3574                 nh = &nhgi->nexthops[i];
3575
3576                 if (nh->should_offload != nh->offloaded) {
3577                         offload_change = true;
3578                         if (nh->should_offload)
3579                                 nh->update = 1;
3580                 }
3581         }
3582         if (!offload_change) {
3583                 /* Nothing was added or removed, so no need to reallocate. Just
3584                  * update MAC on existing adjacency indexes.
3585                  */
3586                 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false);
3587                 if (err) {
3588                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3589                         goto set_trap;
3590                 }
3591                 return;
3592         }
3593         mlxsw_sp_nexthop_group_normalize(nhgi);
3594         if (!nhgi->sum_norm_weight)
3595                 /* No neigh of this group is connected so we just set
3596                  * the trap and let everthing flow through kernel.
3597                  */
3598                 goto set_trap;
3599
3600         ecmp_size = nhgi->sum_norm_weight;
3601         err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size);
3602         if (err)
3603                 /* No valid allocation size available. */
3604                 goto set_trap;
3605
3606         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3607                                   ecmp_size, &adj_index);
3608         if (err) {
3609                 /* We ran out of KVD linear space, just set the
3610                  * trap and let everything flow through kernel.
3611                  */
3612                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
3613                 goto set_trap;
3614         }
3615         old_adj_index_valid = nhgi->adj_index_valid;
3616         old_adj_index = nhgi->adj_index;
3617         old_ecmp_size = nhgi->ecmp_size;
3618         nhgi->adj_index_valid = 1;
3619         nhgi->adj_index = adj_index;
3620         nhgi->ecmp_size = ecmp_size;
3621         mlxsw_sp_nexthop_group_rebalance(nhgi);
3622         err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true);
3623         if (err) {
3624                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3625                 goto set_trap;
3626         }
3627
3628         mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3629
3630         if (!old_adj_index_valid) {
3631                 /* The trap was set for fib entries, so we have to call
3632                  * fib entry update to unset it and use adjacency index.
3633                  */
3634                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3635                 if (err) {
3636                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
3637                         goto set_trap;
3638                 }
3639                 return;
3640         }
3641
3642         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
3643                                              old_adj_index, old_ecmp_size);
3644         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3645                            old_ecmp_size, old_adj_index);
3646         if (err) {
3647                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
3648                 goto set_trap;
3649         }
3650
3651         return;
3652
3653 set_trap:
3654         old_adj_index_valid = nhgi->adj_index_valid;
3655         nhgi->adj_index_valid = 0;
3656         for (i = 0; i < nhgi->count; i++) {
3657                 nh = &nhgi->nexthops[i];
3658                 nh->offloaded = 0;
3659         }
3660         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3661         if (err)
3662                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
3663         mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3664         if (old_adj_index_valid)
3665                 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3666                                    nhgi->ecmp_size, nhgi->adj_index);
3667 }
3668
3669 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
3670                                             bool removing)
3671 {
3672         if (!removing)
3673                 nh->should_offload = 1;
3674         else
3675                 nh->should_offload = 0;
3676         nh->update = 1;
3677 }
3678
3679 static int
3680 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp,
3681                                     struct mlxsw_sp_neigh_entry *neigh_entry)
3682 {
3683         struct neighbour *n, *old_n = neigh_entry->key.n;
3684         struct mlxsw_sp_nexthop *nh;
3685         bool entry_connected;
3686         u8 nud_state, dead;
3687         int err;
3688
3689         nh = list_first_entry(&neigh_entry->nexthop_list,
3690                               struct mlxsw_sp_nexthop, neigh_list_node);
3691
3692         n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3693         if (!n) {
3694                 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3695                 if (IS_ERR(n))
3696                         return PTR_ERR(n);
3697                 neigh_event_send(n, NULL);
3698         }
3699
3700         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
3701         neigh_entry->key.n = n;
3702         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
3703         if (err)
3704                 goto err_neigh_entry_insert;
3705
3706         read_lock_bh(&n->lock);
3707         nud_state = n->nud_state;
3708         dead = n->dead;
3709         read_unlock_bh(&n->lock);
3710         entry_connected = nud_state & NUD_VALID && !dead;
3711
3712         list_for_each_entry(nh, &neigh_entry->nexthop_list,
3713                             neigh_list_node) {
3714                 neigh_release(old_n);
3715                 neigh_clone(n);
3716                 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected);
3717                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
3718         }
3719
3720         neigh_release(n);
3721
3722         return 0;
3723
3724 err_neigh_entry_insert:
3725         neigh_entry->key.n = old_n;
3726         mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
3727         neigh_release(n);
3728         return err;
3729 }
3730
3731 static void
3732 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
3733                               struct mlxsw_sp_neigh_entry *neigh_entry,
3734                               bool removing, bool dead)
3735 {
3736         struct mlxsw_sp_nexthop *nh;
3737
3738         if (list_empty(&neigh_entry->nexthop_list))
3739                 return;
3740
3741         if (dead) {
3742                 int err;
3743
3744                 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp,
3745                                                           neigh_entry);
3746                 if (err)
3747                         dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n");
3748                 return;
3749         }
3750
3751         list_for_each_entry(nh, &neigh_entry->nexthop_list,
3752                             neigh_list_node) {
3753                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
3754                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
3755         }
3756 }
3757
3758 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
3759                                       struct mlxsw_sp_rif *rif)
3760 {
3761         if (nh->rif)
3762                 return;
3763
3764         nh->rif = rif;
3765         list_add(&nh->rif_list_node, &rif->nexthop_list);
3766 }
3767
3768 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
3769 {
3770         if (!nh->rif)
3771                 return;
3772
3773         list_del(&nh->rif_list_node);
3774         nh->rif = NULL;
3775 }
3776
3777 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
3778                                        struct mlxsw_sp_nexthop *nh)
3779 {
3780         struct mlxsw_sp_neigh_entry *neigh_entry;
3781         struct neighbour *n;
3782         u8 nud_state, dead;
3783         int err;
3784
3785         if (!nh->nhgi->gateway || nh->neigh_entry)
3786                 return 0;
3787
3788         /* Take a reference of neigh here ensuring that neigh would
3789          * not be destructed before the nexthop entry is finished.
3790          * The reference is taken either in neigh_lookup() or
3791          * in neigh_create() in case n is not found.
3792          */
3793         n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3794         if (!n) {
3795                 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3796                 if (IS_ERR(n))
3797                         return PTR_ERR(n);
3798                 neigh_event_send(n, NULL);
3799         }
3800         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
3801         if (!neigh_entry) {
3802                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
3803                 if (IS_ERR(neigh_entry)) {
3804                         err = -EINVAL;
3805                         goto err_neigh_entry_create;
3806                 }
3807         }
3808
3809         /* If that is the first nexthop connected to that neigh, add to
3810          * nexthop_neighs_list
3811          */
3812         if (list_empty(&neigh_entry->nexthop_list))
3813                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
3814                               &mlxsw_sp->router->nexthop_neighs_list);
3815
3816         nh->neigh_entry = neigh_entry;
3817         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
3818         read_lock_bh(&n->lock);
3819         nud_state = n->nud_state;
3820         dead = n->dead;
3821         read_unlock_bh(&n->lock);
3822         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
3823
3824         return 0;
3825
3826 err_neigh_entry_create:
3827         neigh_release(n);
3828         return err;
3829 }
3830
3831 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
3832                                         struct mlxsw_sp_nexthop *nh)
3833 {
3834         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3835         struct neighbour *n;
3836
3837         if (!neigh_entry)
3838                 return;
3839         n = neigh_entry->key.n;
3840
3841         __mlxsw_sp_nexthop_neigh_update(nh, true);
3842         list_del(&nh->neigh_list_node);
3843         nh->neigh_entry = NULL;
3844
3845         /* If that is the last nexthop connected to that neigh, remove from
3846          * nexthop_neighs_list
3847          */
3848         if (list_empty(&neigh_entry->nexthop_list))
3849                 list_del(&neigh_entry->nexthop_neighs_list_node);
3850
3851         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
3852                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
3853
3854         neigh_release(n);
3855 }
3856
3857 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev)
3858 {
3859         struct net_device *ul_dev;
3860         bool is_up;
3861
3862         rcu_read_lock();
3863         ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
3864         is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true;
3865         rcu_read_unlock();
3866
3867         return is_up;
3868 }
3869
3870 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
3871                                        struct mlxsw_sp_nexthop *nh,
3872                                        struct mlxsw_sp_ipip_entry *ipip_entry)
3873 {
3874         bool removing;
3875
3876         if (!nh->nhgi->gateway || nh->ipip_entry)
3877                 return;
3878
3879         nh->ipip_entry = ipip_entry;
3880         removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev);
3881         __mlxsw_sp_nexthop_neigh_update(nh, removing);
3882         mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common);
3883 }
3884
3885 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
3886                                        struct mlxsw_sp_nexthop *nh)
3887 {
3888         struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
3889
3890         if (!ipip_entry)
3891                 return;
3892
3893         __mlxsw_sp_nexthop_neigh_update(nh, true);
3894         nh->ipip_entry = NULL;
3895 }
3896
3897 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
3898                                         const struct fib_nh *fib_nh,
3899                                         enum mlxsw_sp_ipip_type *p_ipipt)
3900 {
3901         struct net_device *dev = fib_nh->fib_nh_dev;
3902
3903         return dev &&
3904                fib_nh->nh_parent->fib_type == RTN_UNICAST &&
3905                mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
3906 }
3907
3908 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp,
3909                                       struct mlxsw_sp_nexthop *nh,
3910                                       const struct net_device *dev)
3911 {
3912         const struct mlxsw_sp_ipip_ops *ipip_ops;
3913         struct mlxsw_sp_ipip_entry *ipip_entry;
3914         struct mlxsw_sp_rif *rif;
3915         int err;
3916
3917         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev);
3918         if (ipip_entry) {
3919                 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
3920                 if (ipip_ops->can_offload(mlxsw_sp, dev)) {
3921                         nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
3922                         mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
3923                         return 0;
3924                 }
3925         }
3926
3927         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
3928         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3929         if (!rif)
3930                 return 0;
3931
3932         mlxsw_sp_nexthop_rif_init(nh, rif);
3933         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
3934         if (err)
3935                 goto err_neigh_init;
3936
3937         return 0;
3938
3939 err_neigh_init:
3940         mlxsw_sp_nexthop_rif_fini(nh);
3941         return err;
3942 }
3943
3944 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
3945                                        struct mlxsw_sp_nexthop *nh)
3946 {
3947         switch (nh->type) {
3948         case MLXSW_SP_NEXTHOP_TYPE_ETH:
3949                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
3950                 mlxsw_sp_nexthop_rif_fini(nh);
3951                 break;
3952         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
3953                 mlxsw_sp_nexthop_rif_fini(nh);
3954                 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
3955                 break;
3956         }
3957 }
3958
3959 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
3960                                   struct mlxsw_sp_nexthop_group *nh_grp,
3961                                   struct mlxsw_sp_nexthop *nh,
3962                                   struct fib_nh *fib_nh)
3963 {
3964         struct net_device *dev = fib_nh->fib_nh_dev;
3965         struct in_device *in_dev;
3966         int err;
3967
3968         nh->nhgi = nh_grp->nhgi;
3969         nh->key.fib_nh = fib_nh;
3970 #ifdef CONFIG_IP_ROUTE_MULTIPATH
3971         nh->nh_weight = fib_nh->fib_nh_weight;
3972 #else
3973         nh->nh_weight = 1;
3974 #endif
3975         memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4));
3976         nh->neigh_tbl = &arp_tbl;
3977         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
3978         if (err)
3979                 return err;
3980
3981         mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
3982         list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
3983
3984         if (!dev)
3985                 return 0;
3986         nh->ifindex = dev->ifindex;
3987
3988         rcu_read_lock();
3989         in_dev = __in_dev_get_rcu(dev);
3990         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
3991             fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) {
3992                 rcu_read_unlock();
3993                 return 0;
3994         }
3995         rcu_read_unlock();
3996
3997         err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
3998         if (err)
3999                 goto err_nexthop_neigh_init;
4000
4001         return 0;
4002
4003 err_nexthop_neigh_init:
4004         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4005         return err;
4006 }
4007
4008 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
4009                                    struct mlxsw_sp_nexthop *nh)
4010 {
4011         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4012         list_del(&nh->router_list_node);
4013         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4014         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4015 }
4016
4017 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
4018                                     unsigned long event, struct fib_nh *fib_nh)
4019 {
4020         struct mlxsw_sp_nexthop_key key;
4021         struct mlxsw_sp_nexthop *nh;
4022
4023         if (mlxsw_sp->router->aborted)
4024                 return;
4025
4026         key.fib_nh = fib_nh;
4027         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
4028         if (!nh)
4029                 return;
4030
4031         switch (event) {
4032         case FIB_EVENT_NH_ADD:
4033                 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev);
4034                 break;
4035         case FIB_EVENT_NH_DEL:
4036                 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4037                 break;
4038         }
4039
4040         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4041 }
4042
4043 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
4044                                         struct mlxsw_sp_rif *rif)
4045 {
4046         struct mlxsw_sp_nexthop *nh;
4047         bool removing;
4048
4049         list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) {
4050                 switch (nh->type) {
4051                 case MLXSW_SP_NEXTHOP_TYPE_ETH:
4052                         removing = false;
4053                         break;
4054                 case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4055                         removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev);
4056                         break;
4057                 default:
4058                         WARN_ON(1);
4059                         continue;
4060                 }
4061
4062                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
4063                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4064         }
4065 }
4066
4067 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
4068                                          struct mlxsw_sp_rif *old_rif,
4069                                          struct mlxsw_sp_rif *new_rif)
4070 {
4071         struct mlxsw_sp_nexthop *nh;
4072
4073         list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list);
4074         list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node)
4075                 nh->rif = new_rif;
4076         mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif);
4077 }
4078
4079 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4080                                            struct mlxsw_sp_rif *rif)
4081 {
4082         struct mlxsw_sp_nexthop *nh, *tmp;
4083
4084         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
4085                 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4086                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4087         }
4088 }
4089
4090 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
4091                                    struct fib_info *fi)
4092 {
4093         const struct fib_nh *nh = fib_info_nh(fi, 0);
4094
4095         return nh->fib_nh_scope == RT_SCOPE_LINK ||
4096                mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL);
4097 }
4098
4099 static int
4100 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp,
4101                                   struct mlxsw_sp_nexthop_group *nh_grp)
4102 {
4103         unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi);
4104         struct mlxsw_sp_nexthop_group_info *nhgi;
4105         struct mlxsw_sp_nexthop *nh;
4106         int err, i;
4107
4108         nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL);
4109         if (!nhgi)
4110                 return -ENOMEM;
4111         nh_grp->nhgi = nhgi;
4112         nhgi->nh_grp = nh_grp;
4113         nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi);
4114         nhgi->count = nhs;
4115         for (i = 0; i < nhgi->count; i++) {
4116                 struct fib_nh *fib_nh;
4117
4118                 nh = &nhgi->nexthops[i];
4119                 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i);
4120                 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
4121                 if (err)
4122                         goto err_nexthop4_init;
4123         }
4124         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4125
4126         return 0;
4127
4128 err_nexthop4_init:
4129         for (i--; i >= 0; i--) {
4130                 nh = &nhgi->nexthops[i];
4131                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
4132         }
4133         kfree(nhgi);
4134         return err;
4135 }
4136
4137 static void
4138 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp,
4139                                   struct mlxsw_sp_nexthop_group *nh_grp)
4140 {
4141         struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
4142         int i;
4143
4144         for (i = nhgi->count - 1; i >= 0; i--) {
4145                 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
4146
4147                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
4148         }
4149         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4150         WARN_ON_ONCE(nhgi->adj_index_valid);
4151         kfree(nhgi);
4152 }
4153
4154 static struct mlxsw_sp_nexthop_group *
4155 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
4156 {
4157         struct mlxsw_sp_nexthop_group *nh_grp;
4158         int err;
4159
4160         nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
4161         if (!nh_grp)
4162                 return ERR_PTR(-ENOMEM);
4163         INIT_LIST_HEAD(&nh_grp->fib_list);
4164         nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
4165         nh_grp->ipv4.fi = fi;
4166         fib_info_hold(fi);
4167
4168         err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp);
4169         if (err)
4170                 goto err_nexthop_group_info_init;
4171
4172         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
4173         if (err)
4174                 goto err_nexthop_group_insert;
4175
4176         return nh_grp;
4177
4178 err_nexthop_group_insert:
4179         mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
4180 err_nexthop_group_info_init:
4181         fib_info_put(fi);
4182         kfree(nh_grp);
4183         return ERR_PTR(err);
4184 }
4185
4186 static void
4187 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
4188                                 struct mlxsw_sp_nexthop_group *nh_grp)
4189 {
4190         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
4191         mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
4192         fib_info_put(nh_grp->ipv4.fi);
4193         kfree(nh_grp);
4194 }
4195
4196 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
4197                                        struct mlxsw_sp_fib_entry *fib_entry,
4198                                        struct fib_info *fi)
4199 {
4200         struct mlxsw_sp_nexthop_group *nh_grp;
4201
4202         nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
4203         if (!nh_grp) {
4204                 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
4205                 if (IS_ERR(nh_grp))
4206                         return PTR_ERR(nh_grp);
4207         }
4208         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
4209         fib_entry->nh_group = nh_grp;
4210         return 0;
4211 }
4212
4213 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
4214                                         struct mlxsw_sp_fib_entry *fib_entry)
4215 {
4216         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
4217
4218         list_del(&fib_entry->nexthop_group_node);
4219         if (!list_empty(&nh_grp->fib_list))
4220                 return;
4221         mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
4222 }
4223
4224 static bool
4225 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
4226 {
4227         struct mlxsw_sp_fib4_entry *fib4_entry;
4228
4229         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
4230                                   common);
4231         return !fib4_entry->tos;
4232 }
4233
4234 static bool
4235 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
4236 {
4237         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
4238
4239         switch (fib_entry->fib_node->fib->proto) {
4240         case MLXSW_SP_L3_PROTO_IPV4:
4241                 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
4242                         return false;
4243                 break;
4244         case MLXSW_SP_L3_PROTO_IPV6:
4245                 break;
4246         }
4247
4248         switch (fib_entry->type) {
4249         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
4250                 return !!nh_group->nhgi->adj_index_valid;
4251         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
4252                 return !!nh_group->nhgi->nh_rif;
4253         case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
4254         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
4255         case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
4256                 return true;
4257         default:
4258                 return false;
4259         }
4260 }
4261
4262 static struct mlxsw_sp_nexthop *
4263 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
4264                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
4265 {
4266         int i;
4267
4268         for (i = 0; i < nh_grp->nhgi->count; i++) {
4269                 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
4270                 struct fib6_info *rt = mlxsw_sp_rt6->rt;
4271
4272                 if (nh->rif && nh->rif->dev == rt->fib6_nh->fib_nh_dev &&
4273                     ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
4274                                     &rt->fib6_nh->fib_nh_gw6))
4275                         return nh;
4276                 continue;
4277         }
4278
4279         return NULL;
4280 }
4281
4282 static void
4283 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
4284                                  struct mlxsw_sp_fib_entry *fib_entry)
4285 {
4286         u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
4287         int dst_len = fib_entry->fib_node->key.prefix_len;
4288         struct mlxsw_sp_fib4_entry *fib4_entry;
4289         struct fib_rt_info fri;
4290         bool should_offload;
4291
4292         should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
4293         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
4294                                   common);
4295         fri.fi = fib4_entry->fi;
4296         fri.tb_id = fib4_entry->tb_id;
4297         fri.dst = cpu_to_be32(*p_dst);
4298         fri.dst_len = dst_len;
4299         fri.tos = fib4_entry->tos;
4300         fri.type = fib4_entry->type;
4301         fri.offload = should_offload;
4302         fri.trap = !should_offload;
4303         fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
4304 }
4305
4306 static void
4307 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
4308                                    struct mlxsw_sp_fib_entry *fib_entry)
4309 {
4310         u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
4311         int dst_len = fib_entry->fib_node->key.prefix_len;
4312         struct mlxsw_sp_fib4_entry *fib4_entry;
4313         struct fib_rt_info fri;
4314
4315         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
4316                                   common);
4317         fri.fi = fib4_entry->fi;
4318         fri.tb_id = fib4_entry->tb_id;
4319         fri.dst = cpu_to_be32(*p_dst);
4320         fri.dst_len = dst_len;
4321         fri.tos = fib4_entry->tos;
4322         fri.type = fib4_entry->type;
4323         fri.offload = false;
4324         fri.trap = false;
4325         fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
4326 }
4327
4328 static void
4329 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
4330                                  struct mlxsw_sp_fib_entry *fib_entry)
4331 {
4332         struct mlxsw_sp_fib6_entry *fib6_entry;
4333         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4334         bool should_offload;
4335
4336         should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
4337
4338         /* In IPv6 a multipath route is represented using multiple routes, so
4339          * we need to set the flags on all of them.
4340          */
4341         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
4342                                   common);
4343         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
4344                 fib6_info_hw_flags_set(mlxsw_sp_rt6->rt, should_offload,
4345                                        !should_offload);
4346 }
4347
4348 static void
4349 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
4350                                    struct mlxsw_sp_fib_entry *fib_entry)
4351 {
4352         struct mlxsw_sp_fib6_entry *fib6_entry;
4353         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4354
4355         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
4356                                   common);
4357         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
4358                 fib6_info_hw_flags_set(mlxsw_sp_rt6->rt, false, false);
4359 }
4360
4361 static void
4362 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
4363                                 struct mlxsw_sp_fib_entry *fib_entry)
4364 {
4365         switch (fib_entry->fib_node->fib->proto) {
4366         case MLXSW_SP_L3_PROTO_IPV4:
4367                 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry);
4368                 break;
4369         case MLXSW_SP_L3_PROTO_IPV6:
4370                 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry);
4371                 break;
4372         }
4373 }
4374
4375 static void
4376 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
4377                                   struct mlxsw_sp_fib_entry *fib_entry)
4378 {
4379         switch (fib_entry->fib_node->fib->proto) {
4380         case MLXSW_SP_L3_PROTO_IPV4:
4381                 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry);
4382                 break;
4383         case MLXSW_SP_L3_PROTO_IPV6:
4384                 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry);
4385                 break;
4386         }
4387 }
4388
4389 static void
4390 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp,
4391                                     struct mlxsw_sp_fib_entry *fib_entry,
4392                                     enum mlxsw_sp_fib_entry_op op)
4393 {
4394         switch (op) {
4395         case MLXSW_SP_FIB_ENTRY_OP_WRITE:
4396         case MLXSW_SP_FIB_ENTRY_OP_UPDATE:
4397                 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry);
4398                 break;
4399         case MLXSW_SP_FIB_ENTRY_OP_DELETE:
4400                 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry);
4401                 break;
4402         default:
4403                 break;
4404         }
4405 }
4406
4407 struct mlxsw_sp_fib_entry_op_ctx_basic {
4408         char ralue_pl[MLXSW_REG_RALUE_LEN];
4409 };
4410
4411 static void
4412 mlxsw_sp_router_ll_basic_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4413                                         enum mlxsw_sp_l3proto proto,
4414                                         enum mlxsw_sp_fib_entry_op op,
4415                                         u16 virtual_router, u8 prefix_len,
4416                                         unsigned char *addr,
4417                                         struct mlxsw_sp_fib_entry_priv *priv)
4418 {
4419         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4420         enum mlxsw_reg_ralxx_protocol ralxx_proto;
4421         char *ralue_pl = op_ctx_basic->ralue_pl;
4422         enum mlxsw_reg_ralue_op ralue_op;
4423
4424         ralxx_proto = (enum mlxsw_reg_ralxx_protocol) proto;
4425
4426         switch (op) {
4427         case MLXSW_SP_FIB_ENTRY_OP_WRITE:
4428         case MLXSW_SP_FIB_ENTRY_OP_UPDATE:
4429                 ralue_op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
4430                 break;
4431         case MLXSW_SP_FIB_ENTRY_OP_DELETE:
4432                 ralue_op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
4433                 break;
4434         default:
4435                 WARN_ON_ONCE(1);
4436                 return;
4437         }
4438
4439         switch (proto) {
4440         case MLXSW_SP_L3_PROTO_IPV4:
4441                 mlxsw_reg_ralue_pack4(ralue_pl, ralxx_proto, ralue_op,
4442                                       virtual_router, prefix_len, (u32 *) addr);
4443                 break;
4444         case MLXSW_SP_L3_PROTO_IPV6:
4445                 mlxsw_reg_ralue_pack6(ralue_pl, ralxx_proto, ralue_op,
4446                                       virtual_router, prefix_len, addr);
4447                 break;
4448         }
4449 }
4450
4451 static void
4452 mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4453                                                    enum mlxsw_reg_ralue_trap_action trap_action,
4454                                                    u16 trap_id, u32 adjacency_index, u16 ecmp_size)
4455 {
4456         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4457
4458         mlxsw_reg_ralue_act_remote_pack(op_ctx_basic->ralue_pl, trap_action,
4459                                         trap_id, adjacency_index, ecmp_size);
4460 }
4461
4462 static void
4463 mlxsw_sp_router_ll_basic_fib_entry_act_local_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4464                                                   enum mlxsw_reg_ralue_trap_action trap_action,
4465                                                   u16 trap_id, u16 local_erif)
4466 {
4467         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4468
4469         mlxsw_reg_ralue_act_local_pack(op_ctx_basic->ralue_pl, trap_action,
4470                                        trap_id, local_erif);
4471 }
4472
4473 static void
4474 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx)
4475 {
4476         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4477
4478         mlxsw_reg_ralue_act_ip2me_pack(op_ctx_basic->ralue_pl);
4479 }
4480
4481 static void
4482 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4483                                                       u32 tunnel_ptr)
4484 {
4485         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4486
4487         mlxsw_reg_ralue_act_ip2me_tun_pack(op_ctx_basic->ralue_pl, tunnel_ptr);
4488 }
4489
4490 static int
4491 mlxsw_sp_router_ll_basic_fib_entry_commit(struct mlxsw_sp *mlxsw_sp,
4492                                           struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4493                                           bool *postponed_for_bulk)
4494 {
4495         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4496
4497         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
4498                                op_ctx_basic->ralue_pl);
4499 }
4500
4501 static bool
4502 mlxsw_sp_router_ll_basic_fib_entry_is_committed(struct mlxsw_sp_fib_entry_priv *priv)
4503 {
4504         return true;
4505 }
4506
4507 static void mlxsw_sp_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4508                                     struct mlxsw_sp_fib_entry *fib_entry,
4509                                     enum mlxsw_sp_fib_entry_op op)
4510 {
4511         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
4512
4513         mlxsw_sp_fib_entry_op_ctx_priv_hold(op_ctx, fib_entry->priv);
4514         fib->ll_ops->fib_entry_pack(op_ctx, fib->proto, op, fib->vr->id,
4515                                     fib_entry->fib_node->key.prefix_len,
4516                                     fib_entry->fib_node->key.addr,
4517                                     fib_entry->priv);
4518 }
4519
4520 int mlxsw_sp_fib_entry_commit(struct mlxsw_sp *mlxsw_sp,
4521                               struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4522                               const struct mlxsw_sp_router_ll_ops *ll_ops)
4523 {
4524         bool postponed_for_bulk = false;
4525         int err;
4526
4527         err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, &postponed_for_bulk);
4528         if (!postponed_for_bulk)
4529                 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
4530         return err;
4531 }
4532
4533 static int mlxsw_sp_adj_discard_write(struct mlxsw_sp *mlxsw_sp, u16 rif_index)
4534 {
4535         enum mlxsw_reg_ratr_trap_action trap_action;
4536         char ratr_pl[MLXSW_REG_RATR_LEN];
4537         int err;
4538
4539         if (mlxsw_sp->router->adj_discard_index_valid)
4540                 return 0;
4541
4542         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4543                                   &mlxsw_sp->router->adj_discard_index);
4544         if (err)
4545                 return err;
4546
4547         trap_action = MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS;
4548         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true,
4549                             MLXSW_REG_RATR_TYPE_ETHERNET,
4550                             mlxsw_sp->router->adj_discard_index, rif_index);
4551         mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action);
4552         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
4553         if (err)
4554                 goto err_ratr_write;
4555
4556         mlxsw_sp->router->adj_discard_index_valid = true;
4557
4558         return 0;
4559
4560 err_ratr_write:
4561         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4562                            mlxsw_sp->router->adj_discard_index);
4563         return err;
4564 }
4565
4566 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
4567                                         struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4568                                         struct mlxsw_sp_fib_entry *fib_entry,
4569                                         enum mlxsw_sp_fib_entry_op op)
4570 {
4571         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4572         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
4573         struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi;
4574         enum mlxsw_reg_ralue_trap_action trap_action;
4575         u16 trap_id = 0;
4576         u32 adjacency_index = 0;
4577         u16 ecmp_size = 0;
4578         int err;
4579
4580         /* In case the nexthop group adjacency index is valid, use it
4581          * with provided ECMP size. Otherwise, setup trap and pass
4582          * traffic to kernel.
4583          */
4584         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
4585                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4586                 adjacency_index = nhgi->adj_index;
4587                 ecmp_size = nhgi->ecmp_size;
4588         } else if (!nhgi->adj_index_valid && nhgi->count && nhgi->nh_rif) {
4589                 err = mlxsw_sp_adj_discard_write(mlxsw_sp,
4590                                                  nhgi->nh_rif->rif_index);
4591                 if (err)
4592                         return err;
4593                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4594                 adjacency_index = mlxsw_sp->router->adj_discard_index;
4595                 ecmp_size = 1;
4596         } else {
4597                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4598                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
4599         }
4600
4601         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4602         ll_ops->fib_entry_act_remote_pack(op_ctx, trap_action, trap_id,
4603                                           adjacency_index, ecmp_size);
4604         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4605 }
4606
4607 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
4608                                        struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4609                                        struct mlxsw_sp_fib_entry *fib_entry,
4610                                        enum mlxsw_sp_fib_entry_op op)
4611 {
4612         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4613         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nhgi->nh_rif;
4614         enum mlxsw_reg_ralue_trap_action trap_action;
4615         u16 trap_id = 0;
4616         u16 rif_index = 0;
4617
4618         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
4619                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4620                 rif_index = rif->rif_index;
4621         } else {
4622                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4623                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
4624         }
4625
4626         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4627         ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, rif_index);
4628         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4629 }
4630
4631 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
4632                                       struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4633                                       struct mlxsw_sp_fib_entry *fib_entry,
4634                                       enum mlxsw_sp_fib_entry_op op)
4635 {
4636         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4637
4638         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4639         ll_ops->fib_entry_act_ip2me_pack(op_ctx);
4640         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4641 }
4642
4643 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp,
4644                                            struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4645                                            struct mlxsw_sp_fib_entry *fib_entry,
4646                                            enum mlxsw_sp_fib_entry_op op)
4647 {
4648         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4649         enum mlxsw_reg_ralue_trap_action trap_action;
4650
4651         trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR;
4652         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4653         ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, 0, 0);
4654         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4655 }
4656
4657 static int
4658 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp,
4659                                   struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4660                                   struct mlxsw_sp_fib_entry *fib_entry,
4661                                   enum mlxsw_sp_fib_entry_op op)
4662 {
4663         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4664         enum mlxsw_reg_ralue_trap_action trap_action;
4665         u16 trap_id;
4666
4667         trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4668         trap_id = MLXSW_TRAP_ID_RTR_INGRESS1;
4669
4670         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4671         ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, 0);
4672         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4673 }
4674
4675 static int
4676 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
4677                                  struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4678                                  struct mlxsw_sp_fib_entry *fib_entry,
4679                                  enum mlxsw_sp_fib_entry_op op)
4680 {
4681         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4682         struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
4683         const struct mlxsw_sp_ipip_ops *ipip_ops;
4684
4685         if (WARN_ON(!ipip_entry))
4686                 return -EINVAL;
4687
4688         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
4689         return ipip_ops->fib_entry_op(mlxsw_sp, ll_ops, op_ctx, ipip_entry, op,
4690                                       fib_entry->decap.tunnel_index, fib_entry->priv);
4691 }
4692
4693 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp,
4694                                            struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4695                                            struct mlxsw_sp_fib_entry *fib_entry,
4696                                            enum mlxsw_sp_fib_entry_op op)
4697 {
4698         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4699
4700         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4701         ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx,
4702                                              fib_entry->decap.tunnel_index);
4703         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4704 }
4705
4706 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
4707                                    struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4708                                    struct mlxsw_sp_fib_entry *fib_entry,
4709                                    enum mlxsw_sp_fib_entry_op op)
4710 {
4711         switch (fib_entry->type) {
4712         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
4713                 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, op_ctx, fib_entry, op);
4714         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
4715                 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, op_ctx, fib_entry, op);
4716         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
4717                 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, op_ctx, fib_entry, op);
4718         case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
4719                 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, op_ctx, fib_entry, op);
4720         case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE:
4721                 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, op_ctx, fib_entry, op);
4722         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
4723                 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, op_ctx, fib_entry, op);
4724         case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
4725                 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, op_ctx, fib_entry, op);
4726         }
4727         return -EINVAL;
4728 }
4729
4730 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
4731                                  struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4732                                  struct mlxsw_sp_fib_entry *fib_entry,
4733                                  enum mlxsw_sp_fib_entry_op op)
4734 {
4735         int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, op);
4736
4737         if (err)
4738                 return err;
4739
4740         mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op);
4741
4742         return err;
4743 }
4744
4745 static int __mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
4746                                        struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4747                                        struct mlxsw_sp_fib_entry *fib_entry,
4748                                        bool is_new)
4749 {
4750         return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry,
4751                                      is_new ? MLXSW_SP_FIB_ENTRY_OP_WRITE :
4752                                               MLXSW_SP_FIB_ENTRY_OP_UPDATE);
4753 }
4754
4755 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
4756                                      struct mlxsw_sp_fib_entry *fib_entry)
4757 {
4758         struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx;
4759
4760         mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
4761         return __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, false);
4762 }
4763
4764 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
4765                                   struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4766                                   struct mlxsw_sp_fib_entry *fib_entry)
4767 {
4768         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4769
4770         if (!ll_ops->fib_entry_is_committed(fib_entry->priv))
4771                 return 0;
4772         return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry,
4773                                      MLXSW_SP_FIB_ENTRY_OP_DELETE);
4774 }
4775
4776 static int
4777 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
4778                              const struct fib_entry_notifier_info *fen_info,
4779                              struct mlxsw_sp_fib_entry *fib_entry)
4780 {
4781         struct net_device *dev = fib_info_nh(fen_info->fi, 0)->fib_nh_dev;
4782         union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
4783         struct mlxsw_sp_router *router = mlxsw_sp->router;
4784         u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id);
4785         struct mlxsw_sp_ipip_entry *ipip_entry;
4786         struct fib_info *fi = fen_info->fi;
4787
4788         switch (fen_info->type) {
4789         case RTN_LOCAL:
4790                 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, dev->ifindex,
4791                                                                MLXSW_SP_L3_PROTO_IPV4, dip);
4792                 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
4793                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
4794                         return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
4795                                                              fib_entry,
4796                                                              ipip_entry);
4797                 }
4798                 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id,
4799                                                  MLXSW_SP_L3_PROTO_IPV4,
4800                                                  &dip)) {
4801                         u32 tunnel_index;
4802
4803                         tunnel_index = router->nve_decap_config.tunnel_index;
4804                         fib_entry->decap.tunnel_index = tunnel_index;
4805                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
4806                         return 0;
4807                 }
4808                 fallthrough;
4809         case RTN_BROADCAST:
4810                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
4811                 return 0;
4812         case RTN_BLACKHOLE:
4813                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
4814                 return 0;
4815         case RTN_UNREACHABLE:
4816         case RTN_PROHIBIT:
4817                 /* Packets hitting these routes need to be trapped, but
4818                  * can do so with a lower priority than packets directed
4819                  * at the host, so use action type local instead of trap.
4820                  */
4821                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
4822                 return 0;
4823         case RTN_UNICAST:
4824                 if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi))
4825                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
4826                 else
4827                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4828                 return 0;
4829         default:
4830                 return -EINVAL;
4831         }
4832 }
4833
4834 static void
4835 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
4836                                struct mlxsw_sp_fib_entry *fib_entry)
4837 {
4838         switch (fib_entry->type) {
4839         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
4840                 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
4841                 break;
4842         default:
4843                 break;
4844         }
4845 }
4846
4847 static struct mlxsw_sp_fib4_entry *
4848 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
4849                            struct mlxsw_sp_fib_node *fib_node,
4850                            const struct fib_entry_notifier_info *fen_info)
4851 {
4852         struct mlxsw_sp_fib4_entry *fib4_entry;
4853         struct mlxsw_sp_fib_entry *fib_entry;
4854         int err;
4855
4856         fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
4857         if (!fib4_entry)
4858                 return ERR_PTR(-ENOMEM);
4859         fib_entry = &fib4_entry->common;
4860
4861         fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops);
4862         if (IS_ERR(fib_entry->priv)) {
4863                 err = PTR_ERR(fib_entry->priv);
4864                 goto err_fib_entry_priv_create;
4865         }
4866
4867         err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
4868         if (err)
4869                 goto err_nexthop4_group_get;
4870
4871         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
4872         if (err)
4873                 goto err_fib4_entry_type_set;
4874
4875         fib4_entry->fi = fen_info->fi;
4876         fib_info_hold(fib4_entry->fi);
4877         fib4_entry->tb_id = fen_info->tb_id;
4878         fib4_entry->type = fen_info->type;
4879         fib4_entry->tos = fen_info->tos;
4880
4881         fib_entry->fib_node = fib_node;
4882
4883         return fib4_entry;
4884
4885 err_fib4_entry_type_set:
4886         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
4887 err_nexthop4_group_get:
4888         mlxsw_sp_fib_entry_priv_put(fib_entry->priv);
4889 err_fib_entry_priv_create:
4890         kfree(fib4_entry);
4891         return ERR_PTR(err);
4892 }
4893
4894 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
4895                                         struct mlxsw_sp_fib4_entry *fib4_entry)
4896 {
4897         fib_info_put(fib4_entry->fi);
4898         mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, &fib4_entry->common);
4899         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
4900         mlxsw_sp_fib_entry_priv_put(fib4_entry->common.priv);
4901         kfree(fib4_entry);
4902 }
4903
4904 static struct mlxsw_sp_fib4_entry *
4905 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
4906                            const struct fib_entry_notifier_info *fen_info)
4907 {
4908         struct mlxsw_sp_fib4_entry *fib4_entry;
4909         struct mlxsw_sp_fib_node *fib_node;
4910         struct mlxsw_sp_fib *fib;
4911         struct mlxsw_sp_vr *vr;
4912
4913         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
4914         if (!vr)
4915                 return NULL;
4916         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
4917
4918         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
4919                                             sizeof(fen_info->dst),
4920                                             fen_info->dst_len);
4921         if (!fib_node)
4922                 return NULL;
4923
4924         fib4_entry = container_of(fib_node->fib_entry,
4925                                   struct mlxsw_sp_fib4_entry, common);
4926         if (fib4_entry->tb_id == fen_info->tb_id &&
4927             fib4_entry->tos == fen_info->tos &&
4928             fib4_entry->type == fen_info->type &&
4929             fib4_entry->fi == fen_info->fi)
4930                 return fib4_entry;
4931
4932         return NULL;
4933 }
4934
4935 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
4936         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
4937         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
4938         .key_len = sizeof(struct mlxsw_sp_fib_key),
4939         .automatic_shrinking = true,
4940 };
4941
4942 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
4943                                     struct mlxsw_sp_fib_node *fib_node)
4944 {
4945         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
4946                                       mlxsw_sp_fib_ht_params);
4947 }
4948
4949 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
4950                                      struct mlxsw_sp_fib_node *fib_node)
4951 {
4952         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
4953                                mlxsw_sp_fib_ht_params);
4954 }
4955
4956 static struct mlxsw_sp_fib_node *
4957 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
4958                          size_t addr_len, unsigned char prefix_len)
4959 {
4960         struct mlxsw_sp_fib_key key;
4961
4962         memset(&key, 0, sizeof(key));
4963         memcpy(key.addr, addr, addr_len);
4964         key.prefix_len = prefix_len;
4965         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
4966 }
4967
4968 static struct mlxsw_sp_fib_node *
4969 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
4970                          size_t addr_len, unsigned char prefix_len)
4971 {
4972         struct mlxsw_sp_fib_node *fib_node;
4973
4974         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
4975         if (!fib_node)
4976                 return NULL;
4977
4978         list_add(&fib_node->list, &fib->node_list);
4979         memcpy(fib_node->key.addr, addr, addr_len);
4980         fib_node->key.prefix_len = prefix_len;
4981
4982         return fib_node;
4983 }
4984
4985 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
4986 {
4987         list_del(&fib_node->list);
4988         kfree(fib_node);
4989 }
4990
4991 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
4992                                       struct mlxsw_sp_fib_node *fib_node)
4993 {
4994         struct mlxsw_sp_prefix_usage req_prefix_usage;
4995         struct mlxsw_sp_fib *fib = fib_node->fib;
4996         struct mlxsw_sp_lpm_tree *lpm_tree;
4997         int err;
4998
4999         lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto];
5000         if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
5001                 goto out;
5002
5003         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
5004         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
5005         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
5006                                          fib->proto);
5007         if (IS_ERR(lpm_tree))
5008                 return PTR_ERR(lpm_tree);
5009
5010         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
5011         if (err)
5012                 goto err_lpm_tree_replace;
5013
5014 out:
5015         lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++;
5016         return 0;
5017
5018 err_lpm_tree_replace:
5019         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
5020         return err;
5021 }
5022
5023 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
5024                                          struct mlxsw_sp_fib_node *fib_node)
5025 {
5026         struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
5027         struct mlxsw_sp_prefix_usage req_prefix_usage;
5028         struct mlxsw_sp_fib *fib = fib_node->fib;
5029         int err;
5030
5031         if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
5032                 return;
5033         /* Try to construct a new LPM tree from the current prefix usage
5034          * minus the unused one. If we fail, continue using the old one.
5035          */
5036         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
5037         mlxsw_sp_prefix_usage_clear(&req_prefix_usage,
5038                                     fib_node->key.prefix_len);
5039         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
5040                                          fib->proto);
5041         if (IS_ERR(lpm_tree))
5042                 return;
5043
5044         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
5045         if (err)
5046                 goto err_lpm_tree_replace;
5047
5048         return;
5049
5050 err_lpm_tree_replace:
5051         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
5052 }
5053
5054 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
5055                                   struct mlxsw_sp_fib_node *fib_node,
5056                                   struct mlxsw_sp_fib *fib)
5057 {
5058         int err;
5059
5060         err = mlxsw_sp_fib_node_insert(fib, fib_node);
5061         if (err)
5062                 return err;
5063         fib_node->fib = fib;
5064
5065         err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node);
5066         if (err)
5067                 goto err_fib_lpm_tree_link;
5068
5069         return 0;
5070
5071 err_fib_lpm_tree_link:
5072         fib_node->fib = NULL;
5073         mlxsw_sp_fib_node_remove(fib, fib_node);
5074         return err;
5075 }
5076
5077 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
5078                                    struct mlxsw_sp_fib_node *fib_node)
5079 {
5080         struct mlxsw_sp_fib *fib = fib_node->fib;
5081
5082         mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node);
5083         fib_node->fib = NULL;
5084         mlxsw_sp_fib_node_remove(fib, fib_node);
5085 }
5086
5087 static struct mlxsw_sp_fib_node *
5088 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
5089                       size_t addr_len, unsigned char prefix_len,
5090                       enum mlxsw_sp_l3proto proto)
5091 {
5092         struct mlxsw_sp_fib_node *fib_node;
5093         struct mlxsw_sp_fib *fib;
5094         struct mlxsw_sp_vr *vr;
5095         int err;
5096
5097         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL);
5098         if (IS_ERR(vr))
5099                 return ERR_CAST(vr);
5100         fib = mlxsw_sp_vr_fib(vr, proto);
5101
5102         fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
5103         if (fib_node)
5104                 return fib_node;
5105
5106         fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
5107         if (!fib_node) {
5108                 err = -ENOMEM;
5109                 goto err_fib_node_create;
5110         }
5111
5112         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
5113         if (err)
5114                 goto err_fib_node_init;
5115
5116         return fib_node;
5117
5118 err_fib_node_init:
5119         mlxsw_sp_fib_node_destroy(fib_node);
5120 err_fib_node_create:
5121         mlxsw_sp_vr_put(mlxsw_sp, vr);
5122         return ERR_PTR(err);
5123 }
5124
5125 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
5126                                   struct mlxsw_sp_fib_node *fib_node)
5127 {
5128         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
5129
5130         if (fib_node->fib_entry)
5131                 return;
5132         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
5133         mlxsw_sp_fib_node_destroy(fib_node);
5134         mlxsw_sp_vr_put(mlxsw_sp, vr);
5135 }
5136
5137 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp,
5138                                         struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5139                                         struct mlxsw_sp_fib_entry *fib_entry)
5140 {
5141         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
5142         bool is_new = !fib_node->fib_entry;
5143         int err;
5144
5145         fib_node->fib_entry = fib_entry;
5146
5147         err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, is_new);
5148         if (err)
5149                 goto err_fib_entry_update;
5150
5151         return 0;
5152
5153 err_fib_entry_update:
5154         fib_node->fib_entry = NULL;
5155         return err;
5156 }
5157
5158 static int __mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
5159                                             struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5160                                             struct mlxsw_sp_fib_entry *fib_entry)
5161 {
5162         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
5163         int err;
5164
5165         err = mlxsw_sp_fib_entry_del(mlxsw_sp, op_ctx, fib_entry);
5166         fib_node->fib_entry = NULL;
5167         return err;
5168 }
5169
5170 static void mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
5171                                            struct mlxsw_sp_fib_entry *fib_entry)
5172 {
5173         struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx;
5174
5175         mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
5176         __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, fib_entry);
5177 }
5178
5179 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry)
5180 {
5181         struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
5182         struct mlxsw_sp_fib4_entry *fib4_replaced;
5183
5184         if (!fib_node->fib_entry)
5185                 return true;
5186
5187         fib4_replaced = container_of(fib_node->fib_entry,
5188                                      struct mlxsw_sp_fib4_entry, common);
5189         if (fib4_entry->tb_id == RT_TABLE_MAIN &&
5190             fib4_replaced->tb_id == RT_TABLE_LOCAL)
5191                 return false;
5192
5193         return true;
5194 }
5195
5196 static int
5197 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp,
5198                              struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5199                              const struct fib_entry_notifier_info *fen_info)
5200 {
5201         struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced;
5202         struct mlxsw_sp_fib_entry *replaced;
5203         struct mlxsw_sp_fib_node *fib_node;
5204         int err;
5205
5206         if (mlxsw_sp->router->aborted)
5207                 return 0;
5208
5209         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
5210                                          &fen_info->dst, sizeof(fen_info->dst),
5211                                          fen_info->dst_len,
5212                                          MLXSW_SP_L3_PROTO_IPV4);
5213         if (IS_ERR(fib_node)) {
5214                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
5215                 return PTR_ERR(fib_node);
5216         }
5217
5218         fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
5219         if (IS_ERR(fib4_entry)) {
5220                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
5221                 err = PTR_ERR(fib4_entry);
5222                 goto err_fib4_entry_create;
5223         }
5224
5225         if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) {
5226                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
5227                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5228                 return 0;
5229         }
5230
5231         replaced = fib_node->fib_entry;
5232         err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib4_entry->common);
5233         if (err) {
5234                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
5235                 goto err_fib_node_entry_link;
5236         }
5237
5238         /* Nothing to replace */
5239         if (!replaced)
5240                 return 0;
5241
5242         mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
5243         fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry,
5244                                      common);
5245         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced);
5246
5247         return 0;
5248
5249 err_fib_node_entry_link:
5250         fib_node->fib_entry = replaced;
5251         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
5252 err_fib4_entry_create:
5253         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5254         return err;
5255 }
5256
5257 static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
5258                                     struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5259                                     struct fib_entry_notifier_info *fen_info)
5260 {
5261         struct mlxsw_sp_fib4_entry *fib4_entry;
5262         struct mlxsw_sp_fib_node *fib_node;
5263         int err;
5264
5265         if (mlxsw_sp->router->aborted)
5266                 return 0;
5267
5268         fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
5269         if (!fib4_entry)
5270                 return 0;
5271         fib_node = fib4_entry->common.fib_node;
5272
5273         err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib4_entry->common);
5274         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
5275         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5276         return err;
5277 }
5278
5279 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt)
5280 {
5281         /* Multicast routes aren't supported, so ignore them. Neighbour
5282          * Discovery packets are specifically trapped.
5283          */
5284         if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST)
5285                 return true;
5286
5287         /* Cloned routes are irrelevant in the forwarding path. */
5288         if (rt->fib6_flags & RTF_CACHE)
5289                 return true;
5290
5291         return false;
5292 }
5293
5294 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt)
5295 {
5296         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5297
5298         mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
5299         if (!mlxsw_sp_rt6)
5300                 return ERR_PTR(-ENOMEM);
5301
5302         /* In case of route replace, replaced route is deleted with
5303          * no notification. Take reference to prevent accessing freed
5304          * memory.
5305          */
5306         mlxsw_sp_rt6->rt = rt;
5307         fib6_info_hold(rt);
5308
5309         return mlxsw_sp_rt6;
5310 }
5311
5312 #if IS_ENABLED(CONFIG_IPV6)
5313 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
5314 {
5315         fib6_info_release(rt);
5316 }
5317 #else
5318 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
5319 {
5320 }
5321 #endif
5322
5323 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
5324 {
5325         struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
5326
5327         fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
5328         mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
5329         kfree(mlxsw_sp_rt6);
5330 }
5331
5332 static struct fib6_info *
5333 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
5334 {
5335         return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
5336                                 list)->rt;
5337 }
5338
5339 static struct mlxsw_sp_rt6 *
5340 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
5341                             const struct fib6_info *rt)
5342 {
5343         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5344
5345         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
5346                 if (mlxsw_sp_rt6->rt == rt)
5347                         return mlxsw_sp_rt6;
5348         }
5349
5350         return NULL;
5351 }
5352
5353 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
5354                                         const struct fib6_info *rt,
5355                                         enum mlxsw_sp_ipip_type *ret)
5356 {
5357         return rt->fib6_nh->fib_nh_dev &&
5358                mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret);
5359 }
5360
5361 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
5362                                   struct mlxsw_sp_nexthop_group *nh_grp,
5363                                   struct mlxsw_sp_nexthop *nh,
5364                                   const struct fib6_info *rt)
5365 {
5366         struct net_device *dev = rt->fib6_nh->fib_nh_dev;
5367
5368         nh->nhgi = nh_grp->nhgi;
5369         nh->nh_weight = rt->fib6_nh->fib_nh_weight;
5370         memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr));
5371 #if IS_ENABLED(CONFIG_IPV6)
5372         nh->neigh_tbl = &nd_tbl;
5373 #endif
5374         mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
5375
5376         list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
5377
5378         if (!dev)
5379                 return 0;
5380         nh->ifindex = dev->ifindex;
5381
5382         return mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
5383 }
5384
5385 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
5386                                    struct mlxsw_sp_nexthop *nh)
5387 {
5388         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
5389         list_del(&nh->router_list_node);
5390         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
5391 }
5392
5393 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
5394                                     const struct fib6_info *rt)
5395 {
5396         return rt->fib6_nh->fib_nh_gw_family ||
5397                mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
5398 }
5399
5400 static int
5401 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp,
5402                                   struct mlxsw_sp_nexthop_group *nh_grp,
5403                                   struct mlxsw_sp_fib6_entry *fib6_entry)
5404 {
5405         struct mlxsw_sp_nexthop_group_info *nhgi;
5406         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5407         struct mlxsw_sp_nexthop *nh;
5408         int err, i;
5409
5410         nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6),
5411                        GFP_KERNEL);
5412         if (!nhgi)
5413                 return -ENOMEM;
5414         nh_grp->nhgi = nhgi;
5415         nhgi->nh_grp = nh_grp;
5416         mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
5417                                         struct mlxsw_sp_rt6, list);
5418         nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
5419         nhgi->count = fib6_entry->nrt6;
5420         for (i = 0; i < nhgi->count; i++) {
5421                 struct fib6_info *rt = mlxsw_sp_rt6->rt;
5422
5423                 nh = &nhgi->nexthops[i];
5424                 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
5425                 if (err)
5426                         goto err_nexthop6_init;
5427                 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
5428         }
5429         nh_grp->nhgi = nhgi;
5430         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5431
5432         return 0;
5433
5434 err_nexthop6_init:
5435         for (i--; i >= 0; i--) {
5436                 nh = &nhgi->nexthops[i];
5437                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
5438         }
5439         kfree(nhgi);
5440         return err;
5441 }
5442
5443 static void
5444 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp,
5445                                   struct mlxsw_sp_nexthop_group *nh_grp)
5446 {
5447         struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
5448         int i;
5449
5450         for (i = nhgi->count - 1; i >= 0; i--) {
5451                 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
5452
5453                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
5454         }
5455         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5456         WARN_ON_ONCE(nhgi->adj_index_valid);
5457         kfree(nhgi);
5458 }
5459
5460 static struct mlxsw_sp_nexthop_group *
5461 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
5462                                struct mlxsw_sp_fib6_entry *fib6_entry)
5463 {
5464         struct mlxsw_sp_nexthop_group *nh_grp;
5465         int err;
5466
5467         nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
5468         if (!nh_grp)
5469                 return ERR_PTR(-ENOMEM);
5470         INIT_LIST_HEAD(&nh_grp->fib_list);
5471         nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
5472
5473         err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry);
5474         if (err)
5475                 goto err_nexthop_group_info_init;
5476
5477         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
5478         if (err)
5479                 goto err_nexthop_group_insert;
5480
5481         return nh_grp;
5482
5483 err_nexthop_group_insert:
5484         mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
5485 err_nexthop_group_info_init:
5486         kfree(nh_grp);
5487         return ERR_PTR(err);
5488 }
5489
5490 static void
5491 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
5492                                 struct mlxsw_sp_nexthop_group *nh_grp)
5493 {
5494         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
5495         mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
5496         kfree(nh_grp);
5497 }
5498
5499 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
5500                                        struct mlxsw_sp_fib6_entry *fib6_entry)
5501 {
5502         struct mlxsw_sp_nexthop_group *nh_grp;
5503
5504         nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
5505         if (!nh_grp) {
5506                 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
5507                 if (IS_ERR(nh_grp))
5508                         return PTR_ERR(nh_grp);
5509         }
5510
5511         list_add_tail(&fib6_entry->common.nexthop_group_node,
5512                       &nh_grp->fib_list);
5513         fib6_entry->common.nh_group = nh_grp;
5514
5515         /* The route and the nexthop are described by the same struct, so we
5516          * need to the update the nexthop offload indication for the new route.
5517          */
5518         __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
5519
5520         return 0;
5521 }
5522
5523 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
5524                                         struct mlxsw_sp_fib_entry *fib_entry)
5525 {
5526         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
5527
5528         list_del(&fib_entry->nexthop_group_node);
5529         if (!list_empty(&nh_grp->fib_list))
5530                 return;
5531         mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
5532 }
5533
5534 static int mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
5535                                           struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5536                                           struct mlxsw_sp_fib6_entry *fib6_entry)
5537 {
5538         struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
5539         int err;
5540
5541         fib6_entry->common.nh_group = NULL;
5542         list_del(&fib6_entry->common.nexthop_group_node);
5543
5544         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
5545         if (err)
5546                 goto err_nexthop6_group_get;
5547
5548         /* In case this entry is offloaded, then the adjacency index
5549          * currently associated with it in the device's table is that
5550          * of the old group. Start using the new one instead.
5551          */
5552         err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx,
5553                                           &fib6_entry->common, false);
5554         if (err)
5555                 goto err_fib_entry_update;
5556
5557         if (list_empty(&old_nh_grp->fib_list))
5558                 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
5559
5560         return 0;
5561
5562 err_fib_entry_update:
5563         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
5564 err_nexthop6_group_get:
5565         list_add_tail(&fib6_entry->common.nexthop_group_node,
5566                       &old_nh_grp->fib_list);
5567         fib6_entry->common.nh_group = old_nh_grp;
5568         return err;
5569 }
5570
5571 static int
5572 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
5573                                 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5574                                 struct mlxsw_sp_fib6_entry *fib6_entry,
5575                                 struct fib6_info **rt_arr, unsigned int nrt6)
5576 {
5577         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5578         int err, i;
5579
5580         for (i = 0; i < nrt6; i++) {
5581                 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
5582                 if (IS_ERR(mlxsw_sp_rt6)) {
5583                         err = PTR_ERR(mlxsw_sp_rt6);
5584                         goto err_rt6_create;
5585                 }
5586
5587                 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
5588                 fib6_entry->nrt6++;
5589         }
5590
5591         err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry);
5592         if (err)
5593                 goto err_nexthop6_group_update;
5594
5595         return 0;
5596
5597 err_nexthop6_group_update:
5598         i = nrt6;
5599 err_rt6_create:
5600         for (i--; i >= 0; i--) {
5601                 fib6_entry->nrt6--;
5602                 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
5603                                                struct mlxsw_sp_rt6, list);
5604                 list_del(&mlxsw_sp_rt6->list);
5605                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5606         }
5607         return err;
5608 }
5609
5610 static void
5611 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
5612                                 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5613                                 struct mlxsw_sp_fib6_entry *fib6_entry,
5614                                 struct fib6_info **rt_arr, unsigned int nrt6)
5615 {
5616         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5617         int i;
5618
5619         for (i = 0; i < nrt6; i++) {
5620                 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry,
5621                                                            rt_arr[i]);
5622                 if (WARN_ON_ONCE(!mlxsw_sp_rt6))
5623                         continue;
5624
5625                 fib6_entry->nrt6--;
5626                 list_del(&mlxsw_sp_rt6->list);
5627                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5628         }
5629
5630         mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry);
5631 }
5632
5633 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
5634                                          struct mlxsw_sp_fib_entry *fib_entry,
5635                                          const struct fib6_info *rt)
5636 {
5637         if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))
5638                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
5639         else if (rt->fib6_type == RTN_BLACKHOLE)
5640                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
5641         else if (rt->fib6_flags & RTF_REJECT)
5642                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
5643         else if (mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt))
5644                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
5645         else
5646                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
5647 }
5648
5649 static void
5650 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
5651 {
5652         struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
5653
5654         list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
5655                                  list) {
5656                 fib6_entry->nrt6--;
5657                 list_del(&mlxsw_sp_rt6->list);
5658                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5659         }
5660 }
5661
5662 static struct mlxsw_sp_fib6_entry *
5663 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
5664                            struct mlxsw_sp_fib_node *fib_node,
5665                            struct fib6_info **rt_arr, unsigned int nrt6)
5666 {
5667         struct mlxsw_sp_fib6_entry *fib6_entry;
5668         struct mlxsw_sp_fib_entry *fib_entry;
5669         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5670         int err, i;
5671
5672         fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
5673         if (!fib6_entry)
5674                 return ERR_PTR(-ENOMEM);
5675         fib_entry = &fib6_entry->common;
5676
5677         fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops);
5678         if (IS_ERR(fib_entry->priv)) {
5679                 err = PTR_ERR(fib_entry->priv);
5680                 goto err_fib_entry_priv_create;
5681         }
5682
5683         INIT_LIST_HEAD(&fib6_entry->rt6_list);
5684
5685         for (i = 0; i < nrt6; i++) {
5686                 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
5687                 if (IS_ERR(mlxsw_sp_rt6)) {
5688                         err = PTR_ERR(mlxsw_sp_rt6);
5689                         goto err_rt6_create;
5690                 }
5691                 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
5692                 fib6_entry->nrt6++;
5693         }
5694
5695         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
5696         if (err)
5697                 goto err_nexthop6_group_get;
5698
5699         mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]);
5700
5701         fib_entry->fib_node = fib_node;
5702
5703         return fib6_entry;
5704
5705 err_nexthop6_group_get:
5706         i = nrt6;
5707 err_rt6_create:
5708         for (i--; i >= 0; i--) {
5709                 fib6_entry->nrt6--;
5710                 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
5711                                                struct mlxsw_sp_rt6, list);
5712                 list_del(&mlxsw_sp_rt6->list);
5713                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5714         }
5715         mlxsw_sp_fib_entry_priv_put(fib_entry->priv);
5716 err_fib_entry_priv_create:
5717         kfree(fib6_entry);
5718         return ERR_PTR(err);
5719 }
5720
5721 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
5722                                         struct mlxsw_sp_fib6_entry *fib6_entry)
5723 {
5724         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
5725         mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
5726         WARN_ON(fib6_entry->nrt6);
5727         mlxsw_sp_fib_entry_priv_put(fib6_entry->common.priv);
5728         kfree(fib6_entry);
5729 }
5730
5731 static struct mlxsw_sp_fib6_entry *
5732 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
5733                            const struct fib6_info *rt)
5734 {
5735         struct mlxsw_sp_fib6_entry *fib6_entry;
5736         struct mlxsw_sp_fib_node *fib_node;
5737         struct mlxsw_sp_fib *fib;
5738         struct fib6_info *cmp_rt;
5739         struct mlxsw_sp_vr *vr;
5740
5741         vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id);
5742         if (!vr)
5743                 return NULL;
5744         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
5745
5746         fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr,
5747                                             sizeof(rt->fib6_dst.addr),
5748                                             rt->fib6_dst.plen);
5749         if (!fib_node)
5750                 return NULL;
5751
5752         fib6_entry = container_of(fib_node->fib_entry,
5753                                   struct mlxsw_sp_fib6_entry, common);
5754         cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
5755         if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id &&
5756             rt->fib6_metric == cmp_rt->fib6_metric &&
5757             mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
5758                 return fib6_entry;
5759
5760         return NULL;
5761 }
5762
5763 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry)
5764 {
5765         struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
5766         struct mlxsw_sp_fib6_entry *fib6_replaced;
5767         struct fib6_info *rt, *rt_replaced;
5768
5769         if (!fib_node->fib_entry)
5770                 return true;
5771
5772         fib6_replaced = container_of(fib_node->fib_entry,
5773                                      struct mlxsw_sp_fib6_entry,
5774                                      common);
5775         rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
5776         rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced);
5777         if (rt->fib6_table->tb6_id == RT_TABLE_MAIN &&
5778             rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL)
5779                 return false;
5780
5781         return true;
5782 }
5783
5784 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp,
5785                                         struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5786                                         struct fib6_info **rt_arr, unsigned int nrt6)
5787 {
5788         struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced;
5789         struct mlxsw_sp_fib_entry *replaced;
5790         struct mlxsw_sp_fib_node *fib_node;
5791         struct fib6_info *rt = rt_arr[0];
5792         int err;
5793
5794         if (mlxsw_sp->router->aborted)
5795                 return 0;
5796
5797         if (rt->fib6_src.plen)
5798                 return -EINVAL;
5799
5800         if (mlxsw_sp_fib6_rt_should_ignore(rt))
5801                 return 0;
5802
5803         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
5804                                          &rt->fib6_dst.addr,
5805                                          sizeof(rt->fib6_dst.addr),
5806                                          rt->fib6_dst.plen,
5807                                          MLXSW_SP_L3_PROTO_IPV6);
5808         if (IS_ERR(fib_node))
5809                 return PTR_ERR(fib_node);
5810
5811         fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr,
5812                                                 nrt6);
5813         if (IS_ERR(fib6_entry)) {
5814                 err = PTR_ERR(fib6_entry);
5815                 goto err_fib6_entry_create;
5816         }
5817
5818         if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) {
5819                 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5820                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5821                 return 0;
5822         }
5823
5824         replaced = fib_node->fib_entry;
5825         err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib6_entry->common);
5826         if (err)
5827                 goto err_fib_node_entry_link;
5828
5829         /* Nothing to replace */
5830         if (!replaced)
5831                 return 0;
5832
5833         mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
5834         fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry,
5835                                      common);
5836         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced);
5837
5838         return 0;
5839
5840 err_fib_node_entry_link:
5841         fib_node->fib_entry = replaced;
5842         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5843 err_fib6_entry_create:
5844         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5845         return err;
5846 }
5847
5848 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp,
5849                                        struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5850                                        struct fib6_info **rt_arr, unsigned int nrt6)
5851 {
5852         struct mlxsw_sp_fib6_entry *fib6_entry;
5853         struct mlxsw_sp_fib_node *fib_node;
5854         struct fib6_info *rt = rt_arr[0];
5855         int err;
5856
5857         if (mlxsw_sp->router->aborted)
5858                 return 0;
5859
5860         if (rt->fib6_src.plen)
5861                 return -EINVAL;
5862
5863         if (mlxsw_sp_fib6_rt_should_ignore(rt))
5864                 return 0;
5865
5866         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
5867                                          &rt->fib6_dst.addr,
5868                                          sizeof(rt->fib6_dst.addr),
5869                                          rt->fib6_dst.plen,
5870                                          MLXSW_SP_L3_PROTO_IPV6);
5871         if (IS_ERR(fib_node))
5872                 return PTR_ERR(fib_node);
5873
5874         if (WARN_ON_ONCE(!fib_node->fib_entry)) {
5875                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5876                 return -EINVAL;
5877         }
5878
5879         fib6_entry = container_of(fib_node->fib_entry,
5880                                   struct mlxsw_sp_fib6_entry, common);
5881         err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6);
5882         if (err)
5883                 goto err_fib6_entry_nexthop_add;
5884
5885         return 0;
5886
5887 err_fib6_entry_nexthop_add:
5888         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5889         return err;
5890 }
5891
5892 static int mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
5893                                     struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5894                                     struct fib6_info **rt_arr, unsigned int nrt6)
5895 {
5896         struct mlxsw_sp_fib6_entry *fib6_entry;
5897         struct mlxsw_sp_fib_node *fib_node;
5898         struct fib6_info *rt = rt_arr[0];
5899         int err;
5900
5901         if (mlxsw_sp->router->aborted)
5902                 return 0;
5903
5904         if (mlxsw_sp_fib6_rt_should_ignore(rt))
5905                 return 0;
5906
5907         /* Multipath routes are first added to the FIB trie and only then
5908          * notified. If we vetoed the addition, we will get a delete
5909          * notification for a route we do not have. Therefore, do not warn if
5910          * route was not found.
5911          */
5912         fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
5913         if (!fib6_entry)
5914                 return 0;
5915
5916         /* If not all the nexthops are deleted, then only reduce the nexthop
5917          * group.
5918          */
5919         if (nrt6 != fib6_entry->nrt6) {
5920                 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6);
5921                 return 0;
5922         }
5923
5924         fib_node = fib6_entry->common.fib_node;
5925
5926         err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib6_entry->common);
5927         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5928         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5929         return err;
5930 }
5931
5932 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
5933                                             enum mlxsw_sp_l3proto proto,
5934                                             u8 tree_id)
5935 {
5936         const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
5937         enum mlxsw_reg_ralxx_protocol ralxx_proto =
5938                                 (enum mlxsw_reg_ralxx_protocol) proto;
5939         struct mlxsw_sp_fib_entry_priv *priv;
5940         char xralta_pl[MLXSW_REG_XRALTA_LEN];
5941         char xralst_pl[MLXSW_REG_XRALST_LEN];
5942         int i, err;
5943
5944         mlxsw_reg_xralta_pack(xralta_pl, true, ralxx_proto, tree_id);
5945         err = ll_ops->ralta_write(mlxsw_sp, xralta_pl);
5946         if (err)
5947                 return err;
5948
5949         mlxsw_reg_xralst_pack(xralst_pl, 0xff, tree_id);
5950         err = ll_ops->ralst_write(mlxsw_sp, xralst_pl);
5951         if (err)
5952                 return err;
5953
5954         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
5955                 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx;
5956                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
5957                 char xraltb_pl[MLXSW_REG_XRALTB_LEN];
5958
5959                 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
5960                 mlxsw_reg_xraltb_pack(xraltb_pl, vr->id, ralxx_proto, tree_id);
5961                 err = ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
5962                 if (err)
5963                         return err;
5964
5965                 priv = mlxsw_sp_fib_entry_priv_create(ll_ops);
5966                 if (IS_ERR(priv))
5967                         return PTR_ERR(priv);
5968
5969                 ll_ops->fib_entry_pack(op_ctx, proto, MLXSW_SP_FIB_ENTRY_OP_WRITE,
5970                                        vr->id, 0, NULL, priv);
5971                 ll_ops->fib_entry_act_ip2me_pack(op_ctx);
5972                 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, NULL);
5973                 mlxsw_sp_fib_entry_priv_put(priv);
5974                 if (err)
5975                         return err;
5976         }
5977
5978         return 0;
5979 }
5980
5981 static struct mlxsw_sp_mr_table *
5982 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family)
5983 {
5984         if (family == RTNL_FAMILY_IPMR)
5985                 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4];
5986         else
5987                 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
5988 }
5989
5990 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp,
5991                                      struct mfc_entry_notifier_info *men_info,
5992                                      bool replace)
5993 {
5994         struct mlxsw_sp_mr_table *mrt;
5995         struct mlxsw_sp_vr *vr;
5996
5997         if (mlxsw_sp->router->aborted)
5998                 return 0;
5999
6000         vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL);
6001         if (IS_ERR(vr))
6002                 return PTR_ERR(vr);
6003
6004         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
6005         return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace);
6006 }
6007
6008 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp,
6009                                       struct mfc_entry_notifier_info *men_info)
6010 {
6011         struct mlxsw_sp_mr_table *mrt;
6012         struct mlxsw_sp_vr *vr;
6013
6014         if (mlxsw_sp->router->aborted)
6015                 return;
6016
6017         vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id);
6018         if (WARN_ON(!vr))
6019                 return;
6020
6021         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
6022         mlxsw_sp_mr_route_del(mrt, men_info->mfc);
6023         mlxsw_sp_vr_put(mlxsw_sp, vr);
6024 }
6025
6026 static int
6027 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp,
6028                               struct vif_entry_notifier_info *ven_info)
6029 {
6030         struct mlxsw_sp_mr_table *mrt;
6031         struct mlxsw_sp_rif *rif;
6032         struct mlxsw_sp_vr *vr;
6033
6034         if (mlxsw_sp->router->aborted)
6035                 return 0;
6036
6037         vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL);
6038         if (IS_ERR(vr))
6039                 return PTR_ERR(vr);
6040
6041         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
6042         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev);
6043         return mlxsw_sp_mr_vif_add(mrt, ven_info->dev,
6044                                    ven_info->vif_index,
6045                                    ven_info->vif_flags, rif);
6046 }
6047
6048 static void
6049 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp,
6050                               struct vif_entry_notifier_info *ven_info)
6051 {
6052         struct mlxsw_sp_mr_table *mrt;
6053         struct mlxsw_sp_vr *vr;
6054
6055         if (mlxsw_sp->router->aborted)
6056                 return;
6057
6058         vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id);
6059         if (WARN_ON(!vr))
6060                 return;
6061
6062         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
6063         mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index);
6064         mlxsw_sp_vr_put(mlxsw_sp, vr);
6065 }
6066
6067 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
6068 {
6069         enum mlxsw_sp_l3proto proto = MLXSW_SP_L3_PROTO_IPV4;
6070         int err;
6071
6072         err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
6073                                                MLXSW_SP_LPM_TREE_MIN);
6074         if (err)
6075                 return err;
6076
6077         /* The multicast router code does not need an abort trap as by default,
6078          * packets that don't match any routes are trapped to the CPU.
6079          */
6080
6081         proto = MLXSW_SP_L3_PROTO_IPV6;
6082         return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
6083                                                 MLXSW_SP_LPM_TREE_MIN + 1);
6084 }
6085
6086 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
6087                                      struct mlxsw_sp_fib_node *fib_node)
6088 {
6089         struct mlxsw_sp_fib4_entry *fib4_entry;
6090
6091         fib4_entry = container_of(fib_node->fib_entry,
6092                                   struct mlxsw_sp_fib4_entry, common);
6093         mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
6094         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6095         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6096 }
6097
6098 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
6099                                      struct mlxsw_sp_fib_node *fib_node)
6100 {
6101         struct mlxsw_sp_fib6_entry *fib6_entry;
6102
6103         fib6_entry = container_of(fib_node->fib_entry,
6104                                   struct mlxsw_sp_fib6_entry, common);
6105         mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
6106         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
6107         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6108 }
6109
6110 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
6111                                     struct mlxsw_sp_fib_node *fib_node)
6112 {
6113         switch (fib_node->fib->proto) {
6114         case MLXSW_SP_L3_PROTO_IPV4:
6115                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
6116                 break;
6117         case MLXSW_SP_L3_PROTO_IPV6:
6118                 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
6119                 break;
6120         }
6121 }
6122
6123 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
6124                                   struct mlxsw_sp_vr *vr,
6125                                   enum mlxsw_sp_l3proto proto)
6126 {
6127         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
6128         struct mlxsw_sp_fib_node *fib_node, *tmp;
6129
6130         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
6131                 bool do_break = &tmp->list == &fib->node_list;
6132
6133                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
6134                 if (do_break)
6135                         break;
6136         }
6137 }
6138
6139 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
6140 {
6141         int i, j;
6142
6143         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
6144                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
6145
6146                 if (!mlxsw_sp_vr_is_used(vr))
6147                         continue;
6148
6149                 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++)
6150                         mlxsw_sp_mr_table_flush(vr->mr_table[j]);
6151                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
6152
6153                 /* If virtual router was only used for IPv4, then it's no
6154                  * longer used.
6155                  */
6156                 if (!mlxsw_sp_vr_is_used(vr))
6157                         continue;
6158                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
6159         }
6160
6161         /* After flushing all the routes, it is not possible anyone is still
6162          * using the adjacency index that is discarding packets, so free it in
6163          * case it was allocated.
6164          */
6165         if (!mlxsw_sp->router->adj_discard_index_valid)
6166                 return;
6167         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
6168                            mlxsw_sp->router->adj_discard_index);
6169         mlxsw_sp->router->adj_discard_index_valid = false;
6170 }
6171
6172 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
6173 {
6174         int err;
6175
6176         if (mlxsw_sp->router->aborted)
6177                 return;
6178         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
6179         mlxsw_sp_router_fib_flush(mlxsw_sp);
6180         mlxsw_sp->router->aborted = true;
6181         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
6182         if (err)
6183                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
6184 }
6185
6186 struct mlxsw_sp_fib6_event {
6187         struct fib6_info **rt_arr;
6188         unsigned int nrt6;
6189 };
6190
6191 struct mlxsw_sp_fib_event {
6192         struct list_head list; /* node in fib queue */
6193         union {
6194                 struct mlxsw_sp_fib6_event fib6_event;
6195                 struct fib_entry_notifier_info fen_info;
6196                 struct fib_rule_notifier_info fr_info;
6197                 struct fib_nh_notifier_info fnh_info;
6198                 struct mfc_entry_notifier_info men_info;
6199                 struct vif_entry_notifier_info ven_info;
6200         };
6201         struct mlxsw_sp *mlxsw_sp;
6202         unsigned long event;
6203         int family;
6204 };
6205
6206 static int
6207 mlxsw_sp_router_fib6_event_init(struct mlxsw_sp_fib6_event *fib6_event,
6208                                 struct fib6_entry_notifier_info *fen6_info)
6209 {
6210         struct fib6_info *rt = fen6_info->rt;
6211         struct fib6_info **rt_arr;
6212         struct fib6_info *iter;
6213         unsigned int nrt6;
6214         int i = 0;
6215
6216         nrt6 = fen6_info->nsiblings + 1;
6217
6218         rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC);
6219         if (!rt_arr)
6220                 return -ENOMEM;
6221
6222         fib6_event->rt_arr = rt_arr;
6223         fib6_event->nrt6 = nrt6;
6224
6225         rt_arr[0] = rt;
6226         fib6_info_hold(rt);
6227
6228         if (!fen6_info->nsiblings)
6229                 return 0;
6230
6231         list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
6232                 if (i == fen6_info->nsiblings)
6233                         break;
6234
6235                 rt_arr[i + 1] = iter;
6236                 fib6_info_hold(iter);
6237                 i++;
6238         }
6239         WARN_ON_ONCE(i != fen6_info->nsiblings);
6240
6241         return 0;
6242 }
6243
6244 static void
6245 mlxsw_sp_router_fib6_event_fini(struct mlxsw_sp_fib6_event *fib6_event)
6246 {
6247         int i;
6248
6249         for (i = 0; i < fib6_event->nrt6; i++)
6250                 mlxsw_sp_rt6_release(fib6_event->rt_arr[i]);
6251         kfree(fib6_event->rt_arr);
6252 }
6253
6254 static void mlxsw_sp_router_fib4_event_process(struct mlxsw_sp *mlxsw_sp,
6255                                                struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
6256                                                struct mlxsw_sp_fib_event *fib_event)
6257 {
6258         int err;
6259
6260         mlxsw_sp_span_respin(mlxsw_sp);
6261
6262         switch (fib_event->event) {
6263         case FIB_EVENT_ENTRY_REPLACE:
6264                 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, op_ctx, &fib_event->fen_info);
6265                 if (err) {
6266                         mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
6267                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6268                 }
6269                 fib_info_put(fib_event->fen_info.fi);
6270                 break;
6271         case FIB_EVENT_ENTRY_DEL:
6272                 err = mlxsw_sp_router_fib4_del(mlxsw_sp, op_ctx, &fib_event->fen_info);
6273                 if (err)
6274                         mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
6275                 fib_info_put(fib_event->fen_info.fi);
6276                 break;
6277         case FIB_EVENT_NH_ADD:
6278         case FIB_EVENT_NH_DEL:
6279                 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_event->event, fib_event->fnh_info.fib_nh);
6280                 fib_info_put(fib_event->fnh_info.fib_nh->nh_parent);
6281                 break;
6282         }
6283 }
6284
6285 static void mlxsw_sp_router_fib6_event_process(struct mlxsw_sp *mlxsw_sp,
6286                                                struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
6287                                                struct mlxsw_sp_fib_event *fib_event)
6288 {
6289         int err;
6290
6291         mlxsw_sp_span_respin(mlxsw_sp);
6292
6293         switch (fib_event->event) {
6294         case FIB_EVENT_ENTRY_REPLACE:
6295                 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
6296                                                    fib_event->fib6_event.nrt6);
6297                 if (err) {
6298                         mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
6299                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6300                 }
6301                 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
6302                 break;
6303         case FIB_EVENT_ENTRY_APPEND:
6304                 err = mlxsw_sp_router_fib6_append(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
6305                                                   fib_event->fib6_event.nrt6);
6306                 if (err) {
6307                         mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
6308                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6309                 }
6310                 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
6311                 break;
6312         case FIB_EVENT_ENTRY_DEL:
6313                 err = mlxsw_sp_router_fib6_del(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
6314                                                fib_event->fib6_event.nrt6);
6315                 if (err)
6316                         mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
6317                 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
6318                 break;
6319         }
6320 }
6321
6322 static void mlxsw_sp_router_fibmr_event_process(struct mlxsw_sp *mlxsw_sp,
6323                                                 struct mlxsw_sp_fib_event *fib_event)
6324 {
6325         bool replace;
6326         int err;
6327
6328         rtnl_lock();
6329         mutex_lock(&mlxsw_sp->router->lock);
6330         switch (fib_event->event) {
6331         case FIB_EVENT_ENTRY_REPLACE:
6332         case FIB_EVENT_ENTRY_ADD:
6333                 replace = fib_event->event == FIB_EVENT_ENTRY_REPLACE;
6334
6335                 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_event->men_info, replace);
6336                 if (err)
6337                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6338                 mr_cache_put(fib_event->men_info.mfc);
6339                 break;
6340         case FIB_EVENT_ENTRY_DEL:
6341                 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_event->men_info);
6342                 mr_cache_put(fib_event->men_info.mfc);
6343                 break;
6344         case FIB_EVENT_VIF_ADD:
6345                 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp,
6346                                                     &fib_event->ven_info);
6347                 if (err)
6348                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6349                 dev_put(fib_event->ven_info.dev);
6350                 break;
6351         case FIB_EVENT_VIF_DEL:
6352                 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, &fib_event->ven_info);
6353                 dev_put(fib_event->ven_info.dev);
6354                 break;
6355         }
6356         mutex_unlock(&mlxsw_sp->router->lock);
6357         rtnl_unlock();
6358 }
6359
6360 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
6361 {
6362         struct mlxsw_sp_router *router = container_of(work, struct mlxsw_sp_router, fib_event_work);
6363         struct mlxsw_sp_fib_entry_op_ctx *op_ctx = router->ll_op_ctx;
6364         struct mlxsw_sp *mlxsw_sp = router->mlxsw_sp;
6365         struct mlxsw_sp_fib_event *next_fib_event;
6366         struct mlxsw_sp_fib_event *fib_event;
6367         int last_family = AF_UNSPEC;
6368         LIST_HEAD(fib_event_queue);
6369
6370         spin_lock_bh(&router->fib_event_queue_lock);
6371         list_splice_init(&router->fib_event_queue, &fib_event_queue);
6372         spin_unlock_bh(&router->fib_event_queue_lock);
6373
6374         /* Router lock is held here to make sure per-instance
6375          * operation context is not used in between FIB4/6 events
6376          * processing.
6377          */
6378         mutex_lock(&router->lock);
6379         mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
6380         list_for_each_entry_safe(fib_event, next_fib_event,
6381                                  &fib_event_queue, list) {
6382                 /* Check if the next entry in the queue exists and it is
6383                  * of the same type (family and event) as the currect one.
6384                  * In that case it is permitted to do the bulking
6385                  * of multiple FIB entries to a single register write.
6386                  */
6387                 op_ctx->bulk_ok = !list_is_last(&fib_event->list, &fib_event_queue) &&
6388                                   fib_event->family == next_fib_event->family &&
6389                                   fib_event->event == next_fib_event->event;
6390
6391                 /* In case family of this and the previous entry are different, context
6392                  * reinitialization is going to be needed now, indicate that.
6393                  * Note that since last_family is initialized to AF_UNSPEC, this is always
6394                  * going to happen for the first entry processed in the work.
6395                  */
6396                 if (fib_event->family != last_family)
6397                         op_ctx->initialized = false;
6398
6399                 switch (fib_event->family) {
6400                 case AF_INET:
6401                         mlxsw_sp_router_fib4_event_process(mlxsw_sp, op_ctx,
6402                                                            fib_event);
6403                         break;
6404                 case AF_INET6:
6405                         mlxsw_sp_router_fib6_event_process(mlxsw_sp, op_ctx,
6406                                                            fib_event);
6407                         break;
6408                 case RTNL_FAMILY_IP6MR:
6409                 case RTNL_FAMILY_IPMR:
6410                         /* Unlock here as inside FIBMR the lock is taken again
6411                          * under RTNL. The per-instance operation context
6412                          * is not used by FIBMR.
6413                          */
6414                         mutex_unlock(&router->lock);
6415                         mlxsw_sp_router_fibmr_event_process(mlxsw_sp,
6416                                                             fib_event);
6417                         mutex_lock(&router->lock);
6418                         break;
6419                 default:
6420                         WARN_ON_ONCE(1);
6421                 }
6422                 last_family = fib_event->family;
6423                 kfree(fib_event);
6424                 cond_resched();
6425         }
6426         WARN_ON_ONCE(!list_empty(&router->ll_op_ctx->fib_entry_priv_list));
6427         mutex_unlock(&router->lock);
6428 }
6429
6430 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event *fib_event,
6431                                        struct fib_notifier_info *info)
6432 {
6433         struct fib_entry_notifier_info *fen_info;
6434         struct fib_nh_notifier_info *fnh_info;
6435
6436         switch (fib_event->event) {
6437         case FIB_EVENT_ENTRY_REPLACE:
6438         case FIB_EVENT_ENTRY_DEL:
6439                 fen_info = container_of(info, struct fib_entry_notifier_info,
6440                                         info);
6441                 fib_event->fen_info = *fen_info;
6442                 /* Take reference on fib_info to prevent it from being
6443                  * freed while event is queued. Release it afterwards.
6444                  */
6445                 fib_info_hold(fib_event->fen_info.fi);
6446                 break;
6447         case FIB_EVENT_NH_ADD:
6448         case FIB_EVENT_NH_DEL:
6449                 fnh_info = container_of(info, struct fib_nh_notifier_info,
6450                                         info);
6451                 fib_event->fnh_info = *fnh_info;
6452                 fib_info_hold(fib_event->fnh_info.fib_nh->nh_parent);
6453                 break;
6454         }
6455 }
6456
6457 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event *fib_event,
6458                                       struct fib_notifier_info *info)
6459 {
6460         struct fib6_entry_notifier_info *fen6_info;
6461         int err;
6462
6463         switch (fib_event->event) {
6464         case FIB_EVENT_ENTRY_REPLACE:
6465         case FIB_EVENT_ENTRY_APPEND:
6466         case FIB_EVENT_ENTRY_DEL:
6467                 fen6_info = container_of(info, struct fib6_entry_notifier_info,
6468                                          info);
6469                 err = mlxsw_sp_router_fib6_event_init(&fib_event->fib6_event,
6470                                                       fen6_info);
6471                 if (err)
6472                         return err;
6473                 break;
6474         }
6475
6476         return 0;
6477 }
6478
6479 static void
6480 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event *fib_event,
6481                             struct fib_notifier_info *info)
6482 {
6483         switch (fib_event->event) {
6484         case FIB_EVENT_ENTRY_REPLACE:
6485         case FIB_EVENT_ENTRY_ADD:
6486         case FIB_EVENT_ENTRY_DEL:
6487                 memcpy(&fib_event->men_info, info, sizeof(fib_event->men_info));
6488                 mr_cache_hold(fib_event->men_info.mfc);
6489                 break;
6490         case FIB_EVENT_VIF_ADD:
6491         case FIB_EVENT_VIF_DEL:
6492                 memcpy(&fib_event->ven_info, info, sizeof(fib_event->ven_info));
6493                 dev_hold(fib_event->ven_info.dev);
6494                 break;
6495         }
6496 }
6497
6498 static int mlxsw_sp_router_fib_rule_event(unsigned long event,
6499                                           struct fib_notifier_info *info,
6500                                           struct mlxsw_sp *mlxsw_sp)
6501 {
6502         struct netlink_ext_ack *extack = info->extack;
6503         struct fib_rule_notifier_info *fr_info;
6504         struct fib_rule *rule;
6505         int err = 0;
6506
6507         /* nothing to do at the moment */
6508         if (event == FIB_EVENT_RULE_DEL)
6509                 return 0;
6510
6511         if (mlxsw_sp->router->aborted)
6512                 return 0;
6513
6514         fr_info = container_of(info, struct fib_rule_notifier_info, info);
6515         rule = fr_info->rule;
6516
6517         /* Rule only affects locally generated traffic */
6518         if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex)
6519                 return 0;
6520
6521         switch (info->family) {
6522         case AF_INET:
6523                 if (!fib4_rule_default(rule) && !rule->l3mdev)
6524                         err = -EOPNOTSUPP;
6525                 break;
6526         case AF_INET6:
6527                 if (!fib6_rule_default(rule) && !rule->l3mdev)
6528                         err = -EOPNOTSUPP;
6529                 break;
6530         case RTNL_FAMILY_IPMR:
6531                 if (!ipmr_rule_default(rule) && !rule->l3mdev)
6532                         err = -EOPNOTSUPP;
6533                 break;
6534         case RTNL_FAMILY_IP6MR:
6535                 if (!ip6mr_rule_default(rule) && !rule->l3mdev)
6536                         err = -EOPNOTSUPP;
6537                 break;
6538         }
6539
6540         if (err < 0)
6541                 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported");
6542
6543         return err;
6544 }
6545
6546 /* Called with rcu_read_lock() */
6547 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
6548                                      unsigned long event, void *ptr)
6549 {
6550         struct mlxsw_sp_fib_event *fib_event;
6551         struct fib_notifier_info *info = ptr;
6552         struct mlxsw_sp_router *router;
6553         int err;
6554
6555         if ((info->family != AF_INET && info->family != AF_INET6 &&
6556              info->family != RTNL_FAMILY_IPMR &&
6557              info->family != RTNL_FAMILY_IP6MR))
6558                 return NOTIFY_DONE;
6559
6560         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
6561
6562         switch (event) {
6563         case FIB_EVENT_RULE_ADD:
6564         case FIB_EVENT_RULE_DEL:
6565                 err = mlxsw_sp_router_fib_rule_event(event, info,
6566                                                      router->mlxsw_sp);
6567                 return notifier_from_errno(err);
6568         case FIB_EVENT_ENTRY_ADD:
6569         case FIB_EVENT_ENTRY_REPLACE:
6570         case FIB_EVENT_ENTRY_APPEND:
6571                 if (router->aborted) {
6572                         NL_SET_ERR_MSG_MOD(info->extack, "FIB offload was aborted. Not configuring route");
6573                         return notifier_from_errno(-EINVAL);
6574                 }
6575                 if (info->family == AF_INET) {
6576                         struct fib_entry_notifier_info *fen_info = ptr;
6577
6578                         if (fen_info->fi->fib_nh_is_v6) {
6579                                 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported");
6580                                 return notifier_from_errno(-EINVAL);
6581                         }
6582                         if (fen_info->fi->nh) {
6583                                 NL_SET_ERR_MSG_MOD(info->extack, "IPv4 route with nexthop objects is not supported");
6584                                 return notifier_from_errno(-EINVAL);
6585                         }
6586                 } else if (info->family == AF_INET6) {
6587                         struct fib6_entry_notifier_info *fen6_info;
6588
6589                         fen6_info = container_of(info,
6590                                                  struct fib6_entry_notifier_info,
6591                                                  info);
6592                         if (fen6_info->rt->nh) {
6593                                 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 route with nexthop objects is not supported");
6594                                 return notifier_from_errno(-EINVAL);
6595                         }
6596                 }
6597                 break;
6598         }
6599
6600         fib_event = kzalloc(sizeof(*fib_event), GFP_ATOMIC);
6601         if (!fib_event)
6602                 return NOTIFY_BAD;
6603
6604         fib_event->mlxsw_sp = router->mlxsw_sp;
6605         fib_event->event = event;
6606         fib_event->family = info->family;
6607
6608         switch (info->family) {
6609         case AF_INET:
6610                 mlxsw_sp_router_fib4_event(fib_event, info);
6611                 break;
6612         case AF_INET6:
6613                 err = mlxsw_sp_router_fib6_event(fib_event, info);
6614                 if (err)
6615                         goto err_fib_event;
6616                 break;
6617         case RTNL_FAMILY_IP6MR:
6618         case RTNL_FAMILY_IPMR:
6619                 mlxsw_sp_router_fibmr_event(fib_event, info);
6620                 break;
6621         }
6622
6623         /* Enqueue the event and trigger the work */
6624         spin_lock_bh(&router->fib_event_queue_lock);
6625         list_add_tail(&fib_event->list, &router->fib_event_queue);
6626         spin_unlock_bh(&router->fib_event_queue_lock);
6627         mlxsw_core_schedule_work(&router->fib_event_work);
6628
6629         return NOTIFY_DONE;
6630
6631 err_fib_event:
6632         kfree(fib_event);
6633         return NOTIFY_BAD;
6634 }
6635
6636 static struct mlxsw_sp_rif *
6637 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
6638                          const struct net_device *dev)
6639 {
6640         int i;
6641
6642         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
6643                 if (mlxsw_sp->router->rifs[i] &&
6644                     mlxsw_sp->router->rifs[i]->dev == dev)
6645                         return mlxsw_sp->router->rifs[i];
6646
6647         return NULL;
6648 }
6649
6650 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp,
6651                          const struct net_device *dev)
6652 {
6653         struct mlxsw_sp_rif *rif;
6654
6655         mutex_lock(&mlxsw_sp->router->lock);
6656         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6657         mutex_unlock(&mlxsw_sp->router->lock);
6658
6659         return rif;
6660 }
6661
6662 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev)
6663 {
6664         struct mlxsw_sp_rif *rif;
6665         u16 vid = 0;
6666
6667         mutex_lock(&mlxsw_sp->router->lock);
6668         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6669         if (!rif)
6670                 goto out;
6671
6672         /* We only return the VID for VLAN RIFs. Otherwise we return an
6673          * invalid value (0).
6674          */
6675         if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN)
6676                 goto out;
6677
6678         vid = mlxsw_sp_fid_8021q_vid(rif->fid);
6679
6680 out:
6681         mutex_unlock(&mlxsw_sp->router->lock);
6682         return vid;
6683 }
6684
6685 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
6686 {
6687         char ritr_pl[MLXSW_REG_RITR_LEN];
6688         int err;
6689
6690         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
6691         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6692         if (err)
6693                 return err;
6694
6695         mlxsw_reg_ritr_enable_set(ritr_pl, false);
6696         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6697 }
6698
6699 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
6700                                           struct mlxsw_sp_rif *rif)
6701 {
6702         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
6703         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
6704         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
6705 }
6706
6707 static bool
6708 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
6709                            unsigned long event)
6710 {
6711         struct inet6_dev *inet6_dev;
6712         bool addr_list_empty = true;
6713         struct in_device *idev;
6714
6715         switch (event) {
6716         case NETDEV_UP:
6717                 return rif == NULL;
6718         case NETDEV_DOWN:
6719                 rcu_read_lock();
6720                 idev = __in_dev_get_rcu(dev);
6721                 if (idev && idev->ifa_list)
6722                         addr_list_empty = false;
6723
6724                 inet6_dev = __in6_dev_get(dev);
6725                 if (addr_list_empty && inet6_dev &&
6726                     !list_empty(&inet6_dev->addr_list))
6727                         addr_list_empty = false;
6728                 rcu_read_unlock();
6729
6730                 /* macvlans do not have a RIF, but rather piggy back on the
6731                  * RIF of their lower device.
6732                  */
6733                 if (netif_is_macvlan(dev) && addr_list_empty)
6734                         return true;
6735
6736                 if (rif && addr_list_empty &&
6737                     !netif_is_l3_slave(rif->dev))
6738                         return true;
6739                 /* It is possible we already removed the RIF ourselves
6740                  * if it was assigned to a netdev that is now a bridge
6741                  * or LAG slave.
6742                  */
6743                 return false;
6744         }
6745
6746         return false;
6747 }
6748
6749 static enum mlxsw_sp_rif_type
6750 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
6751                       const struct net_device *dev)
6752 {
6753         enum mlxsw_sp_fid_type type;
6754
6755         if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
6756                 return MLXSW_SP_RIF_TYPE_IPIP_LB;
6757
6758         /* Otherwise RIF type is derived from the type of the underlying FID. */
6759         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
6760                 type = MLXSW_SP_FID_TYPE_8021Q;
6761         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
6762                 type = MLXSW_SP_FID_TYPE_8021Q;
6763         else if (netif_is_bridge_master(dev))
6764                 type = MLXSW_SP_FID_TYPE_8021D;
6765         else
6766                 type = MLXSW_SP_FID_TYPE_RFID;
6767
6768         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
6769 }
6770
6771 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
6772 {
6773         int i;
6774
6775         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
6776                 if (!mlxsw_sp->router->rifs[i]) {
6777                         *p_rif_index = i;
6778                         return 0;
6779                 }
6780         }
6781
6782         return -ENOBUFS;
6783 }
6784
6785 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
6786                                                u16 vr_id,
6787                                                struct net_device *l3_dev)
6788 {
6789         struct mlxsw_sp_rif *rif;
6790
6791         rif = kzalloc(rif_size, GFP_KERNEL);
6792         if (!rif)
6793                 return NULL;
6794
6795         INIT_LIST_HEAD(&rif->nexthop_list);
6796         INIT_LIST_HEAD(&rif->neigh_list);
6797         if (l3_dev) {
6798                 ether_addr_copy(rif->addr, l3_dev->dev_addr);
6799                 rif->mtu = l3_dev->mtu;
6800                 rif->dev = l3_dev;
6801         }
6802         rif->vr_id = vr_id;
6803         rif->rif_index = rif_index;
6804
6805         return rif;
6806 }
6807
6808 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
6809                                            u16 rif_index)
6810 {
6811         return mlxsw_sp->router->rifs[rif_index];
6812 }
6813
6814 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
6815 {
6816         return rif->rif_index;
6817 }
6818
6819 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6820 {
6821         return lb_rif->common.rif_index;
6822 }
6823
6824 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6825 {
6826         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev);
6827         struct mlxsw_sp_vr *ul_vr;
6828
6829         ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL);
6830         if (WARN_ON(IS_ERR(ul_vr)))
6831                 return 0;
6832
6833         return ul_vr->id;
6834 }
6835
6836 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6837 {
6838         return lb_rif->ul_rif_id;
6839 }
6840
6841 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
6842 {
6843         return rif->dev->ifindex;
6844 }
6845
6846 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif)
6847 {
6848         return rif->dev;
6849 }
6850
6851 static struct mlxsw_sp_rif *
6852 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
6853                     const struct mlxsw_sp_rif_params *params,
6854                     struct netlink_ext_ack *extack)
6855 {
6856         u32 tb_id = l3mdev_fib_table(params->dev);
6857         const struct mlxsw_sp_rif_ops *ops;
6858         struct mlxsw_sp_fid *fid = NULL;
6859         enum mlxsw_sp_rif_type type;
6860         struct mlxsw_sp_rif *rif;
6861         struct mlxsw_sp_vr *vr;
6862         u16 rif_index;
6863         int i, err;
6864
6865         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
6866         ops = mlxsw_sp->rif_ops_arr[type];
6867
6868         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack);
6869         if (IS_ERR(vr))
6870                 return ERR_CAST(vr);
6871         vr->rif_count++;
6872
6873         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
6874         if (err) {
6875                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
6876                 goto err_rif_index_alloc;
6877         }
6878
6879         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
6880         if (!rif) {
6881                 err = -ENOMEM;
6882                 goto err_rif_alloc;
6883         }
6884         dev_hold(rif->dev);
6885         mlxsw_sp->router->rifs[rif_index] = rif;
6886         rif->mlxsw_sp = mlxsw_sp;
6887         rif->ops = ops;
6888
6889         if (ops->fid_get) {
6890                 fid = ops->fid_get(rif, extack);
6891                 if (IS_ERR(fid)) {
6892                         err = PTR_ERR(fid);
6893                         goto err_fid_get;
6894                 }
6895                 rif->fid = fid;
6896         }
6897
6898         if (ops->setup)
6899                 ops->setup(rif, params);
6900
6901         err = ops->configure(rif);
6902         if (err)
6903                 goto err_configure;
6904
6905         for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
6906                 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif);
6907                 if (err)
6908                         goto err_mr_rif_add;
6909         }
6910
6911         mlxsw_sp_rif_counters_alloc(rif);
6912
6913         return rif;
6914
6915 err_mr_rif_add:
6916         for (i--; i >= 0; i--)
6917                 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
6918         ops->deconfigure(rif);
6919 err_configure:
6920         if (fid)
6921                 mlxsw_sp_fid_put(fid);
6922 err_fid_get:
6923         mlxsw_sp->router->rifs[rif_index] = NULL;
6924         dev_put(rif->dev);
6925         kfree(rif);
6926 err_rif_alloc:
6927 err_rif_index_alloc:
6928         vr->rif_count--;
6929         mlxsw_sp_vr_put(mlxsw_sp, vr);
6930         return ERR_PTR(err);
6931 }
6932
6933 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
6934 {
6935         const struct mlxsw_sp_rif_ops *ops = rif->ops;
6936         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
6937         struct mlxsw_sp_fid *fid = rif->fid;
6938         struct mlxsw_sp_vr *vr;
6939         int i;
6940
6941         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
6942         vr = &mlxsw_sp->router->vrs[rif->vr_id];
6943
6944         mlxsw_sp_rif_counters_free(rif);
6945         for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
6946                 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
6947         ops->deconfigure(rif);
6948         if (fid)
6949                 /* Loopback RIFs are not associated with a FID. */
6950                 mlxsw_sp_fid_put(fid);
6951         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
6952         dev_put(rif->dev);
6953         kfree(rif);
6954         vr->rif_count--;
6955         mlxsw_sp_vr_put(mlxsw_sp, vr);
6956 }
6957
6958 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp,
6959                                  struct net_device *dev)
6960 {
6961         struct mlxsw_sp_rif *rif;
6962
6963         mutex_lock(&mlxsw_sp->router->lock);
6964         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6965         if (!rif)
6966                 goto out;
6967         mlxsw_sp_rif_destroy(rif);
6968 out:
6969         mutex_unlock(&mlxsw_sp->router->lock);
6970 }
6971
6972 static void
6973 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
6974                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
6975 {
6976         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
6977
6978         params->vid = mlxsw_sp_port_vlan->vid;
6979         params->lag = mlxsw_sp_port->lagged;
6980         if (params->lag)
6981                 params->lag_id = mlxsw_sp_port->lag_id;
6982         else
6983                 params->system_port = mlxsw_sp_port->local_port;
6984 }
6985
6986 static struct mlxsw_sp_rif_subport *
6987 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
6988 {
6989         return container_of(rif, struct mlxsw_sp_rif_subport, common);
6990 }
6991
6992 static struct mlxsw_sp_rif *
6993 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp,
6994                          const struct mlxsw_sp_rif_params *params,
6995                          struct netlink_ext_ack *extack)
6996 {
6997         struct mlxsw_sp_rif_subport *rif_subport;
6998         struct mlxsw_sp_rif *rif;
6999
7000         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev);
7001         if (!rif)
7002                 return mlxsw_sp_rif_create(mlxsw_sp, params, extack);
7003
7004         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7005         refcount_inc(&rif_subport->ref_count);
7006         return rif;
7007 }
7008
7009 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif)
7010 {
7011         struct mlxsw_sp_rif_subport *rif_subport;
7012
7013         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7014         if (!refcount_dec_and_test(&rif_subport->ref_count))
7015                 return;
7016
7017         mlxsw_sp_rif_destroy(rif);
7018 }
7019
7020 static int
7021 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
7022                                struct net_device *l3_dev,
7023                                struct netlink_ext_ack *extack)
7024 {
7025         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
7026         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
7027         struct mlxsw_sp_rif_params params = {
7028                 .dev = l3_dev,
7029         };
7030         u16 vid = mlxsw_sp_port_vlan->vid;
7031         struct mlxsw_sp_rif *rif;
7032         struct mlxsw_sp_fid *fid;
7033         int err;
7034
7035         mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
7036         rif = mlxsw_sp_rif_subport_get(mlxsw_sp, &params, extack);
7037         if (IS_ERR(rif))
7038                 return PTR_ERR(rif);
7039
7040         /* FID was already created, just take a reference */
7041         fid = rif->ops->fid_get(rif, extack);
7042         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
7043         if (err)
7044                 goto err_fid_port_vid_map;
7045
7046         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
7047         if (err)
7048                 goto err_port_vid_learning_set;
7049
7050         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
7051                                         BR_STATE_FORWARDING);
7052         if (err)
7053                 goto err_port_vid_stp_set;
7054
7055         mlxsw_sp_port_vlan->fid = fid;
7056
7057         return 0;
7058
7059 err_port_vid_stp_set:
7060         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
7061 err_port_vid_learning_set:
7062         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
7063 err_fid_port_vid_map:
7064         mlxsw_sp_fid_put(fid);
7065         mlxsw_sp_rif_subport_put(rif);
7066         return err;
7067 }
7068
7069 static void
7070 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
7071 {
7072         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
7073         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
7074         struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid);
7075         u16 vid = mlxsw_sp_port_vlan->vid;
7076
7077         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
7078                 return;
7079
7080         mlxsw_sp_port_vlan->fid = NULL;
7081         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
7082         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
7083         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
7084         mlxsw_sp_fid_put(fid);
7085         mlxsw_sp_rif_subport_put(rif);
7086 }
7087
7088 void
7089 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
7090 {
7091         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
7092
7093         mutex_lock(&mlxsw_sp->router->lock);
7094         __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
7095         mutex_unlock(&mlxsw_sp->router->lock);
7096 }
7097
7098 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
7099                                              struct net_device *port_dev,
7100                                              unsigned long event, u16 vid,
7101                                              struct netlink_ext_ack *extack)
7102 {
7103         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
7104         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
7105
7106         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
7107         if (WARN_ON(!mlxsw_sp_port_vlan))
7108                 return -EINVAL;
7109
7110         switch (event) {
7111         case NETDEV_UP:
7112                 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
7113                                                       l3_dev, extack);
7114         case NETDEV_DOWN:
7115                 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
7116                 break;
7117         }
7118
7119         return 0;
7120 }
7121
7122 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
7123                                         unsigned long event,
7124                                         struct netlink_ext_ack *extack)
7125 {
7126         if (netif_is_bridge_port(port_dev) ||
7127             netif_is_lag_port(port_dev) ||
7128             netif_is_ovs_port(port_dev))
7129                 return 0;
7130
7131         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event,
7132                                                  MLXSW_SP_DEFAULT_VID, extack);
7133 }
7134
7135 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
7136                                          struct net_device *lag_dev,
7137                                          unsigned long event, u16 vid,
7138                                          struct netlink_ext_ack *extack)
7139 {
7140         struct net_device *port_dev;
7141         struct list_head *iter;
7142         int err;
7143
7144         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
7145                 if (mlxsw_sp_port_dev_check(port_dev)) {
7146                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
7147                                                                 port_dev,
7148                                                                 event, vid,
7149                                                                 extack);
7150                         if (err)
7151                                 return err;
7152                 }
7153         }
7154
7155         return 0;
7156 }
7157
7158 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
7159                                        unsigned long event,
7160                                        struct netlink_ext_ack *extack)
7161 {
7162         if (netif_is_bridge_port(lag_dev))
7163                 return 0;
7164
7165         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event,
7166                                              MLXSW_SP_DEFAULT_VID, extack);
7167 }
7168
7169 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp,
7170                                           struct net_device *l3_dev,
7171                                           unsigned long event,
7172                                           struct netlink_ext_ack *extack)
7173 {
7174         struct mlxsw_sp_rif_params params = {
7175                 .dev = l3_dev,
7176         };
7177         struct mlxsw_sp_rif *rif;
7178
7179         switch (event) {
7180         case NETDEV_UP:
7181                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params, extack);
7182                 if (IS_ERR(rif))
7183                         return PTR_ERR(rif);
7184                 break;
7185         case NETDEV_DOWN:
7186                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
7187                 mlxsw_sp_rif_destroy(rif);
7188                 break;
7189         }
7190
7191         return 0;
7192 }
7193
7194 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp,
7195                                         struct net_device *vlan_dev,
7196                                         unsigned long event,
7197                                         struct netlink_ext_ack *extack)
7198 {
7199         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
7200         u16 vid = vlan_dev_vlan_id(vlan_dev);
7201
7202         if (netif_is_bridge_port(vlan_dev))
7203                 return 0;
7204
7205         if (mlxsw_sp_port_dev_check(real_dev))
7206                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
7207                                                          event, vid, extack);
7208         else if (netif_is_lag_master(real_dev))
7209                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
7210                                                      vid, extack);
7211         else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
7212                 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event,
7213                                                       extack);
7214
7215         return 0;
7216 }
7217
7218 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac)
7219 {
7220         u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 };
7221         u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
7222
7223         return ether_addr_equal_masked(mac, vrrp4, mask);
7224 }
7225
7226 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac)
7227 {
7228         u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 };
7229         u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
7230
7231         return ether_addr_equal_masked(mac, vrrp6, mask);
7232 }
7233
7234 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
7235                                 const u8 *mac, bool adding)
7236 {
7237         char ritr_pl[MLXSW_REG_RITR_LEN];
7238         u8 vrrp_id = adding ? mac[5] : 0;
7239         int err;
7240
7241         if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) &&
7242             !mlxsw_sp_rif_macvlan_is_vrrp6(mac))
7243                 return 0;
7244
7245         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
7246         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7247         if (err)
7248                 return err;
7249
7250         if (mlxsw_sp_rif_macvlan_is_vrrp4(mac))
7251                 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id);
7252         else
7253                 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id);
7254
7255         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7256 }
7257
7258 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp,
7259                                     const struct net_device *macvlan_dev,
7260                                     struct netlink_ext_ack *extack)
7261 {
7262         struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
7263         struct mlxsw_sp_rif *rif;
7264         int err;
7265
7266         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
7267         if (!rif) {
7268                 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces");
7269                 return -EOPNOTSUPP;
7270         }
7271
7272         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
7273                                   mlxsw_sp_fid_index(rif->fid), true);
7274         if (err)
7275                 return err;
7276
7277         err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index,
7278                                    macvlan_dev->dev_addr, true);
7279         if (err)
7280                 goto err_rif_vrrp_add;
7281
7282         /* Make sure the bridge driver does not have this MAC pointing at
7283          * some other port.
7284          */
7285         if (rif->ops->fdb_del)
7286                 rif->ops->fdb_del(rif, macvlan_dev->dev_addr);
7287
7288         return 0;
7289
7290 err_rif_vrrp_add:
7291         mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
7292                             mlxsw_sp_fid_index(rif->fid), false);
7293         return err;
7294 }
7295
7296 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
7297                                        const struct net_device *macvlan_dev)
7298 {
7299         struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
7300         struct mlxsw_sp_rif *rif;
7301
7302         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
7303         /* If we do not have a RIF, then we already took care of
7304          * removing the macvlan's MAC during RIF deletion.
7305          */
7306         if (!rif)
7307                 return;
7308         mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr,
7309                              false);
7310         mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
7311                             mlxsw_sp_fid_index(rif->fid), false);
7312 }
7313
7314 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
7315                               const struct net_device *macvlan_dev)
7316 {
7317         mutex_lock(&mlxsw_sp->router->lock);
7318         __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
7319         mutex_unlock(&mlxsw_sp->router->lock);
7320 }
7321
7322 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp,
7323                                            struct net_device *macvlan_dev,
7324                                            unsigned long event,
7325                                            struct netlink_ext_ack *extack)
7326 {
7327         switch (event) {
7328         case NETDEV_UP:
7329                 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack);
7330         case NETDEV_DOWN:
7331                 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
7332                 break;
7333         }
7334
7335         return 0;
7336 }
7337
7338 static int mlxsw_sp_router_port_check_rif_addr(struct mlxsw_sp *mlxsw_sp,
7339                                                struct net_device *dev,
7340                                                const unsigned char *dev_addr,
7341                                                struct netlink_ext_ack *extack)
7342 {
7343         struct mlxsw_sp_rif *rif;
7344         int i;
7345
7346         /* A RIF is not created for macvlan netdevs. Their MAC is used to
7347          * populate the FDB
7348          */
7349         if (netif_is_macvlan(dev) || netif_is_l3_master(dev))
7350                 return 0;
7351
7352         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
7353                 rif = mlxsw_sp->router->rifs[i];
7354                 if (rif && rif->ops &&
7355                     rif->ops->type == MLXSW_SP_RIF_TYPE_IPIP_LB)
7356                         continue;
7357                 if (rif && rif->dev && rif->dev != dev &&
7358                     !ether_addr_equal_masked(rif->dev->dev_addr, dev_addr,
7359                                              mlxsw_sp->mac_mask)) {
7360                         NL_SET_ERR_MSG_MOD(extack, "All router interface MAC addresses must have the same prefix");
7361                         return -EINVAL;
7362                 }
7363         }
7364
7365         return 0;
7366 }
7367
7368 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp,
7369                                      struct net_device *dev,
7370                                      unsigned long event,
7371                                      struct netlink_ext_ack *extack)
7372 {
7373         if (mlxsw_sp_port_dev_check(dev))
7374                 return mlxsw_sp_inetaddr_port_event(dev, event, extack);
7375         else if (netif_is_lag_master(dev))
7376                 return mlxsw_sp_inetaddr_lag_event(dev, event, extack);
7377         else if (netif_is_bridge_master(dev))
7378                 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event,
7379                                                       extack);
7380         else if (is_vlan_dev(dev))
7381                 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event,
7382                                                     extack);
7383         else if (netif_is_macvlan(dev))
7384                 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event,
7385                                                        extack);
7386         else
7387                 return 0;
7388 }
7389
7390 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb,
7391                                    unsigned long event, void *ptr)
7392 {
7393         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
7394         struct net_device *dev = ifa->ifa_dev->dev;
7395         struct mlxsw_sp_router *router;
7396         struct mlxsw_sp_rif *rif;
7397         int err = 0;
7398
7399         /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */
7400         if (event == NETDEV_UP)
7401                 return NOTIFY_DONE;
7402
7403         router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb);
7404         mutex_lock(&router->lock);
7405         rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev);
7406         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7407                 goto out;
7408
7409         err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL);
7410 out:
7411         mutex_unlock(&router->lock);
7412         return notifier_from_errno(err);
7413 }
7414
7415 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused,
7416                                   unsigned long event, void *ptr)
7417 {
7418         struct in_validator_info *ivi = (struct in_validator_info *) ptr;
7419         struct net_device *dev = ivi->ivi_dev->dev;
7420         struct mlxsw_sp *mlxsw_sp;
7421         struct mlxsw_sp_rif *rif;
7422         int err = 0;
7423
7424         mlxsw_sp = mlxsw_sp_lower_get(dev);
7425         if (!mlxsw_sp)
7426                 return NOTIFY_DONE;
7427
7428         mutex_lock(&mlxsw_sp->router->lock);
7429         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7430         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7431                 goto out;
7432
7433         err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr,
7434                                                   ivi->extack);
7435         if (err)
7436                 goto out;
7437
7438         err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack);
7439 out:
7440         mutex_unlock(&mlxsw_sp->router->lock);
7441         return notifier_from_errno(err);
7442 }
7443
7444 struct mlxsw_sp_inet6addr_event_work {
7445         struct work_struct work;
7446         struct mlxsw_sp *mlxsw_sp;
7447         struct net_device *dev;
7448         unsigned long event;
7449 };
7450
7451 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
7452 {
7453         struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
7454                 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
7455         struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp;
7456         struct net_device *dev = inet6addr_work->dev;
7457         unsigned long event = inet6addr_work->event;
7458         struct mlxsw_sp_rif *rif;
7459
7460         rtnl_lock();
7461         mutex_lock(&mlxsw_sp->router->lock);
7462
7463         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7464         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7465                 goto out;
7466
7467         __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL);
7468 out:
7469         mutex_unlock(&mlxsw_sp->router->lock);
7470         rtnl_unlock();
7471         dev_put(dev);
7472         kfree(inet6addr_work);
7473 }
7474
7475 /* Called with rcu_read_lock() */
7476 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb,
7477                                     unsigned long event, void *ptr)
7478 {
7479         struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
7480         struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
7481         struct net_device *dev = if6->idev->dev;
7482         struct mlxsw_sp_router *router;
7483
7484         /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */
7485         if (event == NETDEV_UP)
7486                 return NOTIFY_DONE;
7487
7488         inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
7489         if (!inet6addr_work)
7490                 return NOTIFY_BAD;
7491
7492         router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb);
7493         INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
7494         inet6addr_work->mlxsw_sp = router->mlxsw_sp;
7495         inet6addr_work->dev = dev;
7496         inet6addr_work->event = event;
7497         dev_hold(dev);
7498         mlxsw_core_schedule_work(&inet6addr_work->work);
7499
7500         return NOTIFY_DONE;
7501 }
7502
7503 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused,
7504                                    unsigned long event, void *ptr)
7505 {
7506         struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr;
7507         struct net_device *dev = i6vi->i6vi_dev->dev;
7508         struct mlxsw_sp *mlxsw_sp;
7509         struct mlxsw_sp_rif *rif;
7510         int err = 0;
7511
7512         mlxsw_sp = mlxsw_sp_lower_get(dev);
7513         if (!mlxsw_sp)
7514                 return NOTIFY_DONE;
7515
7516         mutex_lock(&mlxsw_sp->router->lock);
7517         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7518         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7519                 goto out;
7520
7521         err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr,
7522                                                   i6vi->extack);
7523         if (err)
7524                 goto out;
7525
7526         err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack);
7527 out:
7528         mutex_unlock(&mlxsw_sp->router->lock);
7529         return notifier_from_errno(err);
7530 }
7531
7532 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
7533                              const char *mac, int mtu)
7534 {
7535         char ritr_pl[MLXSW_REG_RITR_LEN];
7536         int err;
7537
7538         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
7539         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7540         if (err)
7541                 return err;
7542
7543         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
7544         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
7545         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
7546         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7547 }
7548
7549 static int
7550 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp,
7551                                   struct mlxsw_sp_rif *rif)
7552 {
7553         struct net_device *dev = rif->dev;
7554         u16 fid_index;
7555         int err;
7556
7557         fid_index = mlxsw_sp_fid_index(rif->fid);
7558
7559         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
7560         if (err)
7561                 return err;
7562
7563         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
7564                                 dev->mtu);
7565         if (err)
7566                 goto err_rif_edit;
7567
7568         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
7569         if (err)
7570                 goto err_rif_fdb_op;
7571
7572         if (rif->mtu != dev->mtu) {
7573                 struct mlxsw_sp_vr *vr;
7574                 int i;
7575
7576                 /* The RIF is relevant only to its mr_table instance, as unlike
7577                  * unicast routing, in multicast routing a RIF cannot be shared
7578                  * between several multicast routing tables.
7579                  */
7580                 vr = &mlxsw_sp->router->vrs[rif->vr_id];
7581                 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
7582                         mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i],
7583                                                    rif, dev->mtu);
7584         }
7585
7586         ether_addr_copy(rif->addr, dev->dev_addr);
7587         rif->mtu = dev->mtu;
7588
7589         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
7590
7591         return 0;
7592
7593 err_rif_fdb_op:
7594         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
7595 err_rif_edit:
7596         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
7597         return err;
7598 }
7599
7600 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif,
7601                             struct netdev_notifier_pre_changeaddr_info *info)
7602 {
7603         struct netlink_ext_ack *extack;
7604
7605         extack = netdev_notifier_info_to_extack(&info->info);
7606         return mlxsw_sp_router_port_check_rif_addr(rif->mlxsw_sp, rif->dev,
7607                                                    info->dev_addr, extack);
7608 }
7609
7610 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev,
7611                                          unsigned long event, void *ptr)
7612 {
7613         struct mlxsw_sp *mlxsw_sp;
7614         struct mlxsw_sp_rif *rif;
7615         int err = 0;
7616
7617         mlxsw_sp = mlxsw_sp_lower_get(dev);
7618         if (!mlxsw_sp)
7619                 return 0;
7620
7621         mutex_lock(&mlxsw_sp->router->lock);
7622         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7623         if (!rif)
7624                 goto out;
7625
7626         switch (event) {
7627         case NETDEV_CHANGEMTU:
7628         case NETDEV_CHANGEADDR:
7629                 err = mlxsw_sp_router_port_change_event(mlxsw_sp, rif);
7630                 break;
7631         case NETDEV_PRE_CHANGEADDR:
7632                 err = mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr);
7633                 break;
7634         }
7635
7636 out:
7637         mutex_unlock(&mlxsw_sp->router->lock);
7638         return err;
7639 }
7640
7641 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
7642                                   struct net_device *l3_dev,
7643                                   struct netlink_ext_ack *extack)
7644 {
7645         struct mlxsw_sp_rif *rif;
7646
7647         /* If netdev is already associated with a RIF, then we need to
7648          * destroy it and create a new one with the new virtual router ID.
7649          */
7650         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
7651         if (rif)
7652                 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN,
7653                                           extack);
7654
7655         return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack);
7656 }
7657
7658 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
7659                                     struct net_device *l3_dev)
7660 {
7661         struct mlxsw_sp_rif *rif;
7662
7663         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
7664         if (!rif)
7665                 return;
7666         __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL);
7667 }
7668
7669 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
7670                                  struct netdev_notifier_changeupper_info *info)
7671 {
7672         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
7673         int err = 0;
7674
7675         /* We do not create a RIF for a macvlan, but only use it to
7676          * direct more MAC addresses to the router.
7677          */
7678         if (!mlxsw_sp || netif_is_macvlan(l3_dev))
7679                 return 0;
7680
7681         mutex_lock(&mlxsw_sp->router->lock);
7682         switch (event) {
7683         case NETDEV_PRECHANGEUPPER:
7684                 break;
7685         case NETDEV_CHANGEUPPER:
7686                 if (info->linking) {
7687                         struct netlink_ext_ack *extack;
7688
7689                         extack = netdev_notifier_info_to_extack(&info->info);
7690                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack);
7691                 } else {
7692                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
7693                 }
7694                 break;
7695         }
7696         mutex_unlock(&mlxsw_sp->router->lock);
7697
7698         return err;
7699 }
7700
7701 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev,
7702                                         struct netdev_nested_priv *priv)
7703 {
7704         struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data;
7705
7706         if (!netif_is_macvlan(dev))
7707                 return 0;
7708
7709         return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
7710                                    mlxsw_sp_fid_index(rif->fid), false);
7711 }
7712
7713 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif)
7714 {
7715         struct netdev_nested_priv priv = {
7716                 .data = (void *)rif,
7717         };
7718
7719         if (!netif_is_macvlan_port(rif->dev))
7720                 return 0;
7721
7722         netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n");
7723         return netdev_walk_all_upper_dev_rcu(rif->dev,
7724                                              __mlxsw_sp_rif_macvlan_flush, &priv);
7725 }
7726
7727 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
7728                                        const struct mlxsw_sp_rif_params *params)
7729 {
7730         struct mlxsw_sp_rif_subport *rif_subport;
7731
7732         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7733         refcount_set(&rif_subport->ref_count, 1);
7734         rif_subport->vid = params->vid;
7735         rif_subport->lag = params->lag;
7736         if (params->lag)
7737                 rif_subport->lag_id = params->lag_id;
7738         else
7739                 rif_subport->system_port = params->system_port;
7740 }
7741
7742 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
7743 {
7744         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7745         struct mlxsw_sp_rif_subport *rif_subport;
7746         char ritr_pl[MLXSW_REG_RITR_LEN];
7747
7748         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7749         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
7750                             rif->rif_index, rif->vr_id, rif->dev->mtu);
7751         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
7752         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
7753                                   rif_subport->lag ? rif_subport->lag_id :
7754                                                      rif_subport->system_port,
7755                                   rif_subport->vid);
7756
7757         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7758 }
7759
7760 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
7761 {
7762         int err;
7763
7764         err = mlxsw_sp_rif_subport_op(rif, true);
7765         if (err)
7766                 return err;
7767
7768         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7769                                   mlxsw_sp_fid_index(rif->fid), true);
7770         if (err)
7771                 goto err_rif_fdb_op;
7772
7773         mlxsw_sp_fid_rif_set(rif->fid, rif);
7774         return 0;
7775
7776 err_rif_fdb_op:
7777         mlxsw_sp_rif_subport_op(rif, false);
7778         return err;
7779 }
7780
7781 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
7782 {
7783         struct mlxsw_sp_fid *fid = rif->fid;
7784
7785         mlxsw_sp_fid_rif_set(fid, NULL);
7786         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7787                             mlxsw_sp_fid_index(fid), false);
7788         mlxsw_sp_rif_macvlan_flush(rif);
7789         mlxsw_sp_rif_subport_op(rif, false);
7790 }
7791
7792 static struct mlxsw_sp_fid *
7793 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif,
7794                              struct netlink_ext_ack *extack)
7795 {
7796         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
7797 }
7798
7799 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
7800         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
7801         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
7802         .setup                  = mlxsw_sp_rif_subport_setup,
7803         .configure              = mlxsw_sp_rif_subport_configure,
7804         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
7805         .fid_get                = mlxsw_sp_rif_subport_fid_get,
7806 };
7807
7808 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
7809                                     enum mlxsw_reg_ritr_if_type type,
7810                                     u16 vid_fid, bool enable)
7811 {
7812         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7813         char ritr_pl[MLXSW_REG_RITR_LEN];
7814
7815         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
7816                             rif->dev->mtu);
7817         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
7818         mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
7819
7820         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7821 }
7822
7823 u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
7824 {
7825         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
7826 }
7827
7828 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
7829 {
7830         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7831         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
7832         int err;
7833
7834         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
7835                                        true);
7836         if (err)
7837                 return err;
7838
7839         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7840                                      mlxsw_sp_router_port(mlxsw_sp), true);
7841         if (err)
7842                 goto err_fid_mc_flood_set;
7843
7844         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7845                                      mlxsw_sp_router_port(mlxsw_sp), true);
7846         if (err)
7847                 goto err_fid_bc_flood_set;
7848
7849         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7850                                   mlxsw_sp_fid_index(rif->fid), true);
7851         if (err)
7852                 goto err_rif_fdb_op;
7853
7854         mlxsw_sp_fid_rif_set(rif->fid, rif);
7855         return 0;
7856
7857 err_rif_fdb_op:
7858         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7859                                mlxsw_sp_router_port(mlxsw_sp), false);
7860 err_fid_bc_flood_set:
7861         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7862                                mlxsw_sp_router_port(mlxsw_sp), false);
7863 err_fid_mc_flood_set:
7864         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
7865         return err;
7866 }
7867
7868 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
7869 {
7870         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
7871         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7872         struct mlxsw_sp_fid *fid = rif->fid;
7873
7874         mlxsw_sp_fid_rif_set(fid, NULL);
7875         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7876                             mlxsw_sp_fid_index(fid), false);
7877         mlxsw_sp_rif_macvlan_flush(rif);
7878         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7879                                mlxsw_sp_router_port(mlxsw_sp), false);
7880         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7881                                mlxsw_sp_router_port(mlxsw_sp), false);
7882         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
7883 }
7884
7885 static struct mlxsw_sp_fid *
7886 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif,
7887                          struct netlink_ext_ack *extack)
7888 {
7889         return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
7890 }
7891
7892 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
7893 {
7894         struct switchdev_notifier_fdb_info info;
7895         struct net_device *dev;
7896
7897         dev = br_fdb_find_port(rif->dev, mac, 0);
7898         if (!dev)
7899                 return;
7900
7901         info.addr = mac;
7902         info.vid = 0;
7903         call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
7904                                  NULL);
7905 }
7906
7907 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
7908         .type                   = MLXSW_SP_RIF_TYPE_FID,
7909         .rif_size               = sizeof(struct mlxsw_sp_rif),
7910         .configure              = mlxsw_sp_rif_fid_configure,
7911         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
7912         .fid_get                = mlxsw_sp_rif_fid_fid_get,
7913         .fdb_del                = mlxsw_sp_rif_fid_fdb_del,
7914 };
7915
7916 static struct mlxsw_sp_fid *
7917 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif,
7918                           struct netlink_ext_ack *extack)
7919 {
7920         struct net_device *br_dev;
7921         u16 vid;
7922         int err;
7923
7924         if (is_vlan_dev(rif->dev)) {
7925                 vid = vlan_dev_vlan_id(rif->dev);
7926                 br_dev = vlan_dev_real_dev(rif->dev);
7927                 if (WARN_ON(!netif_is_bridge_master(br_dev)))
7928                         return ERR_PTR(-EINVAL);
7929         } else {
7930                 err = br_vlan_get_pvid(rif->dev, &vid);
7931                 if (err < 0 || !vid) {
7932                         NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID");
7933                         return ERR_PTR(-EINVAL);
7934                 }
7935         }
7936
7937         return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
7938 }
7939
7940 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
7941 {
7942         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
7943         struct switchdev_notifier_fdb_info info;
7944         struct net_device *br_dev;
7945         struct net_device *dev;
7946
7947         br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev;
7948         dev = br_fdb_find_port(br_dev, mac, vid);
7949         if (!dev)
7950                 return;
7951
7952         info.addr = mac;
7953         info.vid = vid;
7954         call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
7955                                  NULL);
7956 }
7957
7958 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = {
7959         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
7960         .rif_size               = sizeof(struct mlxsw_sp_rif),
7961         .configure              = mlxsw_sp_rif_fid_configure,
7962         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
7963         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
7964         .fdb_del                = mlxsw_sp_rif_vlan_fdb_del,
7965 };
7966
7967 static struct mlxsw_sp_rif_ipip_lb *
7968 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
7969 {
7970         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
7971 }
7972
7973 static void
7974 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
7975                            const struct mlxsw_sp_rif_params *params)
7976 {
7977         struct mlxsw_sp_rif_params_ipip_lb *params_lb;
7978         struct mlxsw_sp_rif_ipip_lb *rif_lb;
7979
7980         params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
7981                                  common);
7982         rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
7983         rif_lb->lb_config = params_lb->lb_config;
7984 }
7985
7986 static int
7987 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
7988 {
7989         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
7990         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
7991         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7992         struct mlxsw_sp_vr *ul_vr;
7993         int err;
7994
7995         ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL);
7996         if (IS_ERR(ul_vr))
7997                 return PTR_ERR(ul_vr);
7998
7999         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true);
8000         if (err)
8001                 goto err_loopback_op;
8002
8003         lb_rif->ul_vr_id = ul_vr->id;
8004         lb_rif->ul_rif_id = 0;
8005         ++ul_vr->rif_count;
8006         return 0;
8007
8008 err_loopback_op:
8009         mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
8010         return err;
8011 }
8012
8013 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
8014 {
8015         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
8016         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8017         struct mlxsw_sp_vr *ul_vr;
8018
8019         ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
8020         mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false);
8021
8022         --ul_vr->rif_count;
8023         mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
8024 }
8025
8026 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = {
8027         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
8028         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
8029         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
8030         .configure              = mlxsw_sp1_rif_ipip_lb_configure,
8031         .deconfigure            = mlxsw_sp1_rif_ipip_lb_deconfigure,
8032 };
8033
8034 const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = {
8035         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
8036         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_emu_ops,
8037         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
8038         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp1_rif_ipip_lb_ops,
8039 };
8040
8041 static int
8042 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable)
8043 {
8044         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
8045         char ritr_pl[MLXSW_REG_RITR_LEN];
8046
8047         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
8048                             ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU);
8049         mlxsw_reg_ritr_loopback_protocol_set(ritr_pl,
8050                                              MLXSW_REG_RITR_LOOPBACK_GENERIC);
8051
8052         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
8053 }
8054
8055 static struct mlxsw_sp_rif *
8056 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
8057                        struct netlink_ext_ack *extack)
8058 {
8059         struct mlxsw_sp_rif *ul_rif;
8060         u16 rif_index;
8061         int err;
8062
8063         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
8064         if (err) {
8065                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
8066                 return ERR_PTR(err);
8067         }
8068
8069         ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL);
8070         if (!ul_rif)
8071                 return ERR_PTR(-ENOMEM);
8072
8073         mlxsw_sp->router->rifs[rif_index] = ul_rif;
8074         ul_rif->mlxsw_sp = mlxsw_sp;
8075         err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true);
8076         if (err)
8077                 goto ul_rif_op_err;
8078
8079         return ul_rif;
8080
8081 ul_rif_op_err:
8082         mlxsw_sp->router->rifs[rif_index] = NULL;
8083         kfree(ul_rif);
8084         return ERR_PTR(err);
8085 }
8086
8087 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif)
8088 {
8089         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
8090
8091         mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false);
8092         mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL;
8093         kfree(ul_rif);
8094 }
8095
8096 static struct mlxsw_sp_rif *
8097 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
8098                     struct netlink_ext_ack *extack)
8099 {
8100         struct mlxsw_sp_vr *vr;
8101         int err;
8102
8103         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack);
8104         if (IS_ERR(vr))
8105                 return ERR_CAST(vr);
8106
8107         if (refcount_inc_not_zero(&vr->ul_rif_refcnt))
8108                 return vr->ul_rif;
8109
8110         vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack);
8111         if (IS_ERR(vr->ul_rif)) {
8112                 err = PTR_ERR(vr->ul_rif);
8113                 goto err_ul_rif_create;
8114         }
8115
8116         vr->rif_count++;
8117         refcount_set(&vr->ul_rif_refcnt, 1);
8118
8119         return vr->ul_rif;
8120
8121 err_ul_rif_create:
8122         mlxsw_sp_vr_put(mlxsw_sp, vr);
8123         return ERR_PTR(err);
8124 }
8125
8126 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif)
8127 {
8128         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
8129         struct mlxsw_sp_vr *vr;
8130
8131         vr = &mlxsw_sp->router->vrs[ul_rif->vr_id];
8132
8133         if (!refcount_dec_and_test(&vr->ul_rif_refcnt))
8134                 return;
8135
8136         vr->rif_count--;
8137         mlxsw_sp_ul_rif_destroy(ul_rif);
8138         mlxsw_sp_vr_put(mlxsw_sp, vr);
8139 }
8140
8141 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
8142                                u16 *ul_rif_index)
8143 {
8144         struct mlxsw_sp_rif *ul_rif;
8145         int err = 0;
8146
8147         mutex_lock(&mlxsw_sp->router->lock);
8148         ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
8149         if (IS_ERR(ul_rif)) {
8150                 err = PTR_ERR(ul_rif);
8151                 goto out;
8152         }
8153         *ul_rif_index = ul_rif->rif_index;
8154 out:
8155         mutex_unlock(&mlxsw_sp->router->lock);
8156         return err;
8157 }
8158
8159 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index)
8160 {
8161         struct mlxsw_sp_rif *ul_rif;
8162
8163         mutex_lock(&mlxsw_sp->router->lock);
8164         ul_rif = mlxsw_sp->router->rifs[ul_rif_index];
8165         if (WARN_ON(!ul_rif))
8166                 goto out;
8167
8168         mlxsw_sp_ul_rif_put(ul_rif);
8169 out:
8170         mutex_unlock(&mlxsw_sp->router->lock);
8171 }
8172
8173 static int
8174 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
8175 {
8176         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
8177         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
8178         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8179         struct mlxsw_sp_rif *ul_rif;
8180         int err;
8181
8182         ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
8183         if (IS_ERR(ul_rif))
8184                 return PTR_ERR(ul_rif);
8185
8186         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true);
8187         if (err)
8188                 goto err_loopback_op;
8189
8190         lb_rif->ul_vr_id = 0;
8191         lb_rif->ul_rif_id = ul_rif->rif_index;
8192
8193         return 0;
8194
8195 err_loopback_op:
8196         mlxsw_sp_ul_rif_put(ul_rif);
8197         return err;
8198 }
8199
8200 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
8201 {
8202         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
8203         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8204         struct mlxsw_sp_rif *ul_rif;
8205
8206         ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id);
8207         mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false);
8208         mlxsw_sp_ul_rif_put(ul_rif);
8209 }
8210
8211 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = {
8212         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
8213         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
8214         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
8215         .configure              = mlxsw_sp2_rif_ipip_lb_configure,
8216         .deconfigure            = mlxsw_sp2_rif_ipip_lb_deconfigure,
8217 };
8218
8219 const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = {
8220         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
8221         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_emu_ops,
8222         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
8223         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp2_rif_ipip_lb_ops,
8224 };
8225
8226 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
8227 {
8228         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
8229
8230         mlxsw_sp->router->rifs = kcalloc(max_rifs,
8231                                          sizeof(struct mlxsw_sp_rif *),
8232                                          GFP_KERNEL);
8233         if (!mlxsw_sp->router->rifs)
8234                 return -ENOMEM;
8235
8236         return 0;
8237 }
8238
8239 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
8240 {
8241         int i;
8242
8243         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
8244                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
8245
8246         kfree(mlxsw_sp->router->rifs);
8247 }
8248
8249 static int
8250 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp)
8251 {
8252         char tigcr_pl[MLXSW_REG_TIGCR_LEN];
8253
8254         mlxsw_reg_tigcr_pack(tigcr_pl, true, 0);
8255         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl);
8256 }
8257
8258 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
8259 {
8260         int err;
8261
8262         mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
8263         INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
8264
8265         err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp);
8266         if (err)
8267                 return err;
8268         err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp);
8269         if (err)
8270                 return err;
8271
8272         return mlxsw_sp_ipip_config_tigcr(mlxsw_sp);
8273 }
8274
8275 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
8276 {
8277         WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
8278 }
8279
8280 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
8281 {
8282         struct mlxsw_sp_router *router;
8283
8284         /* Flush pending FIB notifications and then flush the device's
8285          * table before requesting another dump. The FIB notification
8286          * block is unregistered, so no need to take RTNL.
8287          */
8288         mlxsw_core_flush_owq();
8289         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
8290         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
8291 }
8292
8293 #ifdef CONFIG_IP_ROUTE_MULTIPATH
8294 static void mlxsw_sp_mp_hash_header_set(char *recr2_pl, int header)
8295 {
8296         mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, header, true);
8297 }
8298
8299 static void mlxsw_sp_mp_hash_field_set(char *recr2_pl, int field)
8300 {
8301         mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, field, true);
8302 }
8303
8304 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl)
8305 {
8306         struct net *net = mlxsw_sp_net(mlxsw_sp);
8307         bool only_l3 = !net->ipv4.sysctl_fib_multipath_hash_policy;
8308
8309         mlxsw_sp_mp_hash_header_set(recr2_pl,
8310                                     MLXSW_REG_RECR2_IPV4_EN_NOT_TCP_NOT_UDP);
8311         mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV4_EN_TCP_UDP);
8312         mlxsw_reg_recr2_ipv4_sip_enable(recr2_pl);
8313         mlxsw_reg_recr2_ipv4_dip_enable(recr2_pl);
8314         if (only_l3)
8315                 return;
8316         mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_EN_IPV4);
8317         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV4_PROTOCOL);
8318         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_SPORT);
8319         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_DPORT);
8320 }
8321
8322 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl)
8323 {
8324         bool only_l3 = !ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp));
8325
8326         mlxsw_sp_mp_hash_header_set(recr2_pl,
8327                                     MLXSW_REG_RECR2_IPV6_EN_NOT_TCP_NOT_UDP);
8328         mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV6_EN_TCP_UDP);
8329         mlxsw_reg_recr2_ipv6_sip_enable(recr2_pl);
8330         mlxsw_reg_recr2_ipv6_dip_enable(recr2_pl);
8331         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_NEXT_HEADER);
8332         if (only_l3) {
8333                 mlxsw_sp_mp_hash_field_set(recr2_pl,
8334                                            MLXSW_REG_RECR2_IPV6_FLOW_LABEL);
8335         } else {
8336                 mlxsw_sp_mp_hash_header_set(recr2_pl,
8337                                             MLXSW_REG_RECR2_TCP_UDP_EN_IPV6);
8338                 mlxsw_sp_mp_hash_field_set(recr2_pl,
8339                                            MLXSW_REG_RECR2_TCP_UDP_SPORT);
8340                 mlxsw_sp_mp_hash_field_set(recr2_pl,
8341                                            MLXSW_REG_RECR2_TCP_UDP_DPORT);
8342         }
8343 }
8344
8345 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
8346 {
8347         char recr2_pl[MLXSW_REG_RECR2_LEN];
8348         u32 seed;
8349
8350         seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0);
8351         mlxsw_reg_recr2_pack(recr2_pl, seed);
8352         mlxsw_sp_mp4_hash_init(mlxsw_sp, recr2_pl);
8353         mlxsw_sp_mp6_hash_init(mlxsw_sp, recr2_pl);
8354
8355         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl);
8356 }
8357 #else
8358 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
8359 {
8360         return 0;
8361 }
8362 #endif
8363
8364 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp)
8365 {
8366         char rdpm_pl[MLXSW_REG_RDPM_LEN];
8367         unsigned int i;
8368
8369         MLXSW_REG_ZERO(rdpm, rdpm_pl);
8370
8371         /* HW is determining switch priority based on DSCP-bits, but the
8372          * kernel is still doing that based on the ToS. Since there's a
8373          * mismatch in bits we need to make sure to translate the right
8374          * value ToS would observe, skipping the 2 least-significant ECN bits.
8375          */
8376         for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++)
8377                 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2));
8378
8379         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl);
8380 }
8381
8382 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
8383 {
8384         struct net *net = mlxsw_sp_net(mlxsw_sp);
8385         bool usp = net->ipv4.sysctl_ip_fwd_update_priority;
8386         char rgcr_pl[MLXSW_REG_RGCR_LEN];
8387         u64 max_rifs;
8388
8389         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
8390                 return -EIO;
8391         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
8392
8393         mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
8394         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
8395         mlxsw_reg_rgcr_usp_set(rgcr_pl, usp);
8396         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
8397 }
8398
8399 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
8400 {
8401         char rgcr_pl[MLXSW_REG_RGCR_LEN];
8402
8403         mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
8404         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
8405 }
8406
8407 static const struct mlxsw_sp_router_ll_ops mlxsw_sp_router_ll_basic_ops = {
8408         .ralta_write = mlxsw_sp_router_ll_basic_ralta_write,
8409         .ralst_write = mlxsw_sp_router_ll_basic_ralst_write,
8410         .raltb_write = mlxsw_sp_router_ll_basic_raltb_write,
8411         .fib_entry_op_ctx_size = sizeof(struct mlxsw_sp_fib_entry_op_ctx_basic),
8412         .fib_entry_pack = mlxsw_sp_router_ll_basic_fib_entry_pack,
8413         .fib_entry_act_remote_pack = mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack,
8414         .fib_entry_act_local_pack = mlxsw_sp_router_ll_basic_fib_entry_act_local_pack,
8415         .fib_entry_act_ip2me_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack,
8416         .fib_entry_act_ip2me_tun_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack,
8417         .fib_entry_commit = mlxsw_sp_router_ll_basic_fib_entry_commit,
8418         .fib_entry_is_committed = mlxsw_sp_router_ll_basic_fib_entry_is_committed,
8419 };
8420
8421 static int mlxsw_sp_router_ll_op_ctx_init(struct mlxsw_sp_router *router)
8422 {
8423         size_t max_size = 0;
8424         int i;
8425
8426         for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
8427                 size_t size = router->proto_ll_ops[i]->fib_entry_op_ctx_size;
8428
8429                 if (size > max_size)
8430                         max_size = size;
8431         }
8432         router->ll_op_ctx = kzalloc(sizeof(*router->ll_op_ctx) + max_size,
8433                                     GFP_KERNEL);
8434         if (!router->ll_op_ctx)
8435                 return -ENOMEM;
8436         INIT_LIST_HEAD(&router->ll_op_ctx->fib_entry_priv_list);
8437         return 0;
8438 }
8439
8440 static void mlxsw_sp_router_ll_op_ctx_fini(struct mlxsw_sp_router *router)
8441 {
8442         WARN_ON(!list_empty(&router->ll_op_ctx->fib_entry_priv_list));
8443         kfree(router->ll_op_ctx);
8444 }
8445
8446 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp,
8447                          struct netlink_ext_ack *extack)
8448 {
8449         struct mlxsw_sp_router *router;
8450         int err;
8451
8452         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
8453         if (!router)
8454                 return -ENOMEM;
8455         mutex_init(&router->lock);
8456         mlxsw_sp->router = router;
8457         router->mlxsw_sp = mlxsw_sp;
8458
8459         router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV4] = &mlxsw_sp_router_ll_basic_ops;
8460         router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_router_ll_basic_ops;
8461
8462         err = mlxsw_sp_router_ll_op_ctx_init(router);
8463         if (err)
8464                 goto err_ll_op_ctx_init;
8465
8466         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
8467         err = __mlxsw_sp_router_init(mlxsw_sp);
8468         if (err)
8469                 goto err_router_init;
8470
8471         err = mlxsw_sp_rifs_init(mlxsw_sp);
8472         if (err)
8473                 goto err_rifs_init;
8474
8475         err = mlxsw_sp_ipips_init(mlxsw_sp);
8476         if (err)
8477                 goto err_ipips_init;
8478
8479         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
8480                               &mlxsw_sp_nexthop_ht_params);
8481         if (err)
8482                 goto err_nexthop_ht_init;
8483
8484         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
8485                               &mlxsw_sp_nexthop_group_ht_params);
8486         if (err)
8487                 goto err_nexthop_group_ht_init;
8488
8489         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list);
8490         err = mlxsw_sp_lpm_init(mlxsw_sp);
8491         if (err)
8492                 goto err_lpm_init;
8493
8494         err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops);
8495         if (err)
8496                 goto err_mr_init;
8497
8498         err = mlxsw_sp_vrs_init(mlxsw_sp);
8499         if (err)
8500                 goto err_vrs_init;
8501
8502         err = mlxsw_sp_neigh_init(mlxsw_sp);
8503         if (err)
8504                 goto err_neigh_init;
8505
8506         err = mlxsw_sp_mp_hash_init(mlxsw_sp);
8507         if (err)
8508                 goto err_mp_hash_init;
8509
8510         err = mlxsw_sp_dscp_init(mlxsw_sp);
8511         if (err)
8512                 goto err_dscp_init;
8513
8514         INIT_WORK(&router->fib_event_work, mlxsw_sp_router_fib_event_work);
8515         INIT_LIST_HEAD(&router->fib_event_queue);
8516         spin_lock_init(&router->fib_event_queue_lock);
8517
8518         router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event;
8519         err = register_inetaddr_notifier(&router->inetaddr_nb);
8520         if (err)
8521                 goto err_register_inetaddr_notifier;
8522
8523         router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event;
8524         err = register_inet6addr_notifier(&router->inet6addr_nb);
8525         if (err)
8526                 goto err_register_inet6addr_notifier;
8527
8528         mlxsw_sp->router->netevent_nb.notifier_call =
8529                 mlxsw_sp_router_netevent_event;
8530         err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb);
8531         if (err)
8532                 goto err_register_netevent_notifier;
8533
8534         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
8535         err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp),
8536                                     &mlxsw_sp->router->fib_nb,
8537                                     mlxsw_sp_router_fib_dump_flush, extack);
8538         if (err)
8539                 goto err_register_fib_notifier;
8540
8541         return 0;
8542
8543 err_register_fib_notifier:
8544         unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
8545 err_register_netevent_notifier:
8546         unregister_inet6addr_notifier(&router->inet6addr_nb);
8547 err_register_inet6addr_notifier:
8548         unregister_inetaddr_notifier(&router->inetaddr_nb);
8549 err_register_inetaddr_notifier:
8550         mlxsw_core_flush_owq();
8551         WARN_ON(!list_empty(&router->fib_event_queue));
8552 err_dscp_init:
8553 err_mp_hash_init:
8554         mlxsw_sp_neigh_fini(mlxsw_sp);
8555 err_neigh_init:
8556         mlxsw_sp_vrs_fini(mlxsw_sp);
8557 err_vrs_init:
8558         mlxsw_sp_mr_fini(mlxsw_sp);
8559 err_mr_init:
8560         mlxsw_sp_lpm_fini(mlxsw_sp);
8561 err_lpm_init:
8562         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
8563 err_nexthop_group_ht_init:
8564         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
8565 err_nexthop_ht_init:
8566         mlxsw_sp_ipips_fini(mlxsw_sp);
8567 err_ipips_init:
8568         mlxsw_sp_rifs_fini(mlxsw_sp);
8569 err_rifs_init:
8570         __mlxsw_sp_router_fini(mlxsw_sp);
8571 err_router_init:
8572         mlxsw_sp_router_ll_op_ctx_fini(router);
8573 err_ll_op_ctx_init:
8574         mutex_destroy(&mlxsw_sp->router->lock);
8575         kfree(mlxsw_sp->router);
8576         return err;
8577 }
8578
8579 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
8580 {
8581         unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp),
8582                                 &mlxsw_sp->router->fib_nb);
8583         unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
8584         unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb);
8585         unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb);
8586         mlxsw_core_flush_owq();
8587         WARN_ON(!list_empty(&mlxsw_sp->router->fib_event_queue));
8588         mlxsw_sp_neigh_fini(mlxsw_sp);
8589         mlxsw_sp_vrs_fini(mlxsw_sp);
8590         mlxsw_sp_mr_fini(mlxsw_sp);
8591         mlxsw_sp_lpm_fini(mlxsw_sp);
8592         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
8593         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
8594         mlxsw_sp_ipips_fini(mlxsw_sp);
8595         mlxsw_sp_rifs_fini(mlxsw_sp);
8596         __mlxsw_sp_router_fini(mlxsw_sp);
8597         mlxsw_sp_router_ll_op_ctx_fini(mlxsw_sp->router);
8598         mutex_destroy(&mlxsw_sp->router->lock);
8599         kfree(mlxsw_sp->router);
8600 }