drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/types.h>
   6 #include <linux/rhashtable.h>
   7 #include <linux/bitops.h>
   8 #include <linux/in6.h>
   9 #include <linux/notifier.h>
  10 #include <linux/inetdevice.h>
  11 #include <linux/netdevice.h>
  12 #include <linux/if_bridge.h>
  13 #include <linux/socket.h>
  14 #include <linux/route.h>
  15 #include <linux/gcd.h>
  16 #include <linux/if_macvlan.h>
  17 #include <linux/refcount.h>
  18 #include <linux/jhash.h>
  19 #include <linux/net_namespace.h>
  20 #include <linux/mutex.h>
  21 #include <net/netevent.h>
  22 #include <net/neighbour.h>
  23 #include <net/arp.h>
  24 #include <net/ip_fib.h>
  25 #include <net/ip6_fib.h>
  26 #include <net/nexthop.h>
  27 #include <net/fib_rules.h>
  28 #include <net/ip_tunnels.h>
  29 #include <net/l3mdev.h>
  30 #include <net/addrconf.h>
  31 #include <net/ndisc.h>
  32 #include <net/ipv6.h>
  33 #include <net/fib_notifier.h>
  34 #include <net/switchdev.h>
  35
  36 #include "spectrum.h"
  37 #include "core.h"
  38 #include "reg.h"
  39 #include "spectrum_cnt.h"
  40 #include "spectrum_dpipe.h"
  41 #include "spectrum_ipip.h"
  42 #include "spectrum_mr.h"
  43 #include "spectrum_mr_tcam.h"
  44 #include "spectrum_router.h"
  45 #include "spectrum_span.h"
  46
  47 struct mlxsw_sp_fib;
  48 struct mlxsw_sp_vr;
  49 struct mlxsw_sp_lpm_tree;
  50 struct mlxsw_sp_rif_ops;
  51
  52 struct mlxsw_sp_rif {
  53         struct list_head nexthop_list;
  54         struct list_head neigh_list;
  55         struct net_device *dev; /* NULL for underlay RIF */
  56         struct mlxsw_sp_fid *fid;
  57         unsigned char addr[ETH_ALEN];
  58         int mtu;
  59         u16 rif_index;
  60         u16 vr_id;
  61         const struct mlxsw_sp_rif_ops *ops;
  62         struct mlxsw_sp *mlxsw_sp;
  63
  64         unsigned int counter_ingress;
  65         bool counter_ingress_valid;
  66         unsigned int counter_egress;
  67         bool counter_egress_valid;
  68 };
  69
  70 struct mlxsw_sp_rif_params {
  71         struct net_device *dev;
  72         union {
  73                 u16 system_port;
  74                 u16 lag_id;
  75         };
  76         u16 vid;
  77         bool lag;
  78 };
  79
  80 struct mlxsw_sp_rif_subport {
  81         struct mlxsw_sp_rif common;
  82         refcount_t ref_count;
  83         union {
  84                 u16 system_port;
  85                 u16 lag_id;
  86         };
  87         u16 vid;
  88         bool lag;
  89 };
  90
  91 struct mlxsw_sp_rif_ipip_lb {
  92         struct mlxsw_sp_rif common;
  93         struct mlxsw_sp_rif_ipip_lb_config lb_config;
  94         u16 ul_vr_id; /* Reserved for Spectrum-2. */
  95         u16 ul_rif_id; /* Reserved for Spectrum. */
  96 };
  97
  98 struct mlxsw_sp_rif_params_ipip_lb {
  99         struct mlxsw_sp_rif_params common;
 100         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 101 };
 102
 103 struct mlxsw_sp_rif_ops {
 104         enum mlxsw_sp_rif_type type;
 105         size_t rif_size;
 106
 107         void (*setup)(struct mlxsw_sp_rif *rif,
 108                       const struct mlxsw_sp_rif_params *params);
 109         int (*configure)(struct mlxsw_sp_rif *rif);
 110         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 111         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif,
 112                                          struct netlink_ext_ack *extack);
 113         void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac);
 114 };
 115
 116 static struct mlxsw_sp_rif *
 117 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 118                          const struct net_device *dev);
 119 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif);
 120 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree);
 121 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 122                                   struct mlxsw_sp_lpm_tree *lpm_tree);
 123 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 124                                      const struct mlxsw_sp_fib *fib,
 125                                      u8 tree_id);
 126 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 127                                        const struct mlxsw_sp_fib *fib);
 128
 129 static unsigned int *
 130 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 131                            enum mlxsw_sp_rif_counter_dir dir)
 132 {
 133         switch (dir) {
 134         case MLXSW_SP_RIF_COUNTER_EGRESS:
 135                 return &rif->counter_egress;
 136         case MLXSW_SP_RIF_COUNTER_INGRESS:
 137                 return &rif->counter_ingress;
 138         }
 139         return NULL;
 140 }
 141
 142 static bool
 143 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 144                                enum mlxsw_sp_rif_counter_dir dir)
 145 {
 146         switch (dir) {
 147         case MLXSW_SP_RIF_COUNTER_EGRESS:
 148                 return rif->counter_egress_valid;
 149         case MLXSW_SP_RIF_COUNTER_INGRESS:
 150                 return rif->counter_ingress_valid;
 151         }
 152         return false;
 153 }
 154
 155 static void
 156 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 157                                enum mlxsw_sp_rif_counter_dir dir,
 158                                bool valid)
 159 {
 160         switch (dir) {
 161         case MLXSW_SP_RIF_COUNTER_EGRESS:
 162                 rif->counter_egress_valid = valid;
 163                 break;
 164         case MLXSW_SP_RIF_COUNTER_INGRESS:
 165                 rif->counter_ingress_valid = valid;
 166                 break;
 167         }
 168 }
 169
 170 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 171                                      unsigned int counter_index, bool enable,
 172                                      enum mlxsw_sp_rif_counter_dir dir)
 173 {
 174         char ritr_pl[MLXSW_REG_RITR_LEN];
 175         bool is_egress = false;
 176         int err;
 177
 178         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 179                 is_egress = true;
 180         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 181         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 182         if (err)
 183                 return err;
 184
 185         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 186                                     is_egress);
 187         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 188 }
 189
 190 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 191                                    struct mlxsw_sp_rif *rif,
 192                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 193 {
 194         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 195         unsigned int *p_counter_index;
 196         bool valid;
 197         int err;
 198
 199         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 200         if (!valid)
 201                 return -EINVAL;
 202
 203         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 204         if (!p_counter_index)
 205                 return -EINVAL;
 206         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 207                              MLXSW_REG_RICNT_OPCODE_NOP);
 208         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 209         if (err)
 210                 return err;
 211         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 212         return 0;
 213 }
 214
 215 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 216                                       unsigned int counter_index)
 217 {
 218         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 219
 220         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 221                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 222         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 223 }
 224
 225 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 226                                struct mlxsw_sp_rif *rif,
 227                                enum mlxsw_sp_rif_counter_dir dir)
 228 {
 229         unsigned int *p_counter_index;
 230         int err;
 231
 232         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 233         if (!p_counter_index)
 234                 return -EINVAL;
 235         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 236                                      p_counter_index);
 237         if (err)
 238                 return err;
 239
 240         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 241         if (err)
 242                 goto err_counter_clear;
 243
 244         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 245                                         *p_counter_index, true, dir);
 246         if (err)
 247                 goto err_counter_edit;
 248         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 249         return 0;
 250
 251 err_counter_edit:
 252 err_counter_clear:
 253         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 254                               *p_counter_index);
 255         return err;
 256 }
 257
 258 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 259                                struct mlxsw_sp_rif *rif,
 260                                enum mlxsw_sp_rif_counter_dir dir)
 261 {
 262         unsigned int *p_counter_index;
 263
 264         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 265                 return;
 266
 267         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 268         if (WARN_ON(!p_counter_index))
 269                 return;
 270         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 271                                   *p_counter_index, false, dir);
 272         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 273                               *p_counter_index);
 274         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 275 }
 276
 277 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 278 {
 279         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 280         struct devlink *devlink;
 281
 282         devlink = priv_to_devlink(mlxsw_sp->core);
 283         if (!devlink_dpipe_table_counter_enabled(devlink,
 284                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 285                 return;
 286         mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 287 }
 288
 289 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 290 {
 291         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 292
 293         mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 294 }
 295
 296 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
 297
 298 struct mlxsw_sp_prefix_usage {
 299         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 300 };
 301
 302 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 303         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 304
 305 static bool
 306 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 307                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 308 {
 309         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 310 }
 311
 312 static void
 313 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 314                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 315 {
 316         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 317 }
 318
 319 static void
 320 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 321                           unsigned char prefix_len)
 322 {
 323         set_bit(prefix_len, prefix_usage->b);
 324 }
 325
 326 static void
 327 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 328                             unsigned char prefix_len)
 329 {
 330         clear_bit(prefix_len, prefix_usage->b);
 331 }
 332
 333 struct mlxsw_sp_fib_key {
 334         unsigned char addr[sizeof(struct in6_addr)];
 335         unsigned char prefix_len;
 336 };
 337
 338 enum mlxsw_sp_fib_entry_type {
 339         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 340         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 341         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 342         MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE,
 343         MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE,
 344
 345         /* This is a special case of local delivery, where a packet should be
 346          * decapsulated on reception. Note that there is no corresponding ENCAP,
 347          * because that's a type of next hop, not of FIB entry. (There can be
 348          * several next hops in a REMOTE entry, and some of them may be
 349          * encapsulating entries.)
 350          */
 351         MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
 352         MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP,
 353 };
 354
 355 struct mlxsw_sp_nexthop_group_info;
 356 struct mlxsw_sp_nexthop_group;
 357 struct mlxsw_sp_fib_entry;
 358
 359 struct mlxsw_sp_fib_node {
 360         struct mlxsw_sp_fib_entry *fib_entry;
 361         struct list_head list;
 362         struct rhash_head ht_node;
 363         struct mlxsw_sp_fib *fib;
 364         struct mlxsw_sp_fib_key key;
 365 };
 366
 367 struct mlxsw_sp_fib_entry_decap {
 368         struct mlxsw_sp_ipip_entry *ipip_entry;
 369         u32 tunnel_index;
 370 };
 371
 372 static struct mlxsw_sp_fib_entry_priv *
 373 mlxsw_sp_fib_entry_priv_create(const struct mlxsw_sp_router_ll_ops *ll_ops)
 374 {
 375         struct mlxsw_sp_fib_entry_priv *priv;
 376
 377         if (!ll_ops->fib_entry_priv_size)
 378                 /* No need to have priv */
 379                 return NULL;
 380
 381         priv = kzalloc(sizeof(*priv) + ll_ops->fib_entry_priv_size, GFP_KERNEL);
 382         if (!priv)
 383                 return ERR_PTR(-ENOMEM);
 384         refcount_set(&priv->refcnt, 1);
 385         return priv;
 386 }
 387
 388 static void
 389 mlxsw_sp_fib_entry_priv_destroy(struct mlxsw_sp_fib_entry_priv *priv)
 390 {
 391         kfree(priv);
 392 }
 393
 394 static void mlxsw_sp_fib_entry_priv_hold(struct mlxsw_sp_fib_entry_priv *priv)
 395 {
 396         refcount_inc(&priv->refcnt);
 397 }
 398
 399 static void mlxsw_sp_fib_entry_priv_put(struct mlxsw_sp_fib_entry_priv *priv)
 400 {
 401         if (!priv || !refcount_dec_and_test(&priv->refcnt))
 402                 return;
 403         mlxsw_sp_fib_entry_priv_destroy(priv);
 404 }
 405
 406 static void mlxsw_sp_fib_entry_op_ctx_priv_hold(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
 407                                                 struct mlxsw_sp_fib_entry_priv *priv)
 408 {
 409         if (!priv)
 410                 return;
 411         mlxsw_sp_fib_entry_priv_hold(priv);
 412         list_add(&priv->list, &op_ctx->fib_entry_priv_list);
 413 }
 414
 415 static void mlxsw_sp_fib_entry_op_ctx_priv_put_all(struct mlxsw_sp_fib_entry_op_ctx *op_ctx)
 416 {
 417         struct mlxsw_sp_fib_entry_priv *priv, *tmp;
 418
 419         list_for_each_entry_safe(priv, tmp, &op_ctx->fib_entry_priv_list, list)
 420                 mlxsw_sp_fib_entry_priv_put(priv);
 421         INIT_LIST_HEAD(&op_ctx->fib_entry_priv_list);
 422 }
 423
 424 struct mlxsw_sp_fib_entry {
 425         struct mlxsw_sp_fib_node *fib_node;
 426         enum mlxsw_sp_fib_entry_type type;
 427         struct list_head nexthop_group_node;
 428         struct mlxsw_sp_nexthop_group *nh_group;
 429         struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
 430         struct mlxsw_sp_fib_entry_priv *priv;
 431 };
 432
 433 struct mlxsw_sp_fib4_entry {
 434         struct mlxsw_sp_fib_entry common;
 435         struct fib_info *fi;
 436         u32 tb_id;
 437         u8 tos;
 438         u8 type;
 439 };
 440
 441 struct mlxsw_sp_fib6_entry {
 442         struct mlxsw_sp_fib_entry common;
 443         struct list_head rt6_list;
 444         unsigned int nrt6;
 445 };
 446
 447 struct mlxsw_sp_rt6 {
 448         struct list_head list;
 449         struct fib6_info *rt;
 450 };
 451
 452 struct mlxsw_sp_lpm_tree {
 453         u8 id; /* tree ID */
 454         unsigned int ref_count;
 455         enum mlxsw_sp_l3proto proto;
 456         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 457         struct mlxsw_sp_prefix_usage prefix_usage;
 458 };
 459
 460 struct mlxsw_sp_fib {
 461         struct rhashtable ht;
 462         struct list_head node_list;
 463         struct mlxsw_sp_vr *vr;
 464         struct mlxsw_sp_lpm_tree *lpm_tree;
 465         enum mlxsw_sp_l3proto proto;
 466         const struct mlxsw_sp_router_ll_ops *ll_ops;
 467 };
 468
 469 struct mlxsw_sp_vr {
 470         u16 id; /* virtual router ID */
 471         u32 tb_id; /* kernel fib table id */
 472         unsigned int rif_count;
 473         struct mlxsw_sp_fib *fib4;
 474         struct mlxsw_sp_fib *fib6;
 475         struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX];
 476         struct mlxsw_sp_rif *ul_rif;
 477         refcount_t ul_rif_refcnt;
 478 };
 479
 480 static int mlxsw_sp_router_ll_basic_ralta_write(struct mlxsw_sp *mlxsw_sp, char *xralta_pl)
 481 {
 482         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta),
 483                                xralta_pl + MLXSW_REG_XRALTA_RALTA_OFFSET);
 484 }
 485
 486 static int mlxsw_sp_router_ll_basic_ralst_write(struct mlxsw_sp *mlxsw_sp, char *xralst_pl)
 487 {
 488         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst),
 489                                xralst_pl + MLXSW_REG_XRALST_RALST_OFFSET);
 490 }
 491
 492 static int mlxsw_sp_router_ll_basic_raltb_write(struct mlxsw_sp *mlxsw_sp, char *xraltb_pl)
 493 {
 494         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
 495                                xraltb_pl + MLXSW_REG_XRALTB_RALTB_OFFSET);
 496 }
 497
 498 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 499
 500 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp,
 501                                                 struct mlxsw_sp_vr *vr,
 502                                                 enum mlxsw_sp_l3proto proto)
 503 {
 504         const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
 505         struct mlxsw_sp_lpm_tree *lpm_tree;
 506         struct mlxsw_sp_fib *fib;
 507         int err;
 508
 509         lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 510         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 511         if (!fib)
 512                 return ERR_PTR(-ENOMEM);
 513         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 514         if (err)
 515                 goto err_rhashtable_init;
 516         INIT_LIST_HEAD(&fib->node_list);
 517         fib->proto = proto;
 518         fib->vr = vr;
 519         fib->lpm_tree = lpm_tree;
 520         fib->ll_ops = ll_ops;
 521         mlxsw_sp_lpm_tree_hold(lpm_tree);
 522         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id);
 523         if (err)
 524                 goto err_lpm_tree_bind;
 525         return fib;
 526
 527 err_lpm_tree_bind:
 528         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 529 err_rhashtable_init:
 530         kfree(fib);
 531         return ERR_PTR(err);
 532 }
 533
 534 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp,
 535                                  struct mlxsw_sp_fib *fib)
 536 {
 537         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
 538         mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
 539         WARN_ON(!list_empty(&fib->node_list));
 540         rhashtable_destroy(&fib->ht);
 541         kfree(fib);
 542 }
 543
 544 static struct mlxsw_sp_lpm_tree *
 545 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 546 {
 547         static struct mlxsw_sp_lpm_tree *lpm_tree;
 548         int i;
 549
 550         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 551                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 552                 if (lpm_tree->ref_count == 0)
 553                         return lpm_tree;
 554         }
 555         return NULL;
 556 }
 557
 558 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 559                                    const struct mlxsw_sp_router_ll_ops *ll_ops,
 560                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 561 {
 562         char xralta_pl[MLXSW_REG_XRALTA_LEN];
 563
 564         mlxsw_reg_xralta_pack(xralta_pl, true,
 565                               (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 566                               lpm_tree->id);
 567         return ll_ops->ralta_write(mlxsw_sp, xralta_pl);
 568 }
 569
 570 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 571                                    const struct mlxsw_sp_router_ll_ops *ll_ops,
 572                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 573 {
 574         char xralta_pl[MLXSW_REG_XRALTA_LEN];
 575
 576         mlxsw_reg_xralta_pack(xralta_pl, false,
 577                               (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 578                               lpm_tree->id);
 579         ll_ops->ralta_write(mlxsw_sp, xralta_pl);
 580 }
 581
 582 static int
 583 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 584                                   const struct mlxsw_sp_router_ll_ops *ll_ops,
 585                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 586                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 587 {
 588         char xralst_pl[MLXSW_REG_XRALST_LEN];
 589         u8 root_bin = 0;
 590         u8 prefix;
 591         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 592
 593         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 594                 root_bin = prefix;
 595
 596         mlxsw_reg_xralst_pack(xralst_pl, root_bin, lpm_tree->id);
 597         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 598                 if (prefix == 0)
 599                         continue;
 600                 mlxsw_reg_xralst_bin_pack(xralst_pl, prefix, last_prefix,
 601                                           MLXSW_REG_RALST_BIN_NO_CHILD);
 602                 last_prefix = prefix;
 603         }
 604         return ll_ops->ralst_write(mlxsw_sp, xralst_pl);
 605 }
 606
 607 static struct mlxsw_sp_lpm_tree *
 608 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 609                          const struct mlxsw_sp_router_ll_ops *ll_ops,
 610                          struct mlxsw_sp_prefix_usage *prefix_usage,
 611                          enum mlxsw_sp_l3proto proto)
 612 {
 613         struct mlxsw_sp_lpm_tree *lpm_tree;
 614         int err;
 615
 616         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 617         if (!lpm_tree)
 618                 return ERR_PTR(-EBUSY);
 619         lpm_tree->proto = proto;
 620         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, ll_ops, lpm_tree);
 621         if (err)
 622                 return ERR_PTR(err);
 623
 624         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, ll_ops, prefix_usage, lpm_tree);
 625         if (err)
 626                 goto err_left_struct_set;
 627         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 628                sizeof(lpm_tree->prefix_usage));
 629         memset(&lpm_tree->prefix_ref_count, 0,
 630                sizeof(lpm_tree->prefix_ref_count));
 631         lpm_tree->ref_count = 1;
 632         return lpm_tree;
 633
 634 err_left_struct_set:
 635         mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree);
 636         return ERR_PTR(err);
 637 }
 638
 639 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 640                                       const struct mlxsw_sp_router_ll_ops *ll_ops,
 641                                       struct mlxsw_sp_lpm_tree *lpm_tree)
 642 {
 643         mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree);
 644 }
 645
 646 static struct mlxsw_sp_lpm_tree *
 647 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 648                       struct mlxsw_sp_prefix_usage *prefix_usage,
 649                       enum mlxsw_sp_l3proto proto)
 650 {
 651         const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
 652         struct mlxsw_sp_lpm_tree *lpm_tree;
 653         int i;
 654
 655         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 656                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 657                 if (lpm_tree->ref_count != 0 &&
 658                     lpm_tree->proto == proto &&
 659                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 660                                              prefix_usage)) {
 661                         mlxsw_sp_lpm_tree_hold(lpm_tree);
 662                         return lpm_tree;
 663                 }
 664         }
 665         return mlxsw_sp_lpm_tree_create(mlxsw_sp, ll_ops, prefix_usage, proto);
 666 }
 667
 668 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
 669 {
 670         lpm_tree->ref_count++;
 671 }
 672
 673 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 674                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 675 {
 676         const struct mlxsw_sp_router_ll_ops *ll_ops =
 677                                 mlxsw_sp->router->proto_ll_ops[lpm_tree->proto];
 678
 679         if (--lpm_tree->ref_count == 0)
 680                 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, ll_ops, lpm_tree);
 681 }
 682
 683 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 684
 685 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 686 {
 687         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
 688         struct mlxsw_sp_lpm_tree *lpm_tree;
 689         u64 max_trees;
 690         int err, i;
 691
 692         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 693                 return -EIO;
 694
 695         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 696         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 697         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 698                                              sizeof(struct mlxsw_sp_lpm_tree),
 699                                              GFP_KERNEL);
 700         if (!mlxsw_sp->router->lpm.trees)
 701                 return -ENOMEM;
 702
 703         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 704                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 705                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 706         }
 707
 708         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 709                                          MLXSW_SP_L3_PROTO_IPV4);
 710         if (IS_ERR(lpm_tree)) {
 711                 err = PTR_ERR(lpm_tree);
 712                 goto err_ipv4_tree_get;
 713         }
 714         mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree;
 715
 716         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 717                                          MLXSW_SP_L3_PROTO_IPV6);
 718         if (IS_ERR(lpm_tree)) {
 719                 err = PTR_ERR(lpm_tree);
 720                 goto err_ipv6_tree_get;
 721         }
 722         mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree;
 723
 724         return 0;
 725
 726 err_ipv6_tree_get:
 727         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
 728         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 729 err_ipv4_tree_get:
 730         kfree(mlxsw_sp->router->lpm.trees);
 731         return err;
 732 }
 733
 734 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 735 {
 736         struct mlxsw_sp_lpm_tree *lpm_tree;
 737
 738         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6];
 739         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 740
 741         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
 742         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 743
 744         kfree(mlxsw_sp->router->lpm.trees);
 745 }
 746
 747 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 748 {
 749         return !!vr->fib4 || !!vr->fib6 ||
 750                !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] ||
 751                !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
 752 }
 753
 754 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 755 {
 756         struct mlxsw_sp_vr *vr;
 757         int i;
 758
 759         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 760                 vr = &mlxsw_sp->router->vrs[i];
 761                 if (!mlxsw_sp_vr_is_used(vr))
 762                         return vr;
 763         }
 764         return NULL;
 765 }
 766
 767 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 768                                      const struct mlxsw_sp_fib *fib, u8 tree_id)
 769 {
 770         char xraltb_pl[MLXSW_REG_XRALTB_LEN];
 771
 772         mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id,
 773                               (enum mlxsw_reg_ralxx_protocol) fib->proto,
 774                               tree_id);
 775         return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
 776 }
 777
 778 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 779                                        const struct mlxsw_sp_fib *fib)
 780 {
 781         char xraltb_pl[MLXSW_REG_XRALTB_LEN];
 782
 783         /* Bind to tree 0 which is default */
 784         mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id,
 785                               (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 786         return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
 787 }
 788
 789 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 790 {
 791         /* For our purpose, squash main, default and local tables into one */
 792         if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT)
 793                 tb_id = RT_TABLE_MAIN;
 794         return tb_id;
 795 }
 796
 797 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 798                                             u32 tb_id)
 799 {
 800         struct mlxsw_sp_vr *vr;
 801         int i;
 802
 803         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 804
 805         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 806                 vr = &mlxsw_sp->router->vrs[i];
 807                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 808                         return vr;
 809         }
 810         return NULL;
 811 }
 812
 813 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
 814                                 u16 *vr_id)
 815 {
 816         struct mlxsw_sp_vr *vr;
 817         int err = 0;
 818
 819         mutex_lock(&mlxsw_sp->router->lock);
 820         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 821         if (!vr) {
 822                 err = -ESRCH;
 823                 goto out;
 824         }
 825         *vr_id = vr->id;
 826 out:
 827         mutex_unlock(&mlxsw_sp->router->lock);
 828         return err;
 829 }
 830
 831 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 832                                             enum mlxsw_sp_l3proto proto)
 833 {
 834         switch (proto) {
 835         case MLXSW_SP_L3_PROTO_IPV4:
 836                 return vr->fib4;
 837         case MLXSW_SP_L3_PROTO_IPV6:
 838                 return vr->fib6;
 839         }
 840         return NULL;
 841 }
 842
 843 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 844                                               u32 tb_id,
 845                                               struct netlink_ext_ack *extack)
 846 {
 847         struct mlxsw_sp_mr_table *mr4_table, *mr6_table;
 848         struct mlxsw_sp_fib *fib4;
 849         struct mlxsw_sp_fib *fib6;
 850         struct mlxsw_sp_vr *vr;
 851         int err;
 852
 853         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 854         if (!vr) {
 855                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers");
 856                 return ERR_PTR(-EBUSY);
 857         }
 858         fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
 859         if (IS_ERR(fib4))
 860                 return ERR_CAST(fib4);
 861         fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
 862         if (IS_ERR(fib6)) {
 863                 err = PTR_ERR(fib6);
 864                 goto err_fib6_create;
 865         }
 866         mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
 867                                              MLXSW_SP_L3_PROTO_IPV4);
 868         if (IS_ERR(mr4_table)) {
 869                 err = PTR_ERR(mr4_table);
 870                 goto err_mr4_table_create;
 871         }
 872         mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
 873                                              MLXSW_SP_L3_PROTO_IPV6);
 874         if (IS_ERR(mr6_table)) {
 875                 err = PTR_ERR(mr6_table);
 876                 goto err_mr6_table_create;
 877         }
 878
 879         vr->fib4 = fib4;
 880         vr->fib6 = fib6;
 881         vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table;
 882         vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table;
 883         vr->tb_id = tb_id;
 884         return vr;
 885
 886 err_mr6_table_create:
 887         mlxsw_sp_mr_table_destroy(mr4_table);
 888 err_mr4_table_create:
 889         mlxsw_sp_fib_destroy(mlxsw_sp, fib6);
 890 err_fib6_create:
 891         mlxsw_sp_fib_destroy(mlxsw_sp, fib4);
 892         return ERR_PTR(err);
 893 }
 894
 895 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
 896                                 struct mlxsw_sp_vr *vr)
 897 {
 898         mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]);
 899         vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL;
 900         mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]);
 901         vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL;
 902         mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6);
 903         vr->fib6 = NULL;
 904         mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4);
 905         vr->fib4 = NULL;
 906 }
 907
 908 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
 909                                            struct netlink_ext_ack *extack)
 910 {
 911         struct mlxsw_sp_vr *vr;
 912
 913         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 914         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 915         if (!vr)
 916                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack);
 917         return vr;
 918 }
 919
 920 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
 921 {
 922         if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
 923             list_empty(&vr->fib6->node_list) &&
 924             mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) &&
 925             mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]))
 926                 mlxsw_sp_vr_destroy(mlxsw_sp, vr);
 927 }
 928
 929 static bool
 930 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
 931                                     enum mlxsw_sp_l3proto proto, u8 tree_id)
 932 {
 933         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 934
 935         if (!mlxsw_sp_vr_is_used(vr))
 936                 return false;
 937         if (fib->lpm_tree->id == tree_id)
 938                 return true;
 939         return false;
 940 }
 941
 942 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 943                                         struct mlxsw_sp_fib *fib,
 944                                         struct mlxsw_sp_lpm_tree *new_tree)
 945 {
 946         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 947         int err;
 948
 949         fib->lpm_tree = new_tree;
 950         mlxsw_sp_lpm_tree_hold(new_tree);
 951         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 952         if (err)
 953                 goto err_tree_bind;
 954         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 955         return 0;
 956
 957 err_tree_bind:
 958         mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 959         fib->lpm_tree = old_tree;
 960         return err;
 961 }
 962
 963 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 964                                          struct mlxsw_sp_fib *fib,
 965                                          struct mlxsw_sp_lpm_tree *new_tree)
 966 {
 967         enum mlxsw_sp_l3proto proto = fib->proto;
 968         struct mlxsw_sp_lpm_tree *old_tree;
 969         u8 old_id, new_id = new_tree->id;
 970         struct mlxsw_sp_vr *vr;
 971         int i, err;
 972
 973         old_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 974         old_id = old_tree->id;
 975
 976         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 977                 vr = &mlxsw_sp->router->vrs[i];
 978                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
 979                         continue;
 980                 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 981                                                    mlxsw_sp_vr_fib(vr, proto),
 982                                                    new_tree);
 983                 if (err)
 984                         goto err_tree_replace;
 985         }
 986
 987         memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count,
 988                sizeof(new_tree->prefix_ref_count));
 989         mlxsw_sp->router->lpm.proto_trees[proto] = new_tree;
 990         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 991
 992         return 0;
 993
 994 err_tree_replace:
 995         for (i--; i >= 0; i--) {
 996                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
 997                         continue;
 998                 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 999                                              mlxsw_sp_vr_fib(vr, proto),
1000                                              old_tree);
1001         }
1002         return err;
1003 }
1004
1005 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
1006 {
1007         struct mlxsw_sp_vr *vr;
1008         u64 max_vrs;
1009         int i;
1010
1011         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
1012                 return -EIO;
1013
1014         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
1015         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
1016                                         GFP_KERNEL);
1017         if (!mlxsw_sp->router->vrs)
1018                 return -ENOMEM;
1019
1020         for (i = 0; i < max_vrs; i++) {
1021                 vr = &mlxsw_sp->router->vrs[i];
1022                 vr->id = i;
1023         }
1024
1025         return 0;
1026 }
1027
1028 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
1029
1030 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
1031 {
1032         /* At this stage we're guaranteed not to have new incoming
1033          * FIB notifications and the work queue is free from FIBs
1034          * sitting on top of mlxsw netdevs. However, we can still
1035          * have other FIBs queued. Flush the queue before flushing
1036          * the device's tables. No need for locks, as we're the only
1037          * writer.
1038          */
1039         mlxsw_core_flush_owq();
1040         mlxsw_sp_router_fib_flush(mlxsw_sp);
1041         kfree(mlxsw_sp->router->vrs);
1042 }
1043
1044 static struct net_device *
1045 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
1046 {
1047         struct ip_tunnel *tun = netdev_priv(ol_dev);
1048         struct net *net = dev_net(ol_dev);
1049
1050         return dev_get_by_index_rcu(net, tun->parms.link);
1051 }
1052
1053 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
1054 {
1055         struct net_device *d;
1056         u32 tb_id;
1057
1058         rcu_read_lock();
1059         d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1060         if (d)
1061                 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
1062         else
1063                 tb_id = RT_TABLE_MAIN;
1064         rcu_read_unlock();
1065
1066         return tb_id;
1067 }
1068
1069 static struct mlxsw_sp_rif *
1070 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
1071                     const struct mlxsw_sp_rif_params *params,
1072                     struct netlink_ext_ack *extack);
1073
1074 static struct mlxsw_sp_rif_ipip_lb *
1075 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
1076                                 enum mlxsw_sp_ipip_type ipipt,
1077                                 struct net_device *ol_dev,
1078                                 struct netlink_ext_ack *extack)
1079 {
1080         struct mlxsw_sp_rif_params_ipip_lb lb_params;
1081         const struct mlxsw_sp_ipip_ops *ipip_ops;
1082         struct mlxsw_sp_rif *rif;
1083
1084         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1085         lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
1086                 .common.dev = ol_dev,
1087                 .common.lag = false,
1088                 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
1089         };
1090
1091         rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack);
1092         if (IS_ERR(rif))
1093                 return ERR_CAST(rif);
1094         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
1095 }
1096
1097 static struct mlxsw_sp_ipip_entry *
1098 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
1099                           enum mlxsw_sp_ipip_type ipipt,
1100                           struct net_device *ol_dev)
1101 {
1102         const struct mlxsw_sp_ipip_ops *ipip_ops;
1103         struct mlxsw_sp_ipip_entry *ipip_entry;
1104         struct mlxsw_sp_ipip_entry *ret = NULL;
1105
1106         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1107         ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
1108         if (!ipip_entry)
1109                 return ERR_PTR(-ENOMEM);
1110
1111         ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
1112                                                             ol_dev, NULL);
1113         if (IS_ERR(ipip_entry->ol_lb)) {
1114                 ret = ERR_CAST(ipip_entry->ol_lb);
1115                 goto err_ol_ipip_lb_create;
1116         }
1117
1118         ipip_entry->ipipt = ipipt;
1119         ipip_entry->ol_dev = ol_dev;
1120
1121         switch (ipip_ops->ul_proto) {
1122         case MLXSW_SP_L3_PROTO_IPV4:
1123                 ipip_entry->parms4 = mlxsw_sp_ipip_netdev_parms4(ol_dev);
1124                 break;
1125         case MLXSW_SP_L3_PROTO_IPV6:
1126                 WARN_ON(1);
1127                 break;
1128         }
1129
1130         return ipip_entry;
1131
1132 err_ol_ipip_lb_create:
1133         kfree(ipip_entry);
1134         return ret;
1135 }
1136
1137 static void
1138 mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry)
1139 {
1140         mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
1141         kfree(ipip_entry);
1142 }
1143
1144 static bool
1145 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1146                                   const enum mlxsw_sp_l3proto ul_proto,
1147                                   union mlxsw_sp_l3addr saddr,
1148                                   u32 ul_tb_id,
1149                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1150 {
1151         u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1152         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1153         union mlxsw_sp_l3addr tun_saddr;
1154
1155         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1156                 return false;
1157
1158         tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1159         return tun_ul_tb_id == ul_tb_id &&
1160                mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1161 }
1162
1163 static int
1164 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1165                               struct mlxsw_sp_fib_entry *fib_entry,
1166                               struct mlxsw_sp_ipip_entry *ipip_entry)
1167 {
1168         u32 tunnel_index;
1169         int err;
1170
1171         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1172                                   1, &tunnel_index);
1173         if (err)
1174                 return err;
1175
1176         ipip_entry->decap_fib_entry = fib_entry;
1177         fib_entry->decap.ipip_entry = ipip_entry;
1178         fib_entry->decap.tunnel_index = tunnel_index;
1179         return 0;
1180 }
1181
1182 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1183                                           struct mlxsw_sp_fib_entry *fib_entry)
1184 {
1185         /* Unlink this node from the IPIP entry that it's the decap entry of. */
1186         fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1187         fib_entry->decap.ipip_entry = NULL;
1188         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1189                            1, fib_entry->decap.tunnel_index);
1190 }
1191
1192 static struct mlxsw_sp_fib_node *
1193 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1194                          size_t addr_len, unsigned char prefix_len);
1195 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1196                                      struct mlxsw_sp_fib_entry *fib_entry);
1197
1198 static void
1199 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1200                                  struct mlxsw_sp_ipip_entry *ipip_entry)
1201 {
1202         struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1203
1204         mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1205         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1206
1207         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1208 }
1209
1210 static void
1211 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1212                                   struct mlxsw_sp_ipip_entry *ipip_entry,
1213                                   struct mlxsw_sp_fib_entry *decap_fib_entry)
1214 {
1215         if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1216                                           ipip_entry))
1217                 return;
1218         decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1219
1220         if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1221                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1222 }
1223
1224 static struct mlxsw_sp_fib_entry *
1225 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
1226                                      enum mlxsw_sp_l3proto proto,
1227                                      const union mlxsw_sp_l3addr *addr,
1228                                      enum mlxsw_sp_fib_entry_type type)
1229 {
1230         struct mlxsw_sp_fib_node *fib_node;
1231         unsigned char addr_prefix_len;
1232         struct mlxsw_sp_fib *fib;
1233         struct mlxsw_sp_vr *vr;
1234         const void *addrp;
1235         size_t addr_len;
1236         u32 addr4;
1237
1238         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
1239         if (!vr)
1240                 return NULL;
1241         fib = mlxsw_sp_vr_fib(vr, proto);
1242
1243         switch (proto) {
1244         case MLXSW_SP_L3_PROTO_IPV4:
1245                 addr4 = be32_to_cpu(addr->addr4);
1246                 addrp = &addr4;
1247                 addr_len = 4;
1248                 addr_prefix_len = 32;
1249                 break;
1250         case MLXSW_SP_L3_PROTO_IPV6:
1251         default:
1252                 WARN_ON(1);
1253                 return NULL;
1254         }
1255
1256         fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len,
1257                                             addr_prefix_len);
1258         if (!fib_node || fib_node->fib_entry->type != type)
1259                 return NULL;
1260
1261         return fib_node->fib_entry;
1262 }
1263
1264 /* Given an IPIP entry, find the corresponding decap route. */
1265 static struct mlxsw_sp_fib_entry *
1266 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1267                                struct mlxsw_sp_ipip_entry *ipip_entry)
1268 {
1269         static struct mlxsw_sp_fib_node *fib_node;
1270         const struct mlxsw_sp_ipip_ops *ipip_ops;
1271         unsigned char saddr_prefix_len;
1272         union mlxsw_sp_l3addr saddr;
1273         struct mlxsw_sp_fib *ul_fib;
1274         struct mlxsw_sp_vr *ul_vr;
1275         const void *saddrp;
1276         size_t saddr_len;
1277         u32 ul_tb_id;
1278         u32 saddr4;
1279
1280         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1281
1282         ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1283         ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1284         if (!ul_vr)
1285                 return NULL;
1286
1287         ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1288         saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1289                                            ipip_entry->ol_dev);
1290
1291         switch (ipip_ops->ul_proto) {
1292         case MLXSW_SP_L3_PROTO_IPV4:
1293                 saddr4 = be32_to_cpu(saddr.addr4);
1294                 saddrp = &saddr4;
1295                 saddr_len = 4;
1296                 saddr_prefix_len = 32;
1297                 break;
1298         default:
1299                 WARN_ON(1);
1300                 return NULL;
1301         }
1302
1303         fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1304                                             saddr_prefix_len);
1305         if (!fib_node ||
1306             fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1307                 return NULL;
1308
1309         return fib_node->fib_entry;
1310 }
1311
1312 static struct mlxsw_sp_ipip_entry *
1313 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp,
1314                            enum mlxsw_sp_ipip_type ipipt,
1315                            struct net_device *ol_dev)
1316 {
1317         struct mlxsw_sp_ipip_entry *ipip_entry;
1318
1319         ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1320         if (IS_ERR(ipip_entry))
1321                 return ipip_entry;
1322
1323         list_add_tail(&ipip_entry->ipip_list_node,
1324                       &mlxsw_sp->router->ipip_list);
1325
1326         return ipip_entry;
1327 }
1328
1329 static void
1330 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1331                             struct mlxsw_sp_ipip_entry *ipip_entry)
1332 {
1333         list_del(&ipip_entry->ipip_list_node);
1334         mlxsw_sp_ipip_entry_dealloc(ipip_entry);
1335 }
1336
1337 static bool
1338 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1339                                   const struct net_device *ul_dev,
1340                                   enum mlxsw_sp_l3proto ul_proto,
1341                                   union mlxsw_sp_l3addr ul_dip,
1342                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1343 {
1344         u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1345         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1346
1347         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1348                 return false;
1349
1350         return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1351                                                  ul_tb_id, ipip_entry);
1352 }
1353
1354 /* Given decap parameters, find the corresponding IPIP entry. */
1355 static struct mlxsw_sp_ipip_entry *
1356 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex,
1357                                   enum mlxsw_sp_l3proto ul_proto,
1358                                   union mlxsw_sp_l3addr ul_dip)
1359 {
1360         struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1361         struct net_device *ul_dev;
1362
1363         rcu_read_lock();
1364
1365         ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex);
1366         if (!ul_dev)
1367                 goto out_unlock;
1368
1369         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1370                             ipip_list_node)
1371                 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1372                                                       ul_proto, ul_dip,
1373                                                       ipip_entry))
1374                         goto out_unlock;
1375
1376         rcu_read_unlock();
1377
1378         return NULL;
1379
1380 out_unlock:
1381         rcu_read_unlock();
1382         return ipip_entry;
1383 }
1384
1385 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
1386                                       const struct net_device *dev,
1387                                       enum mlxsw_sp_ipip_type *p_type)
1388 {
1389         struct mlxsw_sp_router *router = mlxsw_sp->router;
1390         const struct mlxsw_sp_ipip_ops *ipip_ops;
1391         enum mlxsw_sp_ipip_type ipipt;
1392
1393         for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
1394                 ipip_ops = router->ipip_ops_arr[ipipt];
1395                 if (dev->type == ipip_ops->dev_type) {
1396                         if (p_type)
1397                                 *p_type = ipipt;
1398                         return true;
1399                 }
1400         }
1401         return false;
1402 }
1403
1404 bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp,
1405                                 const struct net_device *dev)
1406 {
1407         return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
1408 }
1409
1410 static struct mlxsw_sp_ipip_entry *
1411 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp,
1412                                    const struct net_device *ol_dev)
1413 {
1414         struct mlxsw_sp_ipip_entry *ipip_entry;
1415
1416         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1417                             ipip_list_node)
1418                 if (ipip_entry->ol_dev == ol_dev)
1419                         return ipip_entry;
1420
1421         return NULL;
1422 }
1423
1424 static struct mlxsw_sp_ipip_entry *
1425 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp,
1426                                    const struct net_device *ul_dev,
1427                                    struct mlxsw_sp_ipip_entry *start)
1428 {
1429         struct mlxsw_sp_ipip_entry *ipip_entry;
1430
1431         ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list,
1432                                         ipip_list_node);
1433         list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list,
1434                                      ipip_list_node) {
1435                 struct net_device *ol_dev = ipip_entry->ol_dev;
1436                 struct net_device *ipip_ul_dev;
1437
1438                 rcu_read_lock();
1439                 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1440                 rcu_read_unlock();
1441
1442                 if (ipip_ul_dev == ul_dev)
1443                         return ipip_entry;
1444         }
1445
1446         return NULL;
1447 }
1448
1449 bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp,
1450                                 const struct net_device *dev)
1451 {
1452         bool is_ipip_ul;
1453
1454         mutex_lock(&mlxsw_sp->router->lock);
1455         is_ipip_ul = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL);
1456         mutex_unlock(&mlxsw_sp->router->lock);
1457
1458         return is_ipip_ul;
1459 }
1460
1461 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp,
1462                                                 const struct net_device *ol_dev,
1463                                                 enum mlxsw_sp_ipip_type ipipt)
1464 {
1465         const struct mlxsw_sp_ipip_ops *ops
1466                 = mlxsw_sp->router->ipip_ops_arr[ipipt];
1467
1468         return ops->can_offload(mlxsw_sp, ol_dev);
1469 }
1470
1471 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp,
1472                                                 struct net_device *ol_dev)
1473 {
1474         enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX;
1475         struct mlxsw_sp_ipip_entry *ipip_entry;
1476         enum mlxsw_sp_l3proto ul_proto;
1477         union mlxsw_sp_l3addr saddr;
1478         u32 ul_tb_id;
1479
1480         mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt);
1481         if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) {
1482                 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1483                 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto;
1484                 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1485                 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1486                                                           saddr, ul_tb_id,
1487                                                           NULL)) {
1488                         ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt,
1489                                                                 ol_dev);
1490                         if (IS_ERR(ipip_entry))
1491                                 return PTR_ERR(ipip_entry);
1492                 }
1493         }
1494
1495         return 0;
1496 }
1497
1498 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp,
1499                                                    struct net_device *ol_dev)
1500 {
1501         struct mlxsw_sp_ipip_entry *ipip_entry;
1502
1503         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1504         if (ipip_entry)
1505                 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1506 }
1507
1508 static void
1509 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1510                                 struct mlxsw_sp_ipip_entry *ipip_entry)
1511 {
1512         struct mlxsw_sp_fib_entry *decap_fib_entry;
1513
1514         decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1515         if (decap_fib_entry)
1516                 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1517                                                   decap_fib_entry);
1518 }
1519
1520 static int
1521 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id,
1522                         u16 ul_rif_id, bool enable)
1523 {
1524         struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
1525         struct mlxsw_sp_rif *rif = &lb_rif->common;
1526         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
1527         char ritr_pl[MLXSW_REG_RITR_LEN];
1528         u32 saddr4;
1529
1530         switch (lb_cf.ul_protocol) {
1531         case MLXSW_SP_L3_PROTO_IPV4:
1532                 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
1533                 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
1534                                     rif->rif_index, rif->vr_id, rif->dev->mtu);
1535                 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
1536                             MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
1537                             ul_vr_id, ul_rif_id, saddr4, lb_cf.okey);
1538                 break;
1539
1540         case MLXSW_SP_L3_PROTO_IPV6:
1541                 return -EAFNOSUPPORT;
1542         }
1543
1544         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
1545 }
1546
1547 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp,
1548                                                  struct net_device *ol_dev)
1549 {
1550         struct mlxsw_sp_ipip_entry *ipip_entry;
1551         struct mlxsw_sp_rif_ipip_lb *lb_rif;
1552         int err = 0;
1553
1554         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1555         if (ipip_entry) {
1556                 lb_rif = ipip_entry->ol_lb;
1557                 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id,
1558                                               lb_rif->ul_rif_id, true);
1559                 if (err)
1560                         goto out;
1561                 lb_rif->common.mtu = ol_dev->mtu;
1562         }
1563
1564 out:
1565         return err;
1566 }
1567
1568 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1569                                                 struct net_device *ol_dev)
1570 {
1571         struct mlxsw_sp_ipip_entry *ipip_entry;
1572
1573         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1574         if (ipip_entry)
1575                 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1576 }
1577
1578 static void
1579 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1580                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1581 {
1582         if (ipip_entry->decap_fib_entry)
1583                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1584 }
1585
1586 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1587                                                   struct net_device *ol_dev)
1588 {
1589         struct mlxsw_sp_ipip_entry *ipip_entry;
1590
1591         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1592         if (ipip_entry)
1593                 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1594 }
1595
1596 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
1597                                          struct mlxsw_sp_rif *old_rif,
1598                                          struct mlxsw_sp_rif *new_rif);
1599 static int
1600 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp,
1601                                  struct mlxsw_sp_ipip_entry *ipip_entry,
1602                                  bool keep_encap,
1603                                  struct netlink_ext_ack *extack)
1604 {
1605         struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb;
1606         struct mlxsw_sp_rif_ipip_lb *new_lb_rif;
1607
1608         new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp,
1609                                                      ipip_entry->ipipt,
1610                                                      ipip_entry->ol_dev,
1611                                                      extack);
1612         if (IS_ERR(new_lb_rif))
1613                 return PTR_ERR(new_lb_rif);
1614         ipip_entry->ol_lb = new_lb_rif;
1615
1616         if (keep_encap)
1617                 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common,
1618                                              &new_lb_rif->common);
1619
1620         mlxsw_sp_rif_destroy(&old_lb_rif->common);
1621
1622         return 0;
1623 }
1624
1625 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
1626                                         struct mlxsw_sp_rif *rif);
1627
1628 /**
1629  * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry.
1630  * @mlxsw_sp: mlxsw_sp.
1631  * @ipip_entry: IPIP entry.
1632  * @recreate_loopback: Recreates the associated loopback RIF.
1633  * @keep_encap: Updates next hops that use the tunnel netdevice. This is only
1634  *              relevant when recreate_loopback is true.
1635  * @update_nexthops: Updates next hops, keeping the current loopback RIF. This
1636  *                   is only relevant when recreate_loopback is false.
1637  * @extack: extack.
1638  *
1639  * Return: Non-zero value on failure.
1640  */
1641 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp,
1642                                         struct mlxsw_sp_ipip_entry *ipip_entry,
1643                                         bool recreate_loopback,
1644                                         bool keep_encap,
1645                                         bool update_nexthops,
1646                                         struct netlink_ext_ack *extack)
1647 {
1648         int err;
1649
1650         /* RIFs can't be edited, so to update loopback, we need to destroy and
1651          * recreate it. That creates a window of opportunity where RALUE and
1652          * RATR registers end up referencing a RIF that's already gone. RATRs
1653          * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care
1654          * of RALUE, demote the decap route back.
1655          */
1656         if (ipip_entry->decap_fib_entry)
1657                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1658
1659         if (recreate_loopback) {
1660                 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry,
1661                                                        keep_encap, extack);
1662                 if (err)
1663                         return err;
1664         } else if (update_nexthops) {
1665                 mlxsw_sp_nexthop_rif_update(mlxsw_sp,
1666                                             &ipip_entry->ol_lb->common);
1667         }
1668
1669         if (ipip_entry->ol_dev->flags & IFF_UP)
1670                 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1671
1672         return 0;
1673 }
1674
1675 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp,
1676                                                 struct net_device *ol_dev,
1677                                                 struct netlink_ext_ack *extack)
1678 {
1679         struct mlxsw_sp_ipip_entry *ipip_entry =
1680                 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1681
1682         if (!ipip_entry)
1683                 return 0;
1684
1685         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1686                                                    true, false, false, extack);
1687 }
1688
1689 static int
1690 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp,
1691                                      struct mlxsw_sp_ipip_entry *ipip_entry,
1692                                      struct net_device *ul_dev,
1693                                      bool *demote_this,
1694                                      struct netlink_ext_ack *extack)
1695 {
1696         u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1697         enum mlxsw_sp_l3proto ul_proto;
1698         union mlxsw_sp_l3addr saddr;
1699
1700         /* Moving underlay to a different VRF might cause local address
1701          * conflict, and the conflicting tunnels need to be demoted.
1702          */
1703         ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1704         saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1705         if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1706                                                  saddr, ul_tb_id,
1707                                                  ipip_entry)) {
1708                 *demote_this = true;
1709                 return 0;
1710         }
1711
1712         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1713                                                    true, true, false, extack);
1714 }
1715
1716 static int
1717 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp,
1718                                     struct mlxsw_sp_ipip_entry *ipip_entry,
1719                                     struct net_device *ul_dev)
1720 {
1721         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1722                                                    false, false, true, NULL);
1723 }
1724
1725 static int
1726 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp,
1727                                       struct mlxsw_sp_ipip_entry *ipip_entry,
1728                                       struct net_device *ul_dev)
1729 {
1730         /* A down underlay device causes encapsulated packets to not be
1731          * forwarded, but decap still works. So refresh next hops without
1732          * touching anything else.
1733          */
1734         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1735                                                    false, false, true, NULL);
1736 }
1737
1738 static int
1739 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp,
1740                                         struct net_device *ol_dev,
1741                                         struct netlink_ext_ack *extack)
1742 {
1743         const struct mlxsw_sp_ipip_ops *ipip_ops;
1744         struct mlxsw_sp_ipip_entry *ipip_entry;
1745         int err;
1746
1747         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1748         if (!ipip_entry)
1749                 /* A change might make a tunnel eligible for offloading, but
1750                  * that is currently not implemented. What falls to slow path
1751                  * stays there.
1752                  */
1753                 return 0;
1754
1755         /* A change might make a tunnel not eligible for offloading. */
1756         if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev,
1757                                                  ipip_entry->ipipt)) {
1758                 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1759                 return 0;
1760         }
1761
1762         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1763         err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack);
1764         return err;
1765 }
1766
1767 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp,
1768                                        struct mlxsw_sp_ipip_entry *ipip_entry)
1769 {
1770         struct net_device *ol_dev = ipip_entry->ol_dev;
1771
1772         if (ol_dev->flags & IFF_UP)
1773                 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1774         mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1775 }
1776
1777 /* The configuration where several tunnels have the same local address in the
1778  * same underlay table needs special treatment in the HW. That is currently not
1779  * implemented in the driver. This function finds and demotes the first tunnel
1780  * with a given source address, except the one passed in in the argument
1781  * `except'.
1782  */
1783 bool
1784 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp,
1785                                      enum mlxsw_sp_l3proto ul_proto,
1786                                      union mlxsw_sp_l3addr saddr,
1787                                      u32 ul_tb_id,
1788                                      const struct mlxsw_sp_ipip_entry *except)
1789 {
1790         struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1791
1792         list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1793                                  ipip_list_node) {
1794                 if (ipip_entry != except &&
1795                     mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1796                                                       ul_tb_id, ipip_entry)) {
1797                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1798                         return true;
1799                 }
1800         }
1801
1802         return false;
1803 }
1804
1805 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp,
1806                                                      struct net_device *ul_dev)
1807 {
1808         struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1809
1810         list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1811                                  ipip_list_node) {
1812                 struct net_device *ol_dev = ipip_entry->ol_dev;
1813                 struct net_device *ipip_ul_dev;
1814
1815                 rcu_read_lock();
1816                 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1817                 rcu_read_unlock();
1818                 if (ipip_ul_dev == ul_dev)
1819                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1820         }
1821 }
1822
1823 int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp,
1824                                      struct net_device *ol_dev,
1825                                      unsigned long event,
1826                                      struct netdev_notifier_info *info)
1827 {
1828         struct netdev_notifier_changeupper_info *chup;
1829         struct netlink_ext_ack *extack;
1830         int err = 0;
1831
1832         mutex_lock(&mlxsw_sp->router->lock);
1833         switch (event) {
1834         case NETDEV_REGISTER:
1835                 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev);
1836                 break;
1837         case NETDEV_UNREGISTER:
1838                 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev);
1839                 break;
1840         case NETDEV_UP:
1841                 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev);
1842                 break;
1843         case NETDEV_DOWN:
1844                 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev);
1845                 break;
1846         case NETDEV_CHANGEUPPER:
1847                 chup = container_of(info, typeof(*chup), info);
1848                 extack = info->extack;
1849                 if (netif_is_l3_master(chup->upper_dev))
1850                         err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp,
1851                                                                    ol_dev,
1852                                                                    extack);
1853                 break;
1854         case NETDEV_CHANGE:
1855                 extack = info->extack;
1856                 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp,
1857                                                               ol_dev, extack);
1858                 break;
1859         case NETDEV_CHANGEMTU:
1860                 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev);
1861                 break;
1862         }
1863         mutex_unlock(&mlxsw_sp->router->lock);
1864         return err;
1865 }
1866
1867 static int
1868 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1869                                    struct mlxsw_sp_ipip_entry *ipip_entry,
1870                                    struct net_device *ul_dev,
1871                                    bool *demote_this,
1872                                    unsigned long event,
1873                                    struct netdev_notifier_info *info)
1874 {
1875         struct netdev_notifier_changeupper_info *chup;
1876         struct netlink_ext_ack *extack;
1877
1878         switch (event) {
1879         case NETDEV_CHANGEUPPER:
1880                 chup = container_of(info, typeof(*chup), info);
1881                 extack = info->extack;
1882                 if (netif_is_l3_master(chup->upper_dev))
1883                         return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp,
1884                                                                     ipip_entry,
1885                                                                     ul_dev,
1886                                                                     demote_this,
1887                                                                     extack);
1888                 break;
1889
1890         case NETDEV_UP:
1891                 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry,
1892                                                            ul_dev);
1893         case NETDEV_DOWN:
1894                 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp,
1895                                                              ipip_entry,
1896                                                              ul_dev);
1897         }
1898         return 0;
1899 }
1900
1901 int
1902 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1903                                  struct net_device *ul_dev,
1904                                  unsigned long event,
1905                                  struct netdev_notifier_info *info)
1906 {
1907         struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1908         int err = 0;
1909
1910         mutex_lock(&mlxsw_sp->router->lock);
1911         while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp,
1912                                                                 ul_dev,
1913                                                                 ipip_entry))) {
1914                 struct mlxsw_sp_ipip_entry *prev;
1915                 bool demote_this = false;
1916
1917                 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry,
1918                                                          ul_dev, &demote_this,
1919                                                          event, info);
1920                 if (err) {
1921                         mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp,
1922                                                                  ul_dev);
1923                         break;
1924                 }
1925
1926                 if (demote_this) {
1927                         if (list_is_first(&ipip_entry->ipip_list_node,
1928                                           &mlxsw_sp->router->ipip_list))
1929                                 prev = NULL;
1930                         else
1931                                 /* This can't be cached from previous iteration,
1932                                  * because that entry could be gone now.
1933                                  */
1934                                 prev = list_prev_entry(ipip_entry,
1935                                                        ipip_list_node);
1936                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1937                         ipip_entry = prev;
1938                 }
1939         }
1940         mutex_unlock(&mlxsw_sp->router->lock);
1941
1942         return err;
1943 }
1944
1945 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
1946                                       enum mlxsw_sp_l3proto ul_proto,
1947                                       const union mlxsw_sp_l3addr *ul_sip,
1948                                       u32 tunnel_index)
1949 {
1950         enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1951         struct mlxsw_sp_router *router = mlxsw_sp->router;
1952         struct mlxsw_sp_fib_entry *fib_entry;
1953         int err = 0;
1954
1955         mutex_lock(&mlxsw_sp->router->lock);
1956
1957         if (WARN_ON_ONCE(router->nve_decap_config.valid)) {
1958                 err = -EINVAL;
1959                 goto out;
1960         }
1961
1962         router->nve_decap_config.ul_tb_id = ul_tb_id;
1963         router->nve_decap_config.tunnel_index = tunnel_index;
1964         router->nve_decap_config.ul_proto = ul_proto;
1965         router->nve_decap_config.ul_sip = *ul_sip;
1966         router->nve_decap_config.valid = true;
1967
1968         /* It is valid to create a tunnel with a local IP and only later
1969          * assign this IP address to a local interface
1970          */
1971         fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
1972                                                          ul_proto, ul_sip,
1973                                                          type);
1974         if (!fib_entry)
1975                 goto out;
1976
1977         fib_entry->decap.tunnel_index = tunnel_index;
1978         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
1979
1980         err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1981         if (err)
1982                 goto err_fib_entry_update;
1983
1984         goto out;
1985
1986 err_fib_entry_update:
1987         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1988         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1989 out:
1990         mutex_unlock(&mlxsw_sp->router->lock);
1991         return err;
1992 }
1993
1994 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
1995                                       enum mlxsw_sp_l3proto ul_proto,
1996                                       const union mlxsw_sp_l3addr *ul_sip)
1997 {
1998         enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
1999         struct mlxsw_sp_router *router = mlxsw_sp->router;
2000         struct mlxsw_sp_fib_entry *fib_entry;
2001
2002         mutex_lock(&mlxsw_sp->router->lock);
2003
2004         if (WARN_ON_ONCE(!router->nve_decap_config.valid))
2005                 goto out;
2006
2007         router->nve_decap_config.valid = false;
2008
2009         fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
2010                                                          ul_proto, ul_sip,
2011                                                          type);
2012         if (!fib_entry)
2013                 goto out;
2014
2015         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2016         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2017 out:
2018         mutex_unlock(&mlxsw_sp->router->lock);
2019 }
2020
2021 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp,
2022                                          u32 ul_tb_id,
2023                                          enum mlxsw_sp_l3proto ul_proto,
2024                                          const union mlxsw_sp_l3addr *ul_sip)
2025 {
2026         struct mlxsw_sp_router *router = mlxsw_sp->router;
2027
2028         return router->nve_decap_config.valid &&
2029                router->nve_decap_config.ul_tb_id == ul_tb_id &&
2030                router->nve_decap_config.ul_proto == ul_proto &&
2031                !memcmp(&router->nve_decap_config.ul_sip, ul_sip,
2032                        sizeof(*ul_sip));
2033 }
2034
2035 struct mlxsw_sp_neigh_key {
2036         struct neighbour *n;
2037 };
2038
2039 struct mlxsw_sp_neigh_entry {
2040         struct list_head rif_list_node;
2041         struct rhash_head ht_node;
2042         struct mlxsw_sp_neigh_key key;
2043         u16 rif;
2044         bool connected;
2045         unsigned char ha[ETH_ALEN];
2046         struct list_head nexthop_list; /* list of nexthops using
2047                                         * this neigh entry
2048                                         */
2049         struct list_head nexthop_neighs_list_node;
2050         unsigned int counter_index;
2051         bool counter_valid;
2052 };
2053
2054 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
2055         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
2056         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
2057         .key_len = sizeof(struct mlxsw_sp_neigh_key),
2058 };
2059
2060 struct mlxsw_sp_neigh_entry *
2061 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
2062                         struct mlxsw_sp_neigh_entry *neigh_entry)
2063 {
2064         if (!neigh_entry) {
2065                 if (list_empty(&rif->neigh_list))
2066                         return NULL;
2067                 else
2068                         return list_first_entry(&rif->neigh_list,
2069                                                 typeof(*neigh_entry),
2070                                                 rif_list_node);
2071         }
2072         if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list))
2073                 return NULL;
2074         return list_next_entry(neigh_entry, rif_list_node);
2075 }
2076
2077 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
2078 {
2079         return neigh_entry->key.n->tbl->family;
2080 }
2081
2082 unsigned char *
2083 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
2084 {
2085         return neigh_entry->ha;
2086 }
2087
2088 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2089 {
2090         struct neighbour *n;
2091
2092         n = neigh_entry->key.n;
2093         return ntohl(*((__be32 *) n->primary_key));
2094 }
2095
2096 struct in6_addr *
2097 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2098 {
2099         struct neighbour *n;
2100
2101         n = neigh_entry->key.n;
2102         return (struct in6_addr *) &n->primary_key;
2103 }
2104
2105 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
2106                                struct mlxsw_sp_neigh_entry *neigh_entry,
2107                                u64 *p_counter)
2108 {
2109         if (!neigh_entry->counter_valid)
2110                 return -EINVAL;
2111
2112         return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
2113                                          p_counter, NULL);
2114 }
2115
2116 static struct mlxsw_sp_neigh_entry *
2117 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
2118                            u16 rif)
2119 {
2120         struct mlxsw_sp_neigh_entry *neigh_entry;
2121
2122         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
2123         if (!neigh_entry)
2124                 return NULL;
2125
2126         neigh_entry->key.n = n;
2127         neigh_entry->rif = rif;
2128         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
2129
2130         return neigh_entry;
2131 }
2132
2133 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
2134 {
2135         kfree(neigh_entry);
2136 }
2137
2138 static int
2139 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
2140                             struct mlxsw_sp_neigh_entry *neigh_entry)
2141 {
2142         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
2143                                       &neigh_entry->ht_node,
2144                                       mlxsw_sp_neigh_ht_params);
2145 }
2146
2147 static void
2148 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
2149                             struct mlxsw_sp_neigh_entry *neigh_entry)
2150 {
2151         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
2152                                &neigh_entry->ht_node,
2153                                mlxsw_sp_neigh_ht_params);
2154 }
2155
2156 static bool
2157 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
2158                                     struct mlxsw_sp_neigh_entry *neigh_entry)
2159 {
2160         struct devlink *devlink;
2161         const char *table_name;
2162
2163         switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
2164         case AF_INET:
2165                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
2166                 break;
2167         case AF_INET6:
2168                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
2169                 break;
2170         default:
2171                 WARN_ON(1);
2172                 return false;
2173         }
2174
2175         devlink = priv_to_devlink(mlxsw_sp->core);
2176         return devlink_dpipe_table_counter_enabled(devlink, table_name);
2177 }
2178
2179 static void
2180 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2181                              struct mlxsw_sp_neigh_entry *neigh_entry)
2182 {
2183         if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
2184                 return;
2185
2186         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
2187                 return;
2188
2189         neigh_entry->counter_valid = true;
2190 }
2191
2192 static void
2193 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
2194                             struct mlxsw_sp_neigh_entry *neigh_entry)
2195 {
2196         if (!neigh_entry->counter_valid)
2197                 return;
2198         mlxsw_sp_flow_counter_free(mlxsw_sp,
2199                                    neigh_entry->counter_index);
2200         neigh_entry->counter_valid = false;
2201 }
2202
2203 static struct mlxsw_sp_neigh_entry *
2204 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2205 {
2206         struct mlxsw_sp_neigh_entry *neigh_entry;
2207         struct mlxsw_sp_rif *rif;
2208         int err;
2209
2210         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
2211         if (!rif)
2212                 return ERR_PTR(-EINVAL);
2213
2214         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
2215         if (!neigh_entry)
2216                 return ERR_PTR(-ENOMEM);
2217
2218         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
2219         if (err)
2220                 goto err_neigh_entry_insert;
2221
2222         mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2223         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
2224
2225         return neigh_entry;
2226
2227 err_neigh_entry_insert:
2228         mlxsw_sp_neigh_entry_free(neigh_entry);
2229         return ERR_PTR(err);
2230 }
2231
2232 static void
2233 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2234                              struct mlxsw_sp_neigh_entry *neigh_entry)
2235 {
2236         list_del(&neigh_entry->rif_list_node);
2237         mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2238         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
2239         mlxsw_sp_neigh_entry_free(neigh_entry);
2240 }
2241
2242 static struct mlxsw_sp_neigh_entry *
2243 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2244 {
2245         struct mlxsw_sp_neigh_key key;
2246
2247         key.n = n;
2248         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
2249                                       &key, mlxsw_sp_neigh_ht_params);
2250 }
2251
2252 static void
2253 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
2254 {
2255         unsigned long interval;
2256
2257 #if IS_ENABLED(CONFIG_IPV6)
2258         interval = min_t(unsigned long,
2259                          NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
2260                          NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
2261 #else
2262         interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
2263 #endif
2264         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
2265 }
2266
2267 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2268                                                    char *rauhtd_pl,
2269                                                    int ent_index)
2270 {
2271         struct net_device *dev;
2272         struct neighbour *n;
2273         __be32 dipn;
2274         u32 dip;
2275         u16 rif;
2276
2277         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
2278
2279         if (!mlxsw_sp->router->rifs[rif]) {
2280                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2281                 return;
2282         }
2283
2284         dipn = htonl(dip);
2285         dev = mlxsw_sp->router->rifs[rif]->dev;
2286         n = neigh_lookup(&arp_tbl, &dipn, dev);
2287         if (!n)
2288                 return;
2289
2290         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
2291         neigh_event_send(n, NULL);
2292         neigh_release(n);
2293 }
2294
2295 #if IS_ENABLED(CONFIG_IPV6)
2296 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2297                                                    char *rauhtd_pl,
2298                                                    int rec_index)
2299 {
2300         struct net_device *dev;
2301         struct neighbour *n;
2302         struct in6_addr dip;
2303         u16 rif;
2304
2305         mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
2306                                          (char *) &dip);
2307
2308         if (!mlxsw_sp->router->rifs[rif]) {
2309                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2310                 return;
2311         }
2312
2313         dev = mlxsw_sp->router->rifs[rif]->dev;
2314         n = neigh_lookup(&nd_tbl, &dip, dev);
2315         if (!n)
2316                 return;
2317
2318         netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
2319         neigh_event_send(n, NULL);
2320         neigh_release(n);
2321 }
2322 #else
2323 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2324                                                    char *rauhtd_pl,
2325                                                    int rec_index)
2326 {
2327 }
2328 #endif
2329
2330 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2331                                                    char *rauhtd_pl,
2332                                                    int rec_index)
2333 {
2334         u8 num_entries;
2335         int i;
2336
2337         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2338                                                                 rec_index);
2339         /* Hardware starts counting at 0, so add 1. */
2340         num_entries++;
2341
2342         /* Each record consists of several neighbour entries. */
2343         for (i = 0; i < num_entries; i++) {
2344                 int ent_index;
2345
2346                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
2347                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
2348                                                        ent_index);
2349         }
2350
2351 }
2352
2353 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2354                                                    char *rauhtd_pl,
2355                                                    int rec_index)
2356 {
2357         /* One record contains one entry. */
2358         mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
2359                                                rec_index);
2360 }
2361
2362 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
2363                                               char *rauhtd_pl, int rec_index)
2364 {
2365         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
2366         case MLXSW_REG_RAUHTD_TYPE_IPV4:
2367                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
2368                                                        rec_index);
2369                 break;
2370         case MLXSW_REG_RAUHTD_TYPE_IPV6:
2371                 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
2372                                                        rec_index);
2373                 break;
2374         }
2375 }
2376
2377 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
2378 {
2379         u8 num_rec, last_rec_index, num_entries;
2380
2381         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2382         last_rec_index = num_rec - 1;
2383
2384         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
2385                 return false;
2386         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
2387             MLXSW_REG_RAUHTD_TYPE_IPV6)
2388                 return true;
2389
2390         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2391                                                                 last_rec_index);
2392         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
2393                 return true;
2394         return false;
2395 }
2396
2397 static int
2398 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
2399                                        char *rauhtd_pl,
2400                                        enum mlxsw_reg_rauhtd_type type)
2401 {
2402         int i, num_rec;
2403         int err;
2404
2405         /* Ensure the RIF we read from the device does not change mid-dump. */
2406         mutex_lock(&mlxsw_sp->router->lock);
2407         do {
2408                 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
2409                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
2410                                       rauhtd_pl);
2411                 if (err) {
2412                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n");
2413                         break;
2414                 }
2415                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2416                 for (i = 0; i < num_rec; i++)
2417                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
2418                                                           i);
2419         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
2420         mutex_unlock(&mlxsw_sp->router->lock);
2421
2422         return err;
2423 }
2424
2425 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
2426 {
2427         enum mlxsw_reg_rauhtd_type type;
2428         char *rauhtd_pl;
2429         int err;
2430
2431         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
2432         if (!rauhtd_pl)
2433                 return -ENOMEM;
2434
2435         type = MLXSW_REG_RAUHTD_TYPE_IPV4;
2436         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2437         if (err)
2438                 goto out;
2439
2440         type = MLXSW_REG_RAUHTD_TYPE_IPV6;
2441         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2442 out:
2443         kfree(rauhtd_pl);
2444         return err;
2445 }
2446
2447 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
2448 {
2449         struct mlxsw_sp_neigh_entry *neigh_entry;
2450
2451         mutex_lock(&mlxsw_sp->router->lock);
2452         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
2453                             nexthop_neighs_list_node)
2454                 /* If this neigh have nexthops, make the kernel think this neigh
2455                  * is active regardless of the traffic.
2456                  */
2457                 neigh_event_send(neigh_entry->key.n, NULL);
2458         mutex_unlock(&mlxsw_sp->router->lock);
2459 }
2460
2461 static void
2462 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
2463 {
2464         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
2465
2466         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
2467                                msecs_to_jiffies(interval));
2468 }
2469
2470 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
2471 {
2472         struct mlxsw_sp_router *router;
2473         int err;
2474
2475         router = container_of(work, struct mlxsw_sp_router,
2476                               neighs_update.dw.work);
2477         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
2478         if (err)
2479                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
2480
2481         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
2482
2483         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
2484 }
2485
2486 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
2487 {
2488         struct mlxsw_sp_neigh_entry *neigh_entry;
2489         struct mlxsw_sp_router *router;
2490
2491         router = container_of(work, struct mlxsw_sp_router,
2492                               nexthop_probe_dw.work);
2493         /* Iterate over nexthop neighbours, find those who are unresolved and
2494          * send arp on them. This solves the chicken-egg problem when
2495          * the nexthop wouldn't get offloaded until the neighbor is resolved
2496          * but it wouldn't get resolved ever in case traffic is flowing in HW
2497          * using different nexthop.
2498          */
2499         mutex_lock(&router->lock);
2500         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
2501                             nexthop_neighs_list_node)
2502                 if (!neigh_entry->connected)
2503                         neigh_event_send(neigh_entry->key.n, NULL);
2504         mutex_unlock(&router->lock);
2505
2506         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
2507                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
2508 }
2509
2510 static void
2511 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2512                               struct mlxsw_sp_neigh_entry *neigh_entry,
2513                               bool removing, bool dead);
2514
2515 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
2516 {
2517         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
2518                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
2519 }
2520
2521 static int
2522 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
2523                                 struct mlxsw_sp_neigh_entry *neigh_entry,
2524                                 enum mlxsw_reg_rauht_op op)
2525 {
2526         struct neighbour *n = neigh_entry->key.n;
2527         u32 dip = ntohl(*((__be32 *) n->primary_key));
2528         char rauht_pl[MLXSW_REG_RAUHT_LEN];
2529
2530         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2531                               dip);
2532         if (neigh_entry->counter_valid)
2533                 mlxsw_reg_rauht_pack_counter(rauht_pl,
2534                                              neigh_entry->counter_index);
2535         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2536 }
2537
2538 static int
2539 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
2540                                 struct mlxsw_sp_neigh_entry *neigh_entry,
2541                                 enum mlxsw_reg_rauht_op op)
2542 {
2543         struct neighbour *n = neigh_entry->key.n;
2544         char rauht_pl[MLXSW_REG_RAUHT_LEN];
2545         const char *dip = n->primary_key;
2546
2547         mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2548                               dip);
2549         if (neigh_entry->counter_valid)
2550                 mlxsw_reg_rauht_pack_counter(rauht_pl,
2551                                              neigh_entry->counter_index);
2552         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2553 }
2554
2555 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
2556 {
2557         struct neighbour *n = neigh_entry->key.n;
2558
2559         /* Packets with a link-local destination address are trapped
2560          * after LPM lookup and never reach the neighbour table, so
2561          * there is no need to program such neighbours to the device.
2562          */
2563         if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
2564             IPV6_ADDR_LINKLOCAL)
2565                 return true;
2566         return false;
2567 }
2568
2569 static void
2570 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
2571                             struct mlxsw_sp_neigh_entry *neigh_entry,
2572                             bool adding)
2573 {
2574         enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding);
2575         int err;
2576
2577         if (!adding && !neigh_entry->connected)
2578                 return;
2579         neigh_entry->connected = adding;
2580         if (neigh_entry->key.n->tbl->family == AF_INET) {
2581                 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
2582                                                       op);
2583                 if (err)
2584                         return;
2585         } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
2586                 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
2587                         return;
2588                 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
2589                                                       op);
2590                 if (err)
2591                         return;
2592         } else {
2593                 WARN_ON_ONCE(1);
2594                 return;
2595         }
2596
2597         if (adding)
2598                 neigh_entry->key.n->flags |= NTF_OFFLOADED;
2599         else
2600                 neigh_entry->key.n->flags &= ~NTF_OFFLOADED;
2601 }
2602
2603 void
2604 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
2605                                     struct mlxsw_sp_neigh_entry *neigh_entry,
2606                                     bool adding)
2607 {
2608         if (adding)
2609                 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2610         else
2611                 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2612         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
2613 }
2614
2615 struct mlxsw_sp_netevent_work {
2616         struct work_struct work;
2617         struct mlxsw_sp *mlxsw_sp;
2618         struct neighbour *n;
2619 };
2620
2621 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
2622 {
2623         struct mlxsw_sp_netevent_work *net_work =
2624                 container_of(work, struct mlxsw_sp_netevent_work, work);
2625         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2626         struct mlxsw_sp_neigh_entry *neigh_entry;
2627         struct neighbour *n = net_work->n;
2628         unsigned char ha[ETH_ALEN];
2629         bool entry_connected;
2630         u8 nud_state, dead;
2631
2632         /* If these parameters are changed after we release the lock,
2633          * then we are guaranteed to receive another event letting us
2634          * know about it.
2635          */
2636         read_lock_bh(&n->lock);
2637         memcpy(ha, n->ha, ETH_ALEN);
2638         nud_state = n->nud_state;
2639         dead = n->dead;
2640         read_unlock_bh(&n->lock);
2641
2642         mutex_lock(&mlxsw_sp->router->lock);
2643         mlxsw_sp_span_respin(mlxsw_sp);
2644
2645         entry_connected = nud_state & NUD_VALID && !dead;
2646         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2647         if (!entry_connected && !neigh_entry)
2648                 goto out;
2649         if (!neigh_entry) {
2650                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2651                 if (IS_ERR(neigh_entry))
2652                         goto out;
2653         }
2654
2655         memcpy(neigh_entry->ha, ha, ETH_ALEN);
2656         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
2657         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected,
2658                                       dead);
2659
2660         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2661                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2662
2663 out:
2664         mutex_unlock(&mlxsw_sp->router->lock);
2665         neigh_release(n);
2666         kfree(net_work);
2667 }
2668
2669 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp);
2670
2671 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work)
2672 {
2673         struct mlxsw_sp_netevent_work *net_work =
2674                 container_of(work, struct mlxsw_sp_netevent_work, work);
2675         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2676
2677         mlxsw_sp_mp_hash_init(mlxsw_sp);
2678         kfree(net_work);
2679 }
2680
2681 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp);
2682
2683 static void mlxsw_sp_router_update_priority_work(struct work_struct *work)
2684 {
2685         struct mlxsw_sp_netevent_work *net_work =
2686                 container_of(work, struct mlxsw_sp_netevent_work, work);
2687         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2688
2689         __mlxsw_sp_router_init(mlxsw_sp);
2690         kfree(net_work);
2691 }
2692
2693 static int mlxsw_sp_router_schedule_work(struct net *net,
2694                                          struct notifier_block *nb,
2695                                          void (*cb)(struct work_struct *))
2696 {
2697         struct mlxsw_sp_netevent_work *net_work;
2698         struct mlxsw_sp_router *router;
2699
2700         router = container_of(nb, struct mlxsw_sp_router, netevent_nb);
2701         if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp)))
2702                 return NOTIFY_DONE;
2703
2704         net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2705         if (!net_work)
2706                 return NOTIFY_BAD;
2707
2708         INIT_WORK(&net_work->work, cb);
2709         net_work->mlxsw_sp = router->mlxsw_sp;
2710         mlxsw_core_schedule_work(&net_work->work);
2711         return NOTIFY_DONE;
2712 }
2713
2714 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb,
2715                                           unsigned long event, void *ptr)
2716 {
2717         struct mlxsw_sp_netevent_work *net_work;
2718         struct mlxsw_sp_port *mlxsw_sp_port;
2719         struct mlxsw_sp *mlxsw_sp;
2720         unsigned long interval;
2721         struct neigh_parms *p;
2722         struct neighbour *n;
2723
2724         switch (event) {
2725         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
2726                 p = ptr;
2727
2728                 /* We don't care about changes in the default table. */
2729                 if (!p->dev || (p->tbl->family != AF_INET &&
2730                                 p->tbl->family != AF_INET6))
2731                         return NOTIFY_DONE;
2732
2733                 /* We are in atomic context and can't take RTNL mutex,
2734                  * so use RCU variant to walk the device chain.
2735                  */
2736                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
2737                 if (!mlxsw_sp_port)
2738                         return NOTIFY_DONE;
2739
2740                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2741                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
2742                 mlxsw_sp->router->neighs_update.interval = interval;
2743
2744                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2745                 break;
2746         case NETEVENT_NEIGH_UPDATE:
2747                 n = ptr;
2748
2749                 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
2750                         return NOTIFY_DONE;
2751
2752                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
2753                 if (!mlxsw_sp_port)
2754                         return NOTIFY_DONE;
2755
2756                 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2757                 if (!net_work) {
2758                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
2759                         return NOTIFY_BAD;
2760                 }
2761
2762                 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work);
2763                 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2764                 net_work->n = n;
2765
2766                 /* Take a reference to ensure the neighbour won't be
2767                  * destructed until we drop the reference in delayed
2768                  * work.
2769                  */
2770                 neigh_clone(n);
2771                 mlxsw_core_schedule_work(&net_work->work);
2772                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2773                 break;
2774         case NETEVENT_IPV4_MPATH_HASH_UPDATE:
2775         case NETEVENT_IPV6_MPATH_HASH_UPDATE:
2776                 return mlxsw_sp_router_schedule_work(ptr, nb,
2777                                 mlxsw_sp_router_mp_hash_event_work);
2778
2779         case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE:
2780                 return mlxsw_sp_router_schedule_work(ptr, nb,
2781                                 mlxsw_sp_router_update_priority_work);
2782         }
2783
2784         return NOTIFY_DONE;
2785 }
2786
2787 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
2788 {
2789         int err;
2790
2791         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
2792                               &mlxsw_sp_neigh_ht_params);
2793         if (err)
2794                 return err;
2795
2796         /* Initialize the polling interval according to the default
2797          * table.
2798          */
2799         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
2800
2801         /* Create the delayed works for the activity_update */
2802         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
2803                           mlxsw_sp_router_neighs_update_work);
2804         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
2805                           mlxsw_sp_router_probe_unresolved_nexthops);
2806         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
2807         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
2808         return 0;
2809 }
2810
2811 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2812 {
2813         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
2814         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
2815         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2816 }
2817
2818 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2819                                          struct mlxsw_sp_rif *rif)
2820 {
2821         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2822
2823         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2824                                  rif_list_node) {
2825                 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2826                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2827         }
2828 }
2829
2830 enum mlxsw_sp_nexthop_type {
2831         MLXSW_SP_NEXTHOP_TYPE_ETH,
2832         MLXSW_SP_NEXTHOP_TYPE_IPIP,
2833 };
2834
2835 struct mlxsw_sp_nexthop_key {
2836         struct fib_nh *fib_nh;
2837 };
2838
2839 struct mlxsw_sp_nexthop {
2840         struct list_head neigh_list_node; /* member of neigh entry list */
2841         struct list_head rif_list_node;
2842         struct list_head router_list_node;
2843         struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group
2844                                                    * this nexthop belongs to
2845                                                    */
2846         struct rhash_head ht_node;
2847         struct neigh_table *neigh_tbl;
2848         struct mlxsw_sp_nexthop_key key;
2849         unsigned char gw_addr[sizeof(struct in6_addr)];
2850         int ifindex;
2851         int nh_weight;
2852         int norm_nh_weight;
2853         int num_adj_entries;
2854         struct mlxsw_sp_rif *rif;
2855         u8 should_offload:1, /* set indicates this neigh is connected and
2856                               * should be put to KVD linear area of this group.
2857                               */
2858            offloaded:1, /* set in case the neigh is actually put into
2859                          * KVD linear area of this group.
2860                          */
2861            update:1; /* set indicates that MAC of this neigh should be
2862                       * updated in HW
2863                       */
2864         enum mlxsw_sp_nexthop_type type;
2865         union {
2866                 struct mlxsw_sp_neigh_entry *neigh_entry;
2867                 struct mlxsw_sp_ipip_entry *ipip_entry;
2868         };
2869         unsigned int counter_index;
2870         bool counter_valid;
2871 };
2872
2873 enum mlxsw_sp_nexthop_group_type {
2874         MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4,
2875         MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6,
2876 };
2877
2878 struct mlxsw_sp_nexthop_group_info {
2879         struct mlxsw_sp_nexthop_group *nh_grp;
2880         u32 adj_index;
2881         u16 ecmp_size;
2882         u16 count;
2883         int sum_norm_weight;
2884         u8 adj_index_valid:1,
2885            gateway:1; /* routes using the group use a gateway */
2886         struct mlxsw_sp_nexthop nexthops[0];
2887 #define nh_rif  nexthops[0].rif
2888 };
2889
2890 struct mlxsw_sp_nexthop_group {
2891         struct rhash_head ht_node;
2892         struct list_head fib_list; /* list of fib entries that use this group */
2893         union {
2894                 struct {
2895                         struct fib_info *fi;
2896                 } ipv4;
2897         };
2898         struct mlxsw_sp_nexthop_group_info *nhgi;
2899         enum mlxsw_sp_nexthop_group_type type;
2900         bool can_destroy;
2901 };
2902
2903 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2904                                     struct mlxsw_sp_nexthop *nh)
2905 {
2906         struct devlink *devlink;
2907
2908         devlink = priv_to_devlink(mlxsw_sp->core);
2909         if (!devlink_dpipe_table_counter_enabled(devlink,
2910                                                  MLXSW_SP_DPIPE_TABLE_NAME_ADJ))
2911                 return;
2912
2913         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index))
2914                 return;
2915
2916         nh->counter_valid = true;
2917 }
2918
2919 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp,
2920                                    struct mlxsw_sp_nexthop *nh)
2921 {
2922         if (!nh->counter_valid)
2923                 return;
2924         mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index);
2925         nh->counter_valid = false;
2926 }
2927
2928 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp,
2929                                  struct mlxsw_sp_nexthop *nh, u64 *p_counter)
2930 {
2931         if (!nh->counter_valid)
2932                 return -EINVAL;
2933
2934         return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index,
2935                                          p_counter, NULL);
2936 }
2937
2938 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router,
2939                                                struct mlxsw_sp_nexthop *nh)
2940 {
2941         if (!nh) {
2942                 if (list_empty(&router->nexthop_list))
2943                         return NULL;
2944                 else
2945                         return list_first_entry(&router->nexthop_list,
2946                                                 typeof(*nh), router_list_node);
2947         }
2948         if (list_is_last(&nh->router_list_node, &router->nexthop_list))
2949                 return NULL;
2950         return list_next_entry(nh, router_list_node);
2951 }
2952
2953 bool mlxsw_sp_nexthop_offload(struct mlxsw_sp_nexthop *nh)
2954 {
2955         return nh->offloaded;
2956 }
2957
2958 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh)
2959 {
2960         if (!nh->offloaded)
2961                 return NULL;
2962         return nh->neigh_entry->ha;
2963 }
2964
2965 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index,
2966                              u32 *p_adj_size, u32 *p_adj_hash_index)
2967 {
2968         struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
2969         u32 adj_hash_index = 0;
2970         int i;
2971
2972         if (!nh->offloaded || !nhgi->adj_index_valid)
2973                 return -EINVAL;
2974
2975         *p_adj_index = nhgi->adj_index;
2976         *p_adj_size = nhgi->ecmp_size;
2977
2978         for (i = 0; i < nhgi->count; i++) {
2979                 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
2980
2981                 if (nh_iter == nh)
2982                         break;
2983                 if (nh_iter->offloaded)
2984                         adj_hash_index += nh_iter->num_adj_entries;
2985         }
2986
2987         *p_adj_hash_index = adj_hash_index;
2988         return 0;
2989 }
2990
2991 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh)
2992 {
2993         return nh->rif;
2994 }
2995
2996 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh)
2997 {
2998         struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
2999         int i;
3000
3001         for (i = 0; i < nhgi->count; i++) {
3002                 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
3003
3004                 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP)
3005                         return true;
3006         }
3007         return false;
3008 }
3009
3010 struct mlxsw_sp_nexthop_group_cmp_arg {
3011         enum mlxsw_sp_nexthop_group_type type;
3012         union {
3013                 struct fib_info *fi;
3014                 struct mlxsw_sp_fib6_entry *fib6_entry;
3015         };
3016 };
3017
3018 static bool
3019 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
3020                                     const struct in6_addr *gw, int ifindex,
3021                                     int weight)
3022 {
3023         int i;
3024
3025         for (i = 0; i < nh_grp->nhgi->count; i++) {
3026                 const struct mlxsw_sp_nexthop *nh;
3027
3028                 nh = &nh_grp->nhgi->nexthops[i];
3029                 if (nh->ifindex == ifindex && nh->nh_weight == weight &&
3030                     ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
3031                         return true;
3032         }
3033
3034         return false;
3035 }
3036
3037 static bool
3038 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
3039                             const struct mlxsw_sp_fib6_entry *fib6_entry)
3040 {
3041         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3042
3043         if (nh_grp->nhgi->count != fib6_entry->nrt6)
3044                 return false;
3045
3046         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3047                 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3048                 struct in6_addr *gw;
3049                 int ifindex, weight;
3050
3051                 ifindex = fib6_nh->fib_nh_dev->ifindex;
3052                 weight = fib6_nh->fib_nh_weight;
3053                 gw = &fib6_nh->fib_nh_gw6;
3054                 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex,
3055                                                          weight))
3056                         return false;
3057         }
3058
3059         return true;
3060 }
3061
3062 static int
3063 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
3064 {
3065         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
3066         const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
3067
3068         if (nh_grp->type != cmp_arg->type)
3069                 return 1;
3070
3071         switch (cmp_arg->type) {
3072         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3073                 return cmp_arg->fi != nh_grp->ipv4.fi;
3074         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3075                 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
3076                                                     cmp_arg->fib6_entry);
3077         default:
3078                 WARN_ON(1);
3079                 return 1;
3080         }
3081 }
3082
3083 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
3084 {
3085         const struct mlxsw_sp_nexthop_group *nh_grp = data;
3086         const struct mlxsw_sp_nexthop *nh;
3087         struct fib_info *fi;
3088         unsigned int val;
3089         int i;
3090
3091         switch (nh_grp->type) {
3092         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3093                 fi = nh_grp->ipv4.fi;
3094                 return jhash(&fi, sizeof(fi), seed);
3095         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3096                 val = nh_grp->nhgi->count;
3097                 for (i = 0; i < nh_grp->nhgi->count; i++) {
3098                         nh = &nh_grp->nhgi->nexthops[i];
3099                         val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed);
3100                         val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed);
3101                 }
3102                 return jhash(&val, sizeof(val), seed);
3103         default:
3104                 WARN_ON(1);
3105                 return 0;
3106         }
3107 }
3108
3109 static u32
3110 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
3111 {
3112         unsigned int val = fib6_entry->nrt6;
3113         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3114
3115         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3116                 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3117                 struct net_device *dev = fib6_nh->fib_nh_dev;
3118                 struct in6_addr *gw = &fib6_nh->fib_nh_gw6;
3119
3120                 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed);
3121                 val ^= jhash(gw, sizeof(*gw), seed);
3122         }
3123
3124         return jhash(&val, sizeof(val), seed);
3125 }
3126
3127 static u32
3128 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
3129 {
3130         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
3131
3132         switch (cmp_arg->type) {
3133         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3134                 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
3135         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3136                 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
3137         default:
3138                 WARN_ON(1);
3139                 return 0;
3140         }
3141 }
3142
3143 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
3144         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
3145         .hashfn      = mlxsw_sp_nexthop_group_hash,
3146         .obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
3147         .obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
3148 };
3149
3150 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
3151                                          struct mlxsw_sp_nexthop_group *nh_grp)
3152 {
3153         if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3154             !nh_grp->nhgi->gateway)
3155                 return 0;
3156
3157         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
3158                                       &nh_grp->ht_node,
3159                                       mlxsw_sp_nexthop_group_ht_params);
3160 }
3161
3162 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
3163                                           struct mlxsw_sp_nexthop_group *nh_grp)
3164 {
3165         if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3166             !nh_grp->nhgi->gateway)
3167                 return;
3168
3169         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
3170                                &nh_grp->ht_node,
3171                                mlxsw_sp_nexthop_group_ht_params);
3172 }
3173
3174 static struct mlxsw_sp_nexthop_group *
3175 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
3176                                struct fib_info *fi)
3177 {
3178         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3179
3180         cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
3181         cmp_arg.fi = fi;
3182         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3183                                       &cmp_arg,
3184                                       mlxsw_sp_nexthop_group_ht_params);
3185 }
3186
3187 static struct mlxsw_sp_nexthop_group *
3188 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
3189                                struct mlxsw_sp_fib6_entry *fib6_entry)
3190 {
3191         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3192
3193         cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
3194         cmp_arg.fib6_entry = fib6_entry;
3195         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3196                                       &cmp_arg,
3197                                       mlxsw_sp_nexthop_group_ht_params);
3198 }
3199
3200 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
3201         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
3202         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
3203         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
3204 };
3205
3206 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
3207                                    struct mlxsw_sp_nexthop *nh)
3208 {
3209         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
3210                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
3211 }
3212
3213 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
3214                                     struct mlxsw_sp_nexthop *nh)
3215 {
3216         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
3217                                mlxsw_sp_nexthop_ht_params);
3218 }
3219
3220 static struct mlxsw_sp_nexthop *
3221 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
3222                         struct mlxsw_sp_nexthop_key key)
3223 {
3224         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
3225                                       mlxsw_sp_nexthop_ht_params);
3226 }
3227
3228 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
3229                                              const struct mlxsw_sp_fib *fib,
3230                                              u32 adj_index, u16 ecmp_size,
3231                                              u32 new_adj_index,
3232                                              u16 new_ecmp_size)
3233 {
3234         char raleu_pl[MLXSW_REG_RALEU_LEN];
3235
3236         mlxsw_reg_raleu_pack(raleu_pl,
3237                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
3238                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
3239                              new_ecmp_size);
3240         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
3241 }
3242
3243 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
3244                                           struct mlxsw_sp_nexthop_group *nh_grp,
3245                                           u32 old_adj_index, u16 old_ecmp_size)
3246 {
3247         struct mlxsw_sp_fib_entry *fib_entry;
3248         struct mlxsw_sp_fib *fib = NULL;
3249         int err;
3250
3251         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3252                 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
3253
3254                 if (fib == fib_entry->fib_node->fib)
3255                         continue;
3256                 fib = fib_entry->fib_node->fib;
3257                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
3258                                                         old_adj_index,
3259                                                         old_ecmp_size,
3260                                                         nhgi->adj_index,
3261                                                         nhgi->ecmp_size);
3262                 if (err)
3263                         return err;
3264         }
3265         return 0;
3266 }
3267
3268 static int __mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3269                                      struct mlxsw_sp_nexthop *nh)
3270 {
3271         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3272         char ratr_pl[MLXSW_REG_RATR_LEN];
3273
3274         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
3275                             true, MLXSW_REG_RATR_TYPE_ETHERNET,
3276                             adj_index, neigh_entry->rif);
3277         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
3278         if (nh->counter_valid)
3279                 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true);
3280         else
3281                 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false);
3282
3283         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
3284 }
3285
3286 int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3287                             struct mlxsw_sp_nexthop *nh)
3288 {
3289         int i;
3290
3291         for (i = 0; i < nh->num_adj_entries; i++) {
3292                 int err;
3293
3294                 err = __mlxsw_sp_nexthop_update(mlxsw_sp, adj_index + i, nh);
3295                 if (err)
3296                         return err;
3297         }
3298
3299         return 0;
3300 }
3301
3302 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3303                                           u32 adj_index,
3304                                           struct mlxsw_sp_nexthop *nh)
3305 {
3306         const struct mlxsw_sp_ipip_ops *ipip_ops;
3307
3308         ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
3309         return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry);
3310 }
3311
3312 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3313                                         u32 adj_index,
3314                                         struct mlxsw_sp_nexthop *nh)
3315 {
3316         int i;
3317
3318         for (i = 0; i < nh->num_adj_entries; i++) {
3319                 int err;
3320
3321                 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i,
3322                                                      nh);
3323                 if (err)
3324                         return err;
3325         }
3326
3327         return 0;
3328 }
3329
3330 static int
3331 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
3332                               struct mlxsw_sp_nexthop_group_info *nhgi,
3333                               bool reallocate)
3334 {
3335         u32 adj_index = nhgi->adj_index; /* base */
3336         struct mlxsw_sp_nexthop *nh;
3337         int i;
3338
3339         for (i = 0; i < nhgi->count; i++) {
3340                 nh = &nhgi->nexthops[i];
3341
3342                 if (!nh->should_offload) {
3343                         nh->offloaded = 0;
3344                         continue;
3345                 }
3346
3347                 if (nh->update || reallocate) {
3348                         int err = 0;
3349
3350                         switch (nh->type) {
3351                         case MLXSW_SP_NEXTHOP_TYPE_ETH:
3352                                 err = mlxsw_sp_nexthop_update
3353                                             (mlxsw_sp, adj_index, nh);
3354                                 break;
3355                         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
3356                                 err = mlxsw_sp_nexthop_ipip_update
3357                                             (mlxsw_sp, adj_index, nh);
3358                                 break;
3359                         }
3360                         if (err)
3361                                 return err;
3362                         nh->update = 0;
3363                         nh->offloaded = 1;
3364                 }
3365                 adj_index += nh->num_adj_entries;
3366         }
3367         return 0;
3368 }
3369
3370 static int
3371 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
3372                                     struct mlxsw_sp_nexthop_group *nh_grp)
3373 {
3374         struct mlxsw_sp_fib_entry *fib_entry;
3375         int err;
3376
3377         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3378                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3379                 if (err)
3380                         return err;
3381         }
3382         return 0;
3383 }
3384
3385 static void mlxsw_sp_adj_grp_size_round_up(u16 *p_adj_grp_size)
3386 {
3387         /* Valid sizes for an adjacency group are:
3388          * 1-64, 512, 1024, 2048 and 4096.
3389          */
3390         if (*p_adj_grp_size <= 64)
3391                 return;
3392         else if (*p_adj_grp_size <= 512)
3393                 *p_adj_grp_size = 512;
3394         else if (*p_adj_grp_size <= 1024)
3395                 *p_adj_grp_size = 1024;
3396         else if (*p_adj_grp_size <= 2048)
3397                 *p_adj_grp_size = 2048;
3398         else
3399                 *p_adj_grp_size = 4096;
3400 }
3401
3402 static void mlxsw_sp_adj_grp_size_round_down(u16 *p_adj_grp_size,
3403                                              unsigned int alloc_size)
3404 {
3405         if (alloc_size >= 4096)
3406                 *p_adj_grp_size = 4096;
3407         else if (alloc_size >= 2048)
3408                 *p_adj_grp_size = 2048;
3409         else if (alloc_size >= 1024)
3410                 *p_adj_grp_size = 1024;
3411         else if (alloc_size >= 512)
3412                 *p_adj_grp_size = 512;
3413 }
3414
3415 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp,
3416                                      u16 *p_adj_grp_size)
3417 {
3418         unsigned int alloc_size;
3419         int err;
3420
3421         /* Round up the requested group size to the next size supported
3422          * by the device and make sure the request can be satisfied.
3423          */
3424         mlxsw_sp_adj_grp_size_round_up(p_adj_grp_size);
3425         err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
3426                                               MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3427                                               *p_adj_grp_size, &alloc_size);
3428         if (err)
3429                 return err;
3430         /* It is possible the allocation results in more allocated
3431          * entries than requested. Try to use as much of them as
3432          * possible.
3433          */
3434         mlxsw_sp_adj_grp_size_round_down(p_adj_grp_size, alloc_size);
3435
3436         return 0;
3437 }
3438
3439 static void
3440 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi)
3441 {
3442         int i, g = 0, sum_norm_weight = 0;
3443         struct mlxsw_sp_nexthop *nh;
3444
3445         for (i = 0; i < nhgi->count; i++) {
3446                 nh = &nhgi->nexthops[i];
3447
3448                 if (!nh->should_offload)
3449                         continue;
3450                 if (g > 0)
3451                         g = gcd(nh->nh_weight, g);
3452                 else
3453                         g = nh->nh_weight;
3454         }
3455
3456         for (i = 0; i < nhgi->count; i++) {
3457                 nh = &nhgi->nexthops[i];
3458
3459                 if (!nh->should_offload)
3460                         continue;
3461                 nh->norm_nh_weight = nh->nh_weight / g;
3462                 sum_norm_weight += nh->norm_nh_weight;
3463         }
3464
3465         nhgi->sum_norm_weight = sum_norm_weight;
3466 }
3467
3468 static void
3469 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi)
3470 {
3471         int i, weight = 0, lower_bound = 0;
3472         int total = nhgi->sum_norm_weight;
3473         u16 ecmp_size = nhgi->ecmp_size;
3474
3475         for (i = 0; i < nhgi->count; i++) {
3476                 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
3477                 int upper_bound;
3478
3479                 if (!nh->should_offload)
3480                         continue;
3481                 weight += nh->norm_nh_weight;
3482                 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total);
3483                 nh->num_adj_entries = upper_bound - lower_bound;
3484                 lower_bound = upper_bound;
3485         }
3486 }
3487
3488 static struct mlxsw_sp_nexthop *
3489 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
3490                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6);
3491
3492 static void
3493 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3494                                         struct mlxsw_sp_nexthop_group *nh_grp)
3495 {
3496         int i;
3497
3498         for (i = 0; i < nh_grp->nhgi->count; i++) {
3499                 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
3500
3501                 if (nh->offloaded)
3502                         nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3503                 else
3504                         nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3505         }
3506 }
3507
3508 static void
3509 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp,
3510                                           struct mlxsw_sp_fib6_entry *fib6_entry)
3511 {
3512         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3513
3514         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3515                 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3516                 struct mlxsw_sp_nexthop *nh;
3517
3518                 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3519                 if (nh && nh->offloaded)
3520                         fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3521                 else
3522                         fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3523         }
3524 }
3525
3526 static void
3527 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3528                                         struct mlxsw_sp_nexthop_group *nh_grp)
3529 {
3530         struct mlxsw_sp_fib6_entry *fib6_entry;
3531
3532         /* Unfortunately, in IPv6 the route and the nexthop are described by
3533          * the same struct, so we need to iterate over all the routes using the
3534          * nexthop group and set / clear the offload indication for them.
3535          */
3536         list_for_each_entry(fib6_entry, &nh_grp->fib_list,
3537                             common.nexthop_group_node)
3538                 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
3539 }
3540
3541 static void
3542 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3543                                        struct mlxsw_sp_nexthop_group *nh_grp)
3544 {
3545         switch (nh_grp->type) {
3546         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3547                 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp);
3548                 break;
3549         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3550                 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp);
3551                 break;
3552         }
3553 }
3554
3555 static void
3556 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
3557                                struct mlxsw_sp_nexthop_group *nh_grp)
3558 {
3559         struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
3560         u16 ecmp_size, old_ecmp_size;
3561         struct mlxsw_sp_nexthop *nh;
3562         bool offload_change = false;
3563         u32 adj_index;
3564         bool old_adj_index_valid;
3565         u32 old_adj_index;
3566         int i;
3567         int err;
3568
3569         if (!nhgi->gateway) {
3570                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3571                 return;
3572         }
3573
3574         for (i = 0; i < nhgi->count; i++) {
3575                 nh = &nhgi->nexthops[i];
3576
3577                 if (nh->should_offload != nh->offloaded) {
3578                         offload_change = true;
3579                         if (nh->should_offload)
3580                                 nh->update = 1;
3581                 }
3582         }
3583         if (!offload_change) {
3584                 /* Nothing was added or removed, so no need to reallocate. Just
3585                  * update MAC on existing adjacency indexes.
3586                  */
3587                 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false);
3588                 if (err) {
3589                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3590                         goto set_trap;
3591                 }
3592                 return;
3593         }
3594         mlxsw_sp_nexthop_group_normalize(nhgi);
3595         if (!nhgi->sum_norm_weight)
3596                 /* No neigh of this group is connected so we just set
3597                  * the trap and let everthing flow through kernel.
3598                  */
3599                 goto set_trap;
3600
3601         ecmp_size = nhgi->sum_norm_weight;
3602         err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size);
3603         if (err)
3604                 /* No valid allocation size available. */
3605                 goto set_trap;
3606
3607         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3608                                   ecmp_size, &adj_index);
3609         if (err) {
3610                 /* We ran out of KVD linear space, just set the
3611                  * trap and let everything flow through kernel.
3612                  */
3613                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
3614                 goto set_trap;
3615         }
3616         old_adj_index_valid = nhgi->adj_index_valid;
3617         old_adj_index = nhgi->adj_index;
3618         old_ecmp_size = nhgi->ecmp_size;
3619         nhgi->adj_index_valid = 1;
3620         nhgi->adj_index = adj_index;
3621         nhgi->ecmp_size = ecmp_size;
3622         mlxsw_sp_nexthop_group_rebalance(nhgi);
3623         err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true);
3624         if (err) {
3625                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3626                 goto set_trap;
3627         }
3628
3629         mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3630
3631         if (!old_adj_index_valid) {
3632                 /* The trap was set for fib entries, so we have to call
3633                  * fib entry update to unset it and use adjacency index.
3634                  */
3635                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3636                 if (err) {
3637                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
3638                         goto set_trap;
3639                 }
3640                 return;
3641         }
3642
3643         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
3644                                              old_adj_index, old_ecmp_size);
3645         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3646                            old_ecmp_size, old_adj_index);
3647         if (err) {
3648                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
3649                 goto set_trap;
3650         }
3651
3652         return;
3653
3654 set_trap:
3655         old_adj_index_valid = nhgi->adj_index_valid;
3656         nhgi->adj_index_valid = 0;
3657         for (i = 0; i < nhgi->count; i++) {
3658                 nh = &nhgi->nexthops[i];
3659                 nh->offloaded = 0;
3660         }
3661         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3662         if (err)
3663                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
3664         mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3665         if (old_adj_index_valid)
3666                 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3667                                    nhgi->ecmp_size, nhgi->adj_index);
3668 }
3669
3670 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
3671                                             bool removing)
3672 {
3673         if (!removing)
3674                 nh->should_offload = 1;
3675         else
3676                 nh->should_offload = 0;
3677         nh->update = 1;
3678 }
3679
3680 static int
3681 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp,
3682                                     struct mlxsw_sp_neigh_entry *neigh_entry)
3683 {
3684         struct neighbour *n, *old_n = neigh_entry->key.n;
3685         struct mlxsw_sp_nexthop *nh;
3686         bool entry_connected;
3687         u8 nud_state, dead;
3688         int err;
3689
3690         nh = list_first_entry(&neigh_entry->nexthop_list,
3691                               struct mlxsw_sp_nexthop, neigh_list_node);
3692
3693         n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3694         if (!n) {
3695                 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3696                 if (IS_ERR(n))
3697                         return PTR_ERR(n);
3698                 neigh_event_send(n, NULL);
3699         }
3700
3701         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
3702         neigh_entry->key.n = n;
3703         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
3704         if (err)
3705                 goto err_neigh_entry_insert;
3706
3707         read_lock_bh(&n->lock);
3708         nud_state = n->nud_state;
3709         dead = n->dead;
3710         read_unlock_bh(&n->lock);
3711         entry_connected = nud_state & NUD_VALID && !dead;
3712
3713         list_for_each_entry(nh, &neigh_entry->nexthop_list,
3714                             neigh_list_node) {
3715                 neigh_release(old_n);
3716                 neigh_clone(n);
3717                 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected);
3718                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
3719         }
3720
3721         neigh_release(n);
3722
3723         return 0;
3724
3725 err_neigh_entry_insert:
3726         neigh_entry->key.n = old_n;
3727         mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
3728         neigh_release(n);
3729         return err;
3730 }
3731
3732 static void
3733 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
3734                               struct mlxsw_sp_neigh_entry *neigh_entry,
3735                               bool removing, bool dead)
3736 {
3737         struct mlxsw_sp_nexthop *nh;
3738
3739         if (list_empty(&neigh_entry->nexthop_list))
3740                 return;
3741
3742         if (dead) {
3743                 int err;
3744
3745                 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp,
3746                                                           neigh_entry);
3747                 if (err)
3748                         dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n");
3749                 return;
3750         }
3751
3752         list_for_each_entry(nh, &neigh_entry->nexthop_list,
3753                             neigh_list_node) {
3754                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
3755                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
3756         }
3757 }
3758
3759 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
3760                                       struct mlxsw_sp_rif *rif)
3761 {
3762         if (nh->rif)
3763                 return;
3764
3765         nh->rif = rif;
3766         list_add(&nh->rif_list_node, &rif->nexthop_list);
3767 }
3768
3769 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
3770 {
3771         if (!nh->rif)
3772                 return;
3773
3774         list_del(&nh->rif_list_node);
3775         nh->rif = NULL;
3776 }
3777
3778 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
3779                                        struct mlxsw_sp_nexthop *nh)
3780 {
3781         struct mlxsw_sp_neigh_entry *neigh_entry;
3782         struct neighbour *n;
3783         u8 nud_state, dead;
3784         int err;
3785
3786         if (!nh->nhgi->gateway || nh->neigh_entry)
3787                 return 0;
3788
3789         /* Take a reference of neigh here ensuring that neigh would
3790          * not be destructed before the nexthop entry is finished.
3791          * The reference is taken either in neigh_lookup() or
3792          * in neigh_create() in case n is not found.
3793          */
3794         n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3795         if (!n) {
3796                 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3797                 if (IS_ERR(n))
3798                         return PTR_ERR(n);
3799                 neigh_event_send(n, NULL);
3800         }
3801         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
3802         if (!neigh_entry) {
3803                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
3804                 if (IS_ERR(neigh_entry)) {
3805                         err = -EINVAL;
3806                         goto err_neigh_entry_create;
3807                 }
3808         }
3809
3810         /* If that is the first nexthop connected to that neigh, add to
3811          * nexthop_neighs_list
3812          */
3813         if (list_empty(&neigh_entry->nexthop_list))
3814                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
3815                               &mlxsw_sp->router->nexthop_neighs_list);
3816
3817         nh->neigh_entry = neigh_entry;
3818         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
3819         read_lock_bh(&n->lock);
3820         nud_state = n->nud_state;
3821         dead = n->dead;
3822         read_unlock_bh(&n->lock);
3823         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
3824
3825         return 0;
3826
3827 err_neigh_entry_create:
3828         neigh_release(n);
3829         return err;
3830 }
3831
3832 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
3833                                         struct mlxsw_sp_nexthop *nh)
3834 {
3835         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3836         struct neighbour *n;
3837
3838         if (!neigh_entry)
3839                 return;
3840         n = neigh_entry->key.n;
3841
3842         __mlxsw_sp_nexthop_neigh_update(nh, true);
3843         list_del(&nh->neigh_list_node);
3844         nh->neigh_entry = NULL;
3845
3846         /* If that is the last nexthop connected to that neigh, remove from
3847          * nexthop_neighs_list
3848          */
3849         if (list_empty(&neigh_entry->nexthop_list))
3850                 list_del(&neigh_entry->nexthop_neighs_list_node);
3851
3852         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
3853                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
3854
3855         neigh_release(n);
3856 }
3857
3858 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev)
3859 {
3860         struct net_device *ul_dev;
3861         bool is_up;
3862
3863         rcu_read_lock();
3864         ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
3865         is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true;
3866         rcu_read_unlock();
3867
3868         return is_up;
3869 }
3870
3871 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
3872                                        struct mlxsw_sp_nexthop *nh,
3873                                        struct mlxsw_sp_ipip_entry *ipip_entry)
3874 {
3875         bool removing;
3876
3877         if (!nh->nhgi->gateway || nh->ipip_entry)
3878                 return;
3879
3880         nh->ipip_entry = ipip_entry;
3881         removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev);
3882         __mlxsw_sp_nexthop_neigh_update(nh, removing);
3883         mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common);
3884 }
3885
3886 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
3887                                        struct mlxsw_sp_nexthop *nh)
3888 {
3889         struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
3890
3891         if (!ipip_entry)
3892                 return;
3893
3894         __mlxsw_sp_nexthop_neigh_update(nh, true);
3895         nh->ipip_entry = NULL;
3896 }
3897
3898 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
3899                                         const struct fib_nh *fib_nh,
3900                                         enum mlxsw_sp_ipip_type *p_ipipt)
3901 {
3902         struct net_device *dev = fib_nh->fib_nh_dev;
3903
3904         return dev &&
3905                fib_nh->nh_parent->fib_type == RTN_UNICAST &&
3906                mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
3907 }
3908
3909 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp,
3910                                       struct mlxsw_sp_nexthop *nh,
3911                                       const struct net_device *dev)
3912 {
3913         const struct mlxsw_sp_ipip_ops *ipip_ops;
3914         struct mlxsw_sp_ipip_entry *ipip_entry;
3915         struct mlxsw_sp_rif *rif;
3916         int err;
3917
3918         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev);
3919         if (ipip_entry) {
3920                 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
3921                 if (ipip_ops->can_offload(mlxsw_sp, dev)) {
3922                         nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
3923                         mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
3924                         return 0;
3925                 }
3926         }
3927
3928         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
3929         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3930         if (!rif)
3931                 return 0;
3932
3933         mlxsw_sp_nexthop_rif_init(nh, rif);
3934         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
3935         if (err)
3936                 goto err_neigh_init;
3937
3938         return 0;
3939
3940 err_neigh_init:
3941         mlxsw_sp_nexthop_rif_fini(nh);
3942         return err;
3943 }
3944
3945 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
3946                                        struct mlxsw_sp_nexthop *nh)
3947 {
3948         switch (nh->type) {
3949         case MLXSW_SP_NEXTHOP_TYPE_ETH:
3950                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
3951                 mlxsw_sp_nexthop_rif_fini(nh);
3952                 break;
3953         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
3954                 mlxsw_sp_nexthop_rif_fini(nh);
3955                 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
3956                 break;
3957         }
3958 }
3959
3960 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
3961                                   struct mlxsw_sp_nexthop_group *nh_grp,
3962                                   struct mlxsw_sp_nexthop *nh,
3963                                   struct fib_nh *fib_nh)
3964 {
3965         struct net_device *dev = fib_nh->fib_nh_dev;
3966         struct in_device *in_dev;
3967         int err;
3968
3969         nh->nhgi = nh_grp->nhgi;
3970         nh->key.fib_nh = fib_nh;
3971 #ifdef CONFIG_IP_ROUTE_MULTIPATH
3972         nh->nh_weight = fib_nh->fib_nh_weight;
3973 #else
3974         nh->nh_weight = 1;
3975 #endif
3976         memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4));
3977         nh->neigh_tbl = &arp_tbl;
3978         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
3979         if (err)
3980                 return err;
3981
3982         mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
3983         list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
3984
3985         if (!dev)
3986                 return 0;
3987         nh->ifindex = dev->ifindex;
3988
3989         rcu_read_lock();
3990         in_dev = __in_dev_get_rcu(dev);
3991         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
3992             fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) {
3993                 rcu_read_unlock();
3994                 return 0;
3995         }
3996         rcu_read_unlock();
3997
3998         err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
3999         if (err)
4000                 goto err_nexthop_neigh_init;
4001
4002         return 0;
4003
4004 err_nexthop_neigh_init:
4005         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4006         return err;
4007 }
4008
4009 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
4010                                    struct mlxsw_sp_nexthop *nh)
4011 {
4012         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4013         list_del(&nh->router_list_node);
4014         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4015         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4016 }
4017
4018 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
4019                                     unsigned long event, struct fib_nh *fib_nh)
4020 {
4021         struct mlxsw_sp_nexthop_key key;
4022         struct mlxsw_sp_nexthop *nh;
4023
4024         if (mlxsw_sp->router->aborted)
4025                 return;
4026
4027         key.fib_nh = fib_nh;
4028         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
4029         if (!nh)
4030                 return;
4031
4032         switch (event) {
4033         case FIB_EVENT_NH_ADD:
4034                 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev);
4035                 break;
4036         case FIB_EVENT_NH_DEL:
4037                 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4038                 break;
4039         }
4040
4041         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4042 }
4043
4044 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
4045                                         struct mlxsw_sp_rif *rif)
4046 {
4047         struct mlxsw_sp_nexthop *nh;
4048         bool removing;
4049
4050         list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) {
4051                 switch (nh->type) {
4052                 case MLXSW_SP_NEXTHOP_TYPE_ETH:
4053                         removing = false;
4054                         break;
4055                 case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4056                         removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev);
4057                         break;
4058                 default:
4059                         WARN_ON(1);
4060                         continue;
4061                 }
4062
4063                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
4064                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4065         }
4066 }
4067
4068 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
4069                                          struct mlxsw_sp_rif *old_rif,
4070                                          struct mlxsw_sp_rif *new_rif)
4071 {
4072         struct mlxsw_sp_nexthop *nh;
4073
4074         list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list);
4075         list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node)
4076                 nh->rif = new_rif;
4077         mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif);
4078 }
4079
4080 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4081                                            struct mlxsw_sp_rif *rif)
4082 {
4083         struct mlxsw_sp_nexthop *nh, *tmp;
4084
4085         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
4086                 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4087                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4088         }
4089 }
4090
4091 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
4092                                    struct fib_info *fi)
4093 {
4094         const struct fib_nh *nh = fib_info_nh(fi, 0);
4095
4096         return nh->fib_nh_scope == RT_SCOPE_LINK ||
4097                mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL);
4098 }
4099
4100 static int
4101 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp,
4102                                   struct mlxsw_sp_nexthop_group *nh_grp)
4103 {
4104         unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi);
4105         struct mlxsw_sp_nexthop_group_info *nhgi;
4106         struct mlxsw_sp_nexthop *nh;
4107         int err, i;
4108
4109         nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL);
4110         if (!nhgi)
4111                 return -ENOMEM;
4112         nh_grp->nhgi = nhgi;
4113         nhgi->nh_grp = nh_grp;
4114         nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi);
4115         nhgi->count = nhs;
4116         for (i = 0; i < nhgi->count; i++) {
4117                 struct fib_nh *fib_nh;
4118
4119                 nh = &nhgi->nexthops[i];
4120                 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i);
4121                 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
4122                 if (err)
4123                         goto err_nexthop4_init;
4124         }
4125         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4126
4127         return 0;
4128
4129 err_nexthop4_init:
4130         for (i--; i >= 0; i--) {
4131                 nh = &nhgi->nexthops[i];
4132                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
4133         }
4134         kfree(nhgi);
4135         return err;
4136 }
4137
4138 static void
4139 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp,
4140                                   struct mlxsw_sp_nexthop_group *nh_grp)
4141 {
4142         struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
4143         int i;
4144
4145         for (i = nhgi->count - 1; i >= 0; i--) {
4146                 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
4147
4148                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
4149         }
4150         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4151         WARN_ON_ONCE(nhgi->adj_index_valid);
4152         kfree(nhgi);
4153 }
4154
4155 static struct mlxsw_sp_nexthop_group *
4156 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
4157 {
4158         struct mlxsw_sp_nexthop_group *nh_grp;
4159         int err;
4160
4161         nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
4162         if (!nh_grp)
4163                 return ERR_PTR(-ENOMEM);
4164         INIT_LIST_HEAD(&nh_grp->fib_list);
4165         nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
4166         nh_grp->ipv4.fi = fi;
4167         fib_info_hold(fi);
4168
4169         err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp);
4170         if (err)
4171                 goto err_nexthop_group_info_init;
4172
4173         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
4174         if (err)
4175                 goto err_nexthop_group_insert;
4176
4177         nh_grp->can_destroy = true;
4178
4179         return nh_grp;
4180
4181 err_nexthop_group_insert:
4182         mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
4183 err_nexthop_group_info_init:
4184         fib_info_put(fi);
4185         kfree(nh_grp);
4186         return ERR_PTR(err);
4187 }
4188
4189 static void
4190 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
4191                                 struct mlxsw_sp_nexthop_group *nh_grp)
4192 {
4193         if (!nh_grp->can_destroy)
4194                 return;
4195         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
4196         mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
4197         fib_info_put(nh_grp->ipv4.fi);
4198         kfree(nh_grp);
4199 }
4200
4201 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
4202                                        struct mlxsw_sp_fib_entry *fib_entry,
4203                                        struct fib_info *fi)
4204 {
4205         struct mlxsw_sp_nexthop_group *nh_grp;
4206
4207         nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
4208         if (!nh_grp) {
4209                 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
4210                 if (IS_ERR(nh_grp))
4211                         return PTR_ERR(nh_grp);
4212         }
4213         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
4214         fib_entry->nh_group = nh_grp;
4215         return 0;
4216 }
4217
4218 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
4219                                         struct mlxsw_sp_fib_entry *fib_entry)
4220 {
4221         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
4222
4223         list_del(&fib_entry->nexthop_group_node);
4224         if (!list_empty(&nh_grp->fib_list))
4225                 return;
4226         mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
4227 }
4228
4229 static bool
4230 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
4231 {
4232         struct mlxsw_sp_fib4_entry *fib4_entry;
4233
4234         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
4235                                   common);
4236         return !fib4_entry->tos;
4237 }
4238
4239 static bool
4240 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
4241 {
4242         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
4243
4244         switch (fib_entry->fib_node->fib->proto) {
4245         case MLXSW_SP_L3_PROTO_IPV4:
4246                 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
4247                         return false;
4248                 break;
4249         case MLXSW_SP_L3_PROTO_IPV6:
4250                 break;
4251         }
4252
4253         switch (fib_entry->type) {
4254         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
4255                 return !!nh_group->nhgi->adj_index_valid;
4256         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
4257                 return !!nh_group->nhgi->nh_rif;
4258         case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
4259         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
4260         case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
4261                 return true;
4262         default:
4263                 return false;
4264         }
4265 }
4266
4267 static struct mlxsw_sp_nexthop *
4268 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
4269                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
4270 {
4271         int i;
4272
4273         for (i = 0; i < nh_grp->nhgi->count; i++) {
4274                 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
4275                 struct fib6_info *rt = mlxsw_sp_rt6->rt;
4276
4277                 if (nh->rif && nh->rif->dev == rt->fib6_nh->fib_nh_dev &&
4278                     ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
4279                                     &rt->fib6_nh->fib_nh_gw6))
4280                         return nh;
4281                 continue;
4282         }
4283
4284         return NULL;
4285 }
4286
4287 static void
4288 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
4289                                  struct mlxsw_sp_fib_entry *fib_entry)
4290 {
4291         u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
4292         int dst_len = fib_entry->fib_node->key.prefix_len;
4293         struct mlxsw_sp_fib4_entry *fib4_entry;
4294         struct fib_rt_info fri;
4295         bool should_offload;
4296
4297         should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
4298         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
4299                                   common);
4300         fri.fi = fib4_entry->fi;
4301         fri.tb_id = fib4_entry->tb_id;
4302         fri.dst = cpu_to_be32(*p_dst);
4303         fri.dst_len = dst_len;
4304         fri.tos = fib4_entry->tos;
4305         fri.type = fib4_entry->type;
4306         fri.offload = should_offload;
4307         fri.trap = !should_offload;
4308         fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
4309 }
4310
4311 static void
4312 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
4313                                    struct mlxsw_sp_fib_entry *fib_entry)
4314 {
4315         u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
4316         int dst_len = fib_entry->fib_node->key.prefix_len;
4317         struct mlxsw_sp_fib4_entry *fib4_entry;
4318         struct fib_rt_info fri;
4319
4320         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
4321                                   common);
4322         fri.fi = fib4_entry->fi;
4323         fri.tb_id = fib4_entry->tb_id;
4324         fri.dst = cpu_to_be32(*p_dst);
4325         fri.dst_len = dst_len;
4326         fri.tos = fib4_entry->tos;
4327         fri.type = fib4_entry->type;
4328         fri.offload = false;
4329         fri.trap = false;
4330         fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
4331 }
4332
4333 static void
4334 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
4335                                  struct mlxsw_sp_fib_entry *fib_entry)
4336 {
4337         struct mlxsw_sp_fib6_entry *fib6_entry;
4338         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4339         bool should_offload;
4340
4341         should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
4342
4343         /* In IPv6 a multipath route is represented using multiple routes, so
4344          * we need to set the flags on all of them.
4345          */
4346         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
4347                                   common);
4348         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
4349                 fib6_info_hw_flags_set(mlxsw_sp_rt6->rt, should_offload,
4350                                        !should_offload);
4351 }
4352
4353 static void
4354 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
4355                                    struct mlxsw_sp_fib_entry *fib_entry)
4356 {
4357         struct mlxsw_sp_fib6_entry *fib6_entry;
4358         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4359
4360         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
4361                                   common);
4362         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
4363                 fib6_info_hw_flags_set(mlxsw_sp_rt6->rt, false, false);
4364 }
4365
4366 static void
4367 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
4368                                 struct mlxsw_sp_fib_entry *fib_entry)
4369 {
4370         switch (fib_entry->fib_node->fib->proto) {
4371         case MLXSW_SP_L3_PROTO_IPV4:
4372                 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry);
4373                 break;
4374         case MLXSW_SP_L3_PROTO_IPV6:
4375                 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry);
4376                 break;
4377         }
4378 }
4379
4380 static void
4381 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
4382                                   struct mlxsw_sp_fib_entry *fib_entry)
4383 {
4384         switch (fib_entry->fib_node->fib->proto) {
4385         case MLXSW_SP_L3_PROTO_IPV4:
4386                 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry);
4387                 break;
4388         case MLXSW_SP_L3_PROTO_IPV6:
4389                 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry);
4390                 break;
4391         }
4392 }
4393
4394 static void
4395 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp,
4396                                     struct mlxsw_sp_fib_entry *fib_entry,
4397                                     enum mlxsw_sp_fib_entry_op op)
4398 {
4399         switch (op) {
4400         case MLXSW_SP_FIB_ENTRY_OP_WRITE:
4401         case MLXSW_SP_FIB_ENTRY_OP_UPDATE:
4402                 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry);
4403                 break;
4404         case MLXSW_SP_FIB_ENTRY_OP_DELETE:
4405                 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry);
4406                 break;
4407         default:
4408                 break;
4409         }
4410 }
4411
4412 struct mlxsw_sp_fib_entry_op_ctx_basic {
4413         char ralue_pl[MLXSW_REG_RALUE_LEN];
4414 };
4415
4416 static void
4417 mlxsw_sp_router_ll_basic_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4418                                         enum mlxsw_sp_l3proto proto,
4419                                         enum mlxsw_sp_fib_entry_op op,
4420                                         u16 virtual_router, u8 prefix_len,
4421                                         unsigned char *addr,
4422                                         struct mlxsw_sp_fib_entry_priv *priv)
4423 {
4424         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4425         enum mlxsw_reg_ralxx_protocol ralxx_proto;
4426         char *ralue_pl = op_ctx_basic->ralue_pl;
4427         enum mlxsw_reg_ralue_op ralue_op;
4428
4429         ralxx_proto = (enum mlxsw_reg_ralxx_protocol) proto;
4430
4431         switch (op) {
4432         case MLXSW_SP_FIB_ENTRY_OP_WRITE:
4433         case MLXSW_SP_FIB_ENTRY_OP_UPDATE:
4434                 ralue_op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
4435                 break;
4436         case MLXSW_SP_FIB_ENTRY_OP_DELETE:
4437                 ralue_op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
4438                 break;
4439         default:
4440                 WARN_ON_ONCE(1);
4441                 return;
4442         }
4443
4444         switch (proto) {
4445         case MLXSW_SP_L3_PROTO_IPV4:
4446                 mlxsw_reg_ralue_pack4(ralue_pl, ralxx_proto, ralue_op,
4447                                       virtual_router, prefix_len, (u32 *) addr);
4448                 break;
4449         case MLXSW_SP_L3_PROTO_IPV6:
4450                 mlxsw_reg_ralue_pack6(ralue_pl, ralxx_proto, ralue_op,
4451                                       virtual_router, prefix_len, addr);
4452                 break;
4453         }
4454 }
4455
4456 static void
4457 mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4458                                                    enum mlxsw_reg_ralue_trap_action trap_action,
4459                                                    u16 trap_id, u32 adjacency_index, u16 ecmp_size)
4460 {
4461         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4462
4463         mlxsw_reg_ralue_act_remote_pack(op_ctx_basic->ralue_pl, trap_action,
4464                                         trap_id, adjacency_index, ecmp_size);
4465 }
4466
4467 static void
4468 mlxsw_sp_router_ll_basic_fib_entry_act_local_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4469                                                   enum mlxsw_reg_ralue_trap_action trap_action,
4470                                                   u16 trap_id, u16 local_erif)
4471 {
4472         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4473
4474         mlxsw_reg_ralue_act_local_pack(op_ctx_basic->ralue_pl, trap_action,
4475                                        trap_id, local_erif);
4476 }
4477
4478 static void
4479 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx)
4480 {
4481         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4482
4483         mlxsw_reg_ralue_act_ip2me_pack(op_ctx_basic->ralue_pl);
4484 }
4485
4486 static void
4487 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4488                                                       u32 tunnel_ptr)
4489 {
4490         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4491
4492         mlxsw_reg_ralue_act_ip2me_tun_pack(op_ctx_basic->ralue_pl, tunnel_ptr);
4493 }
4494
4495 static int
4496 mlxsw_sp_router_ll_basic_fib_entry_commit(struct mlxsw_sp *mlxsw_sp,
4497                                           struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4498                                           bool *postponed_for_bulk)
4499 {
4500         struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
4501
4502         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
4503                                op_ctx_basic->ralue_pl);
4504 }
4505
4506 static bool
4507 mlxsw_sp_router_ll_basic_fib_entry_is_committed(struct mlxsw_sp_fib_entry_priv *priv)
4508 {
4509         return true;
4510 }
4511
4512 static void mlxsw_sp_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4513                                     struct mlxsw_sp_fib_entry *fib_entry,
4514                                     enum mlxsw_sp_fib_entry_op op)
4515 {
4516         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
4517
4518         mlxsw_sp_fib_entry_op_ctx_priv_hold(op_ctx, fib_entry->priv);
4519         fib->ll_ops->fib_entry_pack(op_ctx, fib->proto, op, fib->vr->id,
4520                                     fib_entry->fib_node->key.prefix_len,
4521                                     fib_entry->fib_node->key.addr,
4522                                     fib_entry->priv);
4523 }
4524
4525 int mlxsw_sp_fib_entry_commit(struct mlxsw_sp *mlxsw_sp,
4526                               struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4527                               const struct mlxsw_sp_router_ll_ops *ll_ops)
4528 {
4529         bool postponed_for_bulk = false;
4530         int err;
4531
4532         err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, &postponed_for_bulk);
4533         if (!postponed_for_bulk)
4534                 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
4535         return err;
4536 }
4537
4538 static int mlxsw_sp_adj_discard_write(struct mlxsw_sp *mlxsw_sp, u16 rif_index)
4539 {
4540         enum mlxsw_reg_ratr_trap_action trap_action;
4541         char ratr_pl[MLXSW_REG_RATR_LEN];
4542         int err;
4543
4544         if (mlxsw_sp->router->adj_discard_index_valid)
4545                 return 0;
4546
4547         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4548                                   &mlxsw_sp->router->adj_discard_index);
4549         if (err)
4550                 return err;
4551
4552         trap_action = MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS;
4553         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true,
4554                             MLXSW_REG_RATR_TYPE_ETHERNET,
4555                             mlxsw_sp->router->adj_discard_index, rif_index);
4556         mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action);
4557         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
4558         if (err)
4559                 goto err_ratr_write;
4560
4561         mlxsw_sp->router->adj_discard_index_valid = true;
4562
4563         return 0;
4564
4565 err_ratr_write:
4566         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4567                            mlxsw_sp->router->adj_discard_index);
4568         return err;
4569 }
4570
4571 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
4572                                         struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4573                                         struct mlxsw_sp_fib_entry *fib_entry,
4574                                         enum mlxsw_sp_fib_entry_op op)
4575 {
4576         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4577         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
4578         struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi;
4579         enum mlxsw_reg_ralue_trap_action trap_action;
4580         u16 trap_id = 0;
4581         u32 adjacency_index = 0;
4582         u16 ecmp_size = 0;
4583         int err;
4584
4585         /* In case the nexthop group adjacency index is valid, use it
4586          * with provided ECMP size. Otherwise, setup trap and pass
4587          * traffic to kernel.
4588          */
4589         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
4590                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4591                 adjacency_index = nhgi->adj_index;
4592                 ecmp_size = nhgi->ecmp_size;
4593         } else if (!nhgi->adj_index_valid && nhgi->count && nhgi->nh_rif) {
4594                 err = mlxsw_sp_adj_discard_write(mlxsw_sp,
4595                                                  nhgi->nh_rif->rif_index);
4596                 if (err)
4597                         return err;
4598                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4599                 adjacency_index = mlxsw_sp->router->adj_discard_index;
4600                 ecmp_size = 1;
4601         } else {
4602                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4603                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
4604         }
4605
4606         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4607         ll_ops->fib_entry_act_remote_pack(op_ctx, trap_action, trap_id,
4608                                           adjacency_index, ecmp_size);
4609         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4610 }
4611
4612 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
4613                                        struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4614                                        struct mlxsw_sp_fib_entry *fib_entry,
4615                                        enum mlxsw_sp_fib_entry_op op)
4616 {
4617         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4618         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nhgi->nh_rif;
4619         enum mlxsw_reg_ralue_trap_action trap_action;
4620         u16 trap_id = 0;
4621         u16 rif_index = 0;
4622
4623         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
4624                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4625                 rif_index = rif->rif_index;
4626         } else {
4627                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4628                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
4629         }
4630
4631         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4632         ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, rif_index);
4633         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4634 }
4635
4636 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
4637                                       struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4638                                       struct mlxsw_sp_fib_entry *fib_entry,
4639                                       enum mlxsw_sp_fib_entry_op op)
4640 {
4641         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4642
4643         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4644         ll_ops->fib_entry_act_ip2me_pack(op_ctx);
4645         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4646 }
4647
4648 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp,
4649                                            struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4650                                            struct mlxsw_sp_fib_entry *fib_entry,
4651                                            enum mlxsw_sp_fib_entry_op op)
4652 {
4653         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4654         enum mlxsw_reg_ralue_trap_action trap_action;
4655
4656         trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR;
4657         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4658         ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, 0, 0);
4659         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4660 }
4661
4662 static int
4663 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp,
4664                                   struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4665                                   struct mlxsw_sp_fib_entry *fib_entry,
4666                                   enum mlxsw_sp_fib_entry_op op)
4667 {
4668         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4669         enum mlxsw_reg_ralue_trap_action trap_action;
4670         u16 trap_id;
4671
4672         trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4673         trap_id = MLXSW_TRAP_ID_RTR_INGRESS1;
4674
4675         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4676         ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, 0);
4677         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4678 }
4679
4680 static int
4681 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
4682                                  struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4683                                  struct mlxsw_sp_fib_entry *fib_entry,
4684                                  enum mlxsw_sp_fib_entry_op op)
4685 {
4686         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4687         struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
4688         const struct mlxsw_sp_ipip_ops *ipip_ops;
4689
4690         if (WARN_ON(!ipip_entry))
4691                 return -EINVAL;
4692
4693         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
4694         return ipip_ops->fib_entry_op(mlxsw_sp, ll_ops, op_ctx, ipip_entry, op,
4695                                       fib_entry->decap.tunnel_index, fib_entry->priv);
4696 }
4697
4698 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp,
4699                                            struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4700                                            struct mlxsw_sp_fib_entry *fib_entry,
4701                                            enum mlxsw_sp_fib_entry_op op)
4702 {
4703         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4704
4705         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4706         ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx,
4707                                              fib_entry->decap.tunnel_index);
4708         return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
4709 }
4710
4711 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
4712                                    struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4713                                    struct mlxsw_sp_fib_entry *fib_entry,
4714                                    enum mlxsw_sp_fib_entry_op op)
4715 {
4716         switch (fib_entry->type) {
4717         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
4718                 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, op_ctx, fib_entry, op);
4719         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
4720                 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, op_ctx, fib_entry, op);
4721         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
4722                 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, op_ctx, fib_entry, op);
4723         case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
4724                 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, op_ctx, fib_entry, op);
4725         case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE:
4726                 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, op_ctx, fib_entry, op);
4727         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
4728                 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, op_ctx, fib_entry, op);
4729         case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
4730                 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, op_ctx, fib_entry, op);
4731         }
4732         return -EINVAL;
4733 }
4734
4735 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
4736                                  struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4737                                  struct mlxsw_sp_fib_entry *fib_entry,
4738                                  enum mlxsw_sp_fib_entry_op op)
4739 {
4740         int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, op);
4741
4742         if (err)
4743                 return err;
4744
4745         mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op);
4746
4747         return err;
4748 }
4749
4750 static int __mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
4751                                        struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4752                                        struct mlxsw_sp_fib_entry *fib_entry,
4753                                        bool is_new)
4754 {
4755         return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry,
4756                                      is_new ? MLXSW_SP_FIB_ENTRY_OP_WRITE :
4757                                               MLXSW_SP_FIB_ENTRY_OP_UPDATE);
4758 }
4759
4760 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
4761                                      struct mlxsw_sp_fib_entry *fib_entry)
4762 {
4763         struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx;
4764
4765         mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
4766         return __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, false);
4767 }
4768
4769 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
4770                                   struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4771                                   struct mlxsw_sp_fib_entry *fib_entry)
4772 {
4773         const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
4774
4775         if (!ll_ops->fib_entry_is_committed(fib_entry->priv))
4776                 return 0;
4777         return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry,
4778                                      MLXSW_SP_FIB_ENTRY_OP_DELETE);
4779 }
4780
4781 static int
4782 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
4783                              const struct fib_entry_notifier_info *fen_info,
4784                              struct mlxsw_sp_fib_entry *fib_entry)
4785 {
4786         struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi;
4787         union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
4788         struct mlxsw_sp_router *router = mlxsw_sp->router;
4789         u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id);
4790         int ifindex = nhgi->nexthops[0].ifindex;
4791         struct mlxsw_sp_ipip_entry *ipip_entry;
4792
4793         switch (fen_info->type) {
4794         case RTN_LOCAL:
4795                 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex,
4796                                                                MLXSW_SP_L3_PROTO_IPV4, dip);
4797                 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
4798                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
4799                         return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
4800                                                              fib_entry,
4801                                                              ipip_entry);
4802                 }
4803                 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id,
4804                                                  MLXSW_SP_L3_PROTO_IPV4,
4805                                                  &dip)) {
4806                         u32 tunnel_index;
4807
4808                         tunnel_index = router->nve_decap_config.tunnel_index;
4809                         fib_entry->decap.tunnel_index = tunnel_index;
4810                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
4811                         return 0;
4812                 }
4813                 fallthrough;
4814         case RTN_BROADCAST:
4815                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
4816                 return 0;
4817         case RTN_BLACKHOLE:
4818                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
4819                 return 0;
4820         case RTN_UNREACHABLE:
4821         case RTN_PROHIBIT:
4822                 /* Packets hitting these routes need to be trapped, but
4823                  * can do so with a lower priority than packets directed
4824                  * at the host, so use action type local instead of trap.
4825                  */
4826                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
4827                 return 0;
4828         case RTN_UNICAST:
4829                 if (nhgi->gateway)
4830                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
4831                 else
4832                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4833                 return 0;
4834         default:
4835                 return -EINVAL;
4836         }
4837 }
4838
4839 static void
4840 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
4841                                struct mlxsw_sp_fib_entry *fib_entry)
4842 {
4843         switch (fib_entry->type) {
4844         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
4845                 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
4846                 break;
4847         default:
4848                 break;
4849         }
4850 }
4851
4852 static struct mlxsw_sp_fib4_entry *
4853 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
4854                            struct mlxsw_sp_fib_node *fib_node,
4855                            const struct fib_entry_notifier_info *fen_info)
4856 {
4857         struct mlxsw_sp_fib4_entry *fib4_entry;
4858         struct mlxsw_sp_fib_entry *fib_entry;
4859         int err;
4860
4861         fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
4862         if (!fib4_entry)
4863                 return ERR_PTR(-ENOMEM);
4864         fib_entry = &fib4_entry->common;
4865
4866         fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops);
4867         if (IS_ERR(fib_entry->priv)) {
4868                 err = PTR_ERR(fib_entry->priv);
4869                 goto err_fib_entry_priv_create;
4870         }
4871
4872         err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
4873         if (err)
4874                 goto err_nexthop4_group_get;
4875
4876         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
4877         if (err)
4878                 goto err_fib4_entry_type_set;
4879
4880         fib4_entry->fi = fen_info->fi;
4881         fib_info_hold(fib4_entry->fi);
4882         fib4_entry->tb_id = fen_info->tb_id;
4883         fib4_entry->type = fen_info->type;
4884         fib4_entry->tos = fen_info->tos;
4885
4886         fib_entry->fib_node = fib_node;
4887
4888         return fib4_entry;
4889
4890 err_fib4_entry_type_set:
4891         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
4892 err_nexthop4_group_get:
4893         mlxsw_sp_fib_entry_priv_put(fib_entry->priv);
4894 err_fib_entry_priv_create:
4895         kfree(fib4_entry);
4896         return ERR_PTR(err);
4897 }
4898
4899 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
4900                                         struct mlxsw_sp_fib4_entry *fib4_entry)
4901 {
4902         fib_info_put(fib4_entry->fi);
4903         mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, &fib4_entry->common);
4904         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
4905         mlxsw_sp_fib_entry_priv_put(fib4_entry->common.priv);
4906         kfree(fib4_entry);
4907 }
4908
4909 static struct mlxsw_sp_fib4_entry *
4910 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
4911                            const struct fib_entry_notifier_info *fen_info)
4912 {
4913         struct mlxsw_sp_fib4_entry *fib4_entry;
4914         struct mlxsw_sp_fib_node *fib_node;
4915         struct mlxsw_sp_fib *fib;
4916         struct mlxsw_sp_vr *vr;
4917
4918         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
4919         if (!vr)
4920                 return NULL;
4921         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
4922
4923         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
4924                                             sizeof(fen_info->dst),
4925                                             fen_info->dst_len);
4926         if (!fib_node)
4927                 return NULL;
4928
4929         fib4_entry = container_of(fib_node->fib_entry,
4930                                   struct mlxsw_sp_fib4_entry, common);
4931         if (fib4_entry->tb_id == fen_info->tb_id &&
4932             fib4_entry->tos == fen_info->tos &&
4933             fib4_entry->type == fen_info->type &&
4934             fib4_entry->fi == fen_info->fi)
4935                 return fib4_entry;
4936
4937         return NULL;
4938 }
4939
4940 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
4941         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
4942         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
4943         .key_len = sizeof(struct mlxsw_sp_fib_key),
4944         .automatic_shrinking = true,
4945 };
4946
4947 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
4948                                     struct mlxsw_sp_fib_node *fib_node)
4949 {
4950         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
4951                                       mlxsw_sp_fib_ht_params);
4952 }
4953
4954 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
4955                                      struct mlxsw_sp_fib_node *fib_node)
4956 {
4957         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
4958                                mlxsw_sp_fib_ht_params);
4959 }
4960
4961 static struct mlxsw_sp_fib_node *
4962 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
4963                          size_t addr_len, unsigned char prefix_len)
4964 {
4965         struct mlxsw_sp_fib_key key;
4966
4967         memset(&key, 0, sizeof(key));
4968         memcpy(key.addr, addr, addr_len);
4969         key.prefix_len = prefix_len;
4970         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
4971 }
4972
4973 static struct mlxsw_sp_fib_node *
4974 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
4975                          size_t addr_len, unsigned char prefix_len)
4976 {
4977         struct mlxsw_sp_fib_node *fib_node;
4978
4979         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
4980         if (!fib_node)
4981                 return NULL;
4982
4983         list_add(&fib_node->list, &fib->node_list);
4984         memcpy(fib_node->key.addr, addr, addr_len);
4985         fib_node->key.prefix_len = prefix_len;
4986
4987         return fib_node;
4988 }
4989
4990 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
4991 {
4992         list_del(&fib_node->list);
4993         kfree(fib_node);
4994 }
4995
4996 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
4997                                       struct mlxsw_sp_fib_node *fib_node)
4998 {
4999         struct mlxsw_sp_prefix_usage req_prefix_usage;
5000         struct mlxsw_sp_fib *fib = fib_node->fib;
5001         struct mlxsw_sp_lpm_tree *lpm_tree;
5002         int err;
5003
5004         lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto];
5005         if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
5006                 goto out;
5007
5008         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
5009         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
5010         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
5011                                          fib->proto);
5012         if (IS_ERR(lpm_tree))
5013                 return PTR_ERR(lpm_tree);
5014
5015         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
5016         if (err)
5017                 goto err_lpm_tree_replace;
5018
5019 out:
5020         lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++;
5021         return 0;
5022
5023 err_lpm_tree_replace:
5024         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
5025         return err;
5026 }
5027
5028 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
5029                                          struct mlxsw_sp_fib_node *fib_node)
5030 {
5031         struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
5032         struct mlxsw_sp_prefix_usage req_prefix_usage;
5033         struct mlxsw_sp_fib *fib = fib_node->fib;
5034         int err;
5035
5036         if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
5037                 return;
5038         /* Try to construct a new LPM tree from the current prefix usage
5039          * minus the unused one. If we fail, continue using the old one.
5040          */
5041         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
5042         mlxsw_sp_prefix_usage_clear(&req_prefix_usage,
5043                                     fib_node->key.prefix_len);
5044         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
5045                                          fib->proto);
5046         if (IS_ERR(lpm_tree))
5047                 return;
5048
5049         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
5050         if (err)
5051                 goto err_lpm_tree_replace;
5052
5053         return;
5054
5055 err_lpm_tree_replace:
5056         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
5057 }
5058
5059 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
5060                                   struct mlxsw_sp_fib_node *fib_node,
5061                                   struct mlxsw_sp_fib *fib)
5062 {
5063         int err;
5064
5065         err = mlxsw_sp_fib_node_insert(fib, fib_node);
5066         if (err)
5067                 return err;
5068         fib_node->fib = fib;
5069
5070         err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node);
5071         if (err)
5072                 goto err_fib_lpm_tree_link;
5073
5074         return 0;
5075
5076 err_fib_lpm_tree_link:
5077         fib_node->fib = NULL;
5078         mlxsw_sp_fib_node_remove(fib, fib_node);
5079         return err;
5080 }
5081
5082 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
5083                                    struct mlxsw_sp_fib_node *fib_node)
5084 {
5085         struct mlxsw_sp_fib *fib = fib_node->fib;
5086
5087         mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node);
5088         fib_node->fib = NULL;
5089         mlxsw_sp_fib_node_remove(fib, fib_node);
5090 }
5091
5092 static struct mlxsw_sp_fib_node *
5093 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
5094                       size_t addr_len, unsigned char prefix_len,
5095                       enum mlxsw_sp_l3proto proto)
5096 {
5097         struct mlxsw_sp_fib_node *fib_node;
5098         struct mlxsw_sp_fib *fib;
5099         struct mlxsw_sp_vr *vr;
5100         int err;
5101
5102         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL);
5103         if (IS_ERR(vr))
5104                 return ERR_CAST(vr);
5105         fib = mlxsw_sp_vr_fib(vr, proto);
5106
5107         fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
5108         if (fib_node)
5109                 return fib_node;
5110
5111         fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
5112         if (!fib_node) {
5113                 err = -ENOMEM;
5114                 goto err_fib_node_create;
5115         }
5116
5117         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
5118         if (err)
5119                 goto err_fib_node_init;
5120
5121         return fib_node;
5122
5123 err_fib_node_init:
5124         mlxsw_sp_fib_node_destroy(fib_node);
5125 err_fib_node_create:
5126         mlxsw_sp_vr_put(mlxsw_sp, vr);
5127         return ERR_PTR(err);
5128 }
5129
5130 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
5131                                   struct mlxsw_sp_fib_node *fib_node)
5132 {
5133         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
5134
5135         if (fib_node->fib_entry)
5136                 return;
5137         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
5138         mlxsw_sp_fib_node_destroy(fib_node);
5139         mlxsw_sp_vr_put(mlxsw_sp, vr);
5140 }
5141
5142 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp,
5143                                         struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5144                                         struct mlxsw_sp_fib_entry *fib_entry)
5145 {
5146         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
5147         bool is_new = !fib_node->fib_entry;
5148         int err;
5149
5150         fib_node->fib_entry = fib_entry;
5151
5152         err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, is_new);
5153         if (err)
5154                 goto err_fib_entry_update;
5155
5156         return 0;
5157
5158 err_fib_entry_update:
5159         fib_node->fib_entry = NULL;
5160         return err;
5161 }
5162
5163 static int __mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
5164                                             struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5165                                             struct mlxsw_sp_fib_entry *fib_entry)
5166 {
5167         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
5168         int err;
5169
5170         err = mlxsw_sp_fib_entry_del(mlxsw_sp, op_ctx, fib_entry);
5171         fib_node->fib_entry = NULL;
5172         return err;
5173 }
5174
5175 static void mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
5176                                            struct mlxsw_sp_fib_entry *fib_entry)
5177 {
5178         struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx;
5179
5180         mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
5181         __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, fib_entry);
5182 }
5183
5184 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry)
5185 {
5186         struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
5187         struct mlxsw_sp_fib4_entry *fib4_replaced;
5188
5189         if (!fib_node->fib_entry)
5190                 return true;
5191
5192         fib4_replaced = container_of(fib_node->fib_entry,
5193                                      struct mlxsw_sp_fib4_entry, common);
5194         if (fib4_entry->tb_id == RT_TABLE_MAIN &&
5195             fib4_replaced->tb_id == RT_TABLE_LOCAL)
5196                 return false;
5197
5198         return true;
5199 }
5200
5201 static int
5202 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp,
5203                              struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5204                              const struct fib_entry_notifier_info *fen_info)
5205 {
5206         struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced;
5207         struct mlxsw_sp_fib_entry *replaced;
5208         struct mlxsw_sp_fib_node *fib_node;
5209         int err;
5210
5211         if (mlxsw_sp->router->aborted)
5212                 return 0;
5213
5214         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
5215                                          &fen_info->dst, sizeof(fen_info->dst),
5216                                          fen_info->dst_len,
5217                                          MLXSW_SP_L3_PROTO_IPV4);
5218         if (IS_ERR(fib_node)) {
5219                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
5220                 return PTR_ERR(fib_node);
5221         }
5222
5223         fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
5224         if (IS_ERR(fib4_entry)) {
5225                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
5226                 err = PTR_ERR(fib4_entry);
5227                 goto err_fib4_entry_create;
5228         }
5229
5230         if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) {
5231                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
5232                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5233                 return 0;
5234         }
5235
5236         replaced = fib_node->fib_entry;
5237         err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib4_entry->common);
5238         if (err) {
5239                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
5240                 goto err_fib_node_entry_link;
5241         }
5242
5243         /* Nothing to replace */
5244         if (!replaced)
5245                 return 0;
5246
5247         mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
5248         fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry,
5249                                      common);
5250         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced);
5251
5252         return 0;
5253
5254 err_fib_node_entry_link:
5255         fib_node->fib_entry = replaced;
5256         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
5257 err_fib4_entry_create:
5258         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5259         return err;
5260 }
5261
5262 static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
5263                                     struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5264                                     struct fib_entry_notifier_info *fen_info)
5265 {
5266         struct mlxsw_sp_fib4_entry *fib4_entry;
5267         struct mlxsw_sp_fib_node *fib_node;
5268         int err;
5269
5270         if (mlxsw_sp->router->aborted)
5271                 return 0;
5272
5273         fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
5274         if (!fib4_entry)
5275                 return 0;
5276         fib_node = fib4_entry->common.fib_node;
5277
5278         err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib4_entry->common);
5279         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
5280         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5281         return err;
5282 }
5283
5284 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt)
5285 {
5286         /* Multicast routes aren't supported, so ignore them. Neighbour
5287          * Discovery packets are specifically trapped.
5288          */
5289         if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST)
5290                 return true;
5291
5292         /* Cloned routes are irrelevant in the forwarding path. */
5293         if (rt->fib6_flags & RTF_CACHE)
5294                 return true;
5295
5296         return false;
5297 }
5298
5299 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt)
5300 {
5301         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5302
5303         mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
5304         if (!mlxsw_sp_rt6)
5305                 return ERR_PTR(-ENOMEM);
5306
5307         /* In case of route replace, replaced route is deleted with
5308          * no notification. Take reference to prevent accessing freed
5309          * memory.
5310          */
5311         mlxsw_sp_rt6->rt = rt;
5312         fib6_info_hold(rt);
5313
5314         return mlxsw_sp_rt6;
5315 }
5316
5317 #if IS_ENABLED(CONFIG_IPV6)
5318 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
5319 {
5320         fib6_info_release(rt);
5321 }
5322 #else
5323 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
5324 {
5325 }
5326 #endif
5327
5328 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
5329 {
5330         struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
5331
5332         if (!mlxsw_sp_rt6->rt->nh)
5333                 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
5334         mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
5335         kfree(mlxsw_sp_rt6);
5336 }
5337
5338 static struct fib6_info *
5339 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
5340 {
5341         return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
5342                                 list)->rt;
5343 }
5344
5345 static struct mlxsw_sp_rt6 *
5346 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
5347                             const struct fib6_info *rt)
5348 {
5349         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5350
5351         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
5352                 if (mlxsw_sp_rt6->rt == rt)
5353                         return mlxsw_sp_rt6;
5354         }
5355
5356         return NULL;
5357 }
5358
5359 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
5360                                         const struct fib6_info *rt,
5361                                         enum mlxsw_sp_ipip_type *ret)
5362 {
5363         return rt->fib6_nh->fib_nh_dev &&
5364                mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret);
5365 }
5366
5367 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
5368                                   struct mlxsw_sp_nexthop_group *nh_grp,
5369                                   struct mlxsw_sp_nexthop *nh,
5370                                   const struct fib6_info *rt)
5371 {
5372         struct net_device *dev = rt->fib6_nh->fib_nh_dev;
5373
5374         nh->nhgi = nh_grp->nhgi;
5375         nh->nh_weight = rt->fib6_nh->fib_nh_weight;
5376         memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr));
5377 #if IS_ENABLED(CONFIG_IPV6)
5378         nh->neigh_tbl = &nd_tbl;
5379 #endif
5380         mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
5381
5382         list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
5383
5384         if (!dev)
5385                 return 0;
5386         nh->ifindex = dev->ifindex;
5387
5388         return mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
5389 }
5390
5391 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
5392                                    struct mlxsw_sp_nexthop *nh)
5393 {
5394         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
5395         list_del(&nh->router_list_node);
5396         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
5397 }
5398
5399 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
5400                                     const struct fib6_info *rt)
5401 {
5402         return rt->fib6_nh->fib_nh_gw_family ||
5403                mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
5404 }
5405
5406 static int
5407 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp,
5408                                   struct mlxsw_sp_nexthop_group *nh_grp,
5409                                   struct mlxsw_sp_fib6_entry *fib6_entry)
5410 {
5411         struct mlxsw_sp_nexthop_group_info *nhgi;
5412         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5413         struct mlxsw_sp_nexthop *nh;
5414         int err, i;
5415
5416         nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6),
5417                        GFP_KERNEL);
5418         if (!nhgi)
5419                 return -ENOMEM;
5420         nh_grp->nhgi = nhgi;
5421         nhgi->nh_grp = nh_grp;
5422         mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
5423                                         struct mlxsw_sp_rt6, list);
5424         nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
5425         nhgi->count = fib6_entry->nrt6;
5426         for (i = 0; i < nhgi->count; i++) {
5427                 struct fib6_info *rt = mlxsw_sp_rt6->rt;
5428
5429                 nh = &nhgi->nexthops[i];
5430                 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
5431                 if (err)
5432                         goto err_nexthop6_init;
5433                 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
5434         }
5435         nh_grp->nhgi = nhgi;
5436         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5437
5438         return 0;
5439
5440 err_nexthop6_init:
5441         for (i--; i >= 0; i--) {
5442                 nh = &nhgi->nexthops[i];
5443                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
5444         }
5445         kfree(nhgi);
5446         return err;
5447 }
5448
5449 static void
5450 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp,
5451                                   struct mlxsw_sp_nexthop_group *nh_grp)
5452 {
5453         struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
5454         int i;
5455
5456         for (i = nhgi->count - 1; i >= 0; i--) {
5457                 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
5458
5459                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
5460         }
5461         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5462         WARN_ON_ONCE(nhgi->adj_index_valid);
5463         kfree(nhgi);
5464 }
5465
5466 static struct mlxsw_sp_nexthop_group *
5467 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
5468                                struct mlxsw_sp_fib6_entry *fib6_entry)
5469 {
5470         struct mlxsw_sp_nexthop_group *nh_grp;
5471         int err;
5472
5473         nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
5474         if (!nh_grp)
5475                 return ERR_PTR(-ENOMEM);
5476         INIT_LIST_HEAD(&nh_grp->fib_list);
5477         nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
5478
5479         err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry);
5480         if (err)
5481                 goto err_nexthop_group_info_init;
5482
5483         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
5484         if (err)
5485                 goto err_nexthop_group_insert;
5486
5487         nh_grp->can_destroy = true;
5488
5489         return nh_grp;
5490
5491 err_nexthop_group_insert:
5492         mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
5493 err_nexthop_group_info_init:
5494         kfree(nh_grp);
5495         return ERR_PTR(err);
5496 }
5497
5498 static void
5499 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
5500                                 struct mlxsw_sp_nexthop_group *nh_grp)
5501 {
5502         if (!nh_grp->can_destroy)
5503                 return;
5504         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
5505         mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
5506         kfree(nh_grp);
5507 }
5508
5509 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
5510                                        struct mlxsw_sp_fib6_entry *fib6_entry)
5511 {
5512         struct mlxsw_sp_nexthop_group *nh_grp;
5513
5514         nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
5515         if (!nh_grp) {
5516                 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
5517                 if (IS_ERR(nh_grp))
5518                         return PTR_ERR(nh_grp);
5519         }
5520
5521         /* The route and the nexthop are described by the same struct, so we
5522          * need to the update the nexthop offload indication for the new route.
5523          */
5524         __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
5525
5526         list_add_tail(&fib6_entry->common.nexthop_group_node,
5527                       &nh_grp->fib_list);
5528         fib6_entry->common.nh_group = nh_grp;
5529
5530         return 0;
5531 }
5532
5533 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
5534                                         struct mlxsw_sp_fib_entry *fib_entry)
5535 {
5536         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
5537
5538         list_del(&fib_entry->nexthop_group_node);
5539         if (!list_empty(&nh_grp->fib_list))
5540                 return;
5541         mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
5542 }
5543
5544 static int mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
5545                                           struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5546                                           struct mlxsw_sp_fib6_entry *fib6_entry)
5547 {
5548         struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
5549         int err;
5550
5551         fib6_entry->common.nh_group = NULL;
5552         list_del(&fib6_entry->common.nexthop_group_node);
5553
5554         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
5555         if (err)
5556                 goto err_nexthop6_group_get;
5557
5558         /* In case this entry is offloaded, then the adjacency index
5559          * currently associated with it in the device's table is that
5560          * of the old group. Start using the new one instead.
5561          */
5562         err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx,
5563                                           &fib6_entry->common, false);
5564         if (err)
5565                 goto err_fib_entry_update;
5566
5567         if (list_empty(&old_nh_grp->fib_list))
5568                 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
5569
5570         return 0;
5571
5572 err_fib_entry_update:
5573         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
5574 err_nexthop6_group_get:
5575         list_add_tail(&fib6_entry->common.nexthop_group_node,
5576                       &old_nh_grp->fib_list);
5577         fib6_entry->common.nh_group = old_nh_grp;
5578         return err;
5579 }
5580
5581 static int
5582 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
5583                                 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5584                                 struct mlxsw_sp_fib6_entry *fib6_entry,
5585                                 struct fib6_info **rt_arr, unsigned int nrt6)
5586 {
5587         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5588         int err, i;
5589
5590         for (i = 0; i < nrt6; i++) {
5591                 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
5592                 if (IS_ERR(mlxsw_sp_rt6)) {
5593                         err = PTR_ERR(mlxsw_sp_rt6);
5594                         goto err_rt6_create;
5595                 }
5596
5597                 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
5598                 fib6_entry->nrt6++;
5599         }
5600
5601         err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry);
5602         if (err)
5603                 goto err_nexthop6_group_update;
5604
5605         return 0;
5606
5607 err_nexthop6_group_update:
5608         i = nrt6;
5609 err_rt6_create:
5610         for (i--; i >= 0; i--) {
5611                 fib6_entry->nrt6--;
5612                 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
5613                                                struct mlxsw_sp_rt6, list);
5614                 list_del(&mlxsw_sp_rt6->list);
5615                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5616         }
5617         return err;
5618 }
5619
5620 static void
5621 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
5622                                 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5623                                 struct mlxsw_sp_fib6_entry *fib6_entry,
5624                                 struct fib6_info **rt_arr, unsigned int nrt6)
5625 {
5626         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5627         int i;
5628
5629         for (i = 0; i < nrt6; i++) {
5630                 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry,
5631                                                            rt_arr[i]);
5632                 if (WARN_ON_ONCE(!mlxsw_sp_rt6))
5633                         continue;
5634
5635                 fib6_entry->nrt6--;
5636                 list_del(&mlxsw_sp_rt6->list);
5637                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5638         }
5639
5640         mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry);
5641 }
5642
5643 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
5644                                          struct mlxsw_sp_fib_entry *fib_entry,
5645                                          const struct fib6_info *rt)
5646 {
5647         if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))
5648                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
5649         else if (rt->fib6_type == RTN_BLACKHOLE)
5650                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
5651         else if (rt->fib6_flags & RTF_REJECT)
5652                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
5653         else if (fib_entry->nh_group->nhgi->gateway)
5654                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
5655         else
5656                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
5657 }
5658
5659 static void
5660 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
5661 {
5662         struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
5663
5664         list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
5665                                  list) {
5666                 fib6_entry->nrt6--;
5667                 list_del(&mlxsw_sp_rt6->list);
5668                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5669         }
5670 }
5671
5672 static struct mlxsw_sp_fib6_entry *
5673 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
5674                            struct mlxsw_sp_fib_node *fib_node,
5675                            struct fib6_info **rt_arr, unsigned int nrt6)
5676 {
5677         struct mlxsw_sp_fib6_entry *fib6_entry;
5678         struct mlxsw_sp_fib_entry *fib_entry;
5679         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5680         int err, i;
5681
5682         fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
5683         if (!fib6_entry)
5684                 return ERR_PTR(-ENOMEM);
5685         fib_entry = &fib6_entry->common;
5686
5687         fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops);
5688         if (IS_ERR(fib_entry->priv)) {
5689                 err = PTR_ERR(fib_entry->priv);
5690                 goto err_fib_entry_priv_create;
5691         }
5692
5693         INIT_LIST_HEAD(&fib6_entry->rt6_list);
5694
5695         for (i = 0; i < nrt6; i++) {
5696                 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
5697                 if (IS_ERR(mlxsw_sp_rt6)) {
5698                         err = PTR_ERR(mlxsw_sp_rt6);
5699                         goto err_rt6_create;
5700                 }
5701                 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
5702                 fib6_entry->nrt6++;
5703         }
5704
5705         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
5706         if (err)
5707                 goto err_nexthop6_group_get;
5708
5709         mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]);
5710
5711         fib_entry->fib_node = fib_node;
5712
5713         return fib6_entry;
5714
5715 err_nexthop6_group_get:
5716         i = nrt6;
5717 err_rt6_create:
5718         for (i--; i >= 0; i--) {
5719                 fib6_entry->nrt6--;
5720                 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
5721                                                struct mlxsw_sp_rt6, list);
5722                 list_del(&mlxsw_sp_rt6->list);
5723                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5724         }
5725         mlxsw_sp_fib_entry_priv_put(fib_entry->priv);
5726 err_fib_entry_priv_create:
5727         kfree(fib6_entry);
5728         return ERR_PTR(err);
5729 }
5730
5731 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
5732                                         struct mlxsw_sp_fib6_entry *fib6_entry)
5733 {
5734         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
5735         mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
5736         WARN_ON(fib6_entry->nrt6);
5737         mlxsw_sp_fib_entry_priv_put(fib6_entry->common.priv);
5738         kfree(fib6_entry);
5739 }
5740
5741 static struct mlxsw_sp_fib6_entry *
5742 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
5743                            const struct fib6_info *rt)
5744 {
5745         struct mlxsw_sp_fib6_entry *fib6_entry;
5746         struct mlxsw_sp_fib_node *fib_node;
5747         struct mlxsw_sp_fib *fib;
5748         struct fib6_info *cmp_rt;
5749         struct mlxsw_sp_vr *vr;
5750
5751         vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id);
5752         if (!vr)
5753                 return NULL;
5754         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
5755
5756         fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr,
5757                                             sizeof(rt->fib6_dst.addr),
5758                                             rt->fib6_dst.plen);
5759         if (!fib_node)
5760                 return NULL;
5761
5762         fib6_entry = container_of(fib_node->fib_entry,
5763                                   struct mlxsw_sp_fib6_entry, common);
5764         cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
5765         if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id &&
5766             rt->fib6_metric == cmp_rt->fib6_metric &&
5767             mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
5768                 return fib6_entry;
5769
5770         return NULL;
5771 }
5772
5773 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry)
5774 {
5775         struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
5776         struct mlxsw_sp_fib6_entry *fib6_replaced;
5777         struct fib6_info *rt, *rt_replaced;
5778
5779         if (!fib_node->fib_entry)
5780                 return true;
5781
5782         fib6_replaced = container_of(fib_node->fib_entry,
5783                                      struct mlxsw_sp_fib6_entry,
5784                                      common);
5785         rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
5786         rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced);
5787         if (rt->fib6_table->tb6_id == RT_TABLE_MAIN &&
5788             rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL)
5789                 return false;
5790
5791         return true;
5792 }
5793
5794 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp,
5795                                         struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5796                                         struct fib6_info **rt_arr, unsigned int nrt6)
5797 {
5798         struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced;
5799         struct mlxsw_sp_fib_entry *replaced;
5800         struct mlxsw_sp_fib_node *fib_node;
5801         struct fib6_info *rt = rt_arr[0];
5802         int err;
5803
5804         if (mlxsw_sp->router->aborted)
5805                 return 0;
5806
5807         if (rt->fib6_src.plen)
5808                 return -EINVAL;
5809
5810         if (mlxsw_sp_fib6_rt_should_ignore(rt))
5811                 return 0;
5812
5813         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
5814                                          &rt->fib6_dst.addr,
5815                                          sizeof(rt->fib6_dst.addr),
5816                                          rt->fib6_dst.plen,
5817                                          MLXSW_SP_L3_PROTO_IPV6);
5818         if (IS_ERR(fib_node))
5819                 return PTR_ERR(fib_node);
5820
5821         fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr,
5822                                                 nrt6);
5823         if (IS_ERR(fib6_entry)) {
5824                 err = PTR_ERR(fib6_entry);
5825                 goto err_fib6_entry_create;
5826         }
5827
5828         if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) {
5829                 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5830                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5831                 return 0;
5832         }
5833
5834         replaced = fib_node->fib_entry;
5835         err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib6_entry->common);
5836         if (err)
5837                 goto err_fib_node_entry_link;
5838
5839         /* Nothing to replace */
5840         if (!replaced)
5841                 return 0;
5842
5843         mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
5844         fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry,
5845                                      common);
5846         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced);
5847
5848         return 0;
5849
5850 err_fib_node_entry_link:
5851         fib_node->fib_entry = replaced;
5852         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5853 err_fib6_entry_create:
5854         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5855         return err;
5856 }
5857
5858 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp,
5859                                        struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5860                                        struct fib6_info **rt_arr, unsigned int nrt6)
5861 {
5862         struct mlxsw_sp_fib6_entry *fib6_entry;
5863         struct mlxsw_sp_fib_node *fib_node;
5864         struct fib6_info *rt = rt_arr[0];
5865         int err;
5866
5867         if (mlxsw_sp->router->aborted)
5868                 return 0;
5869
5870         if (rt->fib6_src.plen)
5871                 return -EINVAL;
5872
5873         if (mlxsw_sp_fib6_rt_should_ignore(rt))
5874                 return 0;
5875
5876         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
5877                                          &rt->fib6_dst.addr,
5878                                          sizeof(rt->fib6_dst.addr),
5879                                          rt->fib6_dst.plen,
5880                                          MLXSW_SP_L3_PROTO_IPV6);
5881         if (IS_ERR(fib_node))
5882                 return PTR_ERR(fib_node);
5883
5884         if (WARN_ON_ONCE(!fib_node->fib_entry)) {
5885                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5886                 return -EINVAL;
5887         }
5888
5889         fib6_entry = container_of(fib_node->fib_entry,
5890                                   struct mlxsw_sp_fib6_entry, common);
5891         err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6);
5892         if (err)
5893                 goto err_fib6_entry_nexthop_add;
5894
5895         return 0;
5896
5897 err_fib6_entry_nexthop_add:
5898         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5899         return err;
5900 }
5901
5902 static int mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
5903                                     struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5904                                     struct fib6_info **rt_arr, unsigned int nrt6)
5905 {
5906         struct mlxsw_sp_fib6_entry *fib6_entry;
5907         struct mlxsw_sp_fib_node *fib_node;
5908         struct fib6_info *rt = rt_arr[0];
5909         int err;
5910
5911         if (mlxsw_sp->router->aborted)
5912                 return 0;
5913
5914         if (mlxsw_sp_fib6_rt_should_ignore(rt))
5915                 return 0;
5916
5917         /* Multipath routes are first added to the FIB trie and only then
5918          * notified. If we vetoed the addition, we will get a delete
5919          * notification for a route we do not have. Therefore, do not warn if
5920          * route was not found.
5921          */
5922         fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
5923         if (!fib6_entry)
5924                 return 0;
5925
5926         /* If not all the nexthops are deleted, then only reduce the nexthop
5927          * group.
5928          */
5929         if (nrt6 != fib6_entry->nrt6) {
5930                 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6);
5931                 return 0;
5932         }
5933
5934         fib_node = fib6_entry->common.fib_node;
5935
5936         err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib6_entry->common);
5937         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5938         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5939         return err;
5940 }
5941
5942 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
5943                                             enum mlxsw_sp_l3proto proto,
5944                                             u8 tree_id)
5945 {
5946         const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
5947         enum mlxsw_reg_ralxx_protocol ralxx_proto =
5948                                 (enum mlxsw_reg_ralxx_protocol) proto;
5949         struct mlxsw_sp_fib_entry_priv *priv;
5950         char xralta_pl[MLXSW_REG_XRALTA_LEN];
5951         char xralst_pl[MLXSW_REG_XRALST_LEN];
5952         int i, err;
5953
5954         mlxsw_reg_xralta_pack(xralta_pl, true, ralxx_proto, tree_id);
5955         err = ll_ops->ralta_write(mlxsw_sp, xralta_pl);
5956         if (err)
5957                 return err;
5958
5959         mlxsw_reg_xralst_pack(xralst_pl, 0xff, tree_id);
5960         err = ll_ops->ralst_write(mlxsw_sp, xralst_pl);
5961         if (err)
5962                 return err;
5963
5964         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
5965                 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx;
5966                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
5967                 char xraltb_pl[MLXSW_REG_XRALTB_LEN];
5968
5969                 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
5970                 mlxsw_reg_xraltb_pack(xraltb_pl, vr->id, ralxx_proto, tree_id);
5971                 err = ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
5972                 if (err)
5973                         return err;
5974
5975                 priv = mlxsw_sp_fib_entry_priv_create(ll_ops);
5976                 if (IS_ERR(priv))
5977                         return PTR_ERR(priv);
5978
5979                 ll_ops->fib_entry_pack(op_ctx, proto, MLXSW_SP_FIB_ENTRY_OP_WRITE,
5980                                        vr->id, 0, NULL, priv);
5981                 ll_ops->fib_entry_act_ip2me_pack(op_ctx);
5982                 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, NULL);
5983                 mlxsw_sp_fib_entry_priv_put(priv);
5984                 if (err)
5985                         return err;
5986         }
5987
5988         return 0;
5989 }
5990
5991 static struct mlxsw_sp_mr_table *
5992 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family)
5993 {
5994         if (family == RTNL_FAMILY_IPMR)
5995                 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4];
5996         else
5997                 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
5998 }
5999
6000 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp,
6001                                      struct mfc_entry_notifier_info *men_info,
6002                                      bool replace)
6003 {
6004         struct mlxsw_sp_mr_table *mrt;
6005         struct mlxsw_sp_vr *vr;
6006
6007         if (mlxsw_sp->router->aborted)
6008                 return 0;
6009
6010         vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL);
6011         if (IS_ERR(vr))
6012                 return PTR_ERR(vr);
6013
6014         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
6015         return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace);
6016 }
6017
6018 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp,
6019                                       struct mfc_entry_notifier_info *men_info)
6020 {
6021         struct mlxsw_sp_mr_table *mrt;
6022         struct mlxsw_sp_vr *vr;
6023
6024         if (mlxsw_sp->router->aborted)
6025                 return;
6026
6027         vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id);
6028         if (WARN_ON(!vr))
6029                 return;
6030
6031         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
6032         mlxsw_sp_mr_route_del(mrt, men_info->mfc);
6033         mlxsw_sp_vr_put(mlxsw_sp, vr);
6034 }
6035
6036 static int
6037 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp,
6038                               struct vif_entry_notifier_info *ven_info)
6039 {
6040         struct mlxsw_sp_mr_table *mrt;
6041         struct mlxsw_sp_rif *rif;
6042         struct mlxsw_sp_vr *vr;
6043
6044         if (mlxsw_sp->router->aborted)
6045                 return 0;
6046
6047         vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL);
6048         if (IS_ERR(vr))
6049                 return PTR_ERR(vr);
6050
6051         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
6052         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev);
6053         return mlxsw_sp_mr_vif_add(mrt, ven_info->dev,
6054                                    ven_info->vif_index,
6055                                    ven_info->vif_flags, rif);
6056 }
6057
6058 static void
6059 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp,
6060                               struct vif_entry_notifier_info *ven_info)
6061 {
6062         struct mlxsw_sp_mr_table *mrt;
6063         struct mlxsw_sp_vr *vr;
6064
6065         if (mlxsw_sp->router->aborted)
6066                 return;
6067
6068         vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id);
6069         if (WARN_ON(!vr))
6070                 return;
6071
6072         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
6073         mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index);
6074         mlxsw_sp_vr_put(mlxsw_sp, vr);
6075 }
6076
6077 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
6078 {
6079         enum mlxsw_sp_l3proto proto = MLXSW_SP_L3_PROTO_IPV4;
6080         int err;
6081
6082         err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
6083                                                MLXSW_SP_LPM_TREE_MIN);
6084         if (err)
6085                 return err;
6086
6087         /* The multicast router code does not need an abort trap as by default,
6088          * packets that don't match any routes are trapped to the CPU.
6089          */
6090
6091         proto = MLXSW_SP_L3_PROTO_IPV6;
6092         return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
6093                                                 MLXSW_SP_LPM_TREE_MIN + 1);
6094 }
6095
6096 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
6097                                      struct mlxsw_sp_fib_node *fib_node)
6098 {
6099         struct mlxsw_sp_fib4_entry *fib4_entry;
6100
6101         fib4_entry = container_of(fib_node->fib_entry,
6102                                   struct mlxsw_sp_fib4_entry, common);
6103         mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
6104         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6105         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6106 }
6107
6108 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
6109                                      struct mlxsw_sp_fib_node *fib_node)
6110 {
6111         struct mlxsw_sp_fib6_entry *fib6_entry;
6112
6113         fib6_entry = container_of(fib_node->fib_entry,
6114                                   struct mlxsw_sp_fib6_entry, common);
6115         mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
6116         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
6117         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6118 }
6119
6120 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
6121                                     struct mlxsw_sp_fib_node *fib_node)
6122 {
6123         switch (fib_node->fib->proto) {
6124         case MLXSW_SP_L3_PROTO_IPV4:
6125                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
6126                 break;
6127         case MLXSW_SP_L3_PROTO_IPV6:
6128                 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
6129                 break;
6130         }
6131 }
6132
6133 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
6134                                   struct mlxsw_sp_vr *vr,
6135                                   enum mlxsw_sp_l3proto proto)
6136 {
6137         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
6138         struct mlxsw_sp_fib_node *fib_node, *tmp;
6139
6140         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
6141                 bool do_break = &tmp->list == &fib->node_list;
6142
6143                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
6144                 if (do_break)
6145                         break;
6146         }
6147 }
6148
6149 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
6150 {
6151         int i, j;
6152
6153         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
6154                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
6155
6156                 if (!mlxsw_sp_vr_is_used(vr))
6157                         continue;
6158
6159                 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++)
6160                         mlxsw_sp_mr_table_flush(vr->mr_table[j]);
6161                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
6162
6163                 /* If virtual router was only used for IPv4, then it's no
6164                  * longer used.
6165                  */
6166                 if (!mlxsw_sp_vr_is_used(vr))
6167                         continue;
6168                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
6169         }
6170
6171         /* After flushing all the routes, it is not possible anyone is still
6172          * using the adjacency index that is discarding packets, so free it in
6173          * case it was allocated.
6174          */
6175         if (!mlxsw_sp->router->adj_discard_index_valid)
6176                 return;
6177         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
6178                            mlxsw_sp->router->adj_discard_index);
6179         mlxsw_sp->router->adj_discard_index_valid = false;
6180 }
6181
6182 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
6183 {
6184         int err;
6185
6186         if (mlxsw_sp->router->aborted)
6187                 return;
6188         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
6189         mlxsw_sp_router_fib_flush(mlxsw_sp);
6190         mlxsw_sp->router->aborted = true;
6191         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
6192         if (err)
6193                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
6194 }
6195
6196 struct mlxsw_sp_fib6_event {
6197         struct fib6_info **rt_arr;
6198         unsigned int nrt6;
6199 };
6200
6201 struct mlxsw_sp_fib_event {
6202         struct list_head list; /* node in fib queue */
6203         union {
6204                 struct mlxsw_sp_fib6_event fib6_event;
6205                 struct fib_entry_notifier_info fen_info;
6206                 struct fib_rule_notifier_info fr_info;
6207                 struct fib_nh_notifier_info fnh_info;
6208                 struct mfc_entry_notifier_info men_info;
6209                 struct vif_entry_notifier_info ven_info;
6210         };
6211         struct mlxsw_sp *mlxsw_sp;
6212         unsigned long event;
6213         int family;
6214 };
6215
6216 static int
6217 mlxsw_sp_router_fib6_event_init(struct mlxsw_sp_fib6_event *fib6_event,
6218                                 struct fib6_entry_notifier_info *fen6_info)
6219 {
6220         struct fib6_info *rt = fen6_info->rt;
6221         struct fib6_info **rt_arr;
6222         struct fib6_info *iter;
6223         unsigned int nrt6;
6224         int i = 0;
6225
6226         nrt6 = fen6_info->nsiblings + 1;
6227
6228         rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC);
6229         if (!rt_arr)
6230                 return -ENOMEM;
6231
6232         fib6_event->rt_arr = rt_arr;
6233         fib6_event->nrt6 = nrt6;
6234
6235         rt_arr[0] = rt;
6236         fib6_info_hold(rt);
6237
6238         if (!fen6_info->nsiblings)
6239                 return 0;
6240
6241         list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
6242                 if (i == fen6_info->nsiblings)
6243                         break;
6244
6245                 rt_arr[i + 1] = iter;
6246                 fib6_info_hold(iter);
6247                 i++;
6248         }
6249         WARN_ON_ONCE(i != fen6_info->nsiblings);
6250
6251         return 0;
6252 }
6253
6254 static void
6255 mlxsw_sp_router_fib6_event_fini(struct mlxsw_sp_fib6_event *fib6_event)
6256 {
6257         int i;
6258
6259         for (i = 0; i < fib6_event->nrt6; i++)
6260                 mlxsw_sp_rt6_release(fib6_event->rt_arr[i]);
6261         kfree(fib6_event->rt_arr);
6262 }
6263
6264 static void mlxsw_sp_router_fib4_event_process(struct mlxsw_sp *mlxsw_sp,
6265                                                struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
6266                                                struct mlxsw_sp_fib_event *fib_event)
6267 {
6268         int err;
6269
6270         mlxsw_sp_span_respin(mlxsw_sp);
6271
6272         switch (fib_event->event) {
6273         case FIB_EVENT_ENTRY_REPLACE:
6274                 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, op_ctx, &fib_event->fen_info);
6275                 if (err) {
6276                         mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
6277                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6278                 }
6279                 fib_info_put(fib_event->fen_info.fi);
6280                 break;
6281         case FIB_EVENT_ENTRY_DEL:
6282                 err = mlxsw_sp_router_fib4_del(mlxsw_sp, op_ctx, &fib_event->fen_info);
6283                 if (err)
6284                         mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
6285                 fib_info_put(fib_event->fen_info.fi);
6286                 break;
6287         case FIB_EVENT_NH_ADD:
6288         case FIB_EVENT_NH_DEL:
6289                 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_event->event, fib_event->fnh_info.fib_nh);
6290                 fib_info_put(fib_event->fnh_info.fib_nh->nh_parent);
6291                 break;
6292         }
6293 }
6294
6295 static void mlxsw_sp_router_fib6_event_process(struct mlxsw_sp *mlxsw_sp,
6296                                                struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
6297                                                struct mlxsw_sp_fib_event *fib_event)
6298 {
6299         int err;
6300
6301         mlxsw_sp_span_respin(mlxsw_sp);
6302
6303         switch (fib_event->event) {
6304         case FIB_EVENT_ENTRY_REPLACE:
6305                 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
6306                                                    fib_event->fib6_event.nrt6);
6307                 if (err) {
6308                         mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
6309                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6310                 }
6311                 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
6312                 break;
6313         case FIB_EVENT_ENTRY_APPEND:
6314                 err = mlxsw_sp_router_fib6_append(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
6315                                                   fib_event->fib6_event.nrt6);
6316                 if (err) {
6317                         mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
6318                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6319                 }
6320                 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
6321                 break;
6322         case FIB_EVENT_ENTRY_DEL:
6323                 err = mlxsw_sp_router_fib6_del(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
6324                                                fib_event->fib6_event.nrt6);
6325                 if (err)
6326                         mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
6327                 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
6328                 break;
6329         }
6330 }
6331
6332 static void mlxsw_sp_router_fibmr_event_process(struct mlxsw_sp *mlxsw_sp,
6333                                                 struct mlxsw_sp_fib_event *fib_event)
6334 {
6335         bool replace;
6336         int err;
6337
6338         rtnl_lock();
6339         mutex_lock(&mlxsw_sp->router->lock);
6340         switch (fib_event->event) {
6341         case FIB_EVENT_ENTRY_REPLACE:
6342         case FIB_EVENT_ENTRY_ADD:
6343                 replace = fib_event->event == FIB_EVENT_ENTRY_REPLACE;
6344
6345                 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_event->men_info, replace);
6346                 if (err)
6347                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6348                 mr_cache_put(fib_event->men_info.mfc);
6349                 break;
6350         case FIB_EVENT_ENTRY_DEL:
6351                 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_event->men_info);
6352                 mr_cache_put(fib_event->men_info.mfc);
6353                 break;
6354         case FIB_EVENT_VIF_ADD:
6355                 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp,
6356                                                     &fib_event->ven_info);
6357                 if (err)
6358                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6359                 dev_put(fib_event->ven_info.dev);
6360                 break;
6361         case FIB_EVENT_VIF_DEL:
6362                 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, &fib_event->ven_info);
6363                 dev_put(fib_event->ven_info.dev);
6364                 break;
6365         }
6366         mutex_unlock(&mlxsw_sp->router->lock);
6367         rtnl_unlock();
6368 }
6369
6370 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
6371 {
6372         struct mlxsw_sp_router *router = container_of(work, struct mlxsw_sp_router, fib_event_work);
6373         struct mlxsw_sp_fib_entry_op_ctx *op_ctx = router->ll_op_ctx;
6374         struct mlxsw_sp *mlxsw_sp = router->mlxsw_sp;
6375         struct mlxsw_sp_fib_event *next_fib_event;
6376         struct mlxsw_sp_fib_event *fib_event;
6377         int last_family = AF_UNSPEC;
6378         LIST_HEAD(fib_event_queue);
6379
6380         spin_lock_bh(&router->fib_event_queue_lock);
6381         list_splice_init(&router->fib_event_queue, &fib_event_queue);
6382         spin_unlock_bh(&router->fib_event_queue_lock);
6383
6384         /* Router lock is held here to make sure per-instance
6385          * operation context is not used in between FIB4/6 events
6386          * processing.
6387          */
6388         mutex_lock(&router->lock);
6389         mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
6390         list_for_each_entry_safe(fib_event, next_fib_event,
6391                                  &fib_event_queue, list) {
6392                 /* Check if the next entry in the queue exists and it is
6393                  * of the same type (family and event) as the currect one.
6394                  * In that case it is permitted to do the bulking
6395                  * of multiple FIB entries to a single register write.
6396                  */
6397                 op_ctx->bulk_ok = !list_is_last(&fib_event->list, &fib_event_queue) &&
6398                                   fib_event->family == next_fib_event->family &&
6399                                   fib_event->event == next_fib_event->event;
6400
6401                 /* In case family of this and the previous entry are different, context
6402                  * reinitialization is going to be needed now, indicate that.
6403                  * Note that since last_family is initialized to AF_UNSPEC, this is always
6404                  * going to happen for the first entry processed in the work.
6405                  */
6406                 if (fib_event->family != last_family)
6407                         op_ctx->initialized = false;
6408
6409                 switch (fib_event->family) {
6410                 case AF_INET:
6411                         mlxsw_sp_router_fib4_event_process(mlxsw_sp, op_ctx,
6412                                                            fib_event);
6413                         break;
6414                 case AF_INET6:
6415                         mlxsw_sp_router_fib6_event_process(mlxsw_sp, op_ctx,
6416                                                            fib_event);
6417                         break;
6418                 case RTNL_FAMILY_IP6MR:
6419                 case RTNL_FAMILY_IPMR:
6420                         /* Unlock here as inside FIBMR the lock is taken again
6421                          * under RTNL. The per-instance operation context
6422                          * is not used by FIBMR.
6423                          */
6424                         mutex_unlock(&router->lock);
6425                         mlxsw_sp_router_fibmr_event_process(mlxsw_sp,
6426                                                             fib_event);
6427                         mutex_lock(&router->lock);
6428                         break;
6429                 default:
6430                         WARN_ON_ONCE(1);
6431                 }
6432                 last_family = fib_event->family;
6433                 kfree(fib_event);
6434                 cond_resched();
6435         }
6436         WARN_ON_ONCE(!list_empty(&router->ll_op_ctx->fib_entry_priv_list));
6437         mutex_unlock(&router->lock);
6438 }
6439
6440 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event *fib_event,
6441                                        struct fib_notifier_info *info)
6442 {
6443         struct fib_entry_notifier_info *fen_info;
6444         struct fib_nh_notifier_info *fnh_info;
6445
6446         switch (fib_event->event) {
6447         case FIB_EVENT_ENTRY_REPLACE:
6448         case FIB_EVENT_ENTRY_DEL:
6449                 fen_info = container_of(info, struct fib_entry_notifier_info,
6450                                         info);
6451                 fib_event->fen_info = *fen_info;
6452                 /* Take reference on fib_info to prevent it from being
6453                  * freed while event is queued. Release it afterwards.
6454                  */
6455                 fib_info_hold(fib_event->fen_info.fi);
6456                 break;
6457         case FIB_EVENT_NH_ADD:
6458         case FIB_EVENT_NH_DEL:
6459                 fnh_info = container_of(info, struct fib_nh_notifier_info,
6460                                         info);
6461                 fib_event->fnh_info = *fnh_info;
6462                 fib_info_hold(fib_event->fnh_info.fib_nh->nh_parent);
6463                 break;
6464         }
6465 }
6466
6467 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event *fib_event,
6468                                       struct fib_notifier_info *info)
6469 {
6470         struct fib6_entry_notifier_info *fen6_info;
6471         int err;
6472
6473         switch (fib_event->event) {
6474         case FIB_EVENT_ENTRY_REPLACE:
6475         case FIB_EVENT_ENTRY_APPEND:
6476         case FIB_EVENT_ENTRY_DEL:
6477                 fen6_info = container_of(info, struct fib6_entry_notifier_info,
6478                                          info);
6479                 err = mlxsw_sp_router_fib6_event_init(&fib_event->fib6_event,
6480                                                       fen6_info);
6481                 if (err)
6482                         return err;
6483                 break;
6484         }
6485
6486         return 0;
6487 }
6488
6489 static void
6490 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event *fib_event,
6491                             struct fib_notifier_info *info)
6492 {
6493         switch (fib_event->event) {
6494         case FIB_EVENT_ENTRY_REPLACE:
6495         case FIB_EVENT_ENTRY_ADD:
6496         case FIB_EVENT_ENTRY_DEL:
6497                 memcpy(&fib_event->men_info, info, sizeof(fib_event->men_info));
6498                 mr_cache_hold(fib_event->men_info.mfc);
6499                 break;
6500         case FIB_EVENT_VIF_ADD:
6501         case FIB_EVENT_VIF_DEL:
6502                 memcpy(&fib_event->ven_info, info, sizeof(fib_event->ven_info));
6503                 dev_hold(fib_event->ven_info.dev);
6504                 break;
6505         }
6506 }
6507
6508 static int mlxsw_sp_router_fib_rule_event(unsigned long event,
6509                                           struct fib_notifier_info *info,
6510                                           struct mlxsw_sp *mlxsw_sp)
6511 {
6512         struct netlink_ext_ack *extack = info->extack;
6513         struct fib_rule_notifier_info *fr_info;
6514         struct fib_rule *rule;
6515         int err = 0;
6516
6517         /* nothing to do at the moment */
6518         if (event == FIB_EVENT_RULE_DEL)
6519                 return 0;
6520
6521         if (mlxsw_sp->router->aborted)
6522                 return 0;
6523
6524         fr_info = container_of(info, struct fib_rule_notifier_info, info);
6525         rule = fr_info->rule;
6526
6527         /* Rule only affects locally generated traffic */
6528         if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex)
6529                 return 0;
6530
6531         switch (info->family) {
6532         case AF_INET:
6533                 if (!fib4_rule_default(rule) && !rule->l3mdev)
6534                         err = -EOPNOTSUPP;
6535                 break;
6536         case AF_INET6:
6537                 if (!fib6_rule_default(rule) && !rule->l3mdev)
6538                         err = -EOPNOTSUPP;
6539                 break;
6540         case RTNL_FAMILY_IPMR:
6541                 if (!ipmr_rule_default(rule) && !rule->l3mdev)
6542                         err = -EOPNOTSUPP;
6543                 break;
6544         case RTNL_FAMILY_IP6MR:
6545                 if (!ip6mr_rule_default(rule) && !rule->l3mdev)
6546                         err = -EOPNOTSUPP;
6547                 break;
6548         }
6549
6550         if (err < 0)
6551                 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported");
6552
6553         return err;
6554 }
6555
6556 /* Called with rcu_read_lock() */
6557 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
6558                                      unsigned long event, void *ptr)
6559 {
6560         struct mlxsw_sp_fib_event *fib_event;
6561         struct fib_notifier_info *info = ptr;
6562         struct mlxsw_sp_router *router;
6563         int err;
6564
6565         if ((info->family != AF_INET && info->family != AF_INET6 &&
6566              info->family != RTNL_FAMILY_IPMR &&
6567              info->family != RTNL_FAMILY_IP6MR))
6568                 return NOTIFY_DONE;
6569
6570         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
6571
6572         switch (event) {
6573         case FIB_EVENT_RULE_ADD:
6574         case FIB_EVENT_RULE_DEL:
6575                 err = mlxsw_sp_router_fib_rule_event(event, info,
6576                                                      router->mlxsw_sp);
6577                 return notifier_from_errno(err);
6578         case FIB_EVENT_ENTRY_ADD:
6579         case FIB_EVENT_ENTRY_REPLACE:
6580         case FIB_EVENT_ENTRY_APPEND:
6581                 if (router->aborted) {
6582                         NL_SET_ERR_MSG_MOD(info->extack, "FIB offload was aborted. Not configuring route");
6583                         return notifier_from_errno(-EINVAL);
6584                 }
6585                 if (info->family == AF_INET) {
6586                         struct fib_entry_notifier_info *fen_info = ptr;
6587
6588                         if (fen_info->fi->fib_nh_is_v6) {
6589                                 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported");
6590                                 return notifier_from_errno(-EINVAL);
6591                         }
6592                         if (fen_info->fi->nh) {
6593                                 NL_SET_ERR_MSG_MOD(info->extack, "IPv4 route with nexthop objects is not supported");
6594                                 return notifier_from_errno(-EINVAL);
6595                         }
6596                 } else if (info->family == AF_INET6) {
6597                         struct fib6_entry_notifier_info *fen6_info;
6598
6599                         fen6_info = container_of(info,
6600                                                  struct fib6_entry_notifier_info,
6601                                                  info);
6602                         if (fen6_info->rt->nh) {
6603                                 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 route with nexthop objects is not supported");
6604                                 return notifier_from_errno(-EINVAL);
6605                         }
6606                 }
6607                 break;
6608         }
6609
6610         fib_event = kzalloc(sizeof(*fib_event), GFP_ATOMIC);
6611         if (!fib_event)
6612                 return NOTIFY_BAD;
6613
6614         fib_event->mlxsw_sp = router->mlxsw_sp;
6615         fib_event->event = event;
6616         fib_event->family = info->family;
6617
6618         switch (info->family) {
6619         case AF_INET:
6620                 mlxsw_sp_router_fib4_event(fib_event, info);
6621                 break;
6622         case AF_INET6:
6623                 err = mlxsw_sp_router_fib6_event(fib_event, info);
6624                 if (err)
6625                         goto err_fib_event;
6626                 break;
6627         case RTNL_FAMILY_IP6MR:
6628         case RTNL_FAMILY_IPMR:
6629                 mlxsw_sp_router_fibmr_event(fib_event, info);
6630                 break;
6631         }
6632
6633         /* Enqueue the event and trigger the work */
6634         spin_lock_bh(&router->fib_event_queue_lock);
6635         list_add_tail(&fib_event->list, &router->fib_event_queue);
6636         spin_unlock_bh(&router->fib_event_queue_lock);
6637         mlxsw_core_schedule_work(&router->fib_event_work);
6638
6639         return NOTIFY_DONE;
6640
6641 err_fib_event:
6642         kfree(fib_event);
6643         return NOTIFY_BAD;
6644 }
6645
6646 static struct mlxsw_sp_rif *
6647 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
6648                          const struct net_device *dev)
6649 {
6650         int i;
6651
6652         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
6653                 if (mlxsw_sp->router->rifs[i] &&
6654                     mlxsw_sp->router->rifs[i]->dev == dev)
6655                         return mlxsw_sp->router->rifs[i];
6656
6657         return NULL;
6658 }
6659
6660 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp,
6661                          const struct net_device *dev)
6662 {
6663         struct mlxsw_sp_rif *rif;
6664
6665         mutex_lock(&mlxsw_sp->router->lock);
6666         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6667         mutex_unlock(&mlxsw_sp->router->lock);
6668
6669         return rif;
6670 }
6671
6672 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev)
6673 {
6674         struct mlxsw_sp_rif *rif;
6675         u16 vid = 0;
6676
6677         mutex_lock(&mlxsw_sp->router->lock);
6678         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6679         if (!rif)
6680                 goto out;
6681
6682         /* We only return the VID for VLAN RIFs. Otherwise we return an
6683          * invalid value (0).
6684          */
6685         if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN)
6686                 goto out;
6687
6688         vid = mlxsw_sp_fid_8021q_vid(rif->fid);
6689
6690 out:
6691         mutex_unlock(&mlxsw_sp->router->lock);
6692         return vid;
6693 }
6694
6695 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
6696 {
6697         char ritr_pl[MLXSW_REG_RITR_LEN];
6698         int err;
6699
6700         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
6701         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6702         if (err)
6703                 return err;
6704
6705         mlxsw_reg_ritr_enable_set(ritr_pl, false);
6706         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6707 }
6708
6709 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
6710                                           struct mlxsw_sp_rif *rif)
6711 {
6712         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
6713         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
6714         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
6715 }
6716
6717 static bool
6718 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
6719                            unsigned long event)
6720 {
6721         struct inet6_dev *inet6_dev;
6722         bool addr_list_empty = true;
6723         struct in_device *idev;
6724
6725         switch (event) {
6726         case NETDEV_UP:
6727                 return rif == NULL;
6728         case NETDEV_DOWN:
6729                 rcu_read_lock();
6730                 idev = __in_dev_get_rcu(dev);
6731                 if (idev && idev->ifa_list)
6732                         addr_list_empty = false;
6733
6734                 inet6_dev = __in6_dev_get(dev);
6735                 if (addr_list_empty && inet6_dev &&
6736                     !list_empty(&inet6_dev->addr_list))
6737                         addr_list_empty = false;
6738                 rcu_read_unlock();
6739
6740                 /* macvlans do not have a RIF, but rather piggy back on the
6741                  * RIF of their lower device.
6742                  */
6743                 if (netif_is_macvlan(dev) && addr_list_empty)
6744                         return true;
6745
6746                 if (rif && addr_list_empty &&
6747                     !netif_is_l3_slave(rif->dev))
6748                         return true;
6749                 /* It is possible we already removed the RIF ourselves
6750                  * if it was assigned to a netdev that is now a bridge
6751                  * or LAG slave.
6752                  */
6753                 return false;
6754         }
6755
6756         return false;
6757 }
6758
6759 static enum mlxsw_sp_rif_type
6760 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
6761                       const struct net_device *dev)
6762 {
6763         enum mlxsw_sp_fid_type type;
6764
6765         if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
6766                 return MLXSW_SP_RIF_TYPE_IPIP_LB;
6767
6768         /* Otherwise RIF type is derived from the type of the underlying FID. */
6769         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
6770                 type = MLXSW_SP_FID_TYPE_8021Q;
6771         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
6772                 type = MLXSW_SP_FID_TYPE_8021Q;
6773         else if (netif_is_bridge_master(dev))
6774                 type = MLXSW_SP_FID_TYPE_8021D;
6775         else
6776                 type = MLXSW_SP_FID_TYPE_RFID;
6777
6778         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
6779 }
6780
6781 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
6782 {
6783         int i;
6784
6785         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
6786                 if (!mlxsw_sp->router->rifs[i]) {
6787                         *p_rif_index = i;
6788                         return 0;
6789                 }
6790         }
6791
6792         return -ENOBUFS;
6793 }
6794
6795 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
6796                                                u16 vr_id,
6797                                                struct net_device *l3_dev)
6798 {
6799         struct mlxsw_sp_rif *rif;
6800
6801         rif = kzalloc(rif_size, GFP_KERNEL);
6802         if (!rif)
6803                 return NULL;
6804
6805         INIT_LIST_HEAD(&rif->nexthop_list);
6806         INIT_LIST_HEAD(&rif->neigh_list);
6807         if (l3_dev) {
6808                 ether_addr_copy(rif->addr, l3_dev->dev_addr);
6809                 rif->mtu = l3_dev->mtu;
6810                 rif->dev = l3_dev;
6811         }
6812         rif->vr_id = vr_id;
6813         rif->rif_index = rif_index;
6814
6815         return rif;
6816 }
6817
6818 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
6819                                            u16 rif_index)
6820 {
6821         return mlxsw_sp->router->rifs[rif_index];
6822 }
6823
6824 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
6825 {
6826         return rif->rif_index;
6827 }
6828
6829 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6830 {
6831         return lb_rif->common.rif_index;
6832 }
6833
6834 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6835 {
6836         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev);
6837         struct mlxsw_sp_vr *ul_vr;
6838
6839         ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL);
6840         if (WARN_ON(IS_ERR(ul_vr)))
6841                 return 0;
6842
6843         return ul_vr->id;
6844 }
6845
6846 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6847 {
6848         return lb_rif->ul_rif_id;
6849 }
6850
6851 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
6852 {
6853         return rif->dev->ifindex;
6854 }
6855
6856 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif)
6857 {
6858         return rif->dev;
6859 }
6860
6861 static struct mlxsw_sp_rif *
6862 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
6863                     const struct mlxsw_sp_rif_params *params,
6864                     struct netlink_ext_ack *extack)
6865 {
6866         u32 tb_id = l3mdev_fib_table(params->dev);
6867         const struct mlxsw_sp_rif_ops *ops;
6868         struct mlxsw_sp_fid *fid = NULL;
6869         enum mlxsw_sp_rif_type type;
6870         struct mlxsw_sp_rif *rif;
6871         struct mlxsw_sp_vr *vr;
6872         u16 rif_index;
6873         int i, err;
6874
6875         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
6876         ops = mlxsw_sp->rif_ops_arr[type];
6877
6878         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack);
6879         if (IS_ERR(vr))
6880                 return ERR_CAST(vr);
6881         vr->rif_count++;
6882
6883         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
6884         if (err) {
6885                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
6886                 goto err_rif_index_alloc;
6887         }
6888
6889         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
6890         if (!rif) {
6891                 err = -ENOMEM;
6892                 goto err_rif_alloc;
6893         }
6894         dev_hold(rif->dev);
6895         mlxsw_sp->router->rifs[rif_index] = rif;
6896         rif->mlxsw_sp = mlxsw_sp;
6897         rif->ops = ops;
6898
6899         if (ops->fid_get) {
6900                 fid = ops->fid_get(rif, extack);
6901                 if (IS_ERR(fid)) {
6902                         err = PTR_ERR(fid);
6903                         goto err_fid_get;
6904                 }
6905                 rif->fid = fid;
6906         }
6907
6908         if (ops->setup)
6909                 ops->setup(rif, params);
6910
6911         err = ops->configure(rif);
6912         if (err)
6913                 goto err_configure;
6914
6915         for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
6916                 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif);
6917                 if (err)
6918                         goto err_mr_rif_add;
6919         }
6920
6921         mlxsw_sp_rif_counters_alloc(rif);
6922
6923         return rif;
6924
6925 err_mr_rif_add:
6926         for (i--; i >= 0; i--)
6927                 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
6928         ops->deconfigure(rif);
6929 err_configure:
6930         if (fid)
6931                 mlxsw_sp_fid_put(fid);
6932 err_fid_get:
6933         mlxsw_sp->router->rifs[rif_index] = NULL;
6934         dev_put(rif->dev);
6935         kfree(rif);
6936 err_rif_alloc:
6937 err_rif_index_alloc:
6938         vr->rif_count--;
6939         mlxsw_sp_vr_put(mlxsw_sp, vr);
6940         return ERR_PTR(err);
6941 }
6942
6943 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
6944 {
6945         const struct mlxsw_sp_rif_ops *ops = rif->ops;
6946         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
6947         struct mlxsw_sp_fid *fid = rif->fid;
6948         struct mlxsw_sp_vr *vr;
6949         int i;
6950
6951         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
6952         vr = &mlxsw_sp->router->vrs[rif->vr_id];
6953
6954         mlxsw_sp_rif_counters_free(rif);
6955         for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
6956                 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
6957         ops->deconfigure(rif);
6958         if (fid)
6959                 /* Loopback RIFs are not associated with a FID. */
6960                 mlxsw_sp_fid_put(fid);
6961         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
6962         dev_put(rif->dev);
6963         kfree(rif);
6964         vr->rif_count--;
6965         mlxsw_sp_vr_put(mlxsw_sp, vr);
6966 }
6967
6968 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp,
6969                                  struct net_device *dev)
6970 {
6971         struct mlxsw_sp_rif *rif;
6972
6973         mutex_lock(&mlxsw_sp->router->lock);
6974         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6975         if (!rif)
6976                 goto out;
6977         mlxsw_sp_rif_destroy(rif);
6978 out:
6979         mutex_unlock(&mlxsw_sp->router->lock);
6980 }
6981
6982 static void
6983 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
6984                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
6985 {
6986         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
6987
6988         params->vid = mlxsw_sp_port_vlan->vid;
6989         params->lag = mlxsw_sp_port->lagged;
6990         if (params->lag)
6991                 params->lag_id = mlxsw_sp_port->lag_id;
6992         else
6993                 params->system_port = mlxsw_sp_port->local_port;
6994 }
6995
6996 static struct mlxsw_sp_rif_subport *
6997 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
6998 {
6999         return container_of(rif, struct mlxsw_sp_rif_subport, common);
7000 }
7001
7002 static struct mlxsw_sp_rif *
7003 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp,
7004                          const struct mlxsw_sp_rif_params *params,
7005                          struct netlink_ext_ack *extack)
7006 {
7007         struct mlxsw_sp_rif_subport *rif_subport;
7008         struct mlxsw_sp_rif *rif;
7009
7010         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev);
7011         if (!rif)
7012                 return mlxsw_sp_rif_create(mlxsw_sp, params, extack);
7013
7014         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7015         refcount_inc(&rif_subport->ref_count);
7016         return rif;
7017 }
7018
7019 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif)
7020 {
7021         struct mlxsw_sp_rif_subport *rif_subport;
7022
7023         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7024         if (!refcount_dec_and_test(&rif_subport->ref_count))
7025                 return;
7026
7027         mlxsw_sp_rif_destroy(rif);
7028 }
7029
7030 static int
7031 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
7032                                struct net_device *l3_dev,
7033                                struct netlink_ext_ack *extack)
7034 {
7035         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
7036         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
7037         struct mlxsw_sp_rif_params params = {
7038                 .dev = l3_dev,
7039         };
7040         u16 vid = mlxsw_sp_port_vlan->vid;
7041         struct mlxsw_sp_rif *rif;
7042         struct mlxsw_sp_fid *fid;
7043         int err;
7044
7045         mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
7046         rif = mlxsw_sp_rif_subport_get(mlxsw_sp, &params, extack);
7047         if (IS_ERR(rif))
7048                 return PTR_ERR(rif);
7049
7050         /* FID was already created, just take a reference */
7051         fid = rif->ops->fid_get(rif, extack);
7052         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
7053         if (err)
7054                 goto err_fid_port_vid_map;
7055
7056         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
7057         if (err)
7058                 goto err_port_vid_learning_set;
7059
7060         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
7061                                         BR_STATE_FORWARDING);
7062         if (err)
7063                 goto err_port_vid_stp_set;
7064
7065         mlxsw_sp_port_vlan->fid = fid;
7066
7067         return 0;
7068
7069 err_port_vid_stp_set:
7070         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
7071 err_port_vid_learning_set:
7072         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
7073 err_fid_port_vid_map:
7074         mlxsw_sp_fid_put(fid);
7075         mlxsw_sp_rif_subport_put(rif);
7076         return err;
7077 }
7078
7079 static void
7080 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
7081 {
7082         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
7083         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
7084         struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid);
7085         u16 vid = mlxsw_sp_port_vlan->vid;
7086
7087         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
7088                 return;
7089
7090         mlxsw_sp_port_vlan->fid = NULL;
7091         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
7092         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
7093         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
7094         mlxsw_sp_fid_put(fid);
7095         mlxsw_sp_rif_subport_put(rif);
7096 }
7097
7098 void
7099 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
7100 {
7101         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
7102
7103         mutex_lock(&mlxsw_sp->router->lock);
7104         __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
7105         mutex_unlock(&mlxsw_sp->router->lock);
7106 }
7107
7108 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
7109                                              struct net_device *port_dev,
7110                                              unsigned long event, u16 vid,
7111                                              struct netlink_ext_ack *extack)
7112 {
7113         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
7114         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
7115
7116         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
7117         if (WARN_ON(!mlxsw_sp_port_vlan))
7118                 return -EINVAL;
7119
7120         switch (event) {
7121         case NETDEV_UP:
7122                 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
7123                                                       l3_dev, extack);
7124         case NETDEV_DOWN:
7125                 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
7126                 break;
7127         }
7128
7129         return 0;
7130 }
7131
7132 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
7133                                         unsigned long event,
7134                                         struct netlink_ext_ack *extack)
7135 {
7136         if (netif_is_bridge_port(port_dev) ||
7137             netif_is_lag_port(port_dev) ||
7138             netif_is_ovs_port(port_dev))
7139                 return 0;
7140
7141         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event,
7142                                                  MLXSW_SP_DEFAULT_VID, extack);
7143 }
7144
7145 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
7146                                          struct net_device *lag_dev,
7147                                          unsigned long event, u16 vid,
7148                                          struct netlink_ext_ack *extack)
7149 {
7150         struct net_device *port_dev;
7151         struct list_head *iter;
7152         int err;
7153
7154         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
7155                 if (mlxsw_sp_port_dev_check(port_dev)) {
7156                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
7157                                                                 port_dev,
7158                                                                 event, vid,
7159                                                                 extack);
7160                         if (err)
7161                                 return err;
7162                 }
7163         }
7164
7165         return 0;
7166 }
7167
7168 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
7169                                        unsigned long event,
7170                                        struct netlink_ext_ack *extack)
7171 {
7172         if (netif_is_bridge_port(lag_dev))
7173                 return 0;
7174
7175         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event,
7176                                              MLXSW_SP_DEFAULT_VID, extack);
7177 }
7178
7179 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp,
7180                                           struct net_device *l3_dev,
7181                                           unsigned long event,
7182                                           struct netlink_ext_ack *extack)
7183 {
7184         struct mlxsw_sp_rif_params params = {
7185                 .dev = l3_dev,
7186         };
7187         struct mlxsw_sp_rif *rif;
7188
7189         switch (event) {
7190         case NETDEV_UP:
7191                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params, extack);
7192                 if (IS_ERR(rif))
7193                         return PTR_ERR(rif);
7194                 break;
7195         case NETDEV_DOWN:
7196                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
7197                 mlxsw_sp_rif_destroy(rif);
7198                 break;
7199         }
7200
7201         return 0;
7202 }
7203
7204 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp,
7205                                         struct net_device *vlan_dev,
7206                                         unsigned long event,
7207                                         struct netlink_ext_ack *extack)
7208 {
7209         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
7210         u16 vid = vlan_dev_vlan_id(vlan_dev);
7211
7212         if (netif_is_bridge_port(vlan_dev))
7213                 return 0;
7214
7215         if (mlxsw_sp_port_dev_check(real_dev))
7216                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
7217                                                          event, vid, extack);
7218         else if (netif_is_lag_master(real_dev))
7219                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
7220                                                      vid, extack);
7221         else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
7222                 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event,
7223                                                       extack);
7224
7225         return 0;
7226 }
7227
7228 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac)
7229 {
7230         u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 };
7231         u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
7232
7233         return ether_addr_equal_masked(mac, vrrp4, mask);
7234 }
7235
7236 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac)
7237 {
7238         u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 };
7239         u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
7240
7241         return ether_addr_equal_masked(mac, vrrp6, mask);
7242 }
7243
7244 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
7245                                 const u8 *mac, bool adding)
7246 {
7247         char ritr_pl[MLXSW_REG_RITR_LEN];
7248         u8 vrrp_id = adding ? mac[5] : 0;
7249         int err;
7250
7251         if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) &&
7252             !mlxsw_sp_rif_macvlan_is_vrrp6(mac))
7253                 return 0;
7254
7255         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
7256         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7257         if (err)
7258                 return err;
7259
7260         if (mlxsw_sp_rif_macvlan_is_vrrp4(mac))
7261                 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id);
7262         else
7263                 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id);
7264
7265         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7266 }
7267
7268 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp,
7269                                     const struct net_device *macvlan_dev,
7270                                     struct netlink_ext_ack *extack)
7271 {
7272         struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
7273         struct mlxsw_sp_rif *rif;
7274         int err;
7275
7276         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
7277         if (!rif) {
7278                 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces");
7279                 return -EOPNOTSUPP;
7280         }
7281
7282         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
7283                                   mlxsw_sp_fid_index(rif->fid), true);
7284         if (err)
7285                 return err;
7286
7287         err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index,
7288                                    macvlan_dev->dev_addr, true);
7289         if (err)
7290                 goto err_rif_vrrp_add;
7291
7292         /* Make sure the bridge driver does not have this MAC pointing at
7293          * some other port.
7294          */
7295         if (rif->ops->fdb_del)
7296                 rif->ops->fdb_del(rif, macvlan_dev->dev_addr);
7297
7298         return 0;
7299
7300 err_rif_vrrp_add:
7301         mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
7302                             mlxsw_sp_fid_index(rif->fid), false);
7303         return err;
7304 }
7305
7306 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
7307                                        const struct net_device *macvlan_dev)
7308 {
7309         struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
7310         struct mlxsw_sp_rif *rif;
7311
7312         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
7313         /* If we do not have a RIF, then we already took care of
7314          * removing the macvlan's MAC during RIF deletion.
7315          */
7316         if (!rif)
7317                 return;
7318         mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr,
7319                              false);
7320         mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
7321                             mlxsw_sp_fid_index(rif->fid), false);
7322 }
7323
7324 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
7325                               const struct net_device *macvlan_dev)
7326 {
7327         mutex_lock(&mlxsw_sp->router->lock);
7328         __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
7329         mutex_unlock(&mlxsw_sp->router->lock);
7330 }
7331
7332 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp,
7333                                            struct net_device *macvlan_dev,
7334                                            unsigned long event,
7335                                            struct netlink_ext_ack *extack)
7336 {
7337         switch (event) {
7338         case NETDEV_UP:
7339                 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack);
7340         case NETDEV_DOWN:
7341                 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
7342                 break;
7343         }
7344
7345         return 0;
7346 }
7347
7348 static int mlxsw_sp_router_port_check_rif_addr(struct mlxsw_sp *mlxsw_sp,
7349                                                struct net_device *dev,
7350                                                const unsigned char *dev_addr,
7351                                                struct netlink_ext_ack *extack)
7352 {
7353         struct mlxsw_sp_rif *rif;
7354         int i;
7355
7356         /* A RIF is not created for macvlan netdevs. Their MAC is used to
7357          * populate the FDB
7358          */
7359         if (netif_is_macvlan(dev) || netif_is_l3_master(dev))
7360                 return 0;
7361
7362         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
7363                 rif = mlxsw_sp->router->rifs[i];
7364                 if (rif && rif->ops &&
7365                     rif->ops->type == MLXSW_SP_RIF_TYPE_IPIP_LB)
7366                         continue;
7367                 if (rif && rif->dev && rif->dev != dev &&
7368                     !ether_addr_equal_masked(rif->dev->dev_addr, dev_addr,
7369                                              mlxsw_sp->mac_mask)) {
7370                         NL_SET_ERR_MSG_MOD(extack, "All router interface MAC addresses must have the same prefix");
7371                         return -EINVAL;
7372                 }
7373         }
7374
7375         return 0;
7376 }
7377
7378 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp,
7379                                      struct net_device *dev,
7380                                      unsigned long event,
7381                                      struct netlink_ext_ack *extack)
7382 {
7383         if (mlxsw_sp_port_dev_check(dev))
7384                 return mlxsw_sp_inetaddr_port_event(dev, event, extack);
7385         else if (netif_is_lag_master(dev))
7386                 return mlxsw_sp_inetaddr_lag_event(dev, event, extack);
7387         else if (netif_is_bridge_master(dev))
7388                 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event,
7389                                                       extack);
7390         else if (is_vlan_dev(dev))
7391                 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event,
7392                                                     extack);
7393         else if (netif_is_macvlan(dev))
7394                 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event,
7395                                                        extack);
7396         else
7397                 return 0;
7398 }
7399
7400 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb,
7401                                    unsigned long event, void *ptr)
7402 {
7403         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
7404         struct net_device *dev = ifa->ifa_dev->dev;
7405         struct mlxsw_sp_router *router;
7406         struct mlxsw_sp_rif *rif;
7407         int err = 0;
7408
7409         /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */
7410         if (event == NETDEV_UP)
7411                 return NOTIFY_DONE;
7412
7413         router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb);
7414         mutex_lock(&router->lock);
7415         rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev);
7416         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7417                 goto out;
7418
7419         err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL);
7420 out:
7421         mutex_unlock(&router->lock);
7422         return notifier_from_errno(err);
7423 }
7424
7425 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused,
7426                                   unsigned long event, void *ptr)
7427 {
7428         struct in_validator_info *ivi = (struct in_validator_info *) ptr;
7429         struct net_device *dev = ivi->ivi_dev->dev;
7430         struct mlxsw_sp *mlxsw_sp;
7431         struct mlxsw_sp_rif *rif;
7432         int err = 0;
7433
7434         mlxsw_sp = mlxsw_sp_lower_get(dev);
7435         if (!mlxsw_sp)
7436                 return NOTIFY_DONE;
7437
7438         mutex_lock(&mlxsw_sp->router->lock);
7439         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7440         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7441                 goto out;
7442
7443         err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr,
7444                                                   ivi->extack);
7445         if (err)
7446                 goto out;
7447
7448         err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack);
7449 out:
7450         mutex_unlock(&mlxsw_sp->router->lock);
7451         return notifier_from_errno(err);
7452 }
7453
7454 struct mlxsw_sp_inet6addr_event_work {
7455         struct work_struct work;
7456         struct mlxsw_sp *mlxsw_sp;
7457         struct net_device *dev;
7458         unsigned long event;
7459 };
7460
7461 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
7462 {
7463         struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
7464                 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
7465         struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp;
7466         struct net_device *dev = inet6addr_work->dev;
7467         unsigned long event = inet6addr_work->event;
7468         struct mlxsw_sp_rif *rif;
7469
7470         rtnl_lock();
7471         mutex_lock(&mlxsw_sp->router->lock);
7472
7473         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7474         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7475                 goto out;
7476
7477         __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL);
7478 out:
7479         mutex_unlock(&mlxsw_sp->router->lock);
7480         rtnl_unlock();
7481         dev_put(dev);
7482         kfree(inet6addr_work);
7483 }
7484
7485 /* Called with rcu_read_lock() */
7486 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb,
7487                                     unsigned long event, void *ptr)
7488 {
7489         struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
7490         struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
7491         struct net_device *dev = if6->idev->dev;
7492         struct mlxsw_sp_router *router;
7493
7494         /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */
7495         if (event == NETDEV_UP)
7496                 return NOTIFY_DONE;
7497
7498         inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
7499         if (!inet6addr_work)
7500                 return NOTIFY_BAD;
7501
7502         router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb);
7503         INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
7504         inet6addr_work->mlxsw_sp = router->mlxsw_sp;
7505         inet6addr_work->dev = dev;
7506         inet6addr_work->event = event;
7507         dev_hold(dev);
7508         mlxsw_core_schedule_work(&inet6addr_work->work);
7509
7510         return NOTIFY_DONE;
7511 }
7512
7513 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused,
7514                                    unsigned long event, void *ptr)
7515 {
7516         struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr;
7517         struct net_device *dev = i6vi->i6vi_dev->dev;
7518         struct mlxsw_sp *mlxsw_sp;
7519         struct mlxsw_sp_rif *rif;
7520         int err = 0;
7521
7522         mlxsw_sp = mlxsw_sp_lower_get(dev);
7523         if (!mlxsw_sp)
7524                 return NOTIFY_DONE;
7525
7526         mutex_lock(&mlxsw_sp->router->lock);
7527         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7528         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7529                 goto out;
7530
7531         err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr,
7532                                                   i6vi->extack);
7533         if (err)
7534                 goto out;
7535
7536         err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack);
7537 out:
7538         mutex_unlock(&mlxsw_sp->router->lock);
7539         return notifier_from_errno(err);
7540 }
7541
7542 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
7543                              const char *mac, int mtu)
7544 {
7545         char ritr_pl[MLXSW_REG_RITR_LEN];
7546         int err;
7547
7548         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
7549         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7550         if (err)
7551                 return err;
7552
7553         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
7554         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
7555         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
7556         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7557 }
7558
7559 static int
7560 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp,
7561                                   struct mlxsw_sp_rif *rif)
7562 {
7563         struct net_device *dev = rif->dev;
7564         u16 fid_index;
7565         int err;
7566
7567         fid_index = mlxsw_sp_fid_index(rif->fid);
7568
7569         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
7570         if (err)
7571                 return err;
7572
7573         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
7574                                 dev->mtu);
7575         if (err)
7576                 goto err_rif_edit;
7577
7578         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
7579         if (err)
7580                 goto err_rif_fdb_op;
7581
7582         if (rif->mtu != dev->mtu) {
7583                 struct mlxsw_sp_vr *vr;
7584                 int i;
7585
7586                 /* The RIF is relevant only to its mr_table instance, as unlike
7587                  * unicast routing, in multicast routing a RIF cannot be shared
7588                  * between several multicast routing tables.
7589                  */
7590                 vr = &mlxsw_sp->router->vrs[rif->vr_id];
7591                 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
7592                         mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i],
7593                                                    rif, dev->mtu);
7594         }
7595
7596         ether_addr_copy(rif->addr, dev->dev_addr);
7597         rif->mtu = dev->mtu;
7598
7599         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
7600
7601         return 0;
7602
7603 err_rif_fdb_op:
7604         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
7605 err_rif_edit:
7606         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
7607         return err;
7608 }
7609
7610 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif,
7611                             struct netdev_notifier_pre_changeaddr_info *info)
7612 {
7613         struct netlink_ext_ack *extack;
7614
7615         extack = netdev_notifier_info_to_extack(&info->info);
7616         return mlxsw_sp_router_port_check_rif_addr(rif->mlxsw_sp, rif->dev,
7617                                                    info->dev_addr, extack);
7618 }
7619
7620 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev,
7621                                          unsigned long event, void *ptr)
7622 {
7623         struct mlxsw_sp *mlxsw_sp;
7624         struct mlxsw_sp_rif *rif;
7625         int err = 0;
7626
7627         mlxsw_sp = mlxsw_sp_lower_get(dev);
7628         if (!mlxsw_sp)
7629                 return 0;
7630
7631         mutex_lock(&mlxsw_sp->router->lock);
7632         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7633         if (!rif)
7634                 goto out;
7635
7636         switch (event) {
7637         case NETDEV_CHANGEMTU:
7638         case NETDEV_CHANGEADDR:
7639                 err = mlxsw_sp_router_port_change_event(mlxsw_sp, rif);
7640                 break;
7641         case NETDEV_PRE_CHANGEADDR:
7642                 err = mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr);
7643                 break;
7644         }
7645
7646 out:
7647         mutex_unlock(&mlxsw_sp->router->lock);
7648         return err;
7649 }
7650
7651 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
7652                                   struct net_device *l3_dev,
7653                                   struct netlink_ext_ack *extack)
7654 {
7655         struct mlxsw_sp_rif *rif;
7656
7657         /* If netdev is already associated with a RIF, then we need to
7658          * destroy it and create a new one with the new virtual router ID.
7659          */
7660         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
7661         if (rif)
7662                 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN,
7663                                           extack);
7664
7665         return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack);
7666 }
7667
7668 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
7669                                     struct net_device *l3_dev)
7670 {
7671         struct mlxsw_sp_rif *rif;
7672
7673         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
7674         if (!rif)
7675                 return;
7676         __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL);
7677 }
7678
7679 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
7680                                  struct netdev_notifier_changeupper_info *info)
7681 {
7682         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
7683         int err = 0;
7684
7685         /* We do not create a RIF for a macvlan, but only use it to
7686          * direct more MAC addresses to the router.
7687          */
7688         if (!mlxsw_sp || netif_is_macvlan(l3_dev))
7689                 return 0;
7690
7691         mutex_lock(&mlxsw_sp->router->lock);
7692         switch (event) {
7693         case NETDEV_PRECHANGEUPPER:
7694                 break;
7695         case NETDEV_CHANGEUPPER:
7696                 if (info->linking) {
7697                         struct netlink_ext_ack *extack;
7698
7699                         extack = netdev_notifier_info_to_extack(&info->info);
7700                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack);
7701                 } else {
7702                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
7703                 }
7704                 break;
7705         }
7706         mutex_unlock(&mlxsw_sp->router->lock);
7707
7708         return err;
7709 }
7710
7711 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev,
7712                                         struct netdev_nested_priv *priv)
7713 {
7714         struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data;
7715
7716         if (!netif_is_macvlan(dev))
7717                 return 0;
7718
7719         return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
7720                                    mlxsw_sp_fid_index(rif->fid), false);
7721 }
7722
7723 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif)
7724 {
7725         struct netdev_nested_priv priv = {
7726                 .data = (void *)rif,
7727         };
7728
7729         if (!netif_is_macvlan_port(rif->dev))
7730                 return 0;
7731
7732         netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n");
7733         return netdev_walk_all_upper_dev_rcu(rif->dev,
7734                                              __mlxsw_sp_rif_macvlan_flush, &priv);
7735 }
7736
7737 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
7738                                        const struct mlxsw_sp_rif_params *params)
7739 {
7740         struct mlxsw_sp_rif_subport *rif_subport;
7741
7742         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7743         refcount_set(&rif_subport->ref_count, 1);
7744         rif_subport->vid = params->vid;
7745         rif_subport->lag = params->lag;
7746         if (params->lag)
7747                 rif_subport->lag_id = params->lag_id;
7748         else
7749                 rif_subport->system_port = params->system_port;
7750 }
7751
7752 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
7753 {
7754         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7755         struct mlxsw_sp_rif_subport *rif_subport;
7756         char ritr_pl[MLXSW_REG_RITR_LEN];
7757
7758         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7759         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
7760                             rif->rif_index, rif->vr_id, rif->dev->mtu);
7761         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
7762         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
7763                                   rif_subport->lag ? rif_subport->lag_id :
7764                                                      rif_subport->system_port,
7765                                   rif_subport->vid);
7766
7767         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7768 }
7769
7770 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
7771 {
7772         int err;
7773
7774         err = mlxsw_sp_rif_subport_op(rif, true);
7775         if (err)
7776                 return err;
7777
7778         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7779                                   mlxsw_sp_fid_index(rif->fid), true);
7780         if (err)
7781                 goto err_rif_fdb_op;
7782
7783         mlxsw_sp_fid_rif_set(rif->fid, rif);
7784         return 0;
7785
7786 err_rif_fdb_op:
7787         mlxsw_sp_rif_subport_op(rif, false);
7788         return err;
7789 }
7790
7791 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
7792 {
7793         struct mlxsw_sp_fid *fid = rif->fid;
7794
7795         mlxsw_sp_fid_rif_set(fid, NULL);
7796         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7797                             mlxsw_sp_fid_index(fid), false);
7798         mlxsw_sp_rif_macvlan_flush(rif);
7799         mlxsw_sp_rif_subport_op(rif, false);
7800 }
7801
7802 static struct mlxsw_sp_fid *
7803 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif,
7804                              struct netlink_ext_ack *extack)
7805 {
7806         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
7807 }
7808
7809 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
7810         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
7811         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
7812         .setup                  = mlxsw_sp_rif_subport_setup,
7813         .configure              = mlxsw_sp_rif_subport_configure,
7814         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
7815         .fid_get                = mlxsw_sp_rif_subport_fid_get,
7816 };
7817
7818 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
7819                                     enum mlxsw_reg_ritr_if_type type,
7820                                     u16 vid_fid, bool enable)
7821 {
7822         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7823         char ritr_pl[MLXSW_REG_RITR_LEN];
7824
7825         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
7826                             rif->dev->mtu);
7827         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
7828         mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
7829
7830         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7831 }
7832
7833 u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
7834 {
7835         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
7836 }
7837
7838 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
7839 {
7840         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7841         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
7842         int err;
7843
7844         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
7845                                        true);
7846         if (err)
7847                 return err;
7848
7849         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7850                                      mlxsw_sp_router_port(mlxsw_sp), true);
7851         if (err)
7852                 goto err_fid_mc_flood_set;
7853
7854         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7855                                      mlxsw_sp_router_port(mlxsw_sp), true);
7856         if (err)
7857                 goto err_fid_bc_flood_set;
7858
7859         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7860                                   mlxsw_sp_fid_index(rif->fid), true);
7861         if (err)
7862                 goto err_rif_fdb_op;
7863
7864         mlxsw_sp_fid_rif_set(rif->fid, rif);
7865         return 0;
7866
7867 err_rif_fdb_op:
7868         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7869                                mlxsw_sp_router_port(mlxsw_sp), false);
7870 err_fid_bc_flood_set:
7871         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7872                                mlxsw_sp_router_port(mlxsw_sp), false);
7873 err_fid_mc_flood_set:
7874         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
7875         return err;
7876 }
7877
7878 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
7879 {
7880         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
7881         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7882         struct mlxsw_sp_fid *fid = rif->fid;
7883
7884         mlxsw_sp_fid_rif_set(fid, NULL);
7885         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7886                             mlxsw_sp_fid_index(fid), false);
7887         mlxsw_sp_rif_macvlan_flush(rif);
7888         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7889                                mlxsw_sp_router_port(mlxsw_sp), false);
7890         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7891                                mlxsw_sp_router_port(mlxsw_sp), false);
7892         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
7893 }
7894
7895 static struct mlxsw_sp_fid *
7896 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif,
7897                          struct netlink_ext_ack *extack)
7898 {
7899         return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
7900 }
7901
7902 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
7903 {
7904         struct switchdev_notifier_fdb_info info;
7905         struct net_device *dev;
7906
7907         dev = br_fdb_find_port(rif->dev, mac, 0);
7908         if (!dev)
7909                 return;
7910
7911         info.addr = mac;
7912         info.vid = 0;
7913         call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
7914                                  NULL);
7915 }
7916
7917 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
7918         .type                   = MLXSW_SP_RIF_TYPE_FID,
7919         .rif_size               = sizeof(struct mlxsw_sp_rif),
7920         .configure              = mlxsw_sp_rif_fid_configure,
7921         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
7922         .fid_get                = mlxsw_sp_rif_fid_fid_get,
7923         .fdb_del                = mlxsw_sp_rif_fid_fdb_del,
7924 };
7925
7926 static struct mlxsw_sp_fid *
7927 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif,
7928                           struct netlink_ext_ack *extack)
7929 {
7930         struct net_device *br_dev;
7931         u16 vid;
7932         int err;
7933
7934         if (is_vlan_dev(rif->dev)) {
7935                 vid = vlan_dev_vlan_id(rif->dev);
7936                 br_dev = vlan_dev_real_dev(rif->dev);
7937                 if (WARN_ON(!netif_is_bridge_master(br_dev)))
7938                         return ERR_PTR(-EINVAL);
7939         } else {
7940                 err = br_vlan_get_pvid(rif->dev, &vid);
7941                 if (err < 0 || !vid) {
7942                         NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID");
7943                         return ERR_PTR(-EINVAL);
7944                 }
7945         }
7946
7947         return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
7948 }
7949
7950 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
7951 {
7952         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
7953         struct switchdev_notifier_fdb_info info;
7954         struct net_device *br_dev;
7955         struct net_device *dev;
7956
7957         br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev;
7958         dev = br_fdb_find_port(br_dev, mac, vid);
7959         if (!dev)
7960                 return;
7961
7962         info.addr = mac;
7963         info.vid = vid;
7964         call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
7965                                  NULL);
7966 }
7967
7968 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = {
7969         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
7970         .rif_size               = sizeof(struct mlxsw_sp_rif),
7971         .configure              = mlxsw_sp_rif_fid_configure,
7972         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
7973         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
7974         .fdb_del                = mlxsw_sp_rif_vlan_fdb_del,
7975 };
7976
7977 static struct mlxsw_sp_rif_ipip_lb *
7978 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
7979 {
7980         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
7981 }
7982
7983 static void
7984 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
7985                            const struct mlxsw_sp_rif_params *params)
7986 {
7987         struct mlxsw_sp_rif_params_ipip_lb *params_lb;
7988         struct mlxsw_sp_rif_ipip_lb *rif_lb;
7989
7990         params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
7991                                  common);
7992         rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
7993         rif_lb->lb_config = params_lb->lb_config;
7994 }
7995
7996 static int
7997 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
7998 {
7999         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
8000         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
8001         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8002         struct mlxsw_sp_vr *ul_vr;
8003         int err;
8004
8005         ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL);
8006         if (IS_ERR(ul_vr))
8007                 return PTR_ERR(ul_vr);
8008
8009         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true);
8010         if (err)
8011                 goto err_loopback_op;
8012
8013         lb_rif->ul_vr_id = ul_vr->id;
8014         lb_rif->ul_rif_id = 0;
8015         ++ul_vr->rif_count;
8016         return 0;
8017
8018 err_loopback_op:
8019         mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
8020         return err;
8021 }
8022
8023 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
8024 {
8025         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
8026         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8027         struct mlxsw_sp_vr *ul_vr;
8028
8029         ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
8030         mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false);
8031
8032         --ul_vr->rif_count;
8033         mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
8034 }
8035
8036 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = {
8037         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
8038         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
8039         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
8040         .configure              = mlxsw_sp1_rif_ipip_lb_configure,
8041         .deconfigure            = mlxsw_sp1_rif_ipip_lb_deconfigure,
8042 };
8043
8044 const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = {
8045         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
8046         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_emu_ops,
8047         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
8048         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp1_rif_ipip_lb_ops,
8049 };
8050
8051 static int
8052 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable)
8053 {
8054         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
8055         char ritr_pl[MLXSW_REG_RITR_LEN];
8056
8057         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
8058                             ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU);
8059         mlxsw_reg_ritr_loopback_protocol_set(ritr_pl,
8060                                              MLXSW_REG_RITR_LOOPBACK_GENERIC);
8061
8062         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
8063 }
8064
8065 static struct mlxsw_sp_rif *
8066 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
8067                        struct netlink_ext_ack *extack)
8068 {
8069         struct mlxsw_sp_rif *ul_rif;
8070         u16 rif_index;
8071         int err;
8072
8073         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
8074         if (err) {
8075                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
8076                 return ERR_PTR(err);
8077         }
8078
8079         ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL);
8080         if (!ul_rif)
8081                 return ERR_PTR(-ENOMEM);
8082
8083         mlxsw_sp->router->rifs[rif_index] = ul_rif;
8084         ul_rif->mlxsw_sp = mlxsw_sp;
8085         err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true);
8086         if (err)
8087                 goto ul_rif_op_err;
8088
8089         return ul_rif;
8090
8091 ul_rif_op_err:
8092         mlxsw_sp->router->rifs[rif_index] = NULL;
8093         kfree(ul_rif);
8094         return ERR_PTR(err);
8095 }
8096
8097 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif)
8098 {
8099         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
8100
8101         mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false);
8102         mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL;
8103         kfree(ul_rif);
8104 }
8105
8106 static struct mlxsw_sp_rif *
8107 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
8108                     struct netlink_ext_ack *extack)
8109 {
8110         struct mlxsw_sp_vr *vr;
8111         int err;
8112
8113         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack);
8114         if (IS_ERR(vr))
8115                 return ERR_CAST(vr);
8116
8117         if (refcount_inc_not_zero(&vr->ul_rif_refcnt))
8118                 return vr->ul_rif;
8119
8120         vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack);
8121         if (IS_ERR(vr->ul_rif)) {
8122                 err = PTR_ERR(vr->ul_rif);
8123                 goto err_ul_rif_create;
8124         }
8125
8126         vr->rif_count++;
8127         refcount_set(&vr->ul_rif_refcnt, 1);
8128
8129         return vr->ul_rif;
8130
8131 err_ul_rif_create:
8132         mlxsw_sp_vr_put(mlxsw_sp, vr);
8133         return ERR_PTR(err);
8134 }
8135
8136 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif)
8137 {
8138         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
8139         struct mlxsw_sp_vr *vr;
8140
8141         vr = &mlxsw_sp->router->vrs[ul_rif->vr_id];
8142
8143         if (!refcount_dec_and_test(&vr->ul_rif_refcnt))
8144                 return;
8145
8146         vr->rif_count--;
8147         mlxsw_sp_ul_rif_destroy(ul_rif);
8148         mlxsw_sp_vr_put(mlxsw_sp, vr);
8149 }
8150
8151 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
8152                                u16 *ul_rif_index)
8153 {
8154         struct mlxsw_sp_rif *ul_rif;
8155         int err = 0;
8156
8157         mutex_lock(&mlxsw_sp->router->lock);
8158         ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
8159         if (IS_ERR(ul_rif)) {
8160                 err = PTR_ERR(ul_rif);
8161                 goto out;
8162         }
8163         *ul_rif_index = ul_rif->rif_index;
8164 out:
8165         mutex_unlock(&mlxsw_sp->router->lock);
8166         return err;
8167 }
8168
8169 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index)
8170 {
8171         struct mlxsw_sp_rif *ul_rif;
8172
8173         mutex_lock(&mlxsw_sp->router->lock);
8174         ul_rif = mlxsw_sp->router->rifs[ul_rif_index];
8175         if (WARN_ON(!ul_rif))
8176                 goto out;
8177
8178         mlxsw_sp_ul_rif_put(ul_rif);
8179 out:
8180         mutex_unlock(&mlxsw_sp->router->lock);
8181 }
8182
8183 static int
8184 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
8185 {
8186         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
8187         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
8188         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8189         struct mlxsw_sp_rif *ul_rif;
8190         int err;
8191
8192         ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
8193         if (IS_ERR(ul_rif))
8194                 return PTR_ERR(ul_rif);
8195
8196         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true);
8197         if (err)
8198                 goto err_loopback_op;
8199
8200         lb_rif->ul_vr_id = 0;
8201         lb_rif->ul_rif_id = ul_rif->rif_index;
8202
8203         return 0;
8204
8205 err_loopback_op:
8206         mlxsw_sp_ul_rif_put(ul_rif);
8207         return err;
8208 }
8209
8210 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
8211 {
8212         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
8213         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8214         struct mlxsw_sp_rif *ul_rif;
8215
8216         ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id);
8217         mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false);
8218         mlxsw_sp_ul_rif_put(ul_rif);
8219 }
8220
8221 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = {
8222         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
8223         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
8224         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
8225         .configure              = mlxsw_sp2_rif_ipip_lb_configure,
8226         .deconfigure            = mlxsw_sp2_rif_ipip_lb_deconfigure,
8227 };
8228
8229 const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = {
8230         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
8231         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_emu_ops,
8232         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
8233         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp2_rif_ipip_lb_ops,
8234 };
8235
8236 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
8237 {
8238         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
8239
8240         mlxsw_sp->router->rifs = kcalloc(max_rifs,
8241                                          sizeof(struct mlxsw_sp_rif *),
8242                                          GFP_KERNEL);
8243         if (!mlxsw_sp->router->rifs)
8244                 return -ENOMEM;
8245
8246         return 0;
8247 }
8248
8249 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
8250 {
8251         int i;
8252
8253         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
8254                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
8255
8256         kfree(mlxsw_sp->router->rifs);
8257 }
8258
8259 static int
8260 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp)
8261 {
8262         char tigcr_pl[MLXSW_REG_TIGCR_LEN];
8263
8264         mlxsw_reg_tigcr_pack(tigcr_pl, true, 0);
8265         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl);
8266 }
8267
8268 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
8269 {
8270         int err;
8271
8272         mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
8273         INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
8274
8275         err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp);
8276         if (err)
8277                 return err;
8278         err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp);
8279         if (err)
8280                 return err;
8281
8282         return mlxsw_sp_ipip_config_tigcr(mlxsw_sp);
8283 }
8284
8285 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
8286 {
8287         WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
8288 }
8289
8290 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
8291 {
8292         struct mlxsw_sp_router *router;
8293
8294         /* Flush pending FIB notifications and then flush the device's
8295          * table before requesting another dump. The FIB notification
8296          * block is unregistered, so no need to take RTNL.
8297          */
8298         mlxsw_core_flush_owq();
8299         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
8300         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
8301 }
8302
8303 #ifdef CONFIG_IP_ROUTE_MULTIPATH
8304 static void mlxsw_sp_mp_hash_header_set(char *recr2_pl, int header)
8305 {
8306         mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, header, true);
8307 }
8308
8309 static void mlxsw_sp_mp_hash_field_set(char *recr2_pl, int field)
8310 {
8311         mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, field, true);
8312 }
8313
8314 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl)
8315 {
8316         struct net *net = mlxsw_sp_net(mlxsw_sp);
8317         bool only_l3 = !net->ipv4.sysctl_fib_multipath_hash_policy;
8318
8319         mlxsw_sp_mp_hash_header_set(recr2_pl,
8320                                     MLXSW_REG_RECR2_IPV4_EN_NOT_TCP_NOT_UDP);
8321         mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV4_EN_TCP_UDP);
8322         mlxsw_reg_recr2_ipv4_sip_enable(recr2_pl);
8323         mlxsw_reg_recr2_ipv4_dip_enable(recr2_pl);
8324         if (only_l3)
8325                 return;
8326         mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_EN_IPV4);
8327         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV4_PROTOCOL);
8328         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_SPORT);
8329         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_DPORT);
8330 }
8331
8332 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl)
8333 {
8334         bool only_l3 = !ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp));
8335
8336         mlxsw_sp_mp_hash_header_set(recr2_pl,
8337                                     MLXSW_REG_RECR2_IPV6_EN_NOT_TCP_NOT_UDP);
8338         mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV6_EN_TCP_UDP);
8339         mlxsw_reg_recr2_ipv6_sip_enable(recr2_pl);
8340         mlxsw_reg_recr2_ipv6_dip_enable(recr2_pl);
8341         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_NEXT_HEADER);
8342         if (only_l3) {
8343                 mlxsw_sp_mp_hash_field_set(recr2_pl,
8344                                            MLXSW_REG_RECR2_IPV6_FLOW_LABEL);
8345         } else {
8346                 mlxsw_sp_mp_hash_header_set(recr2_pl,
8347                                             MLXSW_REG_RECR2_TCP_UDP_EN_IPV6);
8348                 mlxsw_sp_mp_hash_field_set(recr2_pl,
8349                                            MLXSW_REG_RECR2_TCP_UDP_SPORT);
8350                 mlxsw_sp_mp_hash_field_set(recr2_pl,
8351                                            MLXSW_REG_RECR2_TCP_UDP_DPORT);
8352         }
8353 }
8354
8355 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
8356 {
8357         char recr2_pl[MLXSW_REG_RECR2_LEN];
8358         u32 seed;
8359
8360         seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0);
8361         mlxsw_reg_recr2_pack(recr2_pl, seed);
8362         mlxsw_sp_mp4_hash_init(mlxsw_sp, recr2_pl);
8363         mlxsw_sp_mp6_hash_init(mlxsw_sp, recr2_pl);
8364
8365         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl);
8366 }
8367 #else
8368 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
8369 {
8370         return 0;
8371 }
8372 #endif
8373
8374 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp)
8375 {
8376         char rdpm_pl[MLXSW_REG_RDPM_LEN];
8377         unsigned int i;
8378
8379         MLXSW_REG_ZERO(rdpm, rdpm_pl);
8380
8381         /* HW is determining switch priority based on DSCP-bits, but the
8382          * kernel is still doing that based on the ToS. Since there's a
8383          * mismatch in bits we need to make sure to translate the right
8384          * value ToS would observe, skipping the 2 least-significant ECN bits.
8385          */
8386         for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++)
8387                 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2));
8388
8389         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl);
8390 }
8391
8392 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
8393 {
8394         struct net *net = mlxsw_sp_net(mlxsw_sp);
8395         bool usp = net->ipv4.sysctl_ip_fwd_update_priority;
8396         char rgcr_pl[MLXSW_REG_RGCR_LEN];
8397         u64 max_rifs;
8398
8399         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
8400                 return -EIO;
8401         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
8402
8403         mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
8404         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
8405         mlxsw_reg_rgcr_usp_set(rgcr_pl, usp);
8406         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
8407 }
8408
8409 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
8410 {
8411         char rgcr_pl[MLXSW_REG_RGCR_LEN];
8412
8413         mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
8414         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
8415 }
8416
8417 static const struct mlxsw_sp_router_ll_ops mlxsw_sp_router_ll_basic_ops = {
8418         .ralta_write = mlxsw_sp_router_ll_basic_ralta_write,
8419         .ralst_write = mlxsw_sp_router_ll_basic_ralst_write,
8420         .raltb_write = mlxsw_sp_router_ll_basic_raltb_write,
8421         .fib_entry_op_ctx_size = sizeof(struct mlxsw_sp_fib_entry_op_ctx_basic),
8422         .fib_entry_pack = mlxsw_sp_router_ll_basic_fib_entry_pack,
8423         .fib_entry_act_remote_pack = mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack,
8424         .fib_entry_act_local_pack = mlxsw_sp_router_ll_basic_fib_entry_act_local_pack,
8425         .fib_entry_act_ip2me_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack,
8426         .fib_entry_act_ip2me_tun_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack,
8427         .fib_entry_commit = mlxsw_sp_router_ll_basic_fib_entry_commit,
8428         .fib_entry_is_committed = mlxsw_sp_router_ll_basic_fib_entry_is_committed,
8429 };
8430
8431 static int mlxsw_sp_router_ll_op_ctx_init(struct mlxsw_sp_router *router)
8432 {
8433         size_t max_size = 0;
8434         int i;
8435
8436         for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
8437                 size_t size = router->proto_ll_ops[i]->fib_entry_op_ctx_size;
8438
8439                 if (size > max_size)
8440                         max_size = size;
8441         }
8442         router->ll_op_ctx = kzalloc(sizeof(*router->ll_op_ctx) + max_size,
8443                                     GFP_KERNEL);
8444         if (!router->ll_op_ctx)
8445                 return -ENOMEM;
8446         INIT_LIST_HEAD(&router->ll_op_ctx->fib_entry_priv_list);
8447         return 0;
8448 }
8449
8450 static void mlxsw_sp_router_ll_op_ctx_fini(struct mlxsw_sp_router *router)
8451 {
8452         WARN_ON(!list_empty(&router->ll_op_ctx->fib_entry_priv_list));
8453         kfree(router->ll_op_ctx);
8454 }
8455
8456 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp,
8457                          struct netlink_ext_ack *extack)
8458 {
8459         struct mlxsw_sp_router *router;
8460         int err;
8461
8462         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
8463         if (!router)
8464                 return -ENOMEM;
8465         mutex_init(&router->lock);
8466         mlxsw_sp->router = router;
8467         router->mlxsw_sp = mlxsw_sp;
8468
8469         router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV4] = &mlxsw_sp_router_ll_basic_ops;
8470         router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_router_ll_basic_ops;
8471
8472         err = mlxsw_sp_router_ll_op_ctx_init(router);
8473         if (err)
8474                 goto err_ll_op_ctx_init;
8475
8476         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
8477         err = __mlxsw_sp_router_init(mlxsw_sp);
8478         if (err)
8479                 goto err_router_init;
8480
8481         err = mlxsw_sp_rifs_init(mlxsw_sp);
8482         if (err)
8483                 goto err_rifs_init;
8484
8485         err = mlxsw_sp_ipips_init(mlxsw_sp);
8486         if (err)
8487                 goto err_ipips_init;
8488
8489         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
8490                               &mlxsw_sp_nexthop_ht_params);
8491         if (err)
8492                 goto err_nexthop_ht_init;
8493
8494         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
8495                               &mlxsw_sp_nexthop_group_ht_params);
8496         if (err)
8497                 goto err_nexthop_group_ht_init;
8498
8499         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list);
8500         err = mlxsw_sp_lpm_init(mlxsw_sp);
8501         if (err)
8502                 goto err_lpm_init;
8503
8504         err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops);
8505         if (err)
8506                 goto err_mr_init;
8507
8508         err = mlxsw_sp_vrs_init(mlxsw_sp);
8509         if (err)
8510                 goto err_vrs_init;
8511
8512         err = mlxsw_sp_neigh_init(mlxsw_sp);
8513         if (err)
8514                 goto err_neigh_init;
8515
8516         err = mlxsw_sp_mp_hash_init(mlxsw_sp);
8517         if (err)
8518                 goto err_mp_hash_init;
8519
8520         err = mlxsw_sp_dscp_init(mlxsw_sp);
8521         if (err)
8522                 goto err_dscp_init;
8523
8524         INIT_WORK(&router->fib_event_work, mlxsw_sp_router_fib_event_work);
8525         INIT_LIST_HEAD(&router->fib_event_queue);
8526         spin_lock_init(&router->fib_event_queue_lock);
8527
8528         router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event;
8529         err = register_inetaddr_notifier(&router->inetaddr_nb);
8530         if (err)
8531                 goto err_register_inetaddr_notifier;
8532
8533         router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event;
8534         err = register_inet6addr_notifier(&router->inet6addr_nb);
8535         if (err)
8536                 goto err_register_inet6addr_notifier;
8537
8538         mlxsw_sp->router->netevent_nb.notifier_call =
8539                 mlxsw_sp_router_netevent_event;
8540         err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb);
8541         if (err)
8542                 goto err_register_netevent_notifier;
8543
8544         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
8545         err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp),
8546                                     &mlxsw_sp->router->fib_nb,
8547                                     mlxsw_sp_router_fib_dump_flush, extack);
8548         if (err)
8549                 goto err_register_fib_notifier;
8550
8551         return 0;
8552
8553 err_register_fib_notifier:
8554         unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
8555 err_register_netevent_notifier:
8556         unregister_inet6addr_notifier(&router->inet6addr_nb);
8557 err_register_inet6addr_notifier:
8558         unregister_inetaddr_notifier(&router->inetaddr_nb);
8559 err_register_inetaddr_notifier:
8560         mlxsw_core_flush_owq();
8561         WARN_ON(!list_empty(&router->fib_event_queue));
8562 err_dscp_init:
8563 err_mp_hash_init:
8564         mlxsw_sp_neigh_fini(mlxsw_sp);
8565 err_neigh_init:
8566         mlxsw_sp_vrs_fini(mlxsw_sp);
8567 err_vrs_init:
8568         mlxsw_sp_mr_fini(mlxsw_sp);
8569 err_mr_init:
8570         mlxsw_sp_lpm_fini(mlxsw_sp);
8571 err_lpm_init:
8572         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
8573 err_nexthop_group_ht_init:
8574         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
8575 err_nexthop_ht_init:
8576         mlxsw_sp_ipips_fini(mlxsw_sp);
8577 err_ipips_init:
8578         mlxsw_sp_rifs_fini(mlxsw_sp);
8579 err_rifs_init:
8580         __mlxsw_sp_router_fini(mlxsw_sp);
8581 err_router_init:
8582         mlxsw_sp_router_ll_op_ctx_fini(router);
8583 err_ll_op_ctx_init:
8584         mutex_destroy(&mlxsw_sp->router->lock);
8585         kfree(mlxsw_sp->router);
8586         return err;
8587 }
8588
8589 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
8590 {
8591         unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp),
8592                                 &mlxsw_sp->router->fib_nb);
8593         unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
8594         unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb);
8595         unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb);
8596         mlxsw_core_flush_owq();
8597         WARN_ON(!list_empty(&mlxsw_sp->router->fib_event_queue));
8598         mlxsw_sp_neigh_fini(mlxsw_sp);
8599         mlxsw_sp_vrs_fini(mlxsw_sp);
8600         mlxsw_sp_mr_fini(mlxsw_sp);
8601         mlxsw_sp_lpm_fini(mlxsw_sp);
8602         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
8603         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
8604         mlxsw_sp_ipips_fini(mlxsw_sp);
8605         mlxsw_sp_rifs_fini(mlxsw_sp);
8606         __mlxsw_sp_router_fini(mlxsw_sp);
8607         mlxsw_sp_router_ll_op_ctx_fini(mlxsw_sp->router);
8608         mutex_destroy(&mlxsw_sp->router->lock);
8609         kfree(mlxsw_sp->router);
8610 }