drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/types.h>
   6 #include <linux/rhashtable.h>
   7 #include <linux/bitops.h>
   8 #include <linux/in6.h>
   9 #include <linux/notifier.h>
  10 #include <linux/inetdevice.h>
  11 #include <linux/netdevice.h>
  12 #include <linux/if_bridge.h>
  13 #include <linux/socket.h>
  14 #include <linux/route.h>
  15 #include <linux/gcd.h>
  16 #include <linux/if_macvlan.h>
  17 #include <linux/refcount.h>
  18 #include <linux/jhash.h>
  19 #include <linux/net_namespace.h>
  20 #include <linux/mutex.h>
  21 #include <net/netevent.h>
  22 #include <net/neighbour.h>
  23 #include <net/arp.h>
  24 #include <net/ip_fib.h>
  25 #include <net/ip6_fib.h>
  26 #include <net/nexthop.h>
  27 #include <net/fib_rules.h>
  28 #include <net/ip_tunnels.h>
  29 #include <net/l3mdev.h>
  30 #include <net/addrconf.h>
  31 #include <net/ndisc.h>
  32 #include <net/ipv6.h>
  33 #include <net/fib_notifier.h>
  34 #include <net/switchdev.h>
  35
  36 #include "spectrum.h"
  37 #include "core.h"
  38 #include "reg.h"
  39 #include "spectrum_cnt.h"
  40 #include "spectrum_dpipe.h"
  41 #include "spectrum_ipip.h"
  42 #include "spectrum_mr.h"
  43 #include "spectrum_mr_tcam.h"
  44 #include "spectrum_router.h"
  45 #include "spectrum_span.h"
  46
  47 struct mlxsw_sp_fib;
  48 struct mlxsw_sp_vr;
  49 struct mlxsw_sp_lpm_tree;
  50 struct mlxsw_sp_rif_ops;
  51
  52 struct mlxsw_sp_rif {
  53         struct list_head nexthop_list;
  54         struct list_head neigh_list;
  55         struct net_device *dev; /* NULL for underlay RIF */
  56         struct mlxsw_sp_fid *fid;
  57         unsigned char addr[ETH_ALEN];
  58         int mtu;
  59         u16 rif_index;
  60         u16 vr_id;
  61         const struct mlxsw_sp_rif_ops *ops;
  62         struct mlxsw_sp *mlxsw_sp;
  63
  64         unsigned int counter_ingress;
  65         bool counter_ingress_valid;
  66         unsigned int counter_egress;
  67         bool counter_egress_valid;
  68 };
  69
  70 struct mlxsw_sp_rif_params {
  71         struct net_device *dev;
  72         union {
  73                 u16 system_port;
  74                 u16 lag_id;
  75         };
  76         u16 vid;
  77         bool lag;
  78 };
  79
  80 struct mlxsw_sp_rif_subport {
  81         struct mlxsw_sp_rif common;
  82         refcount_t ref_count;
  83         union {
  84                 u16 system_port;
  85                 u16 lag_id;
  86         };
  87         u16 vid;
  88         bool lag;
  89 };
  90
  91 struct mlxsw_sp_rif_ipip_lb {
  92         struct mlxsw_sp_rif common;
  93         struct mlxsw_sp_rif_ipip_lb_config lb_config;
  94         u16 ul_vr_id; /* Reserved for Spectrum-2. */
  95         u16 ul_rif_id; /* Reserved for Spectrum. */
  96 };
  97
  98 struct mlxsw_sp_rif_params_ipip_lb {
  99         struct mlxsw_sp_rif_params common;
 100         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 101 };
 102
 103 struct mlxsw_sp_rif_ops {
 104         enum mlxsw_sp_rif_type type;
 105         size_t rif_size;
 106
 107         void (*setup)(struct mlxsw_sp_rif *rif,
 108                       const struct mlxsw_sp_rif_params *params);
 109         int (*configure)(struct mlxsw_sp_rif *rif);
 110         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 111         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif,
 112                                          struct netlink_ext_ack *extack);
 113         void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac);
 114 };
 115
 116 static struct mlxsw_sp_rif *
 117 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 118                          const struct net_device *dev);
 119 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif);
 120 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree);
 121 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 122                                   struct mlxsw_sp_lpm_tree *lpm_tree);
 123 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 124                                      const struct mlxsw_sp_fib *fib,
 125                                      u8 tree_id);
 126 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 127                                        const struct mlxsw_sp_fib *fib);
 128
 129 static unsigned int *
 130 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 131                            enum mlxsw_sp_rif_counter_dir dir)
 132 {
 133         switch (dir) {
 134         case MLXSW_SP_RIF_COUNTER_EGRESS:
 135                 return &rif->counter_egress;
 136         case MLXSW_SP_RIF_COUNTER_INGRESS:
 137                 return &rif->counter_ingress;
 138         }
 139         return NULL;
 140 }
 141
 142 static bool
 143 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 144                                enum mlxsw_sp_rif_counter_dir dir)
 145 {
 146         switch (dir) {
 147         case MLXSW_SP_RIF_COUNTER_EGRESS:
 148                 return rif->counter_egress_valid;
 149         case MLXSW_SP_RIF_COUNTER_INGRESS:
 150                 return rif->counter_ingress_valid;
 151         }
 152         return false;
 153 }
 154
 155 static void
 156 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 157                                enum mlxsw_sp_rif_counter_dir dir,
 158                                bool valid)
 159 {
 160         switch (dir) {
 161         case MLXSW_SP_RIF_COUNTER_EGRESS:
 162                 rif->counter_egress_valid = valid;
 163                 break;
 164         case MLXSW_SP_RIF_COUNTER_INGRESS:
 165                 rif->counter_ingress_valid = valid;
 166                 break;
 167         }
 168 }
 169
 170 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 171                                      unsigned int counter_index, bool enable,
 172                                      enum mlxsw_sp_rif_counter_dir dir)
 173 {
 174         char ritr_pl[MLXSW_REG_RITR_LEN];
 175         bool is_egress = false;
 176         int err;
 177
 178         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 179                 is_egress = true;
 180         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 181         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 182         if (err)
 183                 return err;
 184
 185         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 186                                     is_egress);
 187         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 188 }
 189
 190 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 191                                    struct mlxsw_sp_rif *rif,
 192                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 193 {
 194         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 195         unsigned int *p_counter_index;
 196         bool valid;
 197         int err;
 198
 199         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 200         if (!valid)
 201                 return -EINVAL;
 202
 203         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 204         if (!p_counter_index)
 205                 return -EINVAL;
 206         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 207                              MLXSW_REG_RICNT_OPCODE_NOP);
 208         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 209         if (err)
 210                 return err;
 211         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 212         return 0;
 213 }
 214
 215 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 216                                       unsigned int counter_index)
 217 {
 218         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 219
 220         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 221                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 222         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 223 }
 224
 225 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 226                                struct mlxsw_sp_rif *rif,
 227                                enum mlxsw_sp_rif_counter_dir dir)
 228 {
 229         unsigned int *p_counter_index;
 230         int err;
 231
 232         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 233         if (!p_counter_index)
 234                 return -EINVAL;
 235         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 236                                      p_counter_index);
 237         if (err)
 238                 return err;
 239
 240         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 241         if (err)
 242                 goto err_counter_clear;
 243
 244         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 245                                         *p_counter_index, true, dir);
 246         if (err)
 247                 goto err_counter_edit;
 248         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 249         return 0;
 250
 251 err_counter_edit:
 252 err_counter_clear:
 253         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 254                               *p_counter_index);
 255         return err;
 256 }
 257
 258 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 259                                struct mlxsw_sp_rif *rif,
 260                                enum mlxsw_sp_rif_counter_dir dir)
 261 {
 262         unsigned int *p_counter_index;
 263
 264         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 265                 return;
 266
 267         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 268         if (WARN_ON(!p_counter_index))
 269                 return;
 270         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 271                                   *p_counter_index, false, dir);
 272         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 273                               *p_counter_index);
 274         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 275 }
 276
 277 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 278 {
 279         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 280         struct devlink *devlink;
 281
 282         devlink = priv_to_devlink(mlxsw_sp->core);
 283         if (!devlink_dpipe_table_counter_enabled(devlink,
 284                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 285                 return;
 286         mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 287 }
 288
 289 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 290 {
 291         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 292
 293         mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 294 }
 295
 296 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
 297
 298 struct mlxsw_sp_prefix_usage {
 299         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 300 };
 301
 302 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 303         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 304
 305 static bool
 306 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 307                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 308 {
 309         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 310 }
 311
 312 static void
 313 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 314                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 315 {
 316         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 317 }
 318
 319 static void
 320 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 321                           unsigned char prefix_len)
 322 {
 323         set_bit(prefix_len, prefix_usage->b);
 324 }
 325
 326 static void
 327 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 328                             unsigned char prefix_len)
 329 {
 330         clear_bit(prefix_len, prefix_usage->b);
 331 }
 332
 333 struct mlxsw_sp_fib_key {
 334         unsigned char addr[sizeof(struct in6_addr)];
 335         unsigned char prefix_len;
 336 };
 337
 338 enum mlxsw_sp_fib_entry_type {
 339         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 340         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 341         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 342         MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE,
 343         MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE,
 344
 345         /* This is a special case of local delivery, where a packet should be
 346          * decapsulated on reception. Note that there is no corresponding ENCAP,
 347          * because that's a type of next hop, not of FIB entry. (There can be
 348          * several next hops in a REMOTE entry, and some of them may be
 349          * encapsulating entries.)
 350          */
 351         MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
 352         MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP,
 353 };
 354
 355 struct mlxsw_sp_nexthop_group;
 356 struct mlxsw_sp_fib_entry;
 357
 358 struct mlxsw_sp_fib_node {
 359         struct mlxsw_sp_fib_entry *fib_entry;
 360         struct list_head list;
 361         struct rhash_head ht_node;
 362         struct mlxsw_sp_fib *fib;
 363         struct mlxsw_sp_fib_key key;
 364 };
 365
 366 struct mlxsw_sp_fib_entry_decap {
 367         struct mlxsw_sp_ipip_entry *ipip_entry;
 368         u32 tunnel_index;
 369 };
 370
 371 struct mlxsw_sp_fib_entry {
 372         struct mlxsw_sp_fib_node *fib_node;
 373         enum mlxsw_sp_fib_entry_type type;
 374         struct list_head nexthop_group_node;
 375         struct mlxsw_sp_nexthop_group *nh_group;
 376         struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
 377 };
 378
 379 struct mlxsw_sp_fib4_entry {
 380         struct mlxsw_sp_fib_entry common;
 381         u32 tb_id;
 382         u32 prio;
 383         u8 tos;
 384         u8 type;
 385 };
 386
 387 struct mlxsw_sp_fib6_entry {
 388         struct mlxsw_sp_fib_entry common;
 389         struct list_head rt6_list;
 390         unsigned int nrt6;
 391 };
 392
 393 struct mlxsw_sp_rt6 {
 394         struct list_head list;
 395         struct fib6_info *rt;
 396 };
 397
 398 struct mlxsw_sp_lpm_tree {
 399         u8 id; /* tree ID */
 400         unsigned int ref_count;
 401         enum mlxsw_sp_l3proto proto;
 402         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 403         struct mlxsw_sp_prefix_usage prefix_usage;
 404 };
 405
 406 struct mlxsw_sp_fib {
 407         struct rhashtable ht;
 408         struct list_head node_list;
 409         struct mlxsw_sp_vr *vr;
 410         struct mlxsw_sp_lpm_tree *lpm_tree;
 411         enum mlxsw_sp_l3proto proto;
 412         const struct mlxsw_sp_router_ll_ops *ll_ops;
 413 };
 414
 415 struct mlxsw_sp_vr {
 416         u16 id; /* virtual router ID */
 417         u32 tb_id; /* kernel fib table id */
 418         unsigned int rif_count;
 419         struct mlxsw_sp_fib *fib4;
 420         struct mlxsw_sp_fib *fib6;
 421         struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX];
 422         struct mlxsw_sp_rif *ul_rif;
 423         refcount_t ul_rif_refcnt;
 424 };
 425
 426 static int mlxsw_sp_router_ll_basic_ralta_write(struct mlxsw_sp *mlxsw_sp, char *xralta_pl)
 427 {
 428         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta),
 429                                xralta_pl + MLXSW_REG_XRALTA_RALTA_OFFSET);
 430 }
 431
 432 static int mlxsw_sp_router_ll_basic_ralst_write(struct mlxsw_sp *mlxsw_sp, char *xralst_pl)
 433 {
 434         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst),
 435                                xralst_pl + MLXSW_REG_XRALST_RALST_OFFSET);
 436 }
 437
 438 static int mlxsw_sp_router_ll_basic_raltb_write(struct mlxsw_sp *mlxsw_sp, char *xraltb_pl)
 439 {
 440         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
 441                                xraltb_pl + MLXSW_REG_XRALTB_RALTB_OFFSET);
 442 }
 443
 444 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 445
 446 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp,
 447                                                 struct mlxsw_sp_vr *vr,
 448                                                 enum mlxsw_sp_l3proto proto)
 449 {
 450         const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
 451         struct mlxsw_sp_lpm_tree *lpm_tree;
 452         struct mlxsw_sp_fib *fib;
 453         int err;
 454
 455         lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 456         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 457         if (!fib)
 458                 return ERR_PTR(-ENOMEM);
 459         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 460         if (err)
 461                 goto err_rhashtable_init;
 462         INIT_LIST_HEAD(&fib->node_list);
 463         fib->proto = proto;
 464         fib->vr = vr;
 465         fib->lpm_tree = lpm_tree;
 466         fib->ll_ops = ll_ops;
 467         mlxsw_sp_lpm_tree_hold(lpm_tree);
 468         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id);
 469         if (err)
 470                 goto err_lpm_tree_bind;
 471         return fib;
 472
 473 err_lpm_tree_bind:
 474         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 475 err_rhashtable_init:
 476         kfree(fib);
 477         return ERR_PTR(err);
 478 }
 479
 480 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp,
 481                                  struct mlxsw_sp_fib *fib)
 482 {
 483         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
 484         mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
 485         WARN_ON(!list_empty(&fib->node_list));
 486         rhashtable_destroy(&fib->ht);
 487         kfree(fib);
 488 }
 489
 490 static struct mlxsw_sp_lpm_tree *
 491 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 492 {
 493         static struct mlxsw_sp_lpm_tree *lpm_tree;
 494         int i;
 495
 496         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 497                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 498                 if (lpm_tree->ref_count == 0)
 499                         return lpm_tree;
 500         }
 501         return NULL;
 502 }
 503
 504 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 505                                    const struct mlxsw_sp_router_ll_ops *ll_ops,
 506                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 507 {
 508         char xralta_pl[MLXSW_REG_XRALTA_LEN];
 509
 510         mlxsw_reg_xralta_pack(xralta_pl, true,
 511                               (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 512                               lpm_tree->id);
 513         return ll_ops->ralta_write(mlxsw_sp, xralta_pl);
 514 }
 515
 516 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 517                                    const struct mlxsw_sp_router_ll_ops *ll_ops,
 518                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 519 {
 520         char xralta_pl[MLXSW_REG_XRALTA_LEN];
 521
 522         mlxsw_reg_xralta_pack(xralta_pl, false,
 523                               (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 524                               lpm_tree->id);
 525         ll_ops->ralta_write(mlxsw_sp, xralta_pl);
 526 }
 527
 528 static int
 529 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 530                                   const struct mlxsw_sp_router_ll_ops *ll_ops,
 531                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 532                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 533 {
 534         char xralst_pl[MLXSW_REG_XRALST_LEN];
 535         u8 root_bin = 0;
 536         u8 prefix;
 537         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 538
 539         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 540                 root_bin = prefix;
 541
 542         mlxsw_reg_xralst_pack(xralst_pl, root_bin, lpm_tree->id);
 543         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 544                 if (prefix == 0)
 545                         continue;
 546                 mlxsw_reg_xralst_bin_pack(xralst_pl, prefix, last_prefix,
 547                                           MLXSW_REG_RALST_BIN_NO_CHILD);
 548                 last_prefix = prefix;
 549         }
 550         return ll_ops->ralst_write(mlxsw_sp, xralst_pl);
 551 }
 552
 553 static struct mlxsw_sp_lpm_tree *
 554 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 555                          const struct mlxsw_sp_router_ll_ops *ll_ops,
 556                          struct mlxsw_sp_prefix_usage *prefix_usage,
 557                          enum mlxsw_sp_l3proto proto)
 558 {
 559         struct mlxsw_sp_lpm_tree *lpm_tree;
 560         int err;
 561
 562         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 563         if (!lpm_tree)
 564                 return ERR_PTR(-EBUSY);
 565         lpm_tree->proto = proto;
 566         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, ll_ops, lpm_tree);
 567         if (err)
 568                 return ERR_PTR(err);
 569
 570         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, ll_ops, prefix_usage, lpm_tree);
 571         if (err)
 572                 goto err_left_struct_set;
 573         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 574                sizeof(lpm_tree->prefix_usage));
 575         memset(&lpm_tree->prefix_ref_count, 0,
 576                sizeof(lpm_tree->prefix_ref_count));
 577         lpm_tree->ref_count = 1;
 578         return lpm_tree;
 579
 580 err_left_struct_set:
 581         mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree);
 582         return ERR_PTR(err);
 583 }
 584
 585 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 586                                       const struct mlxsw_sp_router_ll_ops *ll_ops,
 587                                       struct mlxsw_sp_lpm_tree *lpm_tree)
 588 {
 589         mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree);
 590 }
 591
 592 static struct mlxsw_sp_lpm_tree *
 593 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 594                       struct mlxsw_sp_prefix_usage *prefix_usage,
 595                       enum mlxsw_sp_l3proto proto)
 596 {
 597         const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
 598         struct mlxsw_sp_lpm_tree *lpm_tree;
 599         int i;
 600
 601         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 602                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 603                 if (lpm_tree->ref_count != 0 &&
 604                     lpm_tree->proto == proto &&
 605                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 606                                              prefix_usage)) {
 607                         mlxsw_sp_lpm_tree_hold(lpm_tree);
 608                         return lpm_tree;
 609                 }
 610         }
 611         return mlxsw_sp_lpm_tree_create(mlxsw_sp, ll_ops, prefix_usage, proto);
 612 }
 613
 614 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
 615 {
 616         lpm_tree->ref_count++;
 617 }
 618
 619 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 620                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 621 {
 622         const struct mlxsw_sp_router_ll_ops *ll_ops =
 623                                 mlxsw_sp->router->proto_ll_ops[lpm_tree->proto];
 624
 625         if (--lpm_tree->ref_count == 0)
 626                 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, ll_ops, lpm_tree);
 627 }
 628
 629 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 630
 631 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 632 {
 633         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
 634         struct mlxsw_sp_lpm_tree *lpm_tree;
 635         u64 max_trees;
 636         int err, i;
 637
 638         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 639                 return -EIO;
 640
 641         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 642         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 643         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 644                                              sizeof(struct mlxsw_sp_lpm_tree),
 645                                              GFP_KERNEL);
 646         if (!mlxsw_sp->router->lpm.trees)
 647                 return -ENOMEM;
 648
 649         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 650                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 651                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 652         }
 653
 654         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 655                                          MLXSW_SP_L3_PROTO_IPV4);
 656         if (IS_ERR(lpm_tree)) {
 657                 err = PTR_ERR(lpm_tree);
 658                 goto err_ipv4_tree_get;
 659         }
 660         mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree;
 661
 662         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 663                                          MLXSW_SP_L3_PROTO_IPV6);
 664         if (IS_ERR(lpm_tree)) {
 665                 err = PTR_ERR(lpm_tree);
 666                 goto err_ipv6_tree_get;
 667         }
 668         mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree;
 669
 670         return 0;
 671
 672 err_ipv6_tree_get:
 673         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
 674         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 675 err_ipv4_tree_get:
 676         kfree(mlxsw_sp->router->lpm.trees);
 677         return err;
 678 }
 679
 680 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 681 {
 682         struct mlxsw_sp_lpm_tree *lpm_tree;
 683
 684         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6];
 685         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 686
 687         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
 688         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 689
 690         kfree(mlxsw_sp->router->lpm.trees);
 691 }
 692
 693 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 694 {
 695         return !!vr->fib4 || !!vr->fib6 ||
 696                !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] ||
 697                !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
 698 }
 699
 700 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 701 {
 702         struct mlxsw_sp_vr *vr;
 703         int i;
 704
 705         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 706                 vr = &mlxsw_sp->router->vrs[i];
 707                 if (!mlxsw_sp_vr_is_used(vr))
 708                         return vr;
 709         }
 710         return NULL;
 711 }
 712
 713 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 714                                      const struct mlxsw_sp_fib *fib, u8 tree_id)
 715 {
 716         char xraltb_pl[MLXSW_REG_XRALTB_LEN];
 717
 718         mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id,
 719                               (enum mlxsw_reg_ralxx_protocol) fib->proto,
 720                               tree_id);
 721         return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
 722 }
 723
 724 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 725                                        const struct mlxsw_sp_fib *fib)
 726 {
 727         char xraltb_pl[MLXSW_REG_XRALTB_LEN];
 728
 729         /* Bind to tree 0 which is default */
 730         mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id,
 731                               (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 732         return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
 733 }
 734
 735 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 736 {
 737         /* For our purpose, squash main, default and local tables into one */
 738         if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT)
 739                 tb_id = RT_TABLE_MAIN;
 740         return tb_id;
 741 }
 742
 743 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 744                                             u32 tb_id)
 745 {
 746         struct mlxsw_sp_vr *vr;
 747         int i;
 748
 749         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 750
 751         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 752                 vr = &mlxsw_sp->router->vrs[i];
 753                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 754                         return vr;
 755         }
 756         return NULL;
 757 }
 758
 759 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
 760                                 u16 *vr_id)
 761 {
 762         struct mlxsw_sp_vr *vr;
 763         int err = 0;
 764
 765         mutex_lock(&mlxsw_sp->router->lock);
 766         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 767         if (!vr) {
 768                 err = -ESRCH;
 769                 goto out;
 770         }
 771         *vr_id = vr->id;
 772 out:
 773         mutex_unlock(&mlxsw_sp->router->lock);
 774         return err;
 775 }
 776
 777 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 778                                             enum mlxsw_sp_l3proto proto)
 779 {
 780         switch (proto) {
 781         case MLXSW_SP_L3_PROTO_IPV4:
 782                 return vr->fib4;
 783         case MLXSW_SP_L3_PROTO_IPV6:
 784                 return vr->fib6;
 785         }
 786         return NULL;
 787 }
 788
 789 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 790                                               u32 tb_id,
 791                                               struct netlink_ext_ack *extack)
 792 {
 793         struct mlxsw_sp_mr_table *mr4_table, *mr6_table;
 794         struct mlxsw_sp_fib *fib4;
 795         struct mlxsw_sp_fib *fib6;
 796         struct mlxsw_sp_vr *vr;
 797         int err;
 798
 799         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 800         if (!vr) {
 801                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers");
 802                 return ERR_PTR(-EBUSY);
 803         }
 804         fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
 805         if (IS_ERR(fib4))
 806                 return ERR_CAST(fib4);
 807         fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
 808         if (IS_ERR(fib6)) {
 809                 err = PTR_ERR(fib6);
 810                 goto err_fib6_create;
 811         }
 812         mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
 813                                              MLXSW_SP_L3_PROTO_IPV4);
 814         if (IS_ERR(mr4_table)) {
 815                 err = PTR_ERR(mr4_table);
 816                 goto err_mr4_table_create;
 817         }
 818         mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
 819                                              MLXSW_SP_L3_PROTO_IPV6);
 820         if (IS_ERR(mr6_table)) {
 821                 err = PTR_ERR(mr6_table);
 822                 goto err_mr6_table_create;
 823         }
 824
 825         vr->fib4 = fib4;
 826         vr->fib6 = fib6;
 827         vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table;
 828         vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table;
 829         vr->tb_id = tb_id;
 830         return vr;
 831
 832 err_mr6_table_create:
 833         mlxsw_sp_mr_table_destroy(mr4_table);
 834 err_mr4_table_create:
 835         mlxsw_sp_fib_destroy(mlxsw_sp, fib6);
 836 err_fib6_create:
 837         mlxsw_sp_fib_destroy(mlxsw_sp, fib4);
 838         return ERR_PTR(err);
 839 }
 840
 841 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
 842                                 struct mlxsw_sp_vr *vr)
 843 {
 844         mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]);
 845         vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL;
 846         mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]);
 847         vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL;
 848         mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6);
 849         vr->fib6 = NULL;
 850         mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4);
 851         vr->fib4 = NULL;
 852 }
 853
 854 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
 855                                            struct netlink_ext_ack *extack)
 856 {
 857         struct mlxsw_sp_vr *vr;
 858
 859         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 860         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 861         if (!vr)
 862                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack);
 863         return vr;
 864 }
 865
 866 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
 867 {
 868         if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
 869             list_empty(&vr->fib6->node_list) &&
 870             mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) &&
 871             mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]))
 872                 mlxsw_sp_vr_destroy(mlxsw_sp, vr);
 873 }
 874
 875 static bool
 876 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
 877                                     enum mlxsw_sp_l3proto proto, u8 tree_id)
 878 {
 879         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 880
 881         if (!mlxsw_sp_vr_is_used(vr))
 882                 return false;
 883         if (fib->lpm_tree->id == tree_id)
 884                 return true;
 885         return false;
 886 }
 887
 888 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 889                                         struct mlxsw_sp_fib *fib,
 890                                         struct mlxsw_sp_lpm_tree *new_tree)
 891 {
 892         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 893         int err;
 894
 895         fib->lpm_tree = new_tree;
 896         mlxsw_sp_lpm_tree_hold(new_tree);
 897         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 898         if (err)
 899                 goto err_tree_bind;
 900         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 901         return 0;
 902
 903 err_tree_bind:
 904         mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 905         fib->lpm_tree = old_tree;
 906         return err;
 907 }
 908
 909 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 910                                          struct mlxsw_sp_fib *fib,
 911                                          struct mlxsw_sp_lpm_tree *new_tree)
 912 {
 913         enum mlxsw_sp_l3proto proto = fib->proto;
 914         struct mlxsw_sp_lpm_tree *old_tree;
 915         u8 old_id, new_id = new_tree->id;
 916         struct mlxsw_sp_vr *vr;
 917         int i, err;
 918
 919         old_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 920         old_id = old_tree->id;
 921
 922         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 923                 vr = &mlxsw_sp->router->vrs[i];
 924                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
 925                         continue;
 926                 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 927                                                    mlxsw_sp_vr_fib(vr, proto),
 928                                                    new_tree);
 929                 if (err)
 930                         goto err_tree_replace;
 931         }
 932
 933         memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count,
 934                sizeof(new_tree->prefix_ref_count));
 935         mlxsw_sp->router->lpm.proto_trees[proto] = new_tree;
 936         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 937
 938         return 0;
 939
 940 err_tree_replace:
 941         for (i--; i >= 0; i--) {
 942                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
 943                         continue;
 944                 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 945                                              mlxsw_sp_vr_fib(vr, proto),
 946                                              old_tree);
 947         }
 948         return err;
 949 }
 950
 951 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 952 {
 953         struct mlxsw_sp_vr *vr;
 954         u64 max_vrs;
 955         int i;
 956
 957         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 958                 return -EIO;
 959
 960         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 961         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 962                                         GFP_KERNEL);
 963         if (!mlxsw_sp->router->vrs)
 964                 return -ENOMEM;
 965
 966         for (i = 0; i < max_vrs; i++) {
 967                 vr = &mlxsw_sp->router->vrs[i];
 968                 vr->id = i;
 969         }
 970
 971         return 0;
 972 }
 973
 974 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 975
 976 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 977 {
 978         /* At this stage we're guaranteed not to have new incoming
 979          * FIB notifications and the work queue is free from FIBs
 980          * sitting on top of mlxsw netdevs. However, we can still
 981          * have other FIBs queued. Flush the queue before flushing
 982          * the device's tables. No need for locks, as we're the only
 983          * writer.
 984          */
 985         mlxsw_core_flush_owq();
 986         mlxsw_sp_router_fib_flush(mlxsw_sp);
 987         kfree(mlxsw_sp->router->vrs);
 988 }
 989
 990 static struct net_device *
 991 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
 992 {
 993         struct ip_tunnel *tun = netdev_priv(ol_dev);
 994         struct net *net = dev_net(ol_dev);
 995
 996         return dev_get_by_index_rcu(net, tun->parms.link);
 997 }
 998
 999 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
1000 {
1001         struct net_device *d;
1002         u32 tb_id;
1003
1004         rcu_read_lock();
1005         d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1006         if (d)
1007                 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
1008         else
1009                 tb_id = RT_TABLE_MAIN;
1010         rcu_read_unlock();
1011
1012         return tb_id;
1013 }
1014
1015 static struct mlxsw_sp_rif *
1016 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
1017                     const struct mlxsw_sp_rif_params *params,
1018                     struct netlink_ext_ack *extack);
1019
1020 static struct mlxsw_sp_rif_ipip_lb *
1021 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
1022                                 enum mlxsw_sp_ipip_type ipipt,
1023                                 struct net_device *ol_dev,
1024                                 struct netlink_ext_ack *extack)
1025 {
1026         struct mlxsw_sp_rif_params_ipip_lb lb_params;
1027         const struct mlxsw_sp_ipip_ops *ipip_ops;
1028         struct mlxsw_sp_rif *rif;
1029
1030         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1031         lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
1032                 .common.dev = ol_dev,
1033                 .common.lag = false,
1034                 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
1035         };
1036
1037         rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack);
1038         if (IS_ERR(rif))
1039                 return ERR_CAST(rif);
1040         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
1041 }
1042
1043 static struct mlxsw_sp_ipip_entry *
1044 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
1045                           enum mlxsw_sp_ipip_type ipipt,
1046                           struct net_device *ol_dev)
1047 {
1048         const struct mlxsw_sp_ipip_ops *ipip_ops;
1049         struct mlxsw_sp_ipip_entry *ipip_entry;
1050         struct mlxsw_sp_ipip_entry *ret = NULL;
1051
1052         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1053         ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
1054         if (!ipip_entry)
1055                 return ERR_PTR(-ENOMEM);
1056
1057         ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
1058                                                             ol_dev, NULL);
1059         if (IS_ERR(ipip_entry->ol_lb)) {
1060                 ret = ERR_CAST(ipip_entry->ol_lb);
1061                 goto err_ol_ipip_lb_create;
1062         }
1063
1064         ipip_entry->ipipt = ipipt;
1065         ipip_entry->ol_dev = ol_dev;
1066
1067         switch (ipip_ops->ul_proto) {
1068         case MLXSW_SP_L3_PROTO_IPV4:
1069                 ipip_entry->parms4 = mlxsw_sp_ipip_netdev_parms4(ol_dev);
1070                 break;
1071         case MLXSW_SP_L3_PROTO_IPV6:
1072                 WARN_ON(1);
1073                 break;
1074         }
1075
1076         return ipip_entry;
1077
1078 err_ol_ipip_lb_create:
1079         kfree(ipip_entry);
1080         return ret;
1081 }
1082
1083 static void
1084 mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry)
1085 {
1086         mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
1087         kfree(ipip_entry);
1088 }
1089
1090 static bool
1091 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1092                                   const enum mlxsw_sp_l3proto ul_proto,
1093                                   union mlxsw_sp_l3addr saddr,
1094                                   u32 ul_tb_id,
1095                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1096 {
1097         u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1098         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1099         union mlxsw_sp_l3addr tun_saddr;
1100
1101         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1102                 return false;
1103
1104         tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1105         return tun_ul_tb_id == ul_tb_id &&
1106                mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1107 }
1108
1109 static int
1110 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1111                               struct mlxsw_sp_fib_entry *fib_entry,
1112                               struct mlxsw_sp_ipip_entry *ipip_entry)
1113 {
1114         u32 tunnel_index;
1115         int err;
1116
1117         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1118                                   1, &tunnel_index);
1119         if (err)
1120                 return err;
1121
1122         ipip_entry->decap_fib_entry = fib_entry;
1123         fib_entry->decap.ipip_entry = ipip_entry;
1124         fib_entry->decap.tunnel_index = tunnel_index;
1125         return 0;
1126 }
1127
1128 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1129                                           struct mlxsw_sp_fib_entry *fib_entry)
1130 {
1131         /* Unlink this node from the IPIP entry that it's the decap entry of. */
1132         fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1133         fib_entry->decap.ipip_entry = NULL;
1134         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1135                            1, fib_entry->decap.tunnel_index);
1136 }
1137
1138 static struct mlxsw_sp_fib_node *
1139 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1140                          size_t addr_len, unsigned char prefix_len);
1141 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1142                                      struct mlxsw_sp_fib_entry *fib_entry);
1143
1144 static void
1145 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1146                                  struct mlxsw_sp_ipip_entry *ipip_entry)
1147 {
1148         struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1149
1150         mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1151         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1152
1153         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1154 }
1155
1156 static void
1157 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1158                                   struct mlxsw_sp_ipip_entry *ipip_entry,
1159                                   struct mlxsw_sp_fib_entry *decap_fib_entry)
1160 {
1161         if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1162                                           ipip_entry))
1163                 return;
1164         decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1165
1166         if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1167                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1168 }
1169
1170 static struct mlxsw_sp_fib_entry *
1171 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
1172                                      enum mlxsw_sp_l3proto proto,
1173                                      const union mlxsw_sp_l3addr *addr,
1174                                      enum mlxsw_sp_fib_entry_type type)
1175 {
1176         struct mlxsw_sp_fib_node *fib_node;
1177         unsigned char addr_prefix_len;
1178         struct mlxsw_sp_fib *fib;
1179         struct mlxsw_sp_vr *vr;
1180         const void *addrp;
1181         size_t addr_len;
1182         u32 addr4;
1183
1184         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
1185         if (!vr)
1186                 return NULL;
1187         fib = mlxsw_sp_vr_fib(vr, proto);
1188
1189         switch (proto) {
1190         case MLXSW_SP_L3_PROTO_IPV4:
1191                 addr4 = be32_to_cpu(addr->addr4);
1192                 addrp = &addr4;
1193                 addr_len = 4;
1194                 addr_prefix_len = 32;
1195                 break;
1196         case MLXSW_SP_L3_PROTO_IPV6:
1197         default:
1198                 WARN_ON(1);
1199                 return NULL;
1200         }
1201
1202         fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len,
1203                                             addr_prefix_len);
1204         if (!fib_node || fib_node->fib_entry->type != type)
1205                 return NULL;
1206
1207         return fib_node->fib_entry;
1208 }
1209
1210 /* Given an IPIP entry, find the corresponding decap route. */
1211 static struct mlxsw_sp_fib_entry *
1212 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1213                                struct mlxsw_sp_ipip_entry *ipip_entry)
1214 {
1215         static struct mlxsw_sp_fib_node *fib_node;
1216         const struct mlxsw_sp_ipip_ops *ipip_ops;
1217         unsigned char saddr_prefix_len;
1218         union mlxsw_sp_l3addr saddr;
1219         struct mlxsw_sp_fib *ul_fib;
1220         struct mlxsw_sp_vr *ul_vr;
1221         const void *saddrp;
1222         size_t saddr_len;
1223         u32 ul_tb_id;
1224         u32 saddr4;
1225
1226         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1227
1228         ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1229         ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1230         if (!ul_vr)
1231                 return NULL;
1232
1233         ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1234         saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1235                                            ipip_entry->ol_dev);
1236
1237         switch (ipip_ops->ul_proto) {
1238         case MLXSW_SP_L3_PROTO_IPV4:
1239                 saddr4 = be32_to_cpu(saddr.addr4);
1240                 saddrp = &saddr4;
1241                 saddr_len = 4;
1242                 saddr_prefix_len = 32;
1243                 break;
1244         default:
1245                 WARN_ON(1);
1246                 return NULL;
1247         }
1248
1249         fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1250                                             saddr_prefix_len);
1251         if (!fib_node ||
1252             fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1253                 return NULL;
1254
1255         return fib_node->fib_entry;
1256 }
1257
1258 static struct mlxsw_sp_ipip_entry *
1259 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp,
1260                            enum mlxsw_sp_ipip_type ipipt,
1261                            struct net_device *ol_dev)
1262 {
1263         struct mlxsw_sp_ipip_entry *ipip_entry;
1264
1265         ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1266         if (IS_ERR(ipip_entry))
1267                 return ipip_entry;
1268
1269         list_add_tail(&ipip_entry->ipip_list_node,
1270                       &mlxsw_sp->router->ipip_list);
1271
1272         return ipip_entry;
1273 }
1274
1275 static void
1276 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1277                             struct mlxsw_sp_ipip_entry *ipip_entry)
1278 {
1279         list_del(&ipip_entry->ipip_list_node);
1280         mlxsw_sp_ipip_entry_dealloc(ipip_entry);
1281 }
1282
1283 static bool
1284 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1285                                   const struct net_device *ul_dev,
1286                                   enum mlxsw_sp_l3proto ul_proto,
1287                                   union mlxsw_sp_l3addr ul_dip,
1288                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1289 {
1290         u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1291         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1292
1293         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1294                 return false;
1295
1296         return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1297                                                  ul_tb_id, ipip_entry);
1298 }
1299
1300 /* Given decap parameters, find the corresponding IPIP entry. */
1301 static struct mlxsw_sp_ipip_entry *
1302 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp,
1303                                   const struct net_device *ul_dev,
1304                                   enum mlxsw_sp_l3proto ul_proto,
1305                                   union mlxsw_sp_l3addr ul_dip)
1306 {
1307         struct mlxsw_sp_ipip_entry *ipip_entry;
1308
1309         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1310                             ipip_list_node)
1311                 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1312                                                       ul_proto, ul_dip,
1313                                                       ipip_entry))
1314                         return ipip_entry;
1315
1316         return NULL;
1317 }
1318
1319 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
1320                                       const struct net_device *dev,
1321                                       enum mlxsw_sp_ipip_type *p_type)
1322 {
1323         struct mlxsw_sp_router *router = mlxsw_sp->router;
1324         const struct mlxsw_sp_ipip_ops *ipip_ops;
1325         enum mlxsw_sp_ipip_type ipipt;
1326
1327         for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
1328                 ipip_ops = router->ipip_ops_arr[ipipt];
1329                 if (dev->type == ipip_ops->dev_type) {
1330                         if (p_type)
1331                                 *p_type = ipipt;
1332                         return true;
1333                 }
1334         }
1335         return false;
1336 }
1337
1338 bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp,
1339                                 const struct net_device *dev)
1340 {
1341         return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
1342 }
1343
1344 static struct mlxsw_sp_ipip_entry *
1345 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp,
1346                                    const struct net_device *ol_dev)
1347 {
1348         struct mlxsw_sp_ipip_entry *ipip_entry;
1349
1350         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1351                             ipip_list_node)
1352                 if (ipip_entry->ol_dev == ol_dev)
1353                         return ipip_entry;
1354
1355         return NULL;
1356 }
1357
1358 static struct mlxsw_sp_ipip_entry *
1359 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp,
1360                                    const struct net_device *ul_dev,
1361                                    struct mlxsw_sp_ipip_entry *start)
1362 {
1363         struct mlxsw_sp_ipip_entry *ipip_entry;
1364
1365         ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list,
1366                                         ipip_list_node);
1367         list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list,
1368                                      ipip_list_node) {
1369                 struct net_device *ol_dev = ipip_entry->ol_dev;
1370                 struct net_device *ipip_ul_dev;
1371
1372                 rcu_read_lock();
1373                 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1374                 rcu_read_unlock();
1375
1376                 if (ipip_ul_dev == ul_dev)
1377                         return ipip_entry;
1378         }
1379
1380         return NULL;
1381 }
1382
1383 bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp,
1384                                 const struct net_device *dev)
1385 {
1386         bool is_ipip_ul;
1387
1388         mutex_lock(&mlxsw_sp->router->lock);
1389         is_ipip_ul = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL);
1390         mutex_unlock(&mlxsw_sp->router->lock);
1391
1392         return is_ipip_ul;
1393 }
1394
1395 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp,
1396                                                 const struct net_device *ol_dev,
1397                                                 enum mlxsw_sp_ipip_type ipipt)
1398 {
1399         const struct mlxsw_sp_ipip_ops *ops
1400                 = mlxsw_sp->router->ipip_ops_arr[ipipt];
1401
1402         /* For deciding whether decap should be offloaded, we don't care about
1403          * overlay protocol, so ask whether either one is supported.
1404          */
1405         return ops->can_offload(mlxsw_sp, ol_dev, MLXSW_SP_L3_PROTO_IPV4) ||
1406                ops->can_offload(mlxsw_sp, ol_dev, MLXSW_SP_L3_PROTO_IPV6);
1407 }
1408
1409 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp,
1410                                                 struct net_device *ol_dev)
1411 {
1412         enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX;
1413         struct mlxsw_sp_ipip_entry *ipip_entry;
1414         enum mlxsw_sp_l3proto ul_proto;
1415         union mlxsw_sp_l3addr saddr;
1416         u32 ul_tb_id;
1417
1418         mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt);
1419         if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) {
1420                 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1421                 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto;
1422                 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1423                 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1424                                                           saddr, ul_tb_id,
1425                                                           NULL)) {
1426                         ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt,
1427                                                                 ol_dev);
1428                         if (IS_ERR(ipip_entry))
1429                                 return PTR_ERR(ipip_entry);
1430                 }
1431         }
1432
1433         return 0;
1434 }
1435
1436 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp,
1437                                                    struct net_device *ol_dev)
1438 {
1439         struct mlxsw_sp_ipip_entry *ipip_entry;
1440
1441         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1442         if (ipip_entry)
1443                 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1444 }
1445
1446 static void
1447 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1448                                 struct mlxsw_sp_ipip_entry *ipip_entry)
1449 {
1450         struct mlxsw_sp_fib_entry *decap_fib_entry;
1451
1452         decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1453         if (decap_fib_entry)
1454                 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1455                                                   decap_fib_entry);
1456 }
1457
1458 static int
1459 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id,
1460                         u16 ul_rif_id, bool enable)
1461 {
1462         struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
1463         struct mlxsw_sp_rif *rif = &lb_rif->common;
1464         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
1465         char ritr_pl[MLXSW_REG_RITR_LEN];
1466         u32 saddr4;
1467
1468         switch (lb_cf.ul_protocol) {
1469         case MLXSW_SP_L3_PROTO_IPV4:
1470                 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
1471                 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
1472                                     rif->rif_index, rif->vr_id, rif->dev->mtu);
1473                 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
1474                             MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
1475                             ul_vr_id, ul_rif_id, saddr4, lb_cf.okey);
1476                 break;
1477
1478         case MLXSW_SP_L3_PROTO_IPV6:
1479                 return -EAFNOSUPPORT;
1480         }
1481
1482         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
1483 }
1484
1485 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp,
1486                                                  struct net_device *ol_dev)
1487 {
1488         struct mlxsw_sp_ipip_entry *ipip_entry;
1489         struct mlxsw_sp_rif_ipip_lb *lb_rif;
1490         int err = 0;
1491
1492         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1493         if (ipip_entry) {
1494                 lb_rif = ipip_entry->ol_lb;
1495                 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id,
1496                                               lb_rif->ul_rif_id, true);
1497                 if (err)
1498                         goto out;
1499                 lb_rif->common.mtu = ol_dev->mtu;
1500         }
1501
1502 out:
1503         return err;
1504 }
1505
1506 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1507                                                 struct net_device *ol_dev)
1508 {
1509         struct mlxsw_sp_ipip_entry *ipip_entry;
1510
1511         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1512         if (ipip_entry)
1513                 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1514 }
1515
1516 static void
1517 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1518                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1519 {
1520         if (ipip_entry->decap_fib_entry)
1521                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1522 }
1523
1524 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1525                                                   struct net_device *ol_dev)
1526 {
1527         struct mlxsw_sp_ipip_entry *ipip_entry;
1528
1529         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1530         if (ipip_entry)
1531                 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1532 }
1533
1534 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
1535                                          struct mlxsw_sp_rif *old_rif,
1536                                          struct mlxsw_sp_rif *new_rif);
1537 static int
1538 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp,
1539                                  struct mlxsw_sp_ipip_entry *ipip_entry,
1540                                  bool keep_encap,
1541                                  struct netlink_ext_ack *extack)
1542 {
1543         struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb;
1544         struct mlxsw_sp_rif_ipip_lb *new_lb_rif;
1545
1546         new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp,
1547                                                      ipip_entry->ipipt,
1548                                                      ipip_entry->ol_dev,
1549                                                      extack);
1550         if (IS_ERR(new_lb_rif))
1551                 return PTR_ERR(new_lb_rif);
1552         ipip_entry->ol_lb = new_lb_rif;
1553
1554         if (keep_encap)
1555                 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common,
1556                                              &new_lb_rif->common);
1557
1558         mlxsw_sp_rif_destroy(&old_lb_rif->common);
1559
1560         return 0;
1561 }
1562
1563 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
1564                                         struct mlxsw_sp_rif *rif);
1565
1566 /**
1567  * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry.
1568  * @mlxsw_sp: mlxsw_sp.
1569  * @ipip_entry: IPIP entry.
1570  * @recreate_loopback: Recreates the associated loopback RIF.
1571  * @keep_encap: Updates next hops that use the tunnel netdevice. This is only
1572  *              relevant when recreate_loopback is true.
1573  * @update_nexthops: Updates next hops, keeping the current loopback RIF. This
1574  *                   is only relevant when recreate_loopback is false.
1575  * @extack: extack.
1576  *
1577  * Return: Non-zero value on failure.
1578  */
1579 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp,
1580                                         struct mlxsw_sp_ipip_entry *ipip_entry,
1581                                         bool recreate_loopback,
1582                                         bool keep_encap,
1583                                         bool update_nexthops,
1584                                         struct netlink_ext_ack *extack)
1585 {
1586         int err;
1587
1588         /* RIFs can't be edited, so to update loopback, we need to destroy and
1589          * recreate it. That creates a window of opportunity where RALUE and
1590          * RATR registers end up referencing a RIF that's already gone. RATRs
1591          * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care
1592          * of RALUE, demote the decap route back.
1593          */
1594         if (ipip_entry->decap_fib_entry)
1595                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1596
1597         if (recreate_loopback) {
1598                 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry,
1599                                                        keep_encap, extack);
1600                 if (err)
1601                         return err;
1602         } else if (update_nexthops) {
1603                 mlxsw_sp_nexthop_rif_update(mlxsw_sp,
1604                                             &ipip_entry->ol_lb->common);
1605         }
1606
1607         if (ipip_entry->ol_dev->flags & IFF_UP)
1608                 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1609
1610         return 0;
1611 }
1612
1613 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp,
1614                                                 struct net_device *ol_dev,
1615                                                 struct netlink_ext_ack *extack)
1616 {
1617         struct mlxsw_sp_ipip_entry *ipip_entry =
1618                 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1619
1620         if (!ipip_entry)
1621                 return 0;
1622
1623         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1624                                                    true, false, false, extack);
1625 }
1626
1627 static int
1628 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp,
1629                                      struct mlxsw_sp_ipip_entry *ipip_entry,
1630                                      struct net_device *ul_dev,
1631                                      bool *demote_this,
1632                                      struct netlink_ext_ack *extack)
1633 {
1634         u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1635         enum mlxsw_sp_l3proto ul_proto;
1636         union mlxsw_sp_l3addr saddr;
1637
1638         /* Moving underlay to a different VRF might cause local address
1639          * conflict, and the conflicting tunnels need to be demoted.
1640          */
1641         ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1642         saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1643         if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1644                                                  saddr, ul_tb_id,
1645                                                  ipip_entry)) {
1646                 *demote_this = true;
1647                 return 0;
1648         }
1649
1650         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1651                                                    true, true, false, extack);
1652 }
1653
1654 static int
1655 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp,
1656                                     struct mlxsw_sp_ipip_entry *ipip_entry,
1657                                     struct net_device *ul_dev)
1658 {
1659         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1660                                                    false, false, true, NULL);
1661 }
1662
1663 static int
1664 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp,
1665                                       struct mlxsw_sp_ipip_entry *ipip_entry,
1666                                       struct net_device *ul_dev)
1667 {
1668         /* A down underlay device causes encapsulated packets to not be
1669          * forwarded, but decap still works. So refresh next hops without
1670          * touching anything else.
1671          */
1672         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1673                                                    false, false, true, NULL);
1674 }
1675
1676 static int
1677 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp,
1678                                         struct net_device *ol_dev,
1679                                         struct netlink_ext_ack *extack)
1680 {
1681         const struct mlxsw_sp_ipip_ops *ipip_ops;
1682         struct mlxsw_sp_ipip_entry *ipip_entry;
1683         int err;
1684
1685         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1686         if (!ipip_entry)
1687                 /* A change might make a tunnel eligible for offloading, but
1688                  * that is currently not implemented. What falls to slow path
1689                  * stays there.
1690                  */
1691                 return 0;
1692
1693         /* A change might make a tunnel not eligible for offloading. */
1694         if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev,
1695                                                  ipip_entry->ipipt)) {
1696                 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1697                 return 0;
1698         }
1699
1700         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1701         err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack);
1702         return err;
1703 }
1704
1705 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp,
1706                                        struct mlxsw_sp_ipip_entry *ipip_entry)
1707 {
1708         struct net_device *ol_dev = ipip_entry->ol_dev;
1709
1710         if (ol_dev->flags & IFF_UP)
1711                 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1712         mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1713 }
1714
1715 /* The configuration where several tunnels have the same local address in the
1716  * same underlay table needs special treatment in the HW. That is currently not
1717  * implemented in the driver. This function finds and demotes the first tunnel
1718  * with a given source address, except the one passed in in the argument
1719  * `except'.
1720  */
1721 bool
1722 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp,
1723                                      enum mlxsw_sp_l3proto ul_proto,
1724                                      union mlxsw_sp_l3addr saddr,
1725                                      u32 ul_tb_id,
1726                                      const struct mlxsw_sp_ipip_entry *except)
1727 {
1728         struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1729
1730         list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1731                                  ipip_list_node) {
1732                 if (ipip_entry != except &&
1733                     mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1734                                                       ul_tb_id, ipip_entry)) {
1735                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1736                         return true;
1737                 }
1738         }
1739
1740         return false;
1741 }
1742
1743 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp,
1744                                                      struct net_device *ul_dev)
1745 {
1746         struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1747
1748         list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1749                                  ipip_list_node) {
1750                 struct net_device *ol_dev = ipip_entry->ol_dev;
1751                 struct net_device *ipip_ul_dev;
1752
1753                 rcu_read_lock();
1754                 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1755                 rcu_read_unlock();
1756                 if (ipip_ul_dev == ul_dev)
1757                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1758         }
1759 }
1760
1761 int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp,
1762                                      struct net_device *ol_dev,
1763                                      unsigned long event,
1764                                      struct netdev_notifier_info *info)
1765 {
1766         struct netdev_notifier_changeupper_info *chup;
1767         struct netlink_ext_ack *extack;
1768         int err = 0;
1769
1770         mutex_lock(&mlxsw_sp->router->lock);
1771         switch (event) {
1772         case NETDEV_REGISTER:
1773                 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev);
1774                 break;
1775         case NETDEV_UNREGISTER:
1776                 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev);
1777                 break;
1778         case NETDEV_UP:
1779                 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev);
1780                 break;
1781         case NETDEV_DOWN:
1782                 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev);
1783                 break;
1784         case NETDEV_CHANGEUPPER:
1785                 chup = container_of(info, typeof(*chup), info);
1786                 extack = info->extack;
1787                 if (netif_is_l3_master(chup->upper_dev))
1788                         err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp,
1789                                                                    ol_dev,
1790                                                                    extack);
1791                 break;
1792         case NETDEV_CHANGE:
1793                 extack = info->extack;
1794                 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp,
1795                                                               ol_dev, extack);
1796                 break;
1797         case NETDEV_CHANGEMTU:
1798                 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev);
1799                 break;
1800         }
1801         mutex_unlock(&mlxsw_sp->router->lock);
1802         return err;
1803 }
1804
1805 static int
1806 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1807                                    struct mlxsw_sp_ipip_entry *ipip_entry,
1808                                    struct net_device *ul_dev,
1809                                    bool *demote_this,
1810                                    unsigned long event,
1811                                    struct netdev_notifier_info *info)
1812 {
1813         struct netdev_notifier_changeupper_info *chup;
1814         struct netlink_ext_ack *extack;
1815
1816         switch (event) {
1817         case NETDEV_CHANGEUPPER:
1818                 chup = container_of(info, typeof(*chup), info);
1819                 extack = info->extack;
1820                 if (netif_is_l3_master(chup->upper_dev))
1821                         return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp,
1822                                                                     ipip_entry,
1823                                                                     ul_dev,
1824                                                                     demote_this,
1825                                                                     extack);
1826                 break;
1827
1828         case NETDEV_UP:
1829                 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry,
1830                                                            ul_dev);
1831         case NETDEV_DOWN:
1832                 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp,
1833                                                              ipip_entry,
1834                                                              ul_dev);
1835         }
1836         return 0;
1837 }
1838
1839 int
1840 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1841                                  struct net_device *ul_dev,
1842                                  unsigned long event,
1843                                  struct netdev_notifier_info *info)
1844 {
1845         struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1846         int err = 0;
1847
1848         mutex_lock(&mlxsw_sp->router->lock);
1849         while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp,
1850                                                                 ul_dev,
1851                                                                 ipip_entry))) {
1852                 struct mlxsw_sp_ipip_entry *prev;
1853                 bool demote_this = false;
1854
1855                 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry,
1856                                                          ul_dev, &demote_this,
1857                                                          event, info);
1858                 if (err) {
1859                         mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp,
1860                                                                  ul_dev);
1861                         break;
1862                 }
1863
1864                 if (demote_this) {
1865                         if (list_is_first(&ipip_entry->ipip_list_node,
1866                                           &mlxsw_sp->router->ipip_list))
1867                                 prev = NULL;
1868                         else
1869                                 /* This can't be cached from previous iteration,
1870                                  * because that entry could be gone now.
1871                                  */
1872                                 prev = list_prev_entry(ipip_entry,
1873                                                        ipip_list_node);
1874                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1875                         ipip_entry = prev;
1876                 }
1877         }
1878         mutex_unlock(&mlxsw_sp->router->lock);
1879
1880         return err;
1881 }
1882
1883 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
1884                                       enum mlxsw_sp_l3proto ul_proto,
1885                                       const union mlxsw_sp_l3addr *ul_sip,
1886                                       u32 tunnel_index)
1887 {
1888         enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1889         struct mlxsw_sp_router *router = mlxsw_sp->router;
1890         struct mlxsw_sp_fib_entry *fib_entry;
1891         int err = 0;
1892
1893         mutex_lock(&mlxsw_sp->router->lock);
1894
1895         if (WARN_ON_ONCE(router->nve_decap_config.valid)) {
1896                 err = -EINVAL;
1897                 goto out;
1898         }
1899
1900         router->nve_decap_config.ul_tb_id = ul_tb_id;
1901         router->nve_decap_config.tunnel_index = tunnel_index;
1902         router->nve_decap_config.ul_proto = ul_proto;
1903         router->nve_decap_config.ul_sip = *ul_sip;
1904         router->nve_decap_config.valid = true;
1905
1906         /* It is valid to create a tunnel with a local IP and only later
1907          * assign this IP address to a local interface
1908          */
1909         fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
1910                                                          ul_proto, ul_sip,
1911                                                          type);
1912         if (!fib_entry)
1913                 goto out;
1914
1915         fib_entry->decap.tunnel_index = tunnel_index;
1916         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
1917
1918         err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1919         if (err)
1920                 goto err_fib_entry_update;
1921
1922         goto out;
1923
1924 err_fib_entry_update:
1925         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1926         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1927 out:
1928         mutex_unlock(&mlxsw_sp->router->lock);
1929         return err;
1930 }
1931
1932 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
1933                                       enum mlxsw_sp_l3proto ul_proto,
1934                                       const union mlxsw_sp_l3addr *ul_sip)
1935 {
1936         enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
1937         struct mlxsw_sp_router *router = mlxsw_sp->router;
1938         struct mlxsw_sp_fib_entry *fib_entry;
1939
1940         mutex_lock(&mlxsw_sp->router->lock);
1941
1942         if (WARN_ON_ONCE(!router->nve_decap_config.valid))
1943                 goto out;
1944
1945         router->nve_decap_config.valid = false;
1946
1947         fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
1948                                                          ul_proto, ul_sip,
1949                                                          type);
1950         if (!fib_entry)
1951                 goto out;
1952
1953         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1954         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1955 out:
1956         mutex_unlock(&mlxsw_sp->router->lock);
1957 }
1958
1959 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp,
1960                                          u32 ul_tb_id,
1961                                          enum mlxsw_sp_l3proto ul_proto,
1962                                          const union mlxsw_sp_l3addr *ul_sip)
1963 {
1964         struct mlxsw_sp_router *router = mlxsw_sp->router;
1965
1966         return router->nve_decap_config.valid &&
1967                router->nve_decap_config.ul_tb_id == ul_tb_id &&
1968                router->nve_decap_config.ul_proto == ul_proto &&
1969                !memcmp(&router->nve_decap_config.ul_sip, ul_sip,
1970                        sizeof(*ul_sip));
1971 }
1972
1973 struct mlxsw_sp_neigh_key {
1974         struct neighbour *n;
1975 };
1976
1977 struct mlxsw_sp_neigh_entry {
1978         struct list_head rif_list_node;
1979         struct rhash_head ht_node;
1980         struct mlxsw_sp_neigh_key key;
1981         u16 rif;
1982         bool connected;
1983         unsigned char ha[ETH_ALEN];
1984         struct list_head nexthop_list; /* list of nexthops using
1985                                         * this neigh entry
1986                                         */
1987         struct list_head nexthop_neighs_list_node;
1988         unsigned int counter_index;
1989         bool counter_valid;
1990 };
1991
1992 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
1993         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
1994         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
1995         .key_len = sizeof(struct mlxsw_sp_neigh_key),
1996 };
1997
1998 struct mlxsw_sp_neigh_entry *
1999 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
2000                         struct mlxsw_sp_neigh_entry *neigh_entry)
2001 {
2002         if (!neigh_entry) {
2003                 if (list_empty(&rif->neigh_list))
2004                         return NULL;
2005                 else
2006                         return list_first_entry(&rif->neigh_list,
2007                                                 typeof(*neigh_entry),
2008                                                 rif_list_node);
2009         }
2010         if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list))
2011                 return NULL;
2012         return list_next_entry(neigh_entry, rif_list_node);
2013 }
2014
2015 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
2016 {
2017         return neigh_entry->key.n->tbl->family;
2018 }
2019
2020 unsigned char *
2021 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
2022 {
2023         return neigh_entry->ha;
2024 }
2025
2026 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2027 {
2028         struct neighbour *n;
2029
2030         n = neigh_entry->key.n;
2031         return ntohl(*((__be32 *) n->primary_key));
2032 }
2033
2034 struct in6_addr *
2035 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2036 {
2037         struct neighbour *n;
2038
2039         n = neigh_entry->key.n;
2040         return (struct in6_addr *) &n->primary_key;
2041 }
2042
2043 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
2044                                struct mlxsw_sp_neigh_entry *neigh_entry,
2045                                u64 *p_counter)
2046 {
2047         if (!neigh_entry->counter_valid)
2048                 return -EINVAL;
2049
2050         return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
2051                                          p_counter, NULL);
2052 }
2053
2054 static struct mlxsw_sp_neigh_entry *
2055 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
2056                            u16 rif)
2057 {
2058         struct mlxsw_sp_neigh_entry *neigh_entry;
2059
2060         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
2061         if (!neigh_entry)
2062                 return NULL;
2063
2064         neigh_entry->key.n = n;
2065         neigh_entry->rif = rif;
2066         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
2067
2068         return neigh_entry;
2069 }
2070
2071 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
2072 {
2073         kfree(neigh_entry);
2074 }
2075
2076 static int
2077 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
2078                             struct mlxsw_sp_neigh_entry *neigh_entry)
2079 {
2080         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
2081                                       &neigh_entry->ht_node,
2082                                       mlxsw_sp_neigh_ht_params);
2083 }
2084
2085 static void
2086 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
2087                             struct mlxsw_sp_neigh_entry *neigh_entry)
2088 {
2089         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
2090                                &neigh_entry->ht_node,
2091                                mlxsw_sp_neigh_ht_params);
2092 }
2093
2094 static bool
2095 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
2096                                     struct mlxsw_sp_neigh_entry *neigh_entry)
2097 {
2098         struct devlink *devlink;
2099         const char *table_name;
2100
2101         switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
2102         case AF_INET:
2103                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
2104                 break;
2105         case AF_INET6:
2106                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
2107                 break;
2108         default:
2109                 WARN_ON(1);
2110                 return false;
2111         }
2112
2113         devlink = priv_to_devlink(mlxsw_sp->core);
2114         return devlink_dpipe_table_counter_enabled(devlink, table_name);
2115 }
2116
2117 static void
2118 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2119                              struct mlxsw_sp_neigh_entry *neigh_entry)
2120 {
2121         if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
2122                 return;
2123
2124         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
2125                 return;
2126
2127         neigh_entry->counter_valid = true;
2128 }
2129
2130 static void
2131 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
2132                             struct mlxsw_sp_neigh_entry *neigh_entry)
2133 {
2134         if (!neigh_entry->counter_valid)
2135                 return;
2136         mlxsw_sp_flow_counter_free(mlxsw_sp,
2137                                    neigh_entry->counter_index);
2138         neigh_entry->counter_valid = false;
2139 }
2140
2141 static struct mlxsw_sp_neigh_entry *
2142 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2143 {
2144         struct mlxsw_sp_neigh_entry *neigh_entry;
2145         struct mlxsw_sp_rif *rif;
2146         int err;
2147
2148         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
2149         if (!rif)
2150                 return ERR_PTR(-EINVAL);
2151
2152         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
2153         if (!neigh_entry)
2154                 return ERR_PTR(-ENOMEM);
2155
2156         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
2157         if (err)
2158                 goto err_neigh_entry_insert;
2159
2160         mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2161         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
2162
2163         return neigh_entry;
2164
2165 err_neigh_entry_insert:
2166         mlxsw_sp_neigh_entry_free(neigh_entry);
2167         return ERR_PTR(err);
2168 }
2169
2170 static void
2171 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2172                              struct mlxsw_sp_neigh_entry *neigh_entry)
2173 {
2174         list_del(&neigh_entry->rif_list_node);
2175         mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2176         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
2177         mlxsw_sp_neigh_entry_free(neigh_entry);
2178 }
2179
2180 static struct mlxsw_sp_neigh_entry *
2181 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2182 {
2183         struct mlxsw_sp_neigh_key key;
2184
2185         key.n = n;
2186         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
2187                                       &key, mlxsw_sp_neigh_ht_params);
2188 }
2189
2190 static void
2191 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
2192 {
2193         unsigned long interval;
2194
2195 #if IS_ENABLED(CONFIG_IPV6)
2196         interval = min_t(unsigned long,
2197                          NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
2198                          NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
2199 #else
2200         interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
2201 #endif
2202         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
2203 }
2204
2205 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2206                                                    char *rauhtd_pl,
2207                                                    int ent_index)
2208 {
2209         struct net_device *dev;
2210         struct neighbour *n;
2211         __be32 dipn;
2212         u32 dip;
2213         u16 rif;
2214
2215         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
2216
2217         if (!mlxsw_sp->router->rifs[rif]) {
2218                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2219                 return;
2220         }
2221
2222         dipn = htonl(dip);
2223         dev = mlxsw_sp->router->rifs[rif]->dev;
2224         n = neigh_lookup(&arp_tbl, &dipn, dev);
2225         if (!n)
2226                 return;
2227
2228         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
2229         neigh_event_send(n, NULL);
2230         neigh_release(n);
2231 }
2232
2233 #if IS_ENABLED(CONFIG_IPV6)
2234 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2235                                                    char *rauhtd_pl,
2236                                                    int rec_index)
2237 {
2238         struct net_device *dev;
2239         struct neighbour *n;
2240         struct in6_addr dip;
2241         u16 rif;
2242
2243         mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
2244                                          (char *) &dip);
2245
2246         if (!mlxsw_sp->router->rifs[rif]) {
2247                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2248                 return;
2249         }
2250
2251         dev = mlxsw_sp->router->rifs[rif]->dev;
2252         n = neigh_lookup(&nd_tbl, &dip, dev);
2253         if (!n)
2254                 return;
2255
2256         netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
2257         neigh_event_send(n, NULL);
2258         neigh_release(n);
2259 }
2260 #else
2261 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2262                                                    char *rauhtd_pl,
2263                                                    int rec_index)
2264 {
2265 }
2266 #endif
2267
2268 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2269                                                    char *rauhtd_pl,
2270                                                    int rec_index)
2271 {
2272         u8 num_entries;
2273         int i;
2274
2275         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2276                                                                 rec_index);
2277         /* Hardware starts counting at 0, so add 1. */
2278         num_entries++;
2279
2280         /* Each record consists of several neighbour entries. */
2281         for (i = 0; i < num_entries; i++) {
2282                 int ent_index;
2283
2284                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
2285                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
2286                                                        ent_index);
2287         }
2288
2289 }
2290
2291 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2292                                                    char *rauhtd_pl,
2293                                                    int rec_index)
2294 {
2295         /* One record contains one entry. */
2296         mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
2297                                                rec_index);
2298 }
2299
2300 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
2301                                               char *rauhtd_pl, int rec_index)
2302 {
2303         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
2304         case MLXSW_REG_RAUHTD_TYPE_IPV4:
2305                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
2306                                                        rec_index);
2307                 break;
2308         case MLXSW_REG_RAUHTD_TYPE_IPV6:
2309                 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
2310                                                        rec_index);
2311                 break;
2312         }
2313 }
2314
2315 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
2316 {
2317         u8 num_rec, last_rec_index, num_entries;
2318
2319         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2320         last_rec_index = num_rec - 1;
2321
2322         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
2323                 return false;
2324         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
2325             MLXSW_REG_RAUHTD_TYPE_IPV6)
2326                 return true;
2327
2328         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2329                                                                 last_rec_index);
2330         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
2331                 return true;
2332         return false;
2333 }
2334
2335 static int
2336 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
2337                                        char *rauhtd_pl,
2338                                        enum mlxsw_reg_rauhtd_type type)
2339 {
2340         int i, num_rec;
2341         int err;
2342
2343         /* Ensure the RIF we read from the device does not change mid-dump. */
2344         mutex_lock(&mlxsw_sp->router->lock);
2345         do {
2346                 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
2347                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
2348                                       rauhtd_pl);
2349                 if (err) {
2350                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n");
2351                         break;
2352                 }
2353                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2354                 for (i = 0; i < num_rec; i++)
2355                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
2356                                                           i);
2357         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
2358         mutex_unlock(&mlxsw_sp->router->lock);
2359
2360         return err;
2361 }
2362
2363 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
2364 {
2365         enum mlxsw_reg_rauhtd_type type;
2366         char *rauhtd_pl;
2367         int err;
2368
2369         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
2370         if (!rauhtd_pl)
2371                 return -ENOMEM;
2372
2373         type = MLXSW_REG_RAUHTD_TYPE_IPV4;
2374         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2375         if (err)
2376                 goto out;
2377
2378         type = MLXSW_REG_RAUHTD_TYPE_IPV6;
2379         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2380 out:
2381         kfree(rauhtd_pl);
2382         return err;
2383 }
2384
2385 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
2386 {
2387         struct mlxsw_sp_neigh_entry *neigh_entry;
2388
2389         mutex_lock(&mlxsw_sp->router->lock);
2390         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
2391                             nexthop_neighs_list_node)
2392                 /* If this neigh have nexthops, make the kernel think this neigh
2393                  * is active regardless of the traffic.
2394                  */
2395                 neigh_event_send(neigh_entry->key.n, NULL);
2396         mutex_unlock(&mlxsw_sp->router->lock);
2397 }
2398
2399 static void
2400 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
2401 {
2402         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
2403
2404         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
2405                                msecs_to_jiffies(interval));
2406 }
2407
2408 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
2409 {
2410         struct mlxsw_sp_router *router;
2411         int err;
2412
2413         router = container_of(work, struct mlxsw_sp_router,
2414                               neighs_update.dw.work);
2415         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
2416         if (err)
2417                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
2418
2419         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
2420
2421         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
2422 }
2423
2424 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
2425 {
2426         struct mlxsw_sp_neigh_entry *neigh_entry;
2427         struct mlxsw_sp_router *router;
2428
2429         router = container_of(work, struct mlxsw_sp_router,
2430                               nexthop_probe_dw.work);
2431         /* Iterate over nexthop neighbours, find those who are unresolved and
2432          * send arp on them. This solves the chicken-egg problem when
2433          * the nexthop wouldn't get offloaded until the neighbor is resolved
2434          * but it wouldn't get resolved ever in case traffic is flowing in HW
2435          * using different nexthop.
2436          */
2437         mutex_lock(&router->lock);
2438         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
2439                             nexthop_neighs_list_node)
2440                 if (!neigh_entry->connected)
2441                         neigh_event_send(neigh_entry->key.n, NULL);
2442         mutex_unlock(&router->lock);
2443
2444         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
2445                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
2446 }
2447
2448 static void
2449 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2450                               struct mlxsw_sp_neigh_entry *neigh_entry,
2451                               bool removing, bool dead);
2452
2453 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
2454 {
2455         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
2456                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
2457 }
2458
2459 static int
2460 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
2461                                 struct mlxsw_sp_neigh_entry *neigh_entry,
2462                                 enum mlxsw_reg_rauht_op op)
2463 {
2464         struct neighbour *n = neigh_entry->key.n;
2465         u32 dip = ntohl(*((__be32 *) n->primary_key));
2466         char rauht_pl[MLXSW_REG_RAUHT_LEN];
2467
2468         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2469                               dip);
2470         if (neigh_entry->counter_valid)
2471                 mlxsw_reg_rauht_pack_counter(rauht_pl,
2472                                              neigh_entry->counter_index);
2473         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2474 }
2475
2476 static int
2477 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
2478                                 struct mlxsw_sp_neigh_entry *neigh_entry,
2479                                 enum mlxsw_reg_rauht_op op)
2480 {
2481         struct neighbour *n = neigh_entry->key.n;
2482         char rauht_pl[MLXSW_REG_RAUHT_LEN];
2483         const char *dip = n->primary_key;
2484
2485         mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2486                               dip);
2487         if (neigh_entry->counter_valid)
2488                 mlxsw_reg_rauht_pack_counter(rauht_pl,
2489                                              neigh_entry->counter_index);
2490         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2491 }
2492
2493 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
2494 {
2495         struct neighbour *n = neigh_entry->key.n;
2496
2497         /* Packets with a link-local destination address are trapped
2498          * after LPM lookup and never reach the neighbour table, so
2499          * there is no need to program such neighbours to the device.
2500          */
2501         if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
2502             IPV6_ADDR_LINKLOCAL)
2503                 return true;
2504         return false;
2505 }
2506
2507 static void
2508 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
2509                             struct mlxsw_sp_neigh_entry *neigh_entry,
2510                             bool adding)
2511 {
2512         enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding);
2513         int err;
2514
2515         if (!adding && !neigh_entry->connected)
2516                 return;
2517         neigh_entry->connected = adding;
2518         if (neigh_entry->key.n->tbl->family == AF_INET) {
2519                 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
2520                                                       op);
2521                 if (err)
2522                         return;
2523         } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
2524                 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
2525                         return;
2526                 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
2527                                                       op);
2528                 if (err)
2529                         return;
2530         } else {
2531                 WARN_ON_ONCE(1);
2532                 return;
2533         }
2534
2535         if (adding)
2536                 neigh_entry->key.n->flags |= NTF_OFFLOADED;
2537         else
2538                 neigh_entry->key.n->flags &= ~NTF_OFFLOADED;
2539 }
2540
2541 void
2542 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
2543                                     struct mlxsw_sp_neigh_entry *neigh_entry,
2544                                     bool adding)
2545 {
2546         if (adding)
2547                 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2548         else
2549                 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2550         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
2551 }
2552
2553 struct mlxsw_sp_netevent_work {
2554         struct work_struct work;
2555         struct mlxsw_sp *mlxsw_sp;
2556         struct neighbour *n;
2557 };
2558
2559 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
2560 {
2561         struct mlxsw_sp_netevent_work *net_work =
2562                 container_of(work, struct mlxsw_sp_netevent_work, work);
2563         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2564         struct mlxsw_sp_neigh_entry *neigh_entry;
2565         struct neighbour *n = net_work->n;
2566         unsigned char ha[ETH_ALEN];
2567         bool entry_connected;
2568         u8 nud_state, dead;
2569
2570         /* If these parameters are changed after we release the lock,
2571          * then we are guaranteed to receive another event letting us
2572          * know about it.
2573          */
2574         read_lock_bh(&n->lock);
2575         memcpy(ha, n->ha, ETH_ALEN);
2576         nud_state = n->nud_state;
2577         dead = n->dead;
2578         read_unlock_bh(&n->lock);
2579
2580         mutex_lock(&mlxsw_sp->router->lock);
2581         mlxsw_sp_span_respin(mlxsw_sp);
2582
2583         entry_connected = nud_state & NUD_VALID && !dead;
2584         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2585         if (!entry_connected && !neigh_entry)
2586                 goto out;
2587         if (!neigh_entry) {
2588                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2589                 if (IS_ERR(neigh_entry))
2590                         goto out;
2591         }
2592
2593         memcpy(neigh_entry->ha, ha, ETH_ALEN);
2594         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
2595         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected,
2596                                       dead);
2597
2598         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2599                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2600
2601 out:
2602         mutex_unlock(&mlxsw_sp->router->lock);
2603         neigh_release(n);
2604         kfree(net_work);
2605 }
2606
2607 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp);
2608
2609 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work)
2610 {
2611         struct mlxsw_sp_netevent_work *net_work =
2612                 container_of(work, struct mlxsw_sp_netevent_work, work);
2613         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2614
2615         mlxsw_sp_mp_hash_init(mlxsw_sp);
2616         kfree(net_work);
2617 }
2618
2619 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp);
2620
2621 static void mlxsw_sp_router_update_priority_work(struct work_struct *work)
2622 {
2623         struct mlxsw_sp_netevent_work *net_work =
2624                 container_of(work, struct mlxsw_sp_netevent_work, work);
2625         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2626
2627         __mlxsw_sp_router_init(mlxsw_sp);
2628         kfree(net_work);
2629 }
2630
2631 static int mlxsw_sp_router_schedule_work(struct net *net,
2632                                          struct notifier_block *nb,
2633                                          void (*cb)(struct work_struct *))
2634 {
2635         struct mlxsw_sp_netevent_work *net_work;
2636         struct mlxsw_sp_router *router;
2637
2638         router = container_of(nb, struct mlxsw_sp_router, netevent_nb);
2639         if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp)))
2640                 return NOTIFY_DONE;
2641
2642         net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2643         if (!net_work)
2644                 return NOTIFY_BAD;
2645
2646         INIT_WORK(&net_work->work, cb);
2647         net_work->mlxsw_sp = router->mlxsw_sp;
2648         mlxsw_core_schedule_work(&net_work->work);
2649         return NOTIFY_DONE;
2650 }
2651
2652 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb,
2653                                           unsigned long event, void *ptr)
2654 {
2655         struct mlxsw_sp_netevent_work *net_work;
2656         struct mlxsw_sp_port *mlxsw_sp_port;
2657         struct mlxsw_sp *mlxsw_sp;
2658         unsigned long interval;
2659         struct neigh_parms *p;
2660         struct neighbour *n;
2661
2662         switch (event) {
2663         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
2664                 p = ptr;
2665
2666                 /* We don't care about changes in the default table. */
2667                 if (!p->dev || (p->tbl->family != AF_INET &&
2668                                 p->tbl->family != AF_INET6))
2669                         return NOTIFY_DONE;
2670
2671                 /* We are in atomic context and can't take RTNL mutex,
2672                  * so use RCU variant to walk the device chain.
2673                  */
2674                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
2675                 if (!mlxsw_sp_port)
2676                         return NOTIFY_DONE;
2677
2678                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2679                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
2680                 mlxsw_sp->router->neighs_update.interval = interval;
2681
2682                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2683                 break;
2684         case NETEVENT_NEIGH_UPDATE:
2685                 n = ptr;
2686
2687                 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
2688                         return NOTIFY_DONE;
2689
2690                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
2691                 if (!mlxsw_sp_port)
2692                         return NOTIFY_DONE;
2693
2694                 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2695                 if (!net_work) {
2696                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
2697                         return NOTIFY_BAD;
2698                 }
2699
2700                 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work);
2701                 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2702                 net_work->n = n;
2703
2704                 /* Take a reference to ensure the neighbour won't be
2705                  * destructed until we drop the reference in delayed
2706                  * work.
2707                  */
2708                 neigh_clone(n);
2709                 mlxsw_core_schedule_work(&net_work->work);
2710                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2711                 break;
2712         case NETEVENT_IPV4_MPATH_HASH_UPDATE:
2713         case NETEVENT_IPV6_MPATH_HASH_UPDATE:
2714                 return mlxsw_sp_router_schedule_work(ptr, nb,
2715                                 mlxsw_sp_router_mp_hash_event_work);
2716
2717         case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE:
2718                 return mlxsw_sp_router_schedule_work(ptr, nb,
2719                                 mlxsw_sp_router_update_priority_work);
2720         }
2721
2722         return NOTIFY_DONE;
2723 }
2724
2725 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
2726 {
2727         int err;
2728
2729         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
2730                               &mlxsw_sp_neigh_ht_params);
2731         if (err)
2732                 return err;
2733
2734         /* Initialize the polling interval according to the default
2735          * table.
2736          */
2737         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
2738
2739         /* Create the delayed works for the activity_update */
2740         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
2741                           mlxsw_sp_router_neighs_update_work);
2742         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
2743                           mlxsw_sp_router_probe_unresolved_nexthops);
2744         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
2745         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
2746         return 0;
2747 }
2748
2749 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2750 {
2751         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
2752         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
2753         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2754 }
2755
2756 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2757                                          struct mlxsw_sp_rif *rif)
2758 {
2759         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2760
2761         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2762                                  rif_list_node) {
2763                 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2764                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2765         }
2766 }
2767
2768 enum mlxsw_sp_nexthop_type {
2769         MLXSW_SP_NEXTHOP_TYPE_ETH,
2770         MLXSW_SP_NEXTHOP_TYPE_IPIP,
2771 };
2772
2773 struct mlxsw_sp_nexthop_key {
2774         struct fib_nh *fib_nh;
2775 };
2776
2777 struct mlxsw_sp_nexthop {
2778         struct list_head neigh_list_node; /* member of neigh entry list */
2779         struct list_head rif_list_node;
2780         struct list_head router_list_node;
2781         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
2782                                                 * this belongs to
2783                                                 */
2784         struct rhash_head ht_node;
2785         struct mlxsw_sp_nexthop_key key;
2786         unsigned char gw_addr[sizeof(struct in6_addr)];
2787         int ifindex;
2788         int nh_weight;
2789         int norm_nh_weight;
2790         int num_adj_entries;
2791         struct mlxsw_sp_rif *rif;
2792         u8 should_offload:1, /* set indicates this neigh is connected and
2793                               * should be put to KVD linear area of this group.
2794                               */
2795            offloaded:1, /* set in case the neigh is actually put into
2796                          * KVD linear area of this group.
2797                          */
2798            update:1; /* set indicates that MAC of this neigh should be
2799                       * updated in HW
2800                       */
2801         enum mlxsw_sp_nexthop_type type;
2802         union {
2803                 struct mlxsw_sp_neigh_entry *neigh_entry;
2804                 struct mlxsw_sp_ipip_entry *ipip_entry;
2805         };
2806         unsigned int counter_index;
2807         bool counter_valid;
2808 };
2809
2810 struct mlxsw_sp_nexthop_group {
2811         void *priv;
2812         struct rhash_head ht_node;
2813         struct list_head fib_list; /* list of fib entries that use this group */
2814         struct neigh_table *neigh_tbl;
2815         u8 adj_index_valid:1,
2816            gateway:1; /* routes using the group use a gateway */
2817         u32 adj_index;
2818         u16 ecmp_size;
2819         u16 count;
2820         int sum_norm_weight;
2821         struct mlxsw_sp_nexthop nexthops[0];
2822 #define nh_rif  nexthops[0].rif
2823 };
2824
2825 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2826                                     struct mlxsw_sp_nexthop *nh)
2827 {
2828         struct devlink *devlink;
2829
2830         devlink = priv_to_devlink(mlxsw_sp->core);
2831         if (!devlink_dpipe_table_counter_enabled(devlink,
2832                                                  MLXSW_SP_DPIPE_TABLE_NAME_ADJ))
2833                 return;
2834
2835         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index))
2836                 return;
2837
2838         nh->counter_valid = true;
2839 }
2840
2841 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp,
2842                                    struct mlxsw_sp_nexthop *nh)
2843 {
2844         if (!nh->counter_valid)
2845                 return;
2846         mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index);
2847         nh->counter_valid = false;
2848 }
2849
2850 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp,
2851                                  struct mlxsw_sp_nexthop *nh, u64 *p_counter)
2852 {
2853         if (!nh->counter_valid)
2854                 return -EINVAL;
2855
2856         return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index,
2857                                          p_counter, NULL);
2858 }
2859
2860 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router,
2861                                                struct mlxsw_sp_nexthop *nh)
2862 {
2863         if (!nh) {
2864                 if (list_empty(&router->nexthop_list))
2865                         return NULL;
2866                 else
2867                         return list_first_entry(&router->nexthop_list,
2868                                                 typeof(*nh), router_list_node);
2869         }
2870         if (list_is_last(&nh->router_list_node, &router->nexthop_list))
2871                 return NULL;
2872         return list_next_entry(nh, router_list_node);
2873 }
2874
2875 bool mlxsw_sp_nexthop_offload(struct mlxsw_sp_nexthop *nh)
2876 {
2877         return nh->offloaded;
2878 }
2879
2880 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh)
2881 {
2882         if (!nh->offloaded)
2883                 return NULL;
2884         return nh->neigh_entry->ha;
2885 }
2886
2887 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index,
2888                              u32 *p_adj_size, u32 *p_adj_hash_index)
2889 {
2890         struct mlxsw_sp_nexthop_group *nh_grp = nh->nh_grp;
2891         u32 adj_hash_index = 0;
2892         int i;
2893
2894         if (!nh->offloaded || !nh_grp->adj_index_valid)
2895                 return -EINVAL;
2896
2897         *p_adj_index = nh_grp->adj_index;
2898         *p_adj_size = nh_grp->ecmp_size;
2899
2900         for (i = 0; i < nh_grp->count; i++) {
2901                 struct mlxsw_sp_nexthop *nh_iter = &nh_grp->nexthops[i];
2902
2903                 if (nh_iter == nh)
2904                         break;
2905                 if (nh_iter->offloaded)
2906                         adj_hash_index += nh_iter->num_adj_entries;
2907         }
2908
2909         *p_adj_hash_index = adj_hash_index;
2910         return 0;
2911 }
2912
2913 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh)
2914 {
2915         return nh->rif;
2916 }
2917
2918 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh)
2919 {
2920         struct mlxsw_sp_nexthop_group *nh_grp = nh->nh_grp;
2921         int i;
2922
2923         for (i = 0; i < nh_grp->count; i++) {
2924                 struct mlxsw_sp_nexthop *nh_iter = &nh_grp->nexthops[i];
2925
2926                 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP)
2927                         return true;
2928         }
2929         return false;
2930 }
2931
2932 static struct fib_info *
2933 mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp)
2934 {
2935         return nh_grp->priv;
2936 }
2937
2938 struct mlxsw_sp_nexthop_group_cmp_arg {
2939         enum mlxsw_sp_l3proto proto;
2940         union {
2941                 struct fib_info *fi;
2942                 struct mlxsw_sp_fib6_entry *fib6_entry;
2943         };
2944 };
2945
2946 static bool
2947 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
2948                                     const struct in6_addr *gw, int ifindex,
2949                                     int weight)
2950 {
2951         int i;
2952
2953         for (i = 0; i < nh_grp->count; i++) {
2954                 const struct mlxsw_sp_nexthop *nh;
2955
2956                 nh = &nh_grp->nexthops[i];
2957                 if (nh->ifindex == ifindex && nh->nh_weight == weight &&
2958                     ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
2959                         return true;
2960         }
2961
2962         return false;
2963 }
2964
2965 static bool
2966 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
2967                             const struct mlxsw_sp_fib6_entry *fib6_entry)
2968 {
2969         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2970
2971         if (nh_grp->count != fib6_entry->nrt6)
2972                 return false;
2973
2974         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2975                 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
2976                 struct in6_addr *gw;
2977                 int ifindex, weight;
2978
2979                 ifindex = fib6_nh->fib_nh_dev->ifindex;
2980                 weight = fib6_nh->fib_nh_weight;
2981                 gw = &fib6_nh->fib_nh_gw6;
2982                 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex,
2983                                                          weight))
2984                         return false;
2985         }
2986
2987         return true;
2988 }
2989
2990 static int
2991 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
2992 {
2993         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
2994         const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
2995
2996         switch (cmp_arg->proto) {
2997         case MLXSW_SP_L3_PROTO_IPV4:
2998                 return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp);
2999         case MLXSW_SP_L3_PROTO_IPV6:
3000                 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
3001                                                     cmp_arg->fib6_entry);
3002         default:
3003                 WARN_ON(1);
3004                 return 1;
3005         }
3006 }
3007
3008 static int
3009 mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp)
3010 {
3011         return nh_grp->neigh_tbl->family;
3012 }
3013
3014 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
3015 {
3016         const struct mlxsw_sp_nexthop_group *nh_grp = data;
3017         const struct mlxsw_sp_nexthop *nh;
3018         struct fib_info *fi;
3019         unsigned int val;
3020         int i;
3021
3022         switch (mlxsw_sp_nexthop_group_type(nh_grp)) {
3023         case AF_INET:
3024                 fi = mlxsw_sp_nexthop4_group_fi(nh_grp);
3025                 return jhash(&fi, sizeof(fi), seed);
3026         case AF_INET6:
3027                 val = nh_grp->count;
3028                 for (i = 0; i < nh_grp->count; i++) {
3029                         nh = &nh_grp->nexthops[i];
3030                         val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed);
3031                         val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed);
3032                 }
3033                 return jhash(&val, sizeof(val), seed);
3034         default:
3035                 WARN_ON(1);
3036                 return 0;
3037         }
3038 }
3039
3040 static u32
3041 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
3042 {
3043         unsigned int val = fib6_entry->nrt6;
3044         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3045
3046         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3047                 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3048                 struct net_device *dev = fib6_nh->fib_nh_dev;
3049                 struct in6_addr *gw = &fib6_nh->fib_nh_gw6;
3050
3051                 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed);
3052                 val ^= jhash(gw, sizeof(*gw), seed);
3053         }
3054
3055         return jhash(&val, sizeof(val), seed);
3056 }
3057
3058 static u32
3059 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
3060 {
3061         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
3062
3063         switch (cmp_arg->proto) {
3064         case MLXSW_SP_L3_PROTO_IPV4:
3065                 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
3066         case MLXSW_SP_L3_PROTO_IPV6:
3067                 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
3068         default:
3069                 WARN_ON(1);
3070                 return 0;
3071         }
3072 }
3073
3074 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
3075         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
3076         .hashfn      = mlxsw_sp_nexthop_group_hash,
3077         .obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
3078         .obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
3079 };
3080
3081 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
3082                                          struct mlxsw_sp_nexthop_group *nh_grp)
3083 {
3084         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
3085             !nh_grp->gateway)
3086                 return 0;
3087
3088         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
3089                                       &nh_grp->ht_node,
3090                                       mlxsw_sp_nexthop_group_ht_params);
3091 }
3092
3093 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
3094                                           struct mlxsw_sp_nexthop_group *nh_grp)
3095 {
3096         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
3097             !nh_grp->gateway)
3098                 return;
3099
3100         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
3101                                &nh_grp->ht_node,
3102                                mlxsw_sp_nexthop_group_ht_params);
3103 }
3104
3105 static struct mlxsw_sp_nexthop_group *
3106 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
3107                                struct fib_info *fi)
3108 {
3109         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3110
3111         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4;
3112         cmp_arg.fi = fi;
3113         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3114                                       &cmp_arg,
3115                                       mlxsw_sp_nexthop_group_ht_params);
3116 }
3117
3118 static struct mlxsw_sp_nexthop_group *
3119 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
3120                                struct mlxsw_sp_fib6_entry *fib6_entry)
3121 {
3122         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3123
3124         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6;
3125         cmp_arg.fib6_entry = fib6_entry;
3126         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3127                                       &cmp_arg,
3128                                       mlxsw_sp_nexthop_group_ht_params);
3129 }
3130
3131 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
3132         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
3133         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
3134         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
3135 };
3136
3137 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
3138                                    struct mlxsw_sp_nexthop *nh)
3139 {
3140         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
3141                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
3142 }
3143
3144 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
3145                                     struct mlxsw_sp_nexthop *nh)
3146 {
3147         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
3148                                mlxsw_sp_nexthop_ht_params);
3149 }
3150
3151 static struct mlxsw_sp_nexthop *
3152 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
3153                         struct mlxsw_sp_nexthop_key key)
3154 {
3155         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
3156                                       mlxsw_sp_nexthop_ht_params);
3157 }
3158
3159 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
3160                                              const struct mlxsw_sp_fib *fib,
3161                                              u32 adj_index, u16 ecmp_size,
3162                                              u32 new_adj_index,
3163                                              u16 new_ecmp_size)
3164 {
3165         char raleu_pl[MLXSW_REG_RALEU_LEN];
3166
3167         mlxsw_reg_raleu_pack(raleu_pl,
3168                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
3169                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
3170                              new_ecmp_size);
3171         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
3172 }
3173
3174 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
3175                                           struct mlxsw_sp_nexthop_group *nh_grp,
3176                                           u32 old_adj_index, u16 old_ecmp_size)
3177 {
3178         struct mlxsw_sp_fib_entry *fib_entry;
3179         struct mlxsw_sp_fib *fib = NULL;
3180         int err;
3181
3182         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3183                 if (fib == fib_entry->fib_node->fib)
3184                         continue;
3185                 fib = fib_entry->fib_node->fib;
3186                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
3187                                                         old_adj_index,
3188                                                         old_ecmp_size,
3189                                                         nh_grp->adj_index,
3190                                                         nh_grp->ecmp_size);
3191                 if (err)
3192                         return err;
3193         }
3194         return 0;
3195 }
3196
3197 static int __mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3198                                      struct mlxsw_sp_nexthop *nh)
3199 {
3200         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3201         char ratr_pl[MLXSW_REG_RATR_LEN];
3202
3203         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
3204                             true, MLXSW_REG_RATR_TYPE_ETHERNET,
3205                             adj_index, neigh_entry->rif);
3206         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
3207         if (nh->counter_valid)
3208                 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true);
3209         else
3210                 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false);
3211
3212         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
3213 }
3214
3215 int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3216                             struct mlxsw_sp_nexthop *nh)
3217 {
3218         int i;
3219
3220         for (i = 0; i < nh->num_adj_entries; i++) {
3221                 int err;
3222
3223                 err = __mlxsw_sp_nexthop_update(mlxsw_sp, adj_index + i, nh);
3224                 if (err)
3225                         return err;
3226         }
3227
3228         return 0;
3229 }
3230
3231 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3232                                           u32 adj_index,
3233                                           struct mlxsw_sp_nexthop *nh)
3234 {
3235         const struct mlxsw_sp_ipip_ops *ipip_ops;
3236
3237         ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
3238         return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry);
3239 }
3240
3241 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3242                                         u32 adj_index,
3243                                         struct mlxsw_sp_nexthop *nh)
3244 {
3245         int i;
3246
3247         for (i = 0; i < nh->num_adj_entries; i++) {
3248                 int err;
3249
3250                 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i,
3251                                                      nh);
3252                 if (err)
3253                         return err;
3254         }
3255
3256         return 0;
3257 }
3258
3259 static int
3260 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
3261                               struct mlxsw_sp_nexthop_group *nh_grp,
3262                               bool reallocate)
3263 {
3264         u32 adj_index = nh_grp->adj_index; /* base */
3265         struct mlxsw_sp_nexthop *nh;
3266         int i;
3267
3268         for (i = 0; i < nh_grp->count; i++) {
3269                 nh = &nh_grp->nexthops[i];
3270
3271                 if (!nh->should_offload) {
3272                         nh->offloaded = 0;
3273                         continue;
3274                 }
3275
3276                 if (nh->update || reallocate) {
3277                         int err = 0;
3278
3279                         switch (nh->type) {
3280                         case MLXSW_SP_NEXTHOP_TYPE_ETH:
3281                                 err = mlxsw_sp_nexthop_update
3282                                             (mlxsw_sp, adj_index, nh);
3283                                 break;
3284                         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
3285                                 err = mlxsw_sp_nexthop_ipip_update
3286                                             (mlxsw_sp, adj_index, nh);
3287                                 break;
3288                         }
3289                         if (err)
3290                                 return err;
3291                         nh->update = 0;
3292                         nh->offloaded = 1;
3293                 }
3294                 adj_index += nh->num_adj_entries;
3295         }
3296         return 0;
3297 }
3298
3299 static int
3300 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
3301                                     struct mlxsw_sp_nexthop_group *nh_grp)
3302 {
3303         struct mlxsw_sp_fib_entry *fib_entry;
3304         int err;
3305
3306         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3307                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3308                 if (err)
3309                         return err;
3310         }
3311         return 0;
3312 }
3313
3314 static void mlxsw_sp_adj_grp_size_round_up(u16 *p_adj_grp_size)
3315 {
3316         /* Valid sizes for an adjacency group are:
3317          * 1-64, 512, 1024, 2048 and 4096.
3318          */
3319         if (*p_adj_grp_size <= 64)
3320                 return;
3321         else if (*p_adj_grp_size <= 512)
3322                 *p_adj_grp_size = 512;
3323         else if (*p_adj_grp_size <= 1024)
3324                 *p_adj_grp_size = 1024;
3325         else if (*p_adj_grp_size <= 2048)
3326                 *p_adj_grp_size = 2048;
3327         else
3328                 *p_adj_grp_size = 4096;
3329 }
3330
3331 static void mlxsw_sp_adj_grp_size_round_down(u16 *p_adj_grp_size,
3332                                              unsigned int alloc_size)
3333 {
3334         if (alloc_size >= 4096)
3335                 *p_adj_grp_size = 4096;
3336         else if (alloc_size >= 2048)
3337                 *p_adj_grp_size = 2048;
3338         else if (alloc_size >= 1024)
3339                 *p_adj_grp_size = 1024;
3340         else if (alloc_size >= 512)
3341                 *p_adj_grp_size = 512;
3342 }
3343
3344 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp,
3345                                      u16 *p_adj_grp_size)
3346 {
3347         unsigned int alloc_size;
3348         int err;
3349
3350         /* Round up the requested group size to the next size supported
3351          * by the device and make sure the request can be satisfied.
3352          */
3353         mlxsw_sp_adj_grp_size_round_up(p_adj_grp_size);
3354         err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
3355                                               MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3356                                               *p_adj_grp_size, &alloc_size);
3357         if (err)
3358                 return err;
3359         /* It is possible the allocation results in more allocated
3360          * entries than requested. Try to use as much of them as
3361          * possible.
3362          */
3363         mlxsw_sp_adj_grp_size_round_down(p_adj_grp_size, alloc_size);
3364
3365         return 0;
3366 }
3367
3368 static void
3369 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group *nh_grp)
3370 {
3371         int i, g = 0, sum_norm_weight = 0;
3372         struct mlxsw_sp_nexthop *nh;
3373
3374         for (i = 0; i < nh_grp->count; i++) {
3375                 nh = &nh_grp->nexthops[i];
3376
3377                 if (!nh->should_offload)
3378                         continue;
3379                 if (g > 0)
3380                         g = gcd(nh->nh_weight, g);
3381                 else
3382                         g = nh->nh_weight;
3383         }
3384
3385         for (i = 0; i < nh_grp->count; i++) {
3386                 nh = &nh_grp->nexthops[i];
3387
3388                 if (!nh->should_offload)
3389                         continue;
3390                 nh->norm_nh_weight = nh->nh_weight / g;
3391                 sum_norm_weight += nh->norm_nh_weight;
3392         }
3393
3394         nh_grp->sum_norm_weight = sum_norm_weight;
3395 }
3396
3397 static void
3398 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group *nh_grp)
3399 {
3400         int total = nh_grp->sum_norm_weight;
3401         u16 ecmp_size = nh_grp->ecmp_size;
3402         int i, weight = 0, lower_bound = 0;
3403
3404         for (i = 0; i < nh_grp->count; i++) {
3405                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3406                 int upper_bound;
3407
3408                 if (!nh->should_offload)
3409                         continue;
3410                 weight += nh->norm_nh_weight;
3411                 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total);
3412                 nh->num_adj_entries = upper_bound - lower_bound;
3413                 lower_bound = upper_bound;
3414         }
3415 }
3416
3417 static struct mlxsw_sp_nexthop *
3418 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
3419                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6);
3420
3421 static void
3422 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3423                                         struct mlxsw_sp_nexthop_group *nh_grp)
3424 {
3425         int i;
3426
3427         for (i = 0; i < nh_grp->count; i++) {
3428                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3429
3430                 if (nh->offloaded)
3431                         nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3432                 else
3433                         nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3434         }
3435 }
3436
3437 static void
3438 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp,
3439                                           struct mlxsw_sp_fib6_entry *fib6_entry)
3440 {
3441         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3442
3443         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3444                 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3445                 struct mlxsw_sp_nexthop *nh;
3446
3447                 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3448                 if (nh && nh->offloaded)
3449                         fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3450                 else
3451                         fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3452         }
3453 }
3454
3455 static void
3456 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3457                                         struct mlxsw_sp_nexthop_group *nh_grp)
3458 {
3459         struct mlxsw_sp_fib6_entry *fib6_entry;
3460
3461         /* Unfortunately, in IPv6 the route and the nexthop are described by
3462          * the same struct, so we need to iterate over all the routes using the
3463          * nexthop group and set / clear the offload indication for them.
3464          */
3465         list_for_each_entry(fib6_entry, &nh_grp->fib_list,
3466                             common.nexthop_group_node)
3467                 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
3468 }
3469
3470 static void
3471 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3472                                        struct mlxsw_sp_nexthop_group *nh_grp)
3473 {
3474         switch (mlxsw_sp_nexthop_group_type(nh_grp)) {
3475         case AF_INET:
3476                 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp);
3477                 break;
3478         case AF_INET6:
3479                 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp);
3480                 break;
3481         }
3482 }
3483
3484 static void
3485 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
3486                                struct mlxsw_sp_nexthop_group *nh_grp)
3487 {
3488         u16 ecmp_size, old_ecmp_size;
3489         struct mlxsw_sp_nexthop *nh;
3490         bool offload_change = false;
3491         u32 adj_index;
3492         bool old_adj_index_valid;
3493         u32 old_adj_index;
3494         int i;
3495         int err;
3496
3497         if (!nh_grp->gateway) {
3498                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3499                 return;
3500         }
3501
3502         for (i = 0; i < nh_grp->count; i++) {
3503                 nh = &nh_grp->nexthops[i];
3504
3505                 if (nh->should_offload != nh->offloaded) {
3506                         offload_change = true;
3507                         if (nh->should_offload)
3508                                 nh->update = 1;
3509                 }
3510         }
3511         if (!offload_change) {
3512                 /* Nothing was added or removed, so no need to reallocate. Just
3513                  * update MAC on existing adjacency indexes.
3514                  */
3515                 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, false);
3516                 if (err) {
3517                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3518                         goto set_trap;
3519                 }
3520                 return;
3521         }
3522         mlxsw_sp_nexthop_group_normalize(nh_grp);
3523         if (!nh_grp->sum_norm_weight)
3524                 /* No neigh of this group is connected so we just set
3525                  * the trap and let everthing flow through kernel.
3526                  */
3527                 goto set_trap;
3528
3529         ecmp_size = nh_grp->sum_norm_weight;
3530         err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size);
3531         if (err)
3532                 /* No valid allocation size available. */
3533                 goto set_trap;
3534
3535         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3536                                   ecmp_size, &adj_index);
3537         if (err) {
3538                 /* We ran out of KVD linear space, just set the
3539                  * trap and let everything flow through kernel.
3540                  */
3541                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
3542                 goto set_trap;
3543         }
3544         old_adj_index_valid = nh_grp->adj_index_valid;
3545         old_adj_index = nh_grp->adj_index;
3546         old_ecmp_size = nh_grp->ecmp_size;
3547         nh_grp->adj_index_valid = 1;
3548         nh_grp->adj_index = adj_index;
3549         nh_grp->ecmp_size = ecmp_size;
3550         mlxsw_sp_nexthop_group_rebalance(nh_grp);
3551         err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, true);
3552         if (err) {
3553                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3554                 goto set_trap;
3555         }
3556
3557         mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3558
3559         if (!old_adj_index_valid) {
3560                 /* The trap was set for fib entries, so we have to call
3561                  * fib entry update to unset it and use adjacency index.
3562                  */
3563                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3564                 if (err) {
3565                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
3566                         goto set_trap;
3567                 }
3568                 return;
3569         }
3570
3571         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
3572                                              old_adj_index, old_ecmp_size);
3573         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3574                            old_ecmp_size, old_adj_index);
3575         if (err) {
3576                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
3577                 goto set_trap;
3578         }
3579
3580         return;
3581
3582 set_trap:
3583         old_adj_index_valid = nh_grp->adj_index_valid;
3584         nh_grp->adj_index_valid = 0;
3585         for (i = 0; i < nh_grp->count; i++) {
3586                 nh = &nh_grp->nexthops[i];
3587                 nh->offloaded = 0;
3588         }
3589         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3590         if (err)
3591                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
3592         mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3593         if (old_adj_index_valid)
3594                 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3595                                    nh_grp->ecmp_size, nh_grp->adj_index);
3596 }
3597
3598 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
3599                                             bool removing)
3600 {
3601         if (!removing)
3602                 nh->should_offload = 1;
3603         else
3604                 nh->should_offload = 0;
3605         nh->update = 1;
3606 }
3607
3608 static int
3609 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp,
3610                                     struct mlxsw_sp_neigh_entry *neigh_entry)
3611 {
3612         struct neighbour *n, *old_n = neigh_entry->key.n;
3613         struct mlxsw_sp_nexthop *nh;
3614         bool entry_connected;
3615         u8 nud_state, dead;
3616         int err;
3617
3618         nh = list_first_entry(&neigh_entry->nexthop_list,
3619                               struct mlxsw_sp_nexthop, neigh_list_node);
3620
3621         n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3622         if (!n) {
3623                 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
3624                                  nh->rif->dev);
3625                 if (IS_ERR(n))
3626                         return PTR_ERR(n);
3627                 neigh_event_send(n, NULL);
3628         }
3629
3630         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
3631         neigh_entry->key.n = n;
3632         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
3633         if (err)
3634                 goto err_neigh_entry_insert;
3635
3636         read_lock_bh(&n->lock);
3637         nud_state = n->nud_state;
3638         dead = n->dead;
3639         read_unlock_bh(&n->lock);
3640         entry_connected = nud_state & NUD_VALID && !dead;
3641
3642         list_for_each_entry(nh, &neigh_entry->nexthop_list,
3643                             neigh_list_node) {
3644                 neigh_release(old_n);
3645                 neigh_clone(n);
3646                 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected);
3647                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
3648         }
3649
3650         neigh_release(n);
3651
3652         return 0;
3653
3654 err_neigh_entry_insert:
3655         neigh_entry->key.n = old_n;
3656         mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
3657         neigh_release(n);
3658         return err;
3659 }
3660
3661 static void
3662 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
3663                               struct mlxsw_sp_neigh_entry *neigh_entry,
3664                               bool removing, bool dead)
3665 {
3666         struct mlxsw_sp_nexthop *nh;
3667
3668         if (list_empty(&neigh_entry->nexthop_list))
3669                 return;
3670
3671         if (dead) {
3672                 int err;
3673
3674                 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp,
3675                                                           neigh_entry);
3676                 if (err)
3677                         dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n");
3678                 return;
3679         }
3680
3681         list_for_each_entry(nh, &neigh_entry->nexthop_list,
3682                             neigh_list_node) {
3683                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
3684                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
3685         }
3686 }
3687
3688 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
3689                                       struct mlxsw_sp_rif *rif)
3690 {
3691         if (nh->rif)
3692                 return;
3693
3694         nh->rif = rif;
3695         list_add(&nh->rif_list_node, &rif->nexthop_list);
3696 }
3697
3698 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
3699 {
3700         if (!nh->rif)
3701                 return;
3702
3703         list_del(&nh->rif_list_node);
3704         nh->rif = NULL;
3705 }
3706
3707 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
3708                                        struct mlxsw_sp_nexthop *nh)
3709 {
3710         struct mlxsw_sp_neigh_entry *neigh_entry;
3711         struct neighbour *n;
3712         u8 nud_state, dead;
3713         int err;
3714
3715         if (!nh->nh_grp->gateway || nh->neigh_entry)
3716                 return 0;
3717
3718         /* Take a reference of neigh here ensuring that neigh would
3719          * not be destructed before the nexthop entry is finished.
3720          * The reference is taken either in neigh_lookup() or
3721          * in neigh_create() in case n is not found.
3722          */
3723         n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3724         if (!n) {
3725                 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
3726                                  nh->rif->dev);
3727                 if (IS_ERR(n))
3728                         return PTR_ERR(n);
3729                 neigh_event_send(n, NULL);
3730         }
3731         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
3732         if (!neigh_entry) {
3733                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
3734                 if (IS_ERR(neigh_entry)) {
3735                         err = -EINVAL;
3736                         goto err_neigh_entry_create;
3737                 }
3738         }
3739
3740         /* If that is the first nexthop connected to that neigh, add to
3741          * nexthop_neighs_list
3742          */
3743         if (list_empty(&neigh_entry->nexthop_list))
3744                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
3745                               &mlxsw_sp->router->nexthop_neighs_list);
3746
3747         nh->neigh_entry = neigh_entry;
3748         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
3749         read_lock_bh(&n->lock);
3750         nud_state = n->nud_state;
3751         dead = n->dead;
3752         read_unlock_bh(&n->lock);
3753         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
3754
3755         return 0;
3756
3757 err_neigh_entry_create:
3758         neigh_release(n);
3759         return err;
3760 }
3761
3762 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
3763                                         struct mlxsw_sp_nexthop *nh)
3764 {
3765         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3766         struct neighbour *n;
3767
3768         if (!neigh_entry)
3769                 return;
3770         n = neigh_entry->key.n;
3771
3772         __mlxsw_sp_nexthop_neigh_update(nh, true);
3773         list_del(&nh->neigh_list_node);
3774         nh->neigh_entry = NULL;
3775
3776         /* If that is the last nexthop connected to that neigh, remove from
3777          * nexthop_neighs_list
3778          */
3779         if (list_empty(&neigh_entry->nexthop_list))
3780                 list_del(&neigh_entry->nexthop_neighs_list_node);
3781
3782         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
3783                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
3784
3785         neigh_release(n);
3786 }
3787
3788 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev)
3789 {
3790         struct net_device *ul_dev;
3791         bool is_up;
3792
3793         rcu_read_lock();
3794         ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
3795         is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true;
3796         rcu_read_unlock();
3797
3798         return is_up;
3799 }
3800
3801 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
3802                                        struct mlxsw_sp_nexthop *nh,
3803                                        struct mlxsw_sp_ipip_entry *ipip_entry)
3804 {
3805         bool removing;
3806
3807         if (!nh->nh_grp->gateway || nh->ipip_entry)
3808                 return;
3809
3810         nh->ipip_entry = ipip_entry;
3811         removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev);
3812         __mlxsw_sp_nexthop_neigh_update(nh, removing);
3813         mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common);
3814 }
3815
3816 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
3817                                        struct mlxsw_sp_nexthop *nh)
3818 {
3819         struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
3820
3821         if (!ipip_entry)
3822                 return;
3823
3824         __mlxsw_sp_nexthop_neigh_update(nh, true);
3825         nh->ipip_entry = NULL;
3826 }
3827
3828 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
3829                                         const struct fib_nh *fib_nh,
3830                                         enum mlxsw_sp_ipip_type *p_ipipt)
3831 {
3832         struct net_device *dev = fib_nh->fib_nh_dev;
3833
3834         return dev &&
3835                fib_nh->nh_parent->fib_type == RTN_UNICAST &&
3836                mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
3837 }
3838
3839 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
3840                                        struct mlxsw_sp_nexthop *nh)
3841 {
3842         switch (nh->type) {
3843         case MLXSW_SP_NEXTHOP_TYPE_ETH:
3844                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
3845                 mlxsw_sp_nexthop_rif_fini(nh);
3846                 break;
3847         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
3848                 mlxsw_sp_nexthop_rif_fini(nh);
3849                 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
3850                 break;
3851         }
3852 }
3853
3854 static int mlxsw_sp_nexthop4_type_init(struct mlxsw_sp *mlxsw_sp,
3855                                        struct mlxsw_sp_nexthop *nh,
3856                                        struct fib_nh *fib_nh)
3857 {
3858         const struct mlxsw_sp_ipip_ops *ipip_ops;
3859         struct net_device *dev = fib_nh->fib_nh_dev;
3860         struct mlxsw_sp_ipip_entry *ipip_entry;
3861         struct mlxsw_sp_rif *rif;
3862         int err;
3863
3864         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev);
3865         if (ipip_entry) {
3866                 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
3867                 if (ipip_ops->can_offload(mlxsw_sp, dev,
3868                                           MLXSW_SP_L3_PROTO_IPV4)) {
3869                         nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
3870                         mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
3871                         return 0;
3872                 }
3873         }
3874
3875         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
3876         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3877         if (!rif)
3878                 return 0;
3879
3880         mlxsw_sp_nexthop_rif_init(nh, rif);
3881         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
3882         if (err)
3883                 goto err_neigh_init;
3884
3885         return 0;
3886
3887 err_neigh_init:
3888         mlxsw_sp_nexthop_rif_fini(nh);
3889         return err;
3890 }
3891
3892 static void mlxsw_sp_nexthop4_type_fini(struct mlxsw_sp *mlxsw_sp,
3893                                         struct mlxsw_sp_nexthop *nh)
3894 {
3895         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
3896 }
3897
3898 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
3899                                   struct mlxsw_sp_nexthop_group *nh_grp,
3900                                   struct mlxsw_sp_nexthop *nh,
3901                                   struct fib_nh *fib_nh)
3902 {
3903         struct net_device *dev = fib_nh->fib_nh_dev;
3904         struct in_device *in_dev;
3905         int err;
3906
3907         nh->nh_grp = nh_grp;
3908         nh->key.fib_nh = fib_nh;
3909 #ifdef CONFIG_IP_ROUTE_MULTIPATH
3910         nh->nh_weight = fib_nh->fib_nh_weight;
3911 #else
3912         nh->nh_weight = 1;
3913 #endif
3914         memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4));
3915         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
3916         if (err)
3917                 return err;
3918
3919         mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
3920         list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
3921
3922         if (!dev)
3923                 return 0;
3924
3925         rcu_read_lock();
3926         in_dev = __in_dev_get_rcu(dev);
3927         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
3928             fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) {
3929                 rcu_read_unlock();
3930                 return 0;
3931         }
3932         rcu_read_unlock();
3933
3934         err = mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
3935         if (err)
3936                 goto err_nexthop_neigh_init;
3937
3938         return 0;
3939
3940 err_nexthop_neigh_init:
3941         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
3942         return err;
3943 }
3944
3945 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
3946                                    struct mlxsw_sp_nexthop *nh)
3947 {
3948         mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
3949         list_del(&nh->router_list_node);
3950         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
3951         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
3952 }
3953
3954 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
3955                                     unsigned long event, struct fib_nh *fib_nh)
3956 {
3957         struct mlxsw_sp_nexthop_key key;
3958         struct mlxsw_sp_nexthop *nh;
3959
3960         if (mlxsw_sp->router->aborted)
3961                 return;
3962
3963         key.fib_nh = fib_nh;
3964         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
3965         if (!nh)
3966                 return;
3967
3968         switch (event) {
3969         case FIB_EVENT_NH_ADD:
3970                 mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
3971                 break;
3972         case FIB_EVENT_NH_DEL:
3973                 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
3974                 break;
3975         }
3976
3977         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
3978 }
3979
3980 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
3981                                         struct mlxsw_sp_rif *rif)
3982 {
3983         struct mlxsw_sp_nexthop *nh;
3984         bool removing;
3985
3986         list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) {
3987                 switch (nh->type) {
3988                 case MLXSW_SP_NEXTHOP_TYPE_ETH:
3989                         removing = false;
3990                         break;
3991                 case MLXSW_SP_NEXTHOP_TYPE_IPIP:
3992                         removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev);
3993                         break;
3994                 default:
3995                         WARN_ON(1);
3996                         continue;
3997                 }
3998
3999                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
4000                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
4001         }
4002 }
4003
4004 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
4005                                          struct mlxsw_sp_rif *old_rif,
4006                                          struct mlxsw_sp_rif *new_rif)
4007 {
4008         struct mlxsw_sp_nexthop *nh;
4009
4010         list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list);
4011         list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node)
4012                 nh->rif = new_rif;
4013         mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif);
4014 }
4015
4016 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4017                                            struct mlxsw_sp_rif *rif)
4018 {
4019         struct mlxsw_sp_nexthop *nh, *tmp;
4020
4021         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
4022                 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4023                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
4024         }
4025 }
4026
4027 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
4028                                    struct fib_info *fi)
4029 {
4030         const struct fib_nh *nh = fib_info_nh(fi, 0);
4031
4032         return nh->fib_nh_scope == RT_SCOPE_LINK ||
4033                mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL);
4034 }
4035
4036 static struct mlxsw_sp_nexthop_group *
4037 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
4038 {
4039         unsigned int nhs = fib_info_num_path(fi);
4040         struct mlxsw_sp_nexthop_group *nh_grp;
4041         struct mlxsw_sp_nexthop *nh;
4042         struct fib_nh *fib_nh;
4043         int i;
4044         int err;
4045
4046         nh_grp = kzalloc(struct_size(nh_grp, nexthops, nhs), GFP_KERNEL);
4047         if (!nh_grp)
4048                 return ERR_PTR(-ENOMEM);
4049         nh_grp->priv = fi;
4050         INIT_LIST_HEAD(&nh_grp->fib_list);
4051         nh_grp->neigh_tbl = &arp_tbl;
4052
4053         nh_grp->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi);
4054         nh_grp->count = nhs;
4055         fib_info_hold(fi);
4056         for (i = 0; i < nh_grp->count; i++) {
4057                 nh = &nh_grp->nexthops[i];
4058                 fib_nh = fib_info_nh(fi, i);
4059                 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
4060                 if (err)
4061                         goto err_nexthop4_init;
4062         }
4063         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
4064         if (err)
4065                 goto err_nexthop_group_insert;
4066         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4067         return nh_grp;
4068
4069 err_nexthop_group_insert:
4070 err_nexthop4_init:
4071         for (i--; i >= 0; i--) {
4072                 nh = &nh_grp->nexthops[i];
4073                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
4074         }
4075         fib_info_put(fi);
4076         kfree(nh_grp);
4077         return ERR_PTR(err);
4078 }
4079
4080 static void
4081 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
4082                                 struct mlxsw_sp_nexthop_group *nh_grp)
4083 {
4084         struct mlxsw_sp_nexthop *nh;
4085         int i;
4086
4087         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
4088         for (i = 0; i < nh_grp->count; i++) {
4089                 nh = &nh_grp->nexthops[i];
4090                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
4091         }
4092         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4093         WARN_ON_ONCE(nh_grp->adj_index_valid);
4094         fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp));
4095         kfree(nh_grp);
4096 }
4097
4098 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
4099                                        struct mlxsw_sp_fib_entry *fib_entry,
4100                                        struct fib_info *fi)
4101 {
4102         struct mlxsw_sp_nexthop_group *nh_grp;
4103
4104         nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
4105         if (!nh_grp) {
4106                 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
4107                 if (IS_ERR(nh_grp))
4108                         return PTR_ERR(nh_grp);
4109         }
4110         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
4111         fib_entry->nh_group = nh_grp;
4112         return 0;
4113 }
4114
4115 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
4116                                         struct mlxsw_sp_fib_entry *fib_entry)
4117 {
4118         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
4119
4120         list_del(&fib_entry->nexthop_group_node);
4121         if (!list_empty(&nh_grp->fib_list))
4122                 return;
4123         mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
4124 }
4125
4126 static bool
4127 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
4128 {
4129         struct mlxsw_sp_fib4_entry *fib4_entry;
4130
4131         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
4132                                   common);
4133         return !fib4_entry->tos;
4134 }
4135
4136 static bool
4137 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
4138 {
4139         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
4140
4141         switch (fib_entry->fib_node->fib->proto) {
4142         case MLXSW_SP_L3_PROTO_IPV4:
4143                 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
4144                         return false;
4145                 break;
4146         case MLXSW_SP_L3_PROTO_IPV6:
4147                 break;
4148         }
4149
4150         switch (fib_entry->type) {
4151         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
4152                 return !!nh_group->adj_index_valid;
4153         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
4154                 return !!nh_group->nh_rif;
4155         case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
4156         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
4157         case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
4158                 return true;
4159         default:
4160                 return false;
4161         }
4162 }
4163
4164 static struct mlxsw_sp_nexthop *
4165 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
4166                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
4167 {
4168         int i;
4169
4170         for (i = 0; i < nh_grp->count; i++) {
4171                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
4172                 struct fib6_info *rt = mlxsw_sp_rt6->rt;
4173
4174                 if (nh->rif && nh->rif->dev == rt->fib6_nh->fib_nh_dev &&
4175                     ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
4176                                     &rt->fib6_nh->fib_nh_gw6))
4177                         return nh;
4178                 continue;
4179         }
4180
4181         return NULL;
4182 }
4183
4184 static void
4185 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
4186                                  struct mlxsw_sp_fib_entry *fib_entry)
4187 {
4188         struct fib_info *fi = mlxsw_sp_nexthop4_group_fi(fib_entry->nh_group);
4189         u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
4190         int dst_len = fib_entry->fib_node->key.prefix_len;
4191         struct mlxsw_sp_fib4_entry *fib4_entry;
4192         struct fib_rt_info fri;
4193         bool should_offload;
4194
4195         should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
4196         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
4197                                   common);
4198         fri.fi = fi;
4199         fri.tb_id = fib4_entry->tb_id;
4200         fri.dst = cpu_to_be32(*p_dst);
4201         fri.dst_len = dst_len;
4202         fri.tos = fib4_entry->tos;
4203         fri.type = fib4_entry->type;
4204         fri.offload = should_offload;
4205         fri.trap = !should_offload;
4206         fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
4207 }
4208
4209 static void
4210 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
4211                                    struct mlxsw_sp_fib_entry *fib_entry)
4212 {
4213         struct fib_info *fi = mlxsw_sp_nexthop4_group_fi(fib_entry->nh_group);
4214         u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
4215         int dst_len = fib_entry->fib_node->key.prefix_len;
4216         struct mlxsw_sp_fib4_entry *fib4_entry;
4217         struct fib_rt_info fri;
4218
4219         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
4220                                   common);
4221         fri.fi = fi;
4222         fri.tb_id = fib4_entry->tb_id;
4223         fri.dst = cpu_to_be32(*p_dst);
4224         fri.dst_len = dst_len;
4225         fri.tos = fib4_entry->tos;
4226         fri.type = fib4_entry->type;
4227         fri.offload = false;
4228         fri.trap = false;
4229         fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
4230 }
4231
4232 static void
4233 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
4234                                  struct mlxsw_sp_fib_entry *fib_entry)
4235 {
4236         struct mlxsw_sp_fib6_entry *fib6_entry;
4237         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4238         bool should_offload;
4239
4240         should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
4241
4242         /* In IPv6 a multipath route is represented using multiple routes, so
4243          * we need to set the flags on all of them.
4244          */
4245         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
4246                                   common);
4247         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
4248                 fib6_info_hw_flags_set(mlxsw_sp_rt6->rt, should_offload,
4249                                        !should_offload);
4250 }
4251
4252 static void
4253 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
4254                                    struct mlxsw_sp_fib_entry *fib_entry)
4255 {
4256         struct mlxsw_sp_fib6_entry *fib6_entry;
4257         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4258
4259         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
4260                                   common);
4261         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
4262                 fib6_info_hw_flags_set(mlxsw_sp_rt6->rt, false, false);
4263 }
4264
4265 static void
4266 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
4267                                 struct mlxsw_sp_fib_entry *fib_entry)
4268 {
4269         switch (fib_entry->fib_node->fib->proto) {
4270         case MLXSW_SP_L3_PROTO_IPV4:
4271                 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry);
4272                 break;
4273         case MLXSW_SP_L3_PROTO_IPV6:
4274                 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry);
4275                 break;
4276         }
4277 }
4278
4279 static void
4280 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
4281                                   struct mlxsw_sp_fib_entry *fib_entry)
4282 {
4283         switch (fib_entry->fib_node->fib->proto) {
4284         case MLXSW_SP_L3_PROTO_IPV4:
4285                 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry);
4286                 break;
4287         case MLXSW_SP_L3_PROTO_IPV6:
4288                 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry);
4289                 break;
4290         }
4291 }
4292
4293 static void
4294 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp,
4295                                     struct mlxsw_sp_fib_entry *fib_entry,
4296                                     enum mlxsw_sp_fib_entry_op op)
4297 {
4298         switch (op) {
4299         case MLXSW_SP_FIB_ENTRY_OP_WRITE:
4300                 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry);
4301                 break;
4302         case MLXSW_SP_FIB_ENTRY_OP_DELETE:
4303                 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry);
4304                 break;
4305         default:
4306                 break;
4307         }
4308 }
4309
4310 void
4311 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl, enum mlxsw_sp_l3proto proto,
4312                               enum mlxsw_sp_fib_entry_op op, u16 virtual_router,
4313                               u8 prefix_len, unsigned char *addr)
4314 {
4315         enum mlxsw_reg_ralxx_protocol ralxx_proto;
4316         enum mlxsw_reg_ralue_op ralue_op;
4317
4318         ralxx_proto = (enum mlxsw_reg_ralxx_protocol) proto;
4319
4320         switch (op) {
4321         case MLXSW_SP_FIB_ENTRY_OP_WRITE:
4322                 ralue_op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
4323                 break;
4324         case MLXSW_SP_FIB_ENTRY_OP_DELETE:
4325                 ralue_op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
4326                 break;
4327         default:
4328                 WARN_ON_ONCE(1);
4329                 return;
4330         }
4331
4332         switch (proto) {
4333         case MLXSW_SP_L3_PROTO_IPV4:
4334                 mlxsw_reg_ralue_pack4(ralue_pl, ralxx_proto, ralue_op,
4335                                       virtual_router, prefix_len, (u32 *) addr);
4336                 break;
4337         case MLXSW_SP_L3_PROTO_IPV6:
4338                 mlxsw_reg_ralue_pack6(ralue_pl, ralxx_proto, ralue_op,
4339                                       virtual_router, prefix_len, addr);
4340                 break;
4341         }
4342 }
4343
4344 static void mlxsw_sp_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4345                                     struct mlxsw_sp_fib_entry *fib_entry,
4346                                     enum mlxsw_sp_fib_entry_op op)
4347 {
4348         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
4349
4350         mlxsw_sp_fib_entry_ralue_pack(op_ctx->ralue_pl, fib->proto, op,
4351                                       fib->vr->id,
4352                                       fib_entry->fib_node->key.prefix_len,
4353                                       fib_entry->fib_node->key.addr);
4354 }
4355
4356 static int mlxsw_sp_adj_discard_write(struct mlxsw_sp *mlxsw_sp, u16 rif_index)
4357 {
4358         enum mlxsw_reg_ratr_trap_action trap_action;
4359         char ratr_pl[MLXSW_REG_RATR_LEN];
4360         int err;
4361
4362         if (mlxsw_sp->router->adj_discard_index_valid)
4363                 return 0;
4364
4365         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4366                                   &mlxsw_sp->router->adj_discard_index);
4367         if (err)
4368                 return err;
4369
4370         trap_action = MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS;
4371         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true,
4372                             MLXSW_REG_RATR_TYPE_ETHERNET,
4373                             mlxsw_sp->router->adj_discard_index, rif_index);
4374         mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action);
4375         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
4376         if (err)
4377                 goto err_ratr_write;
4378
4379         mlxsw_sp->router->adj_discard_index_valid = true;
4380
4381         return 0;
4382
4383 err_ratr_write:
4384         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4385                            mlxsw_sp->router->adj_discard_index);
4386         return err;
4387 }
4388
4389 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
4390                                         struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4391                                         struct mlxsw_sp_fib_entry *fib_entry,
4392                                         enum mlxsw_sp_fib_entry_op op)
4393 {
4394         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
4395         enum mlxsw_reg_ralue_trap_action trap_action;
4396         char *ralue_pl = op_ctx->ralue_pl;
4397         u16 trap_id = 0;
4398         u32 adjacency_index = 0;
4399         u16 ecmp_size = 0;
4400         int err;
4401
4402         /* In case the nexthop group adjacency index is valid, use it
4403          * with provided ECMP size. Otherwise, setup trap and pass
4404          * traffic to kernel.
4405          */
4406         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
4407                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4408                 adjacency_index = fib_entry->nh_group->adj_index;
4409                 ecmp_size = fib_entry->nh_group->ecmp_size;
4410         } else if (!nh_group->adj_index_valid && nh_group->count &&
4411                    nh_group->nh_rif) {
4412                 err = mlxsw_sp_adj_discard_write(mlxsw_sp,
4413                                                  nh_group->nh_rif->rif_index);
4414                 if (err)
4415                         return err;
4416                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4417                 adjacency_index = mlxsw_sp->router->adj_discard_index;
4418                 ecmp_size = 1;
4419         } else {
4420                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4421                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
4422         }
4423
4424         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4425         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
4426                                         adjacency_index, ecmp_size);
4427         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
4428 }
4429
4430 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
4431                                        struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4432                                        struct mlxsw_sp_fib_entry *fib_entry,
4433                                        enum mlxsw_sp_fib_entry_op op)
4434 {
4435         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
4436         enum mlxsw_reg_ralue_trap_action trap_action;
4437         char *ralue_pl = op_ctx->ralue_pl;
4438         u16 trap_id = 0;
4439         u16 rif_index = 0;
4440
4441         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
4442                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4443                 rif_index = rif->rif_index;
4444         } else {
4445                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4446                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
4447         }
4448
4449         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4450         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
4451                                        rif_index);
4452         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
4453 }
4454
4455 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
4456                                       struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4457                                       struct mlxsw_sp_fib_entry *fib_entry,
4458                                       enum mlxsw_sp_fib_entry_op op)
4459 {
4460         char *ralue_pl = op_ctx->ralue_pl;
4461
4462         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4463         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
4464         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
4465 }
4466
4467 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp,
4468                                            struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4469                                            struct mlxsw_sp_fib_entry *fib_entry,
4470                                            enum mlxsw_sp_fib_entry_op op)
4471 {
4472         enum mlxsw_reg_ralue_trap_action trap_action;
4473         char *ralue_pl = op_ctx->ralue_pl;
4474
4475         trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR;
4476         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4477         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, 0, 0);
4478         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
4479 }
4480
4481 static int
4482 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp,
4483                                   struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4484                                   struct mlxsw_sp_fib_entry *fib_entry,
4485                                   enum mlxsw_sp_fib_entry_op op)
4486 {
4487         enum mlxsw_reg_ralue_trap_action trap_action;
4488         char *ralue_pl = op_ctx->ralue_pl;
4489         u16 trap_id;
4490
4491         trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4492         trap_id = MLXSW_TRAP_ID_RTR_INGRESS1;
4493
4494         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4495         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 0);
4496         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
4497 }
4498
4499 static int
4500 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
4501                                  struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4502                                  struct mlxsw_sp_fib_entry *fib_entry,
4503                                  enum mlxsw_sp_fib_entry_op op)
4504 {
4505         struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
4506         const struct mlxsw_sp_ipip_ops *ipip_ops;
4507
4508         if (WARN_ON(!ipip_entry))
4509                 return -EINVAL;
4510
4511         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
4512         return ipip_ops->fib_entry_op(mlxsw_sp, op_ctx, ipip_entry, op,
4513                                       fib_entry->decap.tunnel_index);
4514 }
4515
4516 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp,
4517                                            struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4518                                            struct mlxsw_sp_fib_entry *fib_entry,
4519                                            enum mlxsw_sp_fib_entry_op op)
4520 {
4521         char *ralue_pl = op_ctx->ralue_pl;
4522
4523         mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
4524         mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl,
4525                                            fib_entry->decap.tunnel_index);
4526         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
4527 }
4528
4529 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
4530                                    struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4531                                    struct mlxsw_sp_fib_entry *fib_entry,
4532                                    enum mlxsw_sp_fib_entry_op op)
4533 {
4534         switch (fib_entry->type) {
4535         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
4536                 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, op_ctx, fib_entry, op);
4537         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
4538                 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, op_ctx, fib_entry, op);
4539         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
4540                 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, op_ctx, fib_entry, op);
4541         case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
4542                 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, op_ctx, fib_entry, op);
4543         case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE:
4544                 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, op_ctx, fib_entry, op);
4545         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
4546                 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, op_ctx, fib_entry, op);
4547         case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
4548                 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, op_ctx, fib_entry, op);
4549         }
4550         return -EINVAL;
4551 }
4552
4553 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
4554                                  struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4555                                  struct mlxsw_sp_fib_entry *fib_entry,
4556                                  enum mlxsw_sp_fib_entry_op op)
4557 {
4558         int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, op);
4559
4560         if (err)
4561                 return err;
4562
4563         mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op);
4564
4565         return err;
4566 }
4567
4568 static int __mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
4569                                        struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4570                                        struct mlxsw_sp_fib_entry *fib_entry)
4571 {
4572         return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry,
4573                                      MLXSW_SP_FIB_ENTRY_OP_WRITE);
4574 }
4575
4576 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
4577                                      struct mlxsw_sp_fib_entry *fib_entry)
4578 {
4579         struct mlxsw_sp_fib_entry_op_ctx op_ctx = {};
4580
4581         return __mlxsw_sp_fib_entry_update(mlxsw_sp, &op_ctx, fib_entry);
4582 }
4583
4584 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
4585                                   struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4586                                   struct mlxsw_sp_fib_entry *fib_entry)
4587 {
4588         return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry,
4589                                      MLXSW_SP_FIB_ENTRY_OP_DELETE);
4590 }
4591
4592 static int
4593 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
4594                              const struct fib_entry_notifier_info *fen_info,
4595                              struct mlxsw_sp_fib_entry *fib_entry)
4596 {
4597         struct net_device *dev = fib_info_nh(fen_info->fi, 0)->fib_nh_dev;
4598         union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
4599         struct mlxsw_sp_router *router = mlxsw_sp->router;
4600         u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id);
4601         struct mlxsw_sp_ipip_entry *ipip_entry;
4602         struct fib_info *fi = fen_info->fi;
4603
4604         switch (fen_info->type) {
4605         case RTN_LOCAL:
4606                 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, dev,
4607                                                  MLXSW_SP_L3_PROTO_IPV4, dip);
4608                 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
4609                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
4610                         return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
4611                                                              fib_entry,
4612                                                              ipip_entry);
4613                 }
4614                 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id,
4615                                                  MLXSW_SP_L3_PROTO_IPV4,
4616                                                  &dip)) {
4617                         u32 tunnel_index;
4618
4619                         tunnel_index = router->nve_decap_config.tunnel_index;
4620                         fib_entry->decap.tunnel_index = tunnel_index;
4621                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
4622                         return 0;
4623                 }
4624                 fallthrough;
4625         case RTN_BROADCAST:
4626                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
4627                 return 0;
4628         case RTN_BLACKHOLE:
4629                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
4630                 return 0;
4631         case RTN_UNREACHABLE:
4632         case RTN_PROHIBIT:
4633                 /* Packets hitting these routes need to be trapped, but
4634                  * can do so with a lower priority than packets directed
4635                  * at the host, so use action type local instead of trap.
4636                  */
4637                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
4638                 return 0;
4639         case RTN_UNICAST:
4640                 if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi))
4641                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
4642                 else
4643                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4644                 return 0;
4645         default:
4646                 return -EINVAL;
4647         }
4648 }
4649
4650 static void
4651 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
4652                                struct mlxsw_sp_fib_entry *fib_entry)
4653 {
4654         switch (fib_entry->type) {
4655         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
4656                 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
4657                 break;
4658         default:
4659                 break;
4660         }
4661 }
4662
4663 static struct mlxsw_sp_fib4_entry *
4664 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
4665                            struct mlxsw_sp_fib_node *fib_node,
4666                            const struct fib_entry_notifier_info *fen_info)
4667 {
4668         struct mlxsw_sp_fib4_entry *fib4_entry;
4669         struct mlxsw_sp_fib_entry *fib_entry;
4670         int err;
4671
4672         fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
4673         if (!fib4_entry)
4674                 return ERR_PTR(-ENOMEM);
4675         fib_entry = &fib4_entry->common;
4676
4677         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
4678         if (err)
4679                 goto err_fib4_entry_type_set;
4680
4681         err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
4682         if (err)
4683                 goto err_nexthop4_group_get;
4684
4685         fib4_entry->prio = fen_info->fi->fib_priority;
4686         fib4_entry->tb_id = fen_info->tb_id;
4687         fib4_entry->type = fen_info->type;
4688         fib4_entry->tos = fen_info->tos;
4689
4690         fib_entry->fib_node = fib_node;
4691
4692         return fib4_entry;
4693
4694 err_nexthop4_group_get:
4695         mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, fib_entry);
4696 err_fib4_entry_type_set:
4697         kfree(fib4_entry);
4698         return ERR_PTR(err);
4699 }
4700
4701 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
4702                                         struct mlxsw_sp_fib4_entry *fib4_entry)
4703 {
4704         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
4705         mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, &fib4_entry->common);
4706         kfree(fib4_entry);
4707 }
4708
4709 static struct mlxsw_sp_fib4_entry *
4710 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
4711                            const struct fib_entry_notifier_info *fen_info)
4712 {
4713         struct mlxsw_sp_fib4_entry *fib4_entry;
4714         struct mlxsw_sp_fib_node *fib_node;
4715         struct mlxsw_sp_fib *fib;
4716         struct mlxsw_sp_vr *vr;
4717
4718         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
4719         if (!vr)
4720                 return NULL;
4721         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
4722
4723         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
4724                                             sizeof(fen_info->dst),
4725                                             fen_info->dst_len);
4726         if (!fib_node)
4727                 return NULL;
4728
4729         fib4_entry = container_of(fib_node->fib_entry,
4730                                   struct mlxsw_sp_fib4_entry, common);
4731         if (fib4_entry->tb_id == fen_info->tb_id &&
4732             fib4_entry->tos == fen_info->tos &&
4733             fib4_entry->type == fen_info->type &&
4734             mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) ==
4735             fen_info->fi)
4736                 return fib4_entry;
4737
4738         return NULL;
4739 }
4740
4741 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
4742         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
4743         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
4744         .key_len = sizeof(struct mlxsw_sp_fib_key),
4745         .automatic_shrinking = true,
4746 };
4747
4748 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
4749                                     struct mlxsw_sp_fib_node *fib_node)
4750 {
4751         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
4752                                       mlxsw_sp_fib_ht_params);
4753 }
4754
4755 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
4756                                      struct mlxsw_sp_fib_node *fib_node)
4757 {
4758         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
4759                                mlxsw_sp_fib_ht_params);
4760 }
4761
4762 static struct mlxsw_sp_fib_node *
4763 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
4764                          size_t addr_len, unsigned char prefix_len)
4765 {
4766         struct mlxsw_sp_fib_key key;
4767
4768         memset(&key, 0, sizeof(key));
4769         memcpy(key.addr, addr, addr_len);
4770         key.prefix_len = prefix_len;
4771         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
4772 }
4773
4774 static struct mlxsw_sp_fib_node *
4775 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
4776                          size_t addr_len, unsigned char prefix_len)
4777 {
4778         struct mlxsw_sp_fib_node *fib_node;
4779
4780         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
4781         if (!fib_node)
4782                 return NULL;
4783
4784         list_add(&fib_node->list, &fib->node_list);
4785         memcpy(fib_node->key.addr, addr, addr_len);
4786         fib_node->key.prefix_len = prefix_len;
4787
4788         return fib_node;
4789 }
4790
4791 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
4792 {
4793         list_del(&fib_node->list);
4794         kfree(fib_node);
4795 }
4796
4797 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
4798                                       struct mlxsw_sp_fib_node *fib_node)
4799 {
4800         struct mlxsw_sp_prefix_usage req_prefix_usage;
4801         struct mlxsw_sp_fib *fib = fib_node->fib;
4802         struct mlxsw_sp_lpm_tree *lpm_tree;
4803         int err;
4804
4805         lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto];
4806         if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
4807                 goto out;
4808
4809         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
4810         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
4811         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
4812                                          fib->proto);
4813         if (IS_ERR(lpm_tree))
4814                 return PTR_ERR(lpm_tree);
4815
4816         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
4817         if (err)
4818                 goto err_lpm_tree_replace;
4819
4820 out:
4821         lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++;
4822         return 0;
4823
4824 err_lpm_tree_replace:
4825         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
4826         return err;
4827 }
4828
4829 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
4830                                          struct mlxsw_sp_fib_node *fib_node)
4831 {
4832         struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
4833         struct mlxsw_sp_prefix_usage req_prefix_usage;
4834         struct mlxsw_sp_fib *fib = fib_node->fib;
4835         int err;
4836
4837         if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
4838                 return;
4839         /* Try to construct a new LPM tree from the current prefix usage
4840          * minus the unused one. If we fail, continue using the old one.
4841          */
4842         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
4843         mlxsw_sp_prefix_usage_clear(&req_prefix_usage,
4844                                     fib_node->key.prefix_len);
4845         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
4846                                          fib->proto);
4847         if (IS_ERR(lpm_tree))
4848                 return;
4849
4850         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
4851         if (err)
4852                 goto err_lpm_tree_replace;
4853
4854         return;
4855
4856 err_lpm_tree_replace:
4857         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
4858 }
4859
4860 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
4861                                   struct mlxsw_sp_fib_node *fib_node,
4862                                   struct mlxsw_sp_fib *fib)
4863 {
4864         int err;
4865
4866         err = mlxsw_sp_fib_node_insert(fib, fib_node);
4867         if (err)
4868                 return err;
4869         fib_node->fib = fib;
4870
4871         err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node);
4872         if (err)
4873                 goto err_fib_lpm_tree_link;
4874
4875         return 0;
4876
4877 err_fib_lpm_tree_link:
4878         fib_node->fib = NULL;
4879         mlxsw_sp_fib_node_remove(fib, fib_node);
4880         return err;
4881 }
4882
4883 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
4884                                    struct mlxsw_sp_fib_node *fib_node)
4885 {
4886         struct mlxsw_sp_fib *fib = fib_node->fib;
4887
4888         mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node);
4889         fib_node->fib = NULL;
4890         mlxsw_sp_fib_node_remove(fib, fib_node);
4891 }
4892
4893 static struct mlxsw_sp_fib_node *
4894 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
4895                       size_t addr_len, unsigned char prefix_len,
4896                       enum mlxsw_sp_l3proto proto)
4897 {
4898         struct mlxsw_sp_fib_node *fib_node;
4899         struct mlxsw_sp_fib *fib;
4900         struct mlxsw_sp_vr *vr;
4901         int err;
4902
4903         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL);
4904         if (IS_ERR(vr))
4905                 return ERR_CAST(vr);
4906         fib = mlxsw_sp_vr_fib(vr, proto);
4907
4908         fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
4909         if (fib_node)
4910                 return fib_node;
4911
4912         fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
4913         if (!fib_node) {
4914                 err = -ENOMEM;
4915                 goto err_fib_node_create;
4916         }
4917
4918         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
4919         if (err)
4920                 goto err_fib_node_init;
4921
4922         return fib_node;
4923
4924 err_fib_node_init:
4925         mlxsw_sp_fib_node_destroy(fib_node);
4926 err_fib_node_create:
4927         mlxsw_sp_vr_put(mlxsw_sp, vr);
4928         return ERR_PTR(err);
4929 }
4930
4931 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
4932                                   struct mlxsw_sp_fib_node *fib_node)
4933 {
4934         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
4935
4936         if (fib_node->fib_entry)
4937                 return;
4938         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
4939         mlxsw_sp_fib_node_destroy(fib_node);
4940         mlxsw_sp_vr_put(mlxsw_sp, vr);
4941 }
4942
4943 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp,
4944                                         struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4945                                         struct mlxsw_sp_fib_entry *fib_entry)
4946 {
4947         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
4948         int err;
4949
4950         fib_node->fib_entry = fib_entry;
4951
4952         err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry);
4953         if (err)
4954                 goto err_fib_entry_update;
4955
4956         return 0;
4957
4958 err_fib_entry_update:
4959         fib_node->fib_entry = NULL;
4960         return err;
4961 }
4962
4963 static void __mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
4964                                              struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
4965                                              struct mlxsw_sp_fib_entry *fib_entry)
4966 {
4967         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
4968
4969         mlxsw_sp_fib_entry_del(mlxsw_sp, op_ctx, fib_entry);
4970         fib_node->fib_entry = NULL;
4971 }
4972
4973 static void mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
4974                                            struct mlxsw_sp_fib_entry *fib_entry)
4975 {
4976         struct mlxsw_sp_fib_entry_op_ctx op_ctx = {};
4977
4978         __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &op_ctx, fib_entry);
4979 }
4980
4981 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry)
4982 {
4983         struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
4984         struct mlxsw_sp_fib4_entry *fib4_replaced;
4985
4986         if (!fib_node->fib_entry)
4987                 return true;
4988
4989         fib4_replaced = container_of(fib_node->fib_entry,
4990                                      struct mlxsw_sp_fib4_entry, common);
4991         if (fib4_entry->tb_id == RT_TABLE_MAIN &&
4992             fib4_replaced->tb_id == RT_TABLE_LOCAL)
4993                 return false;
4994
4995         return true;
4996 }
4997
4998 static int
4999 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp,
5000                              struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5001                              const struct fib_entry_notifier_info *fen_info)
5002 {
5003         struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced;
5004         struct mlxsw_sp_fib_entry *replaced;
5005         struct mlxsw_sp_fib_node *fib_node;
5006         int err;
5007
5008         if (mlxsw_sp->router->aborted)
5009                 return 0;
5010
5011         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
5012                                          &fen_info->dst, sizeof(fen_info->dst),
5013                                          fen_info->dst_len,
5014                                          MLXSW_SP_L3_PROTO_IPV4);
5015         if (IS_ERR(fib_node)) {
5016                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
5017                 return PTR_ERR(fib_node);
5018         }
5019
5020         fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
5021         if (IS_ERR(fib4_entry)) {
5022                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
5023                 err = PTR_ERR(fib4_entry);
5024                 goto err_fib4_entry_create;
5025         }
5026
5027         if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) {
5028                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
5029                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5030                 return 0;
5031         }
5032
5033         replaced = fib_node->fib_entry;
5034         err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib4_entry->common);
5035         if (err) {
5036                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
5037                 goto err_fib_node_entry_link;
5038         }
5039
5040         /* Nothing to replace */
5041         if (!replaced)
5042                 return 0;
5043
5044         mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
5045         fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry,
5046                                      common);
5047         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced);
5048
5049         return 0;
5050
5051 err_fib_node_entry_link:
5052         fib_node->fib_entry = replaced;
5053         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
5054 err_fib4_entry_create:
5055         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5056         return err;
5057 }
5058
5059 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
5060                                      struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5061                                      struct fib_entry_notifier_info *fen_info)
5062 {
5063         struct mlxsw_sp_fib4_entry *fib4_entry;
5064         struct mlxsw_sp_fib_node *fib_node;
5065
5066         if (mlxsw_sp->router->aborted)
5067                 return;
5068
5069         fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
5070         if (!fib4_entry)
5071                 return;
5072         fib_node = fib4_entry->common.fib_node;
5073
5074         __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib4_entry->common);
5075         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
5076         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5077 }
5078
5079 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt)
5080 {
5081         /* Multicast routes aren't supported, so ignore them. Neighbour
5082          * Discovery packets are specifically trapped.
5083          */
5084         if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST)
5085                 return true;
5086
5087         /* Cloned routes are irrelevant in the forwarding path. */
5088         if (rt->fib6_flags & RTF_CACHE)
5089                 return true;
5090
5091         return false;
5092 }
5093
5094 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt)
5095 {
5096         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5097
5098         mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
5099         if (!mlxsw_sp_rt6)
5100                 return ERR_PTR(-ENOMEM);
5101
5102         /* In case of route replace, replaced route is deleted with
5103          * no notification. Take reference to prevent accessing freed
5104          * memory.
5105          */
5106         mlxsw_sp_rt6->rt = rt;
5107         fib6_info_hold(rt);
5108
5109         return mlxsw_sp_rt6;
5110 }
5111
5112 #if IS_ENABLED(CONFIG_IPV6)
5113 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
5114 {
5115         fib6_info_release(rt);
5116 }
5117 #else
5118 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
5119 {
5120 }
5121 #endif
5122
5123 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
5124 {
5125         struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
5126
5127         fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
5128         mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
5129         kfree(mlxsw_sp_rt6);
5130 }
5131
5132 static struct fib6_info *
5133 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
5134 {
5135         return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
5136                                 list)->rt;
5137 }
5138
5139 static struct mlxsw_sp_rt6 *
5140 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
5141                             const struct fib6_info *rt)
5142 {
5143         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5144
5145         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
5146                 if (mlxsw_sp_rt6->rt == rt)
5147                         return mlxsw_sp_rt6;
5148         }
5149
5150         return NULL;
5151 }
5152
5153 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
5154                                         const struct fib6_info *rt,
5155                                         enum mlxsw_sp_ipip_type *ret)
5156 {
5157         return rt->fib6_nh->fib_nh_dev &&
5158                mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret);
5159 }
5160
5161 static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp,
5162                                        struct mlxsw_sp_nexthop_group *nh_grp,
5163                                        struct mlxsw_sp_nexthop *nh,
5164                                        const struct fib6_info *rt)
5165 {
5166         const struct mlxsw_sp_ipip_ops *ipip_ops;
5167         struct mlxsw_sp_ipip_entry *ipip_entry;
5168         struct net_device *dev = rt->fib6_nh->fib_nh_dev;
5169         struct mlxsw_sp_rif *rif;
5170         int err;
5171
5172         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev);
5173         if (ipip_entry) {
5174                 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
5175                 if (ipip_ops->can_offload(mlxsw_sp, dev,
5176                                           MLXSW_SP_L3_PROTO_IPV6)) {
5177                         nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
5178                         mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
5179                         return 0;
5180                 }
5181         }
5182
5183         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
5184         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5185         if (!rif)
5186                 return 0;
5187         mlxsw_sp_nexthop_rif_init(nh, rif);
5188
5189         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
5190         if (err)
5191                 goto err_nexthop_neigh_init;
5192
5193         return 0;
5194
5195 err_nexthop_neigh_init:
5196         mlxsw_sp_nexthop_rif_fini(nh);
5197         return err;
5198 }
5199
5200 static void mlxsw_sp_nexthop6_type_fini(struct mlxsw_sp *mlxsw_sp,
5201                                         struct mlxsw_sp_nexthop *nh)
5202 {
5203         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
5204 }
5205
5206 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
5207                                   struct mlxsw_sp_nexthop_group *nh_grp,
5208                                   struct mlxsw_sp_nexthop *nh,
5209                                   const struct fib6_info *rt)
5210 {
5211         struct net_device *dev = rt->fib6_nh->fib_nh_dev;
5212
5213         nh->nh_grp = nh_grp;
5214         nh->nh_weight = rt->fib6_nh->fib_nh_weight;
5215         memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr));
5216         mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
5217
5218         list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
5219
5220         if (!dev)
5221                 return 0;
5222         nh->ifindex = dev->ifindex;
5223
5224         return mlxsw_sp_nexthop6_type_init(mlxsw_sp, nh_grp, nh, rt);
5225 }
5226
5227 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
5228                                    struct mlxsw_sp_nexthop *nh)
5229 {
5230         mlxsw_sp_nexthop6_type_fini(mlxsw_sp, nh);
5231         list_del(&nh->router_list_node);
5232         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
5233 }
5234
5235 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
5236                                     const struct fib6_info *rt)
5237 {
5238         return rt->fib6_nh->fib_nh_gw_family ||
5239                mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
5240 }
5241
5242 static struct mlxsw_sp_nexthop_group *
5243 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
5244                                struct mlxsw_sp_fib6_entry *fib6_entry)
5245 {
5246         struct mlxsw_sp_nexthop_group *nh_grp;
5247         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5248         struct mlxsw_sp_nexthop *nh;
5249         int i = 0;
5250         int err;
5251
5252         nh_grp = kzalloc(struct_size(nh_grp, nexthops, fib6_entry->nrt6),
5253                          GFP_KERNEL);
5254         if (!nh_grp)
5255                 return ERR_PTR(-ENOMEM);
5256         INIT_LIST_HEAD(&nh_grp->fib_list);
5257 #if IS_ENABLED(CONFIG_IPV6)
5258         nh_grp->neigh_tbl = &nd_tbl;
5259 #endif
5260         mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
5261                                         struct mlxsw_sp_rt6, list);
5262         nh_grp->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
5263         nh_grp->count = fib6_entry->nrt6;
5264         for (i = 0; i < nh_grp->count; i++) {
5265                 struct fib6_info *rt = mlxsw_sp_rt6->rt;
5266
5267                 nh = &nh_grp->nexthops[i];
5268                 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
5269                 if (err)
5270                         goto err_nexthop6_init;
5271                 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
5272         }
5273
5274         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
5275         if (err)
5276                 goto err_nexthop_group_insert;
5277
5278         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5279         return nh_grp;
5280
5281 err_nexthop_group_insert:
5282 err_nexthop6_init:
5283         for (i--; i >= 0; i--) {
5284                 nh = &nh_grp->nexthops[i];
5285                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
5286         }
5287         kfree(nh_grp);
5288         return ERR_PTR(err);
5289 }
5290
5291 static void
5292 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
5293                                 struct mlxsw_sp_nexthop_group *nh_grp)
5294 {
5295         struct mlxsw_sp_nexthop *nh;
5296         int i = nh_grp->count;
5297
5298         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
5299         for (i--; i >= 0; i--) {
5300                 nh = &nh_grp->nexthops[i];
5301                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
5302         }
5303         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5304         WARN_ON(nh_grp->adj_index_valid);
5305         kfree(nh_grp);
5306 }
5307
5308 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
5309                                        struct mlxsw_sp_fib6_entry *fib6_entry)
5310 {
5311         struct mlxsw_sp_nexthop_group *nh_grp;
5312
5313         nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
5314         if (!nh_grp) {
5315                 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
5316                 if (IS_ERR(nh_grp))
5317                         return PTR_ERR(nh_grp);
5318         }
5319
5320         list_add_tail(&fib6_entry->common.nexthop_group_node,
5321                       &nh_grp->fib_list);
5322         fib6_entry->common.nh_group = nh_grp;
5323
5324         /* The route and the nexthop are described by the same struct, so we
5325          * need to the update the nexthop offload indication for the new route.
5326          */
5327         __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
5328
5329         return 0;
5330 }
5331
5332 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
5333                                         struct mlxsw_sp_fib_entry *fib_entry)
5334 {
5335         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
5336
5337         list_del(&fib_entry->nexthop_group_node);
5338         if (!list_empty(&nh_grp->fib_list))
5339                 return;
5340         mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
5341 }
5342
5343 static int mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
5344                                           struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5345                                           struct mlxsw_sp_fib6_entry *fib6_entry)
5346 {
5347         struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
5348         int err;
5349
5350         fib6_entry->common.nh_group = NULL;
5351         list_del(&fib6_entry->common.nexthop_group_node);
5352
5353         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
5354         if (err)
5355                 goto err_nexthop6_group_get;
5356
5357         /* In case this entry is offloaded, then the adjacency index
5358          * currently associated with it in the device's table is that
5359          * of the old group. Start using the new one instead.
5360          */
5361         err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, &fib6_entry->common);
5362         if (err)
5363                 goto err_fib_entry_update;
5364
5365         if (list_empty(&old_nh_grp->fib_list))
5366                 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
5367
5368         return 0;
5369
5370 err_fib_entry_update:
5371         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
5372 err_nexthop6_group_get:
5373         list_add_tail(&fib6_entry->common.nexthop_group_node,
5374                       &old_nh_grp->fib_list);
5375         fib6_entry->common.nh_group = old_nh_grp;
5376         return err;
5377 }
5378
5379 static int
5380 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
5381                                 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5382                                 struct mlxsw_sp_fib6_entry *fib6_entry,
5383                                 struct fib6_info **rt_arr, unsigned int nrt6)
5384 {
5385         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5386         int err, i;
5387
5388         for (i = 0; i < nrt6; i++) {
5389                 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
5390                 if (IS_ERR(mlxsw_sp_rt6)) {
5391                         err = PTR_ERR(mlxsw_sp_rt6);
5392                         goto err_rt6_create;
5393                 }
5394
5395                 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
5396                 fib6_entry->nrt6++;
5397         }
5398
5399         err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry);
5400         if (err)
5401                 goto err_nexthop6_group_update;
5402
5403         return 0;
5404
5405 err_nexthop6_group_update:
5406         i = nrt6;
5407 err_rt6_create:
5408         for (i--; i >= 0; i--) {
5409                 fib6_entry->nrt6--;
5410                 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
5411                                                struct mlxsw_sp_rt6, list);
5412                 list_del(&mlxsw_sp_rt6->list);
5413                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5414         }
5415         return err;
5416 }
5417
5418 static void
5419 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
5420                                 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5421                                 struct mlxsw_sp_fib6_entry *fib6_entry,
5422                                 struct fib6_info **rt_arr, unsigned int nrt6)
5423 {
5424         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5425         int i;
5426
5427         for (i = 0; i < nrt6; i++) {
5428                 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry,
5429                                                            rt_arr[i]);
5430                 if (WARN_ON_ONCE(!mlxsw_sp_rt6))
5431                         continue;
5432
5433                 fib6_entry->nrt6--;
5434                 list_del(&mlxsw_sp_rt6->list);
5435                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5436         }
5437
5438         mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry);
5439 }
5440
5441 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
5442                                          struct mlxsw_sp_fib_entry *fib_entry,
5443                                          const struct fib6_info *rt)
5444 {
5445         /* Packets hitting RTF_REJECT routes need to be discarded by the
5446          * stack. We can rely on their destination device not having a
5447          * RIF (it's the loopback device) and can thus use action type
5448          * local, which will cause them to be trapped with a lower
5449          * priority than packets that need to be locally received.
5450          */
5451         if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))
5452                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
5453         else if (rt->fib6_type == RTN_BLACKHOLE)
5454                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
5455         else if (rt->fib6_flags & RTF_REJECT)
5456                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
5457         else if (mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt))
5458                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
5459         else
5460                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
5461 }
5462
5463 static void
5464 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
5465 {
5466         struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
5467
5468         list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
5469                                  list) {
5470                 fib6_entry->nrt6--;
5471                 list_del(&mlxsw_sp_rt6->list);
5472                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5473         }
5474 }
5475
5476 static struct mlxsw_sp_fib6_entry *
5477 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
5478                            struct mlxsw_sp_fib_node *fib_node,
5479                            struct fib6_info **rt_arr, unsigned int nrt6)
5480 {
5481         struct mlxsw_sp_fib6_entry *fib6_entry;
5482         struct mlxsw_sp_fib_entry *fib_entry;
5483         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5484         int err, i;
5485
5486         fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
5487         if (!fib6_entry)
5488                 return ERR_PTR(-ENOMEM);
5489         fib_entry = &fib6_entry->common;
5490
5491         INIT_LIST_HEAD(&fib6_entry->rt6_list);
5492
5493         for (i = 0; i < nrt6; i++) {
5494                 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
5495                 if (IS_ERR(mlxsw_sp_rt6)) {
5496                         err = PTR_ERR(mlxsw_sp_rt6);
5497                         goto err_rt6_create;
5498                 }
5499                 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
5500                 fib6_entry->nrt6++;
5501         }
5502
5503         mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]);
5504
5505         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
5506         if (err)
5507                 goto err_nexthop6_group_get;
5508
5509         fib_entry->fib_node = fib_node;
5510
5511         return fib6_entry;
5512
5513 err_nexthop6_group_get:
5514         i = nrt6;
5515 err_rt6_create:
5516         for (i--; i >= 0; i--) {
5517                 fib6_entry->nrt6--;
5518                 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
5519                                                struct mlxsw_sp_rt6, list);
5520                 list_del(&mlxsw_sp_rt6->list);
5521                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5522         }
5523         kfree(fib6_entry);
5524         return ERR_PTR(err);
5525 }
5526
5527 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
5528                                         struct mlxsw_sp_fib6_entry *fib6_entry)
5529 {
5530         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
5531         mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
5532         WARN_ON(fib6_entry->nrt6);
5533         kfree(fib6_entry);
5534 }
5535
5536 static struct mlxsw_sp_fib6_entry *
5537 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
5538                            const struct fib6_info *rt)
5539 {
5540         struct mlxsw_sp_fib6_entry *fib6_entry;
5541         struct mlxsw_sp_fib_node *fib_node;
5542         struct mlxsw_sp_fib *fib;
5543         struct fib6_info *cmp_rt;
5544         struct mlxsw_sp_vr *vr;
5545
5546         vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id);
5547         if (!vr)
5548                 return NULL;
5549         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
5550
5551         fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr,
5552                                             sizeof(rt->fib6_dst.addr),
5553                                             rt->fib6_dst.plen);
5554         if (!fib_node)
5555                 return NULL;
5556
5557         fib6_entry = container_of(fib_node->fib_entry,
5558                                   struct mlxsw_sp_fib6_entry, common);
5559         cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
5560         if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id &&
5561             rt->fib6_metric == cmp_rt->fib6_metric &&
5562             mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
5563                 return fib6_entry;
5564
5565         return NULL;
5566 }
5567
5568 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry)
5569 {
5570         struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
5571         struct mlxsw_sp_fib6_entry *fib6_replaced;
5572         struct fib6_info *rt, *rt_replaced;
5573
5574         if (!fib_node->fib_entry)
5575                 return true;
5576
5577         fib6_replaced = container_of(fib_node->fib_entry,
5578                                      struct mlxsw_sp_fib6_entry,
5579                                      common);
5580         rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
5581         rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced);
5582         if (rt->fib6_table->tb6_id == RT_TABLE_MAIN &&
5583             rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL)
5584                 return false;
5585
5586         return true;
5587 }
5588
5589 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp,
5590                                         struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5591                                         struct fib6_info **rt_arr, unsigned int nrt6)
5592 {
5593         struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced;
5594         struct mlxsw_sp_fib_entry *replaced;
5595         struct mlxsw_sp_fib_node *fib_node;
5596         struct fib6_info *rt = rt_arr[0];
5597         int err;
5598
5599         if (mlxsw_sp->router->aborted)
5600                 return 0;
5601
5602         if (rt->fib6_src.plen)
5603                 return -EINVAL;
5604
5605         if (mlxsw_sp_fib6_rt_should_ignore(rt))
5606                 return 0;
5607
5608         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
5609                                          &rt->fib6_dst.addr,
5610                                          sizeof(rt->fib6_dst.addr),
5611                                          rt->fib6_dst.plen,
5612                                          MLXSW_SP_L3_PROTO_IPV6);
5613         if (IS_ERR(fib_node))
5614                 return PTR_ERR(fib_node);
5615
5616         fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr,
5617                                                 nrt6);
5618         if (IS_ERR(fib6_entry)) {
5619                 err = PTR_ERR(fib6_entry);
5620                 goto err_fib6_entry_create;
5621         }
5622
5623         if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) {
5624                 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5625                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5626                 return 0;
5627         }
5628
5629         replaced = fib_node->fib_entry;
5630         err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib6_entry->common);
5631         if (err)
5632                 goto err_fib_node_entry_link;
5633
5634         /* Nothing to replace */
5635         if (!replaced)
5636                 return 0;
5637
5638         mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
5639         fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry,
5640                                      common);
5641         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced);
5642
5643         return 0;
5644
5645 err_fib_node_entry_link:
5646         fib_node->fib_entry = replaced;
5647         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5648 err_fib6_entry_create:
5649         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5650         return err;
5651 }
5652
5653 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp,
5654                                        struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5655                                        struct fib6_info **rt_arr, unsigned int nrt6)
5656 {
5657         struct mlxsw_sp_fib6_entry *fib6_entry;
5658         struct mlxsw_sp_fib_node *fib_node;
5659         struct fib6_info *rt = rt_arr[0];
5660         int err;
5661
5662         if (mlxsw_sp->router->aborted)
5663                 return 0;
5664
5665         if (rt->fib6_src.plen)
5666                 return -EINVAL;
5667
5668         if (mlxsw_sp_fib6_rt_should_ignore(rt))
5669                 return 0;
5670
5671         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
5672                                          &rt->fib6_dst.addr,
5673                                          sizeof(rt->fib6_dst.addr),
5674                                          rt->fib6_dst.plen,
5675                                          MLXSW_SP_L3_PROTO_IPV6);
5676         if (IS_ERR(fib_node))
5677                 return PTR_ERR(fib_node);
5678
5679         if (WARN_ON_ONCE(!fib_node->fib_entry)) {
5680                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5681                 return -EINVAL;
5682         }
5683
5684         fib6_entry = container_of(fib_node->fib_entry,
5685                                   struct mlxsw_sp_fib6_entry, common);
5686         err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6);
5687         if (err)
5688                 goto err_fib6_entry_nexthop_add;
5689
5690         return 0;
5691
5692 err_fib6_entry_nexthop_add:
5693         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5694         return err;
5695 }
5696
5697 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
5698                                      struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5699                                      struct fib6_info **rt_arr, unsigned int nrt6)
5700 {
5701         struct mlxsw_sp_fib6_entry *fib6_entry;
5702         struct mlxsw_sp_fib_node *fib_node;
5703         struct fib6_info *rt = rt_arr[0];
5704
5705         if (mlxsw_sp->router->aborted)
5706                 return;
5707
5708         if (mlxsw_sp_fib6_rt_should_ignore(rt))
5709                 return;
5710
5711         /* Multipath routes are first added to the FIB trie and only then
5712          * notified. If we vetoed the addition, we will get a delete
5713          * notification for a route we do not have. Therefore, do not warn if
5714          * route was not found.
5715          */
5716         fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
5717         if (!fib6_entry)
5718                 return;
5719
5720         /* If not all the nexthops are deleted, then only reduce the nexthop
5721          * group.
5722          */
5723         if (nrt6 != fib6_entry->nrt6) {
5724                 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6);
5725                 return;
5726         }
5727
5728         fib_node = fib6_entry->common.fib_node;
5729
5730         __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib6_entry->common);
5731         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5732         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5733 }
5734
5735 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
5736                                             enum mlxsw_sp_l3proto proto,
5737                                             u8 tree_id)
5738 {
5739         const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
5740         enum mlxsw_reg_ralxx_protocol ralxx_proto =
5741                                 (enum mlxsw_reg_ralxx_protocol) proto;
5742         char xralta_pl[MLXSW_REG_XRALTA_LEN];
5743         char xralst_pl[MLXSW_REG_XRALST_LEN];
5744         int i, err;
5745
5746         mlxsw_reg_xralta_pack(xralta_pl, true, ralxx_proto, tree_id);
5747         err = ll_ops->ralta_write(mlxsw_sp, xralta_pl);
5748         if (err)
5749                 return err;
5750
5751         mlxsw_reg_xralst_pack(xralst_pl, 0xff, tree_id);
5752         err = ll_ops->ralst_write(mlxsw_sp, xralst_pl);
5753         if (err)
5754                 return err;
5755
5756         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
5757                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
5758                 struct mlxsw_sp_fib_entry_op_ctx op_ctx = {};
5759                 char xraltb_pl[MLXSW_REG_XRALTB_LEN];
5760                 char *ralue_pl = op_ctx.ralue_pl;
5761
5762                 mlxsw_reg_xraltb_pack(xraltb_pl, vr->id, ralxx_proto, tree_id);
5763                 err = ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
5764                 if (err)
5765                         return err;
5766
5767                 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, proto,
5768                                               MLXSW_SP_FIB_ENTRY_OP_WRITE, vr->id, 0, NULL);
5769                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
5770                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
5771                                       ralue_pl);
5772                 if (err)
5773                         return err;
5774         }
5775
5776         return 0;
5777 }
5778
5779 static struct mlxsw_sp_mr_table *
5780 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family)
5781 {
5782         if (family == RTNL_FAMILY_IPMR)
5783                 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4];
5784         else
5785                 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
5786 }
5787
5788 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp,
5789                                      struct mfc_entry_notifier_info *men_info,
5790                                      bool replace)
5791 {
5792         struct mlxsw_sp_mr_table *mrt;
5793         struct mlxsw_sp_vr *vr;
5794
5795         if (mlxsw_sp->router->aborted)
5796                 return 0;
5797
5798         vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL);
5799         if (IS_ERR(vr))
5800                 return PTR_ERR(vr);
5801
5802         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
5803         return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace);
5804 }
5805
5806 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp,
5807                                       struct mfc_entry_notifier_info *men_info)
5808 {
5809         struct mlxsw_sp_mr_table *mrt;
5810         struct mlxsw_sp_vr *vr;
5811
5812         if (mlxsw_sp->router->aborted)
5813                 return;
5814
5815         vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id);
5816         if (WARN_ON(!vr))
5817                 return;
5818
5819         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
5820         mlxsw_sp_mr_route_del(mrt, men_info->mfc);
5821         mlxsw_sp_vr_put(mlxsw_sp, vr);
5822 }
5823
5824 static int
5825 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp,
5826                               struct vif_entry_notifier_info *ven_info)
5827 {
5828         struct mlxsw_sp_mr_table *mrt;
5829         struct mlxsw_sp_rif *rif;
5830         struct mlxsw_sp_vr *vr;
5831
5832         if (mlxsw_sp->router->aborted)
5833                 return 0;
5834
5835         vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL);
5836         if (IS_ERR(vr))
5837                 return PTR_ERR(vr);
5838
5839         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
5840         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev);
5841         return mlxsw_sp_mr_vif_add(mrt, ven_info->dev,
5842                                    ven_info->vif_index,
5843                                    ven_info->vif_flags, rif);
5844 }
5845
5846 static void
5847 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp,
5848                               struct vif_entry_notifier_info *ven_info)
5849 {
5850         struct mlxsw_sp_mr_table *mrt;
5851         struct mlxsw_sp_vr *vr;
5852
5853         if (mlxsw_sp->router->aborted)
5854                 return;
5855
5856         vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id);
5857         if (WARN_ON(!vr))
5858                 return;
5859
5860         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
5861         mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index);
5862         mlxsw_sp_vr_put(mlxsw_sp, vr);
5863 }
5864
5865 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
5866 {
5867         enum mlxsw_sp_l3proto proto = MLXSW_SP_L3_PROTO_IPV4;
5868         int err;
5869
5870         err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
5871                                                MLXSW_SP_LPM_TREE_MIN);
5872         if (err)
5873                 return err;
5874
5875         /* The multicast router code does not need an abort trap as by default,
5876          * packets that don't match any routes are trapped to the CPU.
5877          */
5878
5879         proto = MLXSW_SP_L3_PROTO_IPV6;
5880         return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
5881                                                 MLXSW_SP_LPM_TREE_MIN + 1);
5882 }
5883
5884 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
5885                                      struct mlxsw_sp_fib_node *fib_node)
5886 {
5887         struct mlxsw_sp_fib4_entry *fib4_entry;
5888
5889         fib4_entry = container_of(fib_node->fib_entry,
5890                                   struct mlxsw_sp_fib4_entry, common);
5891         mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
5892         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
5893         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5894 }
5895
5896 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
5897                                      struct mlxsw_sp_fib_node *fib_node)
5898 {
5899         struct mlxsw_sp_fib6_entry *fib6_entry;
5900
5901         fib6_entry = container_of(fib_node->fib_entry,
5902                                   struct mlxsw_sp_fib6_entry, common);
5903         mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
5904         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5905         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5906 }
5907
5908 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
5909                                     struct mlxsw_sp_fib_node *fib_node)
5910 {
5911         switch (fib_node->fib->proto) {
5912         case MLXSW_SP_L3_PROTO_IPV4:
5913                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
5914                 break;
5915         case MLXSW_SP_L3_PROTO_IPV6:
5916                 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
5917                 break;
5918         }
5919 }
5920
5921 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
5922                                   struct mlxsw_sp_vr *vr,
5923                                   enum mlxsw_sp_l3proto proto)
5924 {
5925         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
5926         struct mlxsw_sp_fib_node *fib_node, *tmp;
5927
5928         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
5929                 bool do_break = &tmp->list == &fib->node_list;
5930
5931                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
5932                 if (do_break)
5933                         break;
5934         }
5935 }
5936
5937 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
5938 {
5939         int i, j;
5940
5941         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
5942                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
5943
5944                 if (!mlxsw_sp_vr_is_used(vr))
5945                         continue;
5946
5947                 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++)
5948                         mlxsw_sp_mr_table_flush(vr->mr_table[j]);
5949                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
5950
5951                 /* If virtual router was only used for IPv4, then it's no
5952                  * longer used.
5953                  */
5954                 if (!mlxsw_sp_vr_is_used(vr))
5955                         continue;
5956                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
5957         }
5958
5959         /* After flushing all the routes, it is not possible anyone is still
5960          * using the adjacency index that is discarding packets, so free it in
5961          * case it was allocated.
5962          */
5963         if (!mlxsw_sp->router->adj_discard_index_valid)
5964                 return;
5965         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
5966                            mlxsw_sp->router->adj_discard_index);
5967         mlxsw_sp->router->adj_discard_index_valid = false;
5968 }
5969
5970 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
5971 {
5972         int err;
5973
5974         if (mlxsw_sp->router->aborted)
5975                 return;
5976         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
5977         mlxsw_sp_router_fib_flush(mlxsw_sp);
5978         mlxsw_sp->router->aborted = true;
5979         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
5980         if (err)
5981                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
5982 }
5983
5984 struct mlxsw_sp_fib6_event {
5985         struct fib6_info **rt_arr;
5986         unsigned int nrt6;
5987 };
5988
5989 struct mlxsw_sp_fib_event {
5990         struct list_head list; /* node in fib queue */
5991         union {
5992                 struct mlxsw_sp_fib6_event fib6_event;
5993                 struct fib_entry_notifier_info fen_info;
5994                 struct fib_rule_notifier_info fr_info;
5995                 struct fib_nh_notifier_info fnh_info;
5996                 struct mfc_entry_notifier_info men_info;
5997                 struct vif_entry_notifier_info ven_info;
5998         };
5999         struct mlxsw_sp *mlxsw_sp;
6000         unsigned long event;
6001         int family;
6002 };
6003
6004 static int
6005 mlxsw_sp_router_fib6_event_init(struct mlxsw_sp_fib6_event *fib6_event,
6006                                 struct fib6_entry_notifier_info *fen6_info)
6007 {
6008         struct fib6_info *rt = fen6_info->rt;
6009         struct fib6_info **rt_arr;
6010         struct fib6_info *iter;
6011         unsigned int nrt6;
6012         int i = 0;
6013
6014         nrt6 = fen6_info->nsiblings + 1;
6015
6016         rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC);
6017         if (!rt_arr)
6018                 return -ENOMEM;
6019
6020         fib6_event->rt_arr = rt_arr;
6021         fib6_event->nrt6 = nrt6;
6022
6023         rt_arr[0] = rt;
6024         fib6_info_hold(rt);
6025
6026         if (!fen6_info->nsiblings)
6027                 return 0;
6028
6029         list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
6030                 if (i == fen6_info->nsiblings)
6031                         break;
6032
6033                 rt_arr[i + 1] = iter;
6034                 fib6_info_hold(iter);
6035                 i++;
6036         }
6037         WARN_ON_ONCE(i != fen6_info->nsiblings);
6038
6039         return 0;
6040 }
6041
6042 static void
6043 mlxsw_sp_router_fib6_event_fini(struct mlxsw_sp_fib6_event *fib6_event)
6044 {
6045         int i;
6046
6047         for (i = 0; i < fib6_event->nrt6; i++)
6048                 mlxsw_sp_rt6_release(fib6_event->rt_arr[i]);
6049         kfree(fib6_event->rt_arr);
6050 }
6051
6052 static void mlxsw_sp_router_fib4_event_process(struct mlxsw_sp *mlxsw_sp,
6053                                                struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
6054                                                struct mlxsw_sp_fib_event *fib_event)
6055 {
6056         int err;
6057
6058         mutex_lock(&mlxsw_sp->router->lock);
6059         mlxsw_sp_span_respin(mlxsw_sp);
6060
6061         switch (fib_event->event) {
6062         case FIB_EVENT_ENTRY_REPLACE:
6063                 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, op_ctx, &fib_event->fen_info);
6064                 if (err)
6065                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6066                 fib_info_put(fib_event->fen_info.fi);
6067                 break;
6068         case FIB_EVENT_ENTRY_DEL:
6069                 mlxsw_sp_router_fib4_del(mlxsw_sp, op_ctx, &fib_event->fen_info);
6070                 fib_info_put(fib_event->fen_info.fi);
6071                 break;
6072         case FIB_EVENT_NH_ADD:
6073         case FIB_EVENT_NH_DEL:
6074                 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_event->event, fib_event->fnh_info.fib_nh);
6075                 fib_info_put(fib_event->fnh_info.fib_nh->nh_parent);
6076                 break;
6077         }
6078         mutex_unlock(&mlxsw_sp->router->lock);
6079 }
6080
6081 static void mlxsw_sp_router_fib6_event_process(struct mlxsw_sp *mlxsw_sp,
6082                                                struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
6083                                                struct mlxsw_sp_fib_event *fib_event)
6084 {
6085         int err;
6086
6087         mutex_lock(&mlxsw_sp->router->lock);
6088         mlxsw_sp_span_respin(mlxsw_sp);
6089
6090         switch (fib_event->event) {
6091         case FIB_EVENT_ENTRY_REPLACE:
6092                 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
6093                                                    fib_event->fib6_event.nrt6);
6094                 if (err)
6095                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6096                 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
6097                 break;
6098         case FIB_EVENT_ENTRY_APPEND:
6099                 err = mlxsw_sp_router_fib6_append(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
6100                                                   fib_event->fib6_event.nrt6);
6101                 if (err)
6102                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6103                 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
6104                 break;
6105         case FIB_EVENT_ENTRY_DEL:
6106                 mlxsw_sp_router_fib6_del(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
6107                                          fib_event->fib6_event.nrt6);
6108                 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
6109                 break;
6110         }
6111         mutex_unlock(&mlxsw_sp->router->lock);
6112 }
6113
6114 static void mlxsw_sp_router_fibmr_event_process(struct mlxsw_sp *mlxsw_sp,
6115                                                 struct mlxsw_sp_fib_event *fib_event)
6116 {
6117         bool replace;
6118         int err;
6119
6120         rtnl_lock();
6121         mutex_lock(&mlxsw_sp->router->lock);
6122         switch (fib_event->event) {
6123         case FIB_EVENT_ENTRY_REPLACE:
6124         case FIB_EVENT_ENTRY_ADD:
6125                 replace = fib_event->event == FIB_EVENT_ENTRY_REPLACE;
6126
6127                 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_event->men_info, replace);
6128                 if (err)
6129                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6130                 mr_cache_put(fib_event->men_info.mfc);
6131                 break;
6132         case FIB_EVENT_ENTRY_DEL:
6133                 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_event->men_info);
6134                 mr_cache_put(fib_event->men_info.mfc);
6135                 break;
6136         case FIB_EVENT_VIF_ADD:
6137                 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp,
6138                                                     &fib_event->ven_info);
6139                 if (err)
6140                         mlxsw_sp_router_fib_abort(mlxsw_sp);
6141                 dev_put(fib_event->ven_info.dev);
6142                 break;
6143         case FIB_EVENT_VIF_DEL:
6144                 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, &fib_event->ven_info);
6145                 dev_put(fib_event->ven_info.dev);
6146                 break;
6147         }
6148         mutex_unlock(&mlxsw_sp->router->lock);
6149         rtnl_unlock();
6150 }
6151
6152 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
6153 {
6154         struct mlxsw_sp_router *router = container_of(work, struct mlxsw_sp_router, fib_event_work);
6155         struct mlxsw_sp_fib_entry_op_ctx op_ctx = {};
6156         struct mlxsw_sp *mlxsw_sp = router->mlxsw_sp;
6157         struct mlxsw_sp_fib_event *fib_event, *tmp;
6158         LIST_HEAD(fib_event_queue);
6159
6160         spin_lock_bh(&router->fib_event_queue_lock);
6161         list_splice_init(&router->fib_event_queue, &fib_event_queue);
6162         spin_unlock_bh(&router->fib_event_queue_lock);
6163
6164         list_for_each_entry_safe(fib_event, tmp, &fib_event_queue, list) {
6165                 switch (fib_event->family) {
6166                 case AF_INET:
6167                         mlxsw_sp_router_fib4_event_process(mlxsw_sp, &op_ctx,
6168                                                            fib_event);
6169                         break;
6170                 case AF_INET6:
6171                         mlxsw_sp_router_fib6_event_process(mlxsw_sp, &op_ctx,
6172                                                            fib_event);
6173                         break;
6174                 case RTNL_FAMILY_IP6MR:
6175                 case RTNL_FAMILY_IPMR:
6176                         mlxsw_sp_router_fibmr_event_process(mlxsw_sp,
6177                                                             fib_event);
6178                         break;
6179                 default:
6180                         WARN_ON_ONCE(1);
6181                 }
6182                 kfree(fib_event);
6183                 cond_resched();
6184         }
6185 }
6186
6187 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event *fib_event,
6188                                        struct fib_notifier_info *info)
6189 {
6190         struct fib_entry_notifier_info *fen_info;
6191         struct fib_nh_notifier_info *fnh_info;
6192
6193         switch (fib_event->event) {
6194         case FIB_EVENT_ENTRY_REPLACE:
6195         case FIB_EVENT_ENTRY_DEL:
6196                 fen_info = container_of(info, struct fib_entry_notifier_info,
6197                                         info);
6198                 fib_event->fen_info = *fen_info;
6199                 /* Take reference on fib_info to prevent it from being
6200                  * freed while event is queued. Release it afterwards.
6201                  */
6202                 fib_info_hold(fib_event->fen_info.fi);
6203                 break;
6204         case FIB_EVENT_NH_ADD:
6205         case FIB_EVENT_NH_DEL:
6206                 fnh_info = container_of(info, struct fib_nh_notifier_info,
6207                                         info);
6208                 fib_event->fnh_info = *fnh_info;
6209                 fib_info_hold(fib_event->fnh_info.fib_nh->nh_parent);
6210                 break;
6211         }
6212 }
6213
6214 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event *fib_event,
6215                                       struct fib_notifier_info *info)
6216 {
6217         struct fib6_entry_notifier_info *fen6_info;
6218         int err;
6219
6220         switch (fib_event->event) {
6221         case FIB_EVENT_ENTRY_REPLACE:
6222         case FIB_EVENT_ENTRY_APPEND:
6223         case FIB_EVENT_ENTRY_DEL:
6224                 fen6_info = container_of(info, struct fib6_entry_notifier_info,
6225                                          info);
6226                 err = mlxsw_sp_router_fib6_event_init(&fib_event->fib6_event,
6227                                                       fen6_info);
6228                 if (err)
6229                         return err;
6230                 break;
6231         }
6232
6233         return 0;
6234 }
6235
6236 static void
6237 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event *fib_event,
6238                             struct fib_notifier_info *info)
6239 {
6240         switch (fib_event->event) {
6241         case FIB_EVENT_ENTRY_REPLACE:
6242         case FIB_EVENT_ENTRY_ADD:
6243         case FIB_EVENT_ENTRY_DEL:
6244                 memcpy(&fib_event->men_info, info, sizeof(fib_event->men_info));
6245                 mr_cache_hold(fib_event->men_info.mfc);
6246                 break;
6247         case FIB_EVENT_VIF_ADD:
6248         case FIB_EVENT_VIF_DEL:
6249                 memcpy(&fib_event->ven_info, info, sizeof(fib_event->ven_info));
6250                 dev_hold(fib_event->ven_info.dev);
6251                 break;
6252         }
6253 }
6254
6255 static int mlxsw_sp_router_fib_rule_event(unsigned long event,
6256                                           struct fib_notifier_info *info,
6257                                           struct mlxsw_sp *mlxsw_sp)
6258 {
6259         struct netlink_ext_ack *extack = info->extack;
6260         struct fib_rule_notifier_info *fr_info;
6261         struct fib_rule *rule;
6262         int err = 0;
6263
6264         /* nothing to do at the moment */
6265         if (event == FIB_EVENT_RULE_DEL)
6266                 return 0;
6267
6268         if (mlxsw_sp->router->aborted)
6269                 return 0;
6270
6271         fr_info = container_of(info, struct fib_rule_notifier_info, info);
6272         rule = fr_info->rule;
6273
6274         /* Rule only affects locally generated traffic */
6275         if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex)
6276                 return 0;
6277
6278         switch (info->family) {
6279         case AF_INET:
6280                 if (!fib4_rule_default(rule) && !rule->l3mdev)
6281                         err = -EOPNOTSUPP;
6282                 break;
6283         case AF_INET6:
6284                 if (!fib6_rule_default(rule) && !rule->l3mdev)
6285                         err = -EOPNOTSUPP;
6286                 break;
6287         case RTNL_FAMILY_IPMR:
6288                 if (!ipmr_rule_default(rule) && !rule->l3mdev)
6289                         err = -EOPNOTSUPP;
6290                 break;
6291         case RTNL_FAMILY_IP6MR:
6292                 if (!ip6mr_rule_default(rule) && !rule->l3mdev)
6293                         err = -EOPNOTSUPP;
6294                 break;
6295         }
6296
6297         if (err < 0)
6298                 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported");
6299
6300         return err;
6301 }
6302
6303 /* Called with rcu_read_lock() */
6304 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
6305                                      unsigned long event, void *ptr)
6306 {
6307         struct mlxsw_sp_fib_event *fib_event;
6308         struct fib_notifier_info *info = ptr;
6309         struct mlxsw_sp_router *router;
6310         int err;
6311
6312         if ((info->family != AF_INET && info->family != AF_INET6 &&
6313              info->family != RTNL_FAMILY_IPMR &&
6314              info->family != RTNL_FAMILY_IP6MR))
6315                 return NOTIFY_DONE;
6316
6317         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
6318
6319         switch (event) {
6320         case FIB_EVENT_RULE_ADD:
6321         case FIB_EVENT_RULE_DEL:
6322                 err = mlxsw_sp_router_fib_rule_event(event, info,
6323                                                      router->mlxsw_sp);
6324                 return notifier_from_errno(err);
6325         case FIB_EVENT_ENTRY_ADD:
6326         case FIB_EVENT_ENTRY_REPLACE:
6327         case FIB_EVENT_ENTRY_APPEND:
6328                 if (router->aborted) {
6329                         NL_SET_ERR_MSG_MOD(info->extack, "FIB offload was aborted. Not configuring route");
6330                         return notifier_from_errno(-EINVAL);
6331                 }
6332                 if (info->family == AF_INET) {
6333                         struct fib_entry_notifier_info *fen_info = ptr;
6334
6335                         if (fen_info->fi->fib_nh_is_v6) {
6336                                 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported");
6337                                 return notifier_from_errno(-EINVAL);
6338                         }
6339                         if (fen_info->fi->nh) {
6340                                 NL_SET_ERR_MSG_MOD(info->extack, "IPv4 route with nexthop objects is not supported");
6341                                 return notifier_from_errno(-EINVAL);
6342                         }
6343                 } else if (info->family == AF_INET6) {
6344                         struct fib6_entry_notifier_info *fen6_info;
6345
6346                         fen6_info = container_of(info,
6347                                                  struct fib6_entry_notifier_info,
6348                                                  info);
6349                         if (fen6_info->rt->nh) {
6350                                 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 route with nexthop objects is not supported");
6351                                 return notifier_from_errno(-EINVAL);
6352                         }
6353                 }
6354                 break;
6355         }
6356
6357         fib_event = kzalloc(sizeof(*fib_event), GFP_ATOMIC);
6358         if (!fib_event)
6359                 return NOTIFY_BAD;
6360
6361         fib_event->mlxsw_sp = router->mlxsw_sp;
6362         fib_event->event = event;
6363         fib_event->family = info->family;
6364
6365         switch (info->family) {
6366         case AF_INET:
6367                 mlxsw_sp_router_fib4_event(fib_event, info);
6368                 break;
6369         case AF_INET6:
6370                 err = mlxsw_sp_router_fib6_event(fib_event, info);
6371                 if (err)
6372                         goto err_fib_event;
6373                 break;
6374         case RTNL_FAMILY_IP6MR:
6375         case RTNL_FAMILY_IPMR:
6376                 mlxsw_sp_router_fibmr_event(fib_event, info);
6377                 break;
6378         }
6379
6380         /* Enqueue the event and trigger the work */
6381         spin_lock_bh(&router->fib_event_queue_lock);
6382         list_add_tail(&fib_event->list, &router->fib_event_queue);
6383         spin_unlock_bh(&router->fib_event_queue_lock);
6384         mlxsw_core_schedule_work(&router->fib_event_work);
6385
6386         return NOTIFY_DONE;
6387
6388 err_fib_event:
6389         kfree(fib_event);
6390         return NOTIFY_BAD;
6391 }
6392
6393 static struct mlxsw_sp_rif *
6394 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
6395                          const struct net_device *dev)
6396 {
6397         int i;
6398
6399         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
6400                 if (mlxsw_sp->router->rifs[i] &&
6401                     mlxsw_sp->router->rifs[i]->dev == dev)
6402                         return mlxsw_sp->router->rifs[i];
6403
6404         return NULL;
6405 }
6406
6407 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp,
6408                          const struct net_device *dev)
6409 {
6410         struct mlxsw_sp_rif *rif;
6411
6412         mutex_lock(&mlxsw_sp->router->lock);
6413         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6414         mutex_unlock(&mlxsw_sp->router->lock);
6415
6416         return rif;
6417 }
6418
6419 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev)
6420 {
6421         struct mlxsw_sp_rif *rif;
6422         u16 vid = 0;
6423
6424         mutex_lock(&mlxsw_sp->router->lock);
6425         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6426         if (!rif)
6427                 goto out;
6428
6429         /* We only return the VID for VLAN RIFs. Otherwise we return an
6430          * invalid value (0).
6431          */
6432         if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN)
6433                 goto out;
6434
6435         vid = mlxsw_sp_fid_8021q_vid(rif->fid);
6436
6437 out:
6438         mutex_unlock(&mlxsw_sp->router->lock);
6439         return vid;
6440 }
6441
6442 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
6443 {
6444         char ritr_pl[MLXSW_REG_RITR_LEN];
6445         int err;
6446
6447         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
6448         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6449         if (err)
6450                 return err;
6451
6452         mlxsw_reg_ritr_enable_set(ritr_pl, false);
6453         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6454 }
6455
6456 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
6457                                           struct mlxsw_sp_rif *rif)
6458 {
6459         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
6460         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
6461         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
6462 }
6463
6464 static bool
6465 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
6466                            unsigned long event)
6467 {
6468         struct inet6_dev *inet6_dev;
6469         bool addr_list_empty = true;
6470         struct in_device *idev;
6471
6472         switch (event) {
6473         case NETDEV_UP:
6474                 return rif == NULL;
6475         case NETDEV_DOWN:
6476                 rcu_read_lock();
6477                 idev = __in_dev_get_rcu(dev);
6478                 if (idev && idev->ifa_list)
6479                         addr_list_empty = false;
6480
6481                 inet6_dev = __in6_dev_get(dev);
6482                 if (addr_list_empty && inet6_dev &&
6483                     !list_empty(&inet6_dev->addr_list))
6484                         addr_list_empty = false;
6485                 rcu_read_unlock();
6486
6487                 /* macvlans do not have a RIF, but rather piggy back on the
6488                  * RIF of their lower device.
6489                  */
6490                 if (netif_is_macvlan(dev) && addr_list_empty)
6491                         return true;
6492
6493                 if (rif && addr_list_empty &&
6494                     !netif_is_l3_slave(rif->dev))
6495                         return true;
6496                 /* It is possible we already removed the RIF ourselves
6497                  * if it was assigned to a netdev that is now a bridge
6498                  * or LAG slave.
6499                  */
6500                 return false;
6501         }
6502
6503         return false;
6504 }
6505
6506 static enum mlxsw_sp_rif_type
6507 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
6508                       const struct net_device *dev)
6509 {
6510         enum mlxsw_sp_fid_type type;
6511
6512         if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
6513                 return MLXSW_SP_RIF_TYPE_IPIP_LB;
6514
6515         /* Otherwise RIF type is derived from the type of the underlying FID. */
6516         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
6517                 type = MLXSW_SP_FID_TYPE_8021Q;
6518         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
6519                 type = MLXSW_SP_FID_TYPE_8021Q;
6520         else if (netif_is_bridge_master(dev))
6521                 type = MLXSW_SP_FID_TYPE_8021D;
6522         else
6523                 type = MLXSW_SP_FID_TYPE_RFID;
6524
6525         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
6526 }
6527
6528 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
6529 {
6530         int i;
6531
6532         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
6533                 if (!mlxsw_sp->router->rifs[i]) {
6534                         *p_rif_index = i;
6535                         return 0;
6536                 }
6537         }
6538
6539         return -ENOBUFS;
6540 }
6541
6542 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
6543                                                u16 vr_id,
6544                                                struct net_device *l3_dev)
6545 {
6546         struct mlxsw_sp_rif *rif;
6547
6548         rif = kzalloc(rif_size, GFP_KERNEL);
6549         if (!rif)
6550                 return NULL;
6551
6552         INIT_LIST_HEAD(&rif->nexthop_list);
6553         INIT_LIST_HEAD(&rif->neigh_list);
6554         if (l3_dev) {
6555                 ether_addr_copy(rif->addr, l3_dev->dev_addr);
6556                 rif->mtu = l3_dev->mtu;
6557                 rif->dev = l3_dev;
6558         }
6559         rif->vr_id = vr_id;
6560         rif->rif_index = rif_index;
6561
6562         return rif;
6563 }
6564
6565 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
6566                                            u16 rif_index)
6567 {
6568         return mlxsw_sp->router->rifs[rif_index];
6569 }
6570
6571 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
6572 {
6573         return rif->rif_index;
6574 }
6575
6576 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6577 {
6578         return lb_rif->common.rif_index;
6579 }
6580
6581 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6582 {
6583         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev);
6584         struct mlxsw_sp_vr *ul_vr;
6585
6586         ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL);
6587         if (WARN_ON(IS_ERR(ul_vr)))
6588                 return 0;
6589
6590         return ul_vr->id;
6591 }
6592
6593 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6594 {
6595         return lb_rif->ul_rif_id;
6596 }
6597
6598 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
6599 {
6600         return rif->dev->ifindex;
6601 }
6602
6603 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif)
6604 {
6605         return rif->dev;
6606 }
6607
6608 static struct mlxsw_sp_rif *
6609 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
6610                     const struct mlxsw_sp_rif_params *params,
6611                     struct netlink_ext_ack *extack)
6612 {
6613         u32 tb_id = l3mdev_fib_table(params->dev);
6614         const struct mlxsw_sp_rif_ops *ops;
6615         struct mlxsw_sp_fid *fid = NULL;
6616         enum mlxsw_sp_rif_type type;
6617         struct mlxsw_sp_rif *rif;
6618         struct mlxsw_sp_vr *vr;
6619         u16 rif_index;
6620         int i, err;
6621
6622         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
6623         ops = mlxsw_sp->rif_ops_arr[type];
6624
6625         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack);
6626         if (IS_ERR(vr))
6627                 return ERR_CAST(vr);
6628         vr->rif_count++;
6629
6630         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
6631         if (err) {
6632                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
6633                 goto err_rif_index_alloc;
6634         }
6635
6636         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
6637         if (!rif) {
6638                 err = -ENOMEM;
6639                 goto err_rif_alloc;
6640         }
6641         dev_hold(rif->dev);
6642         mlxsw_sp->router->rifs[rif_index] = rif;
6643         rif->mlxsw_sp = mlxsw_sp;
6644         rif->ops = ops;
6645
6646         if (ops->fid_get) {
6647                 fid = ops->fid_get(rif, extack);
6648                 if (IS_ERR(fid)) {
6649                         err = PTR_ERR(fid);
6650                         goto err_fid_get;
6651                 }
6652                 rif->fid = fid;
6653         }
6654
6655         if (ops->setup)
6656                 ops->setup(rif, params);
6657
6658         err = ops->configure(rif);
6659         if (err)
6660                 goto err_configure;
6661
6662         for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
6663                 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif);
6664                 if (err)
6665                         goto err_mr_rif_add;
6666         }
6667
6668         mlxsw_sp_rif_counters_alloc(rif);
6669
6670         return rif;
6671
6672 err_mr_rif_add:
6673         for (i--; i >= 0; i--)
6674                 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
6675         ops->deconfigure(rif);
6676 err_configure:
6677         if (fid)
6678                 mlxsw_sp_fid_put(fid);
6679 err_fid_get:
6680         mlxsw_sp->router->rifs[rif_index] = NULL;
6681         dev_put(rif->dev);
6682         kfree(rif);
6683 err_rif_alloc:
6684 err_rif_index_alloc:
6685         vr->rif_count--;
6686         mlxsw_sp_vr_put(mlxsw_sp, vr);
6687         return ERR_PTR(err);
6688 }
6689
6690 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
6691 {
6692         const struct mlxsw_sp_rif_ops *ops = rif->ops;
6693         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
6694         struct mlxsw_sp_fid *fid = rif->fid;
6695         struct mlxsw_sp_vr *vr;
6696         int i;
6697
6698         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
6699         vr = &mlxsw_sp->router->vrs[rif->vr_id];
6700
6701         mlxsw_sp_rif_counters_free(rif);
6702         for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
6703                 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
6704         ops->deconfigure(rif);
6705         if (fid)
6706                 /* Loopback RIFs are not associated with a FID. */
6707                 mlxsw_sp_fid_put(fid);
6708         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
6709         dev_put(rif->dev);
6710         kfree(rif);
6711         vr->rif_count--;
6712         mlxsw_sp_vr_put(mlxsw_sp, vr);
6713 }
6714
6715 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp,
6716                                  struct net_device *dev)
6717 {
6718         struct mlxsw_sp_rif *rif;
6719
6720         mutex_lock(&mlxsw_sp->router->lock);
6721         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6722         if (!rif)
6723                 goto out;
6724         mlxsw_sp_rif_destroy(rif);
6725 out:
6726         mutex_unlock(&mlxsw_sp->router->lock);
6727 }
6728
6729 static void
6730 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
6731                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
6732 {
6733         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
6734
6735         params->vid = mlxsw_sp_port_vlan->vid;
6736         params->lag = mlxsw_sp_port->lagged;
6737         if (params->lag)
6738                 params->lag_id = mlxsw_sp_port->lag_id;
6739         else
6740                 params->system_port = mlxsw_sp_port->local_port;
6741 }
6742
6743 static struct mlxsw_sp_rif_subport *
6744 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
6745 {
6746         return container_of(rif, struct mlxsw_sp_rif_subport, common);
6747 }
6748
6749 static struct mlxsw_sp_rif *
6750 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp,
6751                          const struct mlxsw_sp_rif_params *params,
6752                          struct netlink_ext_ack *extack)
6753 {
6754         struct mlxsw_sp_rif_subport *rif_subport;
6755         struct mlxsw_sp_rif *rif;
6756
6757         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev);
6758         if (!rif)
6759                 return mlxsw_sp_rif_create(mlxsw_sp, params, extack);
6760
6761         rif_subport = mlxsw_sp_rif_subport_rif(rif);
6762         refcount_inc(&rif_subport->ref_count);
6763         return rif;
6764 }
6765
6766 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif)
6767 {
6768         struct mlxsw_sp_rif_subport *rif_subport;
6769
6770         rif_subport = mlxsw_sp_rif_subport_rif(rif);
6771         if (!refcount_dec_and_test(&rif_subport->ref_count))
6772                 return;
6773
6774         mlxsw_sp_rif_destroy(rif);
6775 }
6776
6777 static int
6778 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
6779                                struct net_device *l3_dev,
6780                                struct netlink_ext_ack *extack)
6781 {
6782         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
6783         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
6784         struct mlxsw_sp_rif_params params = {
6785                 .dev = l3_dev,
6786         };
6787         u16 vid = mlxsw_sp_port_vlan->vid;
6788         struct mlxsw_sp_rif *rif;
6789         struct mlxsw_sp_fid *fid;
6790         int err;
6791
6792         mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
6793         rif = mlxsw_sp_rif_subport_get(mlxsw_sp, &params, extack);
6794         if (IS_ERR(rif))
6795                 return PTR_ERR(rif);
6796
6797         /* FID was already created, just take a reference */
6798         fid = rif->ops->fid_get(rif, extack);
6799         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
6800         if (err)
6801                 goto err_fid_port_vid_map;
6802
6803         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
6804         if (err)
6805                 goto err_port_vid_learning_set;
6806
6807         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
6808                                         BR_STATE_FORWARDING);
6809         if (err)
6810                 goto err_port_vid_stp_set;
6811
6812         mlxsw_sp_port_vlan->fid = fid;
6813
6814         return 0;
6815
6816 err_port_vid_stp_set:
6817         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
6818 err_port_vid_learning_set:
6819         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
6820 err_fid_port_vid_map:
6821         mlxsw_sp_fid_put(fid);
6822         mlxsw_sp_rif_subport_put(rif);
6823         return err;
6824 }
6825
6826 static void
6827 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
6828 {
6829         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
6830         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
6831         struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid);
6832         u16 vid = mlxsw_sp_port_vlan->vid;
6833
6834         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
6835                 return;
6836
6837         mlxsw_sp_port_vlan->fid = NULL;
6838         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
6839         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
6840         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
6841         mlxsw_sp_fid_put(fid);
6842         mlxsw_sp_rif_subport_put(rif);
6843 }
6844
6845 void
6846 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
6847 {
6848         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
6849
6850         mutex_lock(&mlxsw_sp->router->lock);
6851         __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
6852         mutex_unlock(&mlxsw_sp->router->lock);
6853 }
6854
6855 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
6856                                              struct net_device *port_dev,
6857                                              unsigned long event, u16 vid,
6858                                              struct netlink_ext_ack *extack)
6859 {
6860         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
6861         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
6862
6863         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
6864         if (WARN_ON(!mlxsw_sp_port_vlan))
6865                 return -EINVAL;
6866
6867         switch (event) {
6868         case NETDEV_UP:
6869                 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
6870                                                       l3_dev, extack);
6871         case NETDEV_DOWN:
6872                 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
6873                 break;
6874         }
6875
6876         return 0;
6877 }
6878
6879 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
6880                                         unsigned long event,
6881                                         struct netlink_ext_ack *extack)
6882 {
6883         if (netif_is_bridge_port(port_dev) ||
6884             netif_is_lag_port(port_dev) ||
6885             netif_is_ovs_port(port_dev))
6886                 return 0;
6887
6888         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event,
6889                                                  MLXSW_SP_DEFAULT_VID, extack);
6890 }
6891
6892 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
6893                                          struct net_device *lag_dev,
6894                                          unsigned long event, u16 vid,
6895                                          struct netlink_ext_ack *extack)
6896 {
6897         struct net_device *port_dev;
6898         struct list_head *iter;
6899         int err;
6900
6901         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
6902                 if (mlxsw_sp_port_dev_check(port_dev)) {
6903                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
6904                                                                 port_dev,
6905                                                                 event, vid,
6906                                                                 extack);
6907                         if (err)
6908                                 return err;
6909                 }
6910         }
6911
6912         return 0;
6913 }
6914
6915 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
6916                                        unsigned long event,
6917                                        struct netlink_ext_ack *extack)
6918 {
6919         if (netif_is_bridge_port(lag_dev))
6920                 return 0;
6921
6922         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event,
6923                                              MLXSW_SP_DEFAULT_VID, extack);
6924 }
6925
6926 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp,
6927                                           struct net_device *l3_dev,
6928                                           unsigned long event,
6929                                           struct netlink_ext_ack *extack)
6930 {
6931         struct mlxsw_sp_rif_params params = {
6932                 .dev = l3_dev,
6933         };
6934         struct mlxsw_sp_rif *rif;
6935
6936         switch (event) {
6937         case NETDEV_UP:
6938                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params, extack);
6939                 if (IS_ERR(rif))
6940                         return PTR_ERR(rif);
6941                 break;
6942         case NETDEV_DOWN:
6943                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
6944                 mlxsw_sp_rif_destroy(rif);
6945                 break;
6946         }
6947
6948         return 0;
6949 }
6950
6951 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp,
6952                                         struct net_device *vlan_dev,
6953                                         unsigned long event,
6954                                         struct netlink_ext_ack *extack)
6955 {
6956         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
6957         u16 vid = vlan_dev_vlan_id(vlan_dev);
6958
6959         if (netif_is_bridge_port(vlan_dev))
6960                 return 0;
6961
6962         if (mlxsw_sp_port_dev_check(real_dev))
6963                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
6964                                                          event, vid, extack);
6965         else if (netif_is_lag_master(real_dev))
6966                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
6967                                                      vid, extack);
6968         else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
6969                 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event,
6970                                                       extack);
6971
6972         return 0;
6973 }
6974
6975 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac)
6976 {
6977         u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 };
6978         u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
6979
6980         return ether_addr_equal_masked(mac, vrrp4, mask);
6981 }
6982
6983 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac)
6984 {
6985         u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 };
6986         u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
6987
6988         return ether_addr_equal_masked(mac, vrrp6, mask);
6989 }
6990
6991 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
6992                                 const u8 *mac, bool adding)
6993 {
6994         char ritr_pl[MLXSW_REG_RITR_LEN];
6995         u8 vrrp_id = adding ? mac[5] : 0;
6996         int err;
6997
6998         if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) &&
6999             !mlxsw_sp_rif_macvlan_is_vrrp6(mac))
7000                 return 0;
7001
7002         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
7003         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7004         if (err)
7005                 return err;
7006
7007         if (mlxsw_sp_rif_macvlan_is_vrrp4(mac))
7008                 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id);
7009         else
7010                 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id);
7011
7012         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7013 }
7014
7015 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp,
7016                                     const struct net_device *macvlan_dev,
7017                                     struct netlink_ext_ack *extack)
7018 {
7019         struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
7020         struct mlxsw_sp_rif *rif;
7021         int err;
7022
7023         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
7024         if (!rif) {
7025                 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces");
7026                 return -EOPNOTSUPP;
7027         }
7028
7029         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
7030                                   mlxsw_sp_fid_index(rif->fid), true);
7031         if (err)
7032                 return err;
7033
7034         err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index,
7035                                    macvlan_dev->dev_addr, true);
7036         if (err)
7037                 goto err_rif_vrrp_add;
7038
7039         /* Make sure the bridge driver does not have this MAC pointing at
7040          * some other port.
7041          */
7042         if (rif->ops->fdb_del)
7043                 rif->ops->fdb_del(rif, macvlan_dev->dev_addr);
7044
7045         return 0;
7046
7047 err_rif_vrrp_add:
7048         mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
7049                             mlxsw_sp_fid_index(rif->fid), false);
7050         return err;
7051 }
7052
7053 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
7054                                        const struct net_device *macvlan_dev)
7055 {
7056         struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
7057         struct mlxsw_sp_rif *rif;
7058
7059         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
7060         /* If we do not have a RIF, then we already took care of
7061          * removing the macvlan's MAC during RIF deletion.
7062          */
7063         if (!rif)
7064                 return;
7065         mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr,
7066                              false);
7067         mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
7068                             mlxsw_sp_fid_index(rif->fid), false);
7069 }
7070
7071 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
7072                               const struct net_device *macvlan_dev)
7073 {
7074         mutex_lock(&mlxsw_sp->router->lock);
7075         __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
7076         mutex_unlock(&mlxsw_sp->router->lock);
7077 }
7078
7079 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp,
7080                                            struct net_device *macvlan_dev,
7081                                            unsigned long event,
7082                                            struct netlink_ext_ack *extack)
7083 {
7084         switch (event) {
7085         case NETDEV_UP:
7086                 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack);
7087         case NETDEV_DOWN:
7088                 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
7089                 break;
7090         }
7091
7092         return 0;
7093 }
7094
7095 static int mlxsw_sp_router_port_check_rif_addr(struct mlxsw_sp *mlxsw_sp,
7096                                                struct net_device *dev,
7097                                                const unsigned char *dev_addr,
7098                                                struct netlink_ext_ack *extack)
7099 {
7100         struct mlxsw_sp_rif *rif;
7101         int i;
7102
7103         /* A RIF is not created for macvlan netdevs. Their MAC is used to
7104          * populate the FDB
7105          */
7106         if (netif_is_macvlan(dev) || netif_is_l3_master(dev))
7107                 return 0;
7108
7109         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
7110                 rif = mlxsw_sp->router->rifs[i];
7111                 if (rif && rif->ops &&
7112                     rif->ops->type == MLXSW_SP_RIF_TYPE_IPIP_LB)
7113                         continue;
7114                 if (rif && rif->dev && rif->dev != dev &&
7115                     !ether_addr_equal_masked(rif->dev->dev_addr, dev_addr,
7116                                              mlxsw_sp->mac_mask)) {
7117                         NL_SET_ERR_MSG_MOD(extack, "All router interface MAC addresses must have the same prefix");
7118                         return -EINVAL;
7119                 }
7120         }
7121
7122         return 0;
7123 }
7124
7125 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp,
7126                                      struct net_device *dev,
7127                                      unsigned long event,
7128                                      struct netlink_ext_ack *extack)
7129 {
7130         if (mlxsw_sp_port_dev_check(dev))
7131                 return mlxsw_sp_inetaddr_port_event(dev, event, extack);
7132         else if (netif_is_lag_master(dev))
7133                 return mlxsw_sp_inetaddr_lag_event(dev, event, extack);
7134         else if (netif_is_bridge_master(dev))
7135                 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event,
7136                                                       extack);
7137         else if (is_vlan_dev(dev))
7138                 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event,
7139                                                     extack);
7140         else if (netif_is_macvlan(dev))
7141                 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event,
7142                                                        extack);
7143         else
7144                 return 0;
7145 }
7146
7147 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb,
7148                                    unsigned long event, void *ptr)
7149 {
7150         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
7151         struct net_device *dev = ifa->ifa_dev->dev;
7152         struct mlxsw_sp_router *router;
7153         struct mlxsw_sp_rif *rif;
7154         int err = 0;
7155
7156         /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */
7157         if (event == NETDEV_UP)
7158                 return NOTIFY_DONE;
7159
7160         router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb);
7161         mutex_lock(&router->lock);
7162         rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev);
7163         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7164                 goto out;
7165
7166         err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL);
7167 out:
7168         mutex_unlock(&router->lock);
7169         return notifier_from_errno(err);
7170 }
7171
7172 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused,
7173                                   unsigned long event, void *ptr)
7174 {
7175         struct in_validator_info *ivi = (struct in_validator_info *) ptr;
7176         struct net_device *dev = ivi->ivi_dev->dev;
7177         struct mlxsw_sp *mlxsw_sp;
7178         struct mlxsw_sp_rif *rif;
7179         int err = 0;
7180
7181         mlxsw_sp = mlxsw_sp_lower_get(dev);
7182         if (!mlxsw_sp)
7183                 return NOTIFY_DONE;
7184
7185         mutex_lock(&mlxsw_sp->router->lock);
7186         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7187         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7188                 goto out;
7189
7190         err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr,
7191                                                   ivi->extack);
7192         if (err)
7193                 goto out;
7194
7195         err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack);
7196 out:
7197         mutex_unlock(&mlxsw_sp->router->lock);
7198         return notifier_from_errno(err);
7199 }
7200
7201 struct mlxsw_sp_inet6addr_event_work {
7202         struct work_struct work;
7203         struct mlxsw_sp *mlxsw_sp;
7204         struct net_device *dev;
7205         unsigned long event;
7206 };
7207
7208 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
7209 {
7210         struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
7211                 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
7212         struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp;
7213         struct net_device *dev = inet6addr_work->dev;
7214         unsigned long event = inet6addr_work->event;
7215         struct mlxsw_sp_rif *rif;
7216
7217         rtnl_lock();
7218         mutex_lock(&mlxsw_sp->router->lock);
7219
7220         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7221         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7222                 goto out;
7223
7224         __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL);
7225 out:
7226         mutex_unlock(&mlxsw_sp->router->lock);
7227         rtnl_unlock();
7228         dev_put(dev);
7229         kfree(inet6addr_work);
7230 }
7231
7232 /* Called with rcu_read_lock() */
7233 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb,
7234                                     unsigned long event, void *ptr)
7235 {
7236         struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
7237         struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
7238         struct net_device *dev = if6->idev->dev;
7239         struct mlxsw_sp_router *router;
7240
7241         /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */
7242         if (event == NETDEV_UP)
7243                 return NOTIFY_DONE;
7244
7245         inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
7246         if (!inet6addr_work)
7247                 return NOTIFY_BAD;
7248
7249         router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb);
7250         INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
7251         inet6addr_work->mlxsw_sp = router->mlxsw_sp;
7252         inet6addr_work->dev = dev;
7253         inet6addr_work->event = event;
7254         dev_hold(dev);
7255         mlxsw_core_schedule_work(&inet6addr_work->work);
7256
7257         return NOTIFY_DONE;
7258 }
7259
7260 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused,
7261                                    unsigned long event, void *ptr)
7262 {
7263         struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr;
7264         struct net_device *dev = i6vi->i6vi_dev->dev;
7265         struct mlxsw_sp *mlxsw_sp;
7266         struct mlxsw_sp_rif *rif;
7267         int err = 0;
7268
7269         mlxsw_sp = mlxsw_sp_lower_get(dev);
7270         if (!mlxsw_sp)
7271                 return NOTIFY_DONE;
7272
7273         mutex_lock(&mlxsw_sp->router->lock);
7274         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7275         if (!mlxsw_sp_rif_should_config(rif, dev, event))
7276                 goto out;
7277
7278         err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr,
7279                                                   i6vi->extack);
7280         if (err)
7281                 goto out;
7282
7283         err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack);
7284 out:
7285         mutex_unlock(&mlxsw_sp->router->lock);
7286         return notifier_from_errno(err);
7287 }
7288
7289 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
7290                              const char *mac, int mtu)
7291 {
7292         char ritr_pl[MLXSW_REG_RITR_LEN];
7293         int err;
7294
7295         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
7296         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7297         if (err)
7298                 return err;
7299
7300         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
7301         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
7302         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
7303         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7304 }
7305
7306 static int
7307 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp,
7308                                   struct mlxsw_sp_rif *rif)
7309 {
7310         struct net_device *dev = rif->dev;
7311         u16 fid_index;
7312         int err;
7313
7314         fid_index = mlxsw_sp_fid_index(rif->fid);
7315
7316         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
7317         if (err)
7318                 return err;
7319
7320         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
7321                                 dev->mtu);
7322         if (err)
7323                 goto err_rif_edit;
7324
7325         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
7326         if (err)
7327                 goto err_rif_fdb_op;
7328
7329         if (rif->mtu != dev->mtu) {
7330                 struct mlxsw_sp_vr *vr;
7331                 int i;
7332
7333                 /* The RIF is relevant only to its mr_table instance, as unlike
7334                  * unicast routing, in multicast routing a RIF cannot be shared
7335                  * between several multicast routing tables.
7336                  */
7337                 vr = &mlxsw_sp->router->vrs[rif->vr_id];
7338                 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
7339                         mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i],
7340                                                    rif, dev->mtu);
7341         }
7342
7343         ether_addr_copy(rif->addr, dev->dev_addr);
7344         rif->mtu = dev->mtu;
7345
7346         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
7347
7348         return 0;
7349
7350 err_rif_fdb_op:
7351         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
7352 err_rif_edit:
7353         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
7354         return err;
7355 }
7356
7357 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif,
7358                             struct netdev_notifier_pre_changeaddr_info *info)
7359 {
7360         struct netlink_ext_ack *extack;
7361
7362         extack = netdev_notifier_info_to_extack(&info->info);
7363         return mlxsw_sp_router_port_check_rif_addr(rif->mlxsw_sp, rif->dev,
7364                                                    info->dev_addr, extack);
7365 }
7366
7367 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev,
7368                                          unsigned long event, void *ptr)
7369 {
7370         struct mlxsw_sp *mlxsw_sp;
7371         struct mlxsw_sp_rif *rif;
7372         int err = 0;
7373
7374         mlxsw_sp = mlxsw_sp_lower_get(dev);
7375         if (!mlxsw_sp)
7376                 return 0;
7377
7378         mutex_lock(&mlxsw_sp->router->lock);
7379         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7380         if (!rif)
7381                 goto out;
7382
7383         switch (event) {
7384         case NETDEV_CHANGEMTU:
7385         case NETDEV_CHANGEADDR:
7386                 err = mlxsw_sp_router_port_change_event(mlxsw_sp, rif);
7387                 break;
7388         case NETDEV_PRE_CHANGEADDR:
7389                 err = mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr);
7390                 break;
7391         }
7392
7393 out:
7394         mutex_unlock(&mlxsw_sp->router->lock);
7395         return err;
7396 }
7397
7398 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
7399                                   struct net_device *l3_dev,
7400                                   struct netlink_ext_ack *extack)
7401 {
7402         struct mlxsw_sp_rif *rif;
7403
7404         /* If netdev is already associated with a RIF, then we need to
7405          * destroy it and create a new one with the new virtual router ID.
7406          */
7407         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
7408         if (rif)
7409                 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN,
7410                                           extack);
7411
7412         return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack);
7413 }
7414
7415 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
7416                                     struct net_device *l3_dev)
7417 {
7418         struct mlxsw_sp_rif *rif;
7419
7420         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
7421         if (!rif)
7422                 return;
7423         __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL);
7424 }
7425
7426 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
7427                                  struct netdev_notifier_changeupper_info *info)
7428 {
7429         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
7430         int err = 0;
7431
7432         /* We do not create a RIF for a macvlan, but only use it to
7433          * direct more MAC addresses to the router.
7434          */
7435         if (!mlxsw_sp || netif_is_macvlan(l3_dev))
7436                 return 0;
7437
7438         mutex_lock(&mlxsw_sp->router->lock);
7439         switch (event) {
7440         case NETDEV_PRECHANGEUPPER:
7441                 break;
7442         case NETDEV_CHANGEUPPER:
7443                 if (info->linking) {
7444                         struct netlink_ext_ack *extack;
7445
7446                         extack = netdev_notifier_info_to_extack(&info->info);
7447                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack);
7448                 } else {
7449                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
7450                 }
7451                 break;
7452         }
7453         mutex_unlock(&mlxsw_sp->router->lock);
7454
7455         return err;
7456 }
7457
7458 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev,
7459                                         struct netdev_nested_priv *priv)
7460 {
7461         struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data;
7462
7463         if (!netif_is_macvlan(dev))
7464                 return 0;
7465
7466         return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
7467                                    mlxsw_sp_fid_index(rif->fid), false);
7468 }
7469
7470 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif)
7471 {
7472         struct netdev_nested_priv priv = {
7473                 .data = (void *)rif,
7474         };
7475
7476         if (!netif_is_macvlan_port(rif->dev))
7477                 return 0;
7478
7479         netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n");
7480         return netdev_walk_all_upper_dev_rcu(rif->dev,
7481                                              __mlxsw_sp_rif_macvlan_flush, &priv);
7482 }
7483
7484 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
7485                                        const struct mlxsw_sp_rif_params *params)
7486 {
7487         struct mlxsw_sp_rif_subport *rif_subport;
7488
7489         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7490         refcount_set(&rif_subport->ref_count, 1);
7491         rif_subport->vid = params->vid;
7492         rif_subport->lag = params->lag;
7493         if (params->lag)
7494                 rif_subport->lag_id = params->lag_id;
7495         else
7496                 rif_subport->system_port = params->system_port;
7497 }
7498
7499 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
7500 {
7501         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7502         struct mlxsw_sp_rif_subport *rif_subport;
7503         char ritr_pl[MLXSW_REG_RITR_LEN];
7504
7505         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7506         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
7507                             rif->rif_index, rif->vr_id, rif->dev->mtu);
7508         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
7509         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
7510                                   rif_subport->lag ? rif_subport->lag_id :
7511                                                      rif_subport->system_port,
7512                                   rif_subport->vid);
7513
7514         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7515 }
7516
7517 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
7518 {
7519         int err;
7520
7521         err = mlxsw_sp_rif_subport_op(rif, true);
7522         if (err)
7523                 return err;
7524
7525         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7526                                   mlxsw_sp_fid_index(rif->fid), true);
7527         if (err)
7528                 goto err_rif_fdb_op;
7529
7530         mlxsw_sp_fid_rif_set(rif->fid, rif);
7531         return 0;
7532
7533 err_rif_fdb_op:
7534         mlxsw_sp_rif_subport_op(rif, false);
7535         return err;
7536 }
7537
7538 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
7539 {
7540         struct mlxsw_sp_fid *fid = rif->fid;
7541
7542         mlxsw_sp_fid_rif_set(fid, NULL);
7543         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7544                             mlxsw_sp_fid_index(fid), false);
7545         mlxsw_sp_rif_macvlan_flush(rif);
7546         mlxsw_sp_rif_subport_op(rif, false);
7547 }
7548
7549 static struct mlxsw_sp_fid *
7550 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif,
7551                              struct netlink_ext_ack *extack)
7552 {
7553         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
7554 }
7555
7556 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
7557         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
7558         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
7559         .setup                  = mlxsw_sp_rif_subport_setup,
7560         .configure              = mlxsw_sp_rif_subport_configure,
7561         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
7562         .fid_get                = mlxsw_sp_rif_subport_fid_get,
7563 };
7564
7565 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
7566                                     enum mlxsw_reg_ritr_if_type type,
7567                                     u16 vid_fid, bool enable)
7568 {
7569         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7570         char ritr_pl[MLXSW_REG_RITR_LEN];
7571
7572         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
7573                             rif->dev->mtu);
7574         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
7575         mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
7576
7577         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7578 }
7579
7580 u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
7581 {
7582         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
7583 }
7584
7585 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
7586 {
7587         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7588         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
7589         int err;
7590
7591         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
7592                                        true);
7593         if (err)
7594                 return err;
7595
7596         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7597                                      mlxsw_sp_router_port(mlxsw_sp), true);
7598         if (err)
7599                 goto err_fid_mc_flood_set;
7600
7601         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7602                                      mlxsw_sp_router_port(mlxsw_sp), true);
7603         if (err)
7604                 goto err_fid_bc_flood_set;
7605
7606         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7607                                   mlxsw_sp_fid_index(rif->fid), true);
7608         if (err)
7609                 goto err_rif_fdb_op;
7610
7611         mlxsw_sp_fid_rif_set(rif->fid, rif);
7612         return 0;
7613
7614 err_rif_fdb_op:
7615         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7616                                mlxsw_sp_router_port(mlxsw_sp), false);
7617 err_fid_bc_flood_set:
7618         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7619                                mlxsw_sp_router_port(mlxsw_sp), false);
7620 err_fid_mc_flood_set:
7621         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
7622         return err;
7623 }
7624
7625 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
7626 {
7627         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
7628         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7629         struct mlxsw_sp_fid *fid = rif->fid;
7630
7631         mlxsw_sp_fid_rif_set(fid, NULL);
7632         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7633                             mlxsw_sp_fid_index(fid), false);
7634         mlxsw_sp_rif_macvlan_flush(rif);
7635         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7636                                mlxsw_sp_router_port(mlxsw_sp), false);
7637         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7638                                mlxsw_sp_router_port(mlxsw_sp), false);
7639         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
7640 }
7641
7642 static struct mlxsw_sp_fid *
7643 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif,
7644                          struct netlink_ext_ack *extack)
7645 {
7646         return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
7647 }
7648
7649 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
7650 {
7651         struct switchdev_notifier_fdb_info info;
7652         struct net_device *dev;
7653
7654         dev = br_fdb_find_port(rif->dev, mac, 0);
7655         if (!dev)
7656                 return;
7657
7658         info.addr = mac;
7659         info.vid = 0;
7660         call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
7661                                  NULL);
7662 }
7663
7664 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
7665         .type                   = MLXSW_SP_RIF_TYPE_FID,
7666         .rif_size               = sizeof(struct mlxsw_sp_rif),
7667         .configure              = mlxsw_sp_rif_fid_configure,
7668         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
7669         .fid_get                = mlxsw_sp_rif_fid_fid_get,
7670         .fdb_del                = mlxsw_sp_rif_fid_fdb_del,
7671 };
7672
7673 static struct mlxsw_sp_fid *
7674 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif,
7675                           struct netlink_ext_ack *extack)
7676 {
7677         struct net_device *br_dev;
7678         u16 vid;
7679         int err;
7680
7681         if (is_vlan_dev(rif->dev)) {
7682                 vid = vlan_dev_vlan_id(rif->dev);
7683                 br_dev = vlan_dev_real_dev(rif->dev);
7684                 if (WARN_ON(!netif_is_bridge_master(br_dev)))
7685                         return ERR_PTR(-EINVAL);
7686         } else {
7687                 err = br_vlan_get_pvid(rif->dev, &vid);
7688                 if (err < 0 || !vid) {
7689                         NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID");
7690                         return ERR_PTR(-EINVAL);
7691                 }
7692         }
7693
7694         return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
7695 }
7696
7697 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
7698 {
7699         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
7700         struct switchdev_notifier_fdb_info info;
7701         struct net_device *br_dev;
7702         struct net_device *dev;
7703
7704         br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev;
7705         dev = br_fdb_find_port(br_dev, mac, vid);
7706         if (!dev)
7707                 return;
7708
7709         info.addr = mac;
7710         info.vid = vid;
7711         call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
7712                                  NULL);
7713 }
7714
7715 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = {
7716         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
7717         .rif_size               = sizeof(struct mlxsw_sp_rif),
7718         .configure              = mlxsw_sp_rif_fid_configure,
7719         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
7720         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
7721         .fdb_del                = mlxsw_sp_rif_vlan_fdb_del,
7722 };
7723
7724 static struct mlxsw_sp_rif_ipip_lb *
7725 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
7726 {
7727         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
7728 }
7729
7730 static void
7731 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
7732                            const struct mlxsw_sp_rif_params *params)
7733 {
7734         struct mlxsw_sp_rif_params_ipip_lb *params_lb;
7735         struct mlxsw_sp_rif_ipip_lb *rif_lb;
7736
7737         params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
7738                                  common);
7739         rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
7740         rif_lb->lb_config = params_lb->lb_config;
7741 }
7742
7743 static int
7744 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
7745 {
7746         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
7747         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
7748         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7749         struct mlxsw_sp_vr *ul_vr;
7750         int err;
7751
7752         ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL);
7753         if (IS_ERR(ul_vr))
7754                 return PTR_ERR(ul_vr);
7755
7756         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true);
7757         if (err)
7758                 goto err_loopback_op;
7759
7760         lb_rif->ul_vr_id = ul_vr->id;
7761         lb_rif->ul_rif_id = 0;
7762         ++ul_vr->rif_count;
7763         return 0;
7764
7765 err_loopback_op:
7766         mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
7767         return err;
7768 }
7769
7770 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
7771 {
7772         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
7773         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7774         struct mlxsw_sp_vr *ul_vr;
7775
7776         ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
7777         mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false);
7778
7779         --ul_vr->rif_count;
7780         mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
7781 }
7782
7783 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = {
7784         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
7785         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
7786         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
7787         .configure              = mlxsw_sp1_rif_ipip_lb_configure,
7788         .deconfigure            = mlxsw_sp1_rif_ipip_lb_deconfigure,
7789 };
7790
7791 const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = {
7792         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
7793         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_emu_ops,
7794         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
7795         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp1_rif_ipip_lb_ops,
7796 };
7797
7798 static int
7799 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable)
7800 {
7801         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
7802         char ritr_pl[MLXSW_REG_RITR_LEN];
7803
7804         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
7805                             ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU);
7806         mlxsw_reg_ritr_loopback_protocol_set(ritr_pl,
7807                                              MLXSW_REG_RITR_LOOPBACK_GENERIC);
7808
7809         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7810 }
7811
7812 static struct mlxsw_sp_rif *
7813 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
7814                        struct netlink_ext_ack *extack)
7815 {
7816         struct mlxsw_sp_rif *ul_rif;
7817         u16 rif_index;
7818         int err;
7819
7820         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
7821         if (err) {
7822                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
7823                 return ERR_PTR(err);
7824         }
7825
7826         ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL);
7827         if (!ul_rif)
7828                 return ERR_PTR(-ENOMEM);
7829
7830         mlxsw_sp->router->rifs[rif_index] = ul_rif;
7831         ul_rif->mlxsw_sp = mlxsw_sp;
7832         err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true);
7833         if (err)
7834                 goto ul_rif_op_err;
7835
7836         return ul_rif;
7837
7838 ul_rif_op_err:
7839         mlxsw_sp->router->rifs[rif_index] = NULL;
7840         kfree(ul_rif);
7841         return ERR_PTR(err);
7842 }
7843
7844 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif)
7845 {
7846         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
7847
7848         mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false);
7849         mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL;
7850         kfree(ul_rif);
7851 }
7852
7853 static struct mlxsw_sp_rif *
7854 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
7855                     struct netlink_ext_ack *extack)
7856 {
7857         struct mlxsw_sp_vr *vr;
7858         int err;
7859
7860         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack);
7861         if (IS_ERR(vr))
7862                 return ERR_CAST(vr);
7863
7864         if (refcount_inc_not_zero(&vr->ul_rif_refcnt))
7865                 return vr->ul_rif;
7866
7867         vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack);
7868         if (IS_ERR(vr->ul_rif)) {
7869                 err = PTR_ERR(vr->ul_rif);
7870                 goto err_ul_rif_create;
7871         }
7872
7873         vr->rif_count++;
7874         refcount_set(&vr->ul_rif_refcnt, 1);
7875
7876         return vr->ul_rif;
7877
7878 err_ul_rif_create:
7879         mlxsw_sp_vr_put(mlxsw_sp, vr);
7880         return ERR_PTR(err);
7881 }
7882
7883 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif)
7884 {
7885         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
7886         struct mlxsw_sp_vr *vr;
7887
7888         vr = &mlxsw_sp->router->vrs[ul_rif->vr_id];
7889
7890         if (!refcount_dec_and_test(&vr->ul_rif_refcnt))
7891                 return;
7892
7893         vr->rif_count--;
7894         mlxsw_sp_ul_rif_destroy(ul_rif);
7895         mlxsw_sp_vr_put(mlxsw_sp, vr);
7896 }
7897
7898 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
7899                                u16 *ul_rif_index)
7900 {
7901         struct mlxsw_sp_rif *ul_rif;
7902         int err = 0;
7903
7904         mutex_lock(&mlxsw_sp->router->lock);
7905         ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
7906         if (IS_ERR(ul_rif)) {
7907                 err = PTR_ERR(ul_rif);
7908                 goto out;
7909         }
7910         *ul_rif_index = ul_rif->rif_index;
7911 out:
7912         mutex_unlock(&mlxsw_sp->router->lock);
7913         return err;
7914 }
7915
7916 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index)
7917 {
7918         struct mlxsw_sp_rif *ul_rif;
7919
7920         mutex_lock(&mlxsw_sp->router->lock);
7921         ul_rif = mlxsw_sp->router->rifs[ul_rif_index];
7922         if (WARN_ON(!ul_rif))
7923                 goto out;
7924
7925         mlxsw_sp_ul_rif_put(ul_rif);
7926 out:
7927         mutex_unlock(&mlxsw_sp->router->lock);
7928 }
7929
7930 static int
7931 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
7932 {
7933         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
7934         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
7935         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7936         struct mlxsw_sp_rif *ul_rif;
7937         int err;
7938
7939         ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
7940         if (IS_ERR(ul_rif))
7941                 return PTR_ERR(ul_rif);
7942
7943         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true);
7944         if (err)
7945                 goto err_loopback_op;
7946
7947         lb_rif->ul_vr_id = 0;
7948         lb_rif->ul_rif_id = ul_rif->rif_index;
7949
7950         return 0;
7951
7952 err_loopback_op:
7953         mlxsw_sp_ul_rif_put(ul_rif);
7954         return err;
7955 }
7956
7957 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
7958 {
7959         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
7960         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7961         struct mlxsw_sp_rif *ul_rif;
7962
7963         ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id);
7964         mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false);
7965         mlxsw_sp_ul_rif_put(ul_rif);
7966 }
7967
7968 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = {
7969         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
7970         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
7971         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
7972         .configure              = mlxsw_sp2_rif_ipip_lb_configure,
7973         .deconfigure            = mlxsw_sp2_rif_ipip_lb_deconfigure,
7974 };
7975
7976 const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = {
7977         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
7978         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_emu_ops,
7979         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
7980         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp2_rif_ipip_lb_ops,
7981 };
7982
7983 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
7984 {
7985         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
7986
7987         mlxsw_sp->router->rifs = kcalloc(max_rifs,
7988                                          sizeof(struct mlxsw_sp_rif *),
7989                                          GFP_KERNEL);
7990         if (!mlxsw_sp->router->rifs)
7991                 return -ENOMEM;
7992
7993         return 0;
7994 }
7995
7996 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
7997 {
7998         int i;
7999
8000         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
8001                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
8002
8003         kfree(mlxsw_sp->router->rifs);
8004 }
8005
8006 static int
8007 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp)
8008 {
8009         char tigcr_pl[MLXSW_REG_TIGCR_LEN];
8010
8011         mlxsw_reg_tigcr_pack(tigcr_pl, true, 0);
8012         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl);
8013 }
8014
8015 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
8016 {
8017         int err;
8018
8019         mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
8020         INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
8021
8022         err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp);
8023         if (err)
8024                 return err;
8025         err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp);
8026         if (err)
8027                 return err;
8028
8029         return mlxsw_sp_ipip_config_tigcr(mlxsw_sp);
8030 }
8031
8032 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
8033 {
8034         WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
8035 }
8036
8037 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
8038 {
8039         struct mlxsw_sp_router *router;
8040
8041         /* Flush pending FIB notifications and then flush the device's
8042          * table before requesting another dump. The FIB notification
8043          * block is unregistered, so no need to take RTNL.
8044          */
8045         mlxsw_core_flush_owq();
8046         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
8047         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
8048 }
8049
8050 #ifdef CONFIG_IP_ROUTE_MULTIPATH
8051 static void mlxsw_sp_mp_hash_header_set(char *recr2_pl, int header)
8052 {
8053         mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, header, true);
8054 }
8055
8056 static void mlxsw_sp_mp_hash_field_set(char *recr2_pl, int field)
8057 {
8058         mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, field, true);
8059 }
8060
8061 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl)
8062 {
8063         struct net *net = mlxsw_sp_net(mlxsw_sp);
8064         bool only_l3 = !net->ipv4.sysctl_fib_multipath_hash_policy;
8065
8066         mlxsw_sp_mp_hash_header_set(recr2_pl,
8067                                     MLXSW_REG_RECR2_IPV4_EN_NOT_TCP_NOT_UDP);
8068         mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV4_EN_TCP_UDP);
8069         mlxsw_reg_recr2_ipv4_sip_enable(recr2_pl);
8070         mlxsw_reg_recr2_ipv4_dip_enable(recr2_pl);
8071         if (only_l3)
8072                 return;
8073         mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_EN_IPV4);
8074         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV4_PROTOCOL);
8075         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_SPORT);
8076         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_DPORT);
8077 }
8078
8079 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl)
8080 {
8081         bool only_l3 = !ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp));
8082
8083         mlxsw_sp_mp_hash_header_set(recr2_pl,
8084                                     MLXSW_REG_RECR2_IPV6_EN_NOT_TCP_NOT_UDP);
8085         mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV6_EN_TCP_UDP);
8086         mlxsw_reg_recr2_ipv6_sip_enable(recr2_pl);
8087         mlxsw_reg_recr2_ipv6_dip_enable(recr2_pl);
8088         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_NEXT_HEADER);
8089         if (only_l3) {
8090                 mlxsw_sp_mp_hash_field_set(recr2_pl,
8091                                            MLXSW_REG_RECR2_IPV6_FLOW_LABEL);
8092         } else {
8093                 mlxsw_sp_mp_hash_header_set(recr2_pl,
8094                                             MLXSW_REG_RECR2_TCP_UDP_EN_IPV6);
8095                 mlxsw_sp_mp_hash_field_set(recr2_pl,
8096                                            MLXSW_REG_RECR2_TCP_UDP_SPORT);
8097                 mlxsw_sp_mp_hash_field_set(recr2_pl,
8098                                            MLXSW_REG_RECR2_TCP_UDP_DPORT);
8099         }
8100 }
8101
8102 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
8103 {
8104         char recr2_pl[MLXSW_REG_RECR2_LEN];
8105         u32 seed;
8106
8107         seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0);
8108         mlxsw_reg_recr2_pack(recr2_pl, seed);
8109         mlxsw_sp_mp4_hash_init(mlxsw_sp, recr2_pl);
8110         mlxsw_sp_mp6_hash_init(mlxsw_sp, recr2_pl);
8111
8112         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl);
8113 }
8114 #else
8115 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
8116 {
8117         return 0;
8118 }
8119 #endif
8120
8121 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp)
8122 {
8123         char rdpm_pl[MLXSW_REG_RDPM_LEN];
8124         unsigned int i;
8125
8126         MLXSW_REG_ZERO(rdpm, rdpm_pl);
8127
8128         /* HW is determining switch priority based on DSCP-bits, but the
8129          * kernel is still doing that based on the ToS. Since there's a
8130          * mismatch in bits we need to make sure to translate the right
8131          * value ToS would observe, skipping the 2 least-significant ECN bits.
8132          */
8133         for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++)
8134                 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2));
8135
8136         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl);
8137 }
8138
8139 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
8140 {
8141         struct net *net = mlxsw_sp_net(mlxsw_sp);
8142         bool usp = net->ipv4.sysctl_ip_fwd_update_priority;
8143         char rgcr_pl[MLXSW_REG_RGCR_LEN];
8144         u64 max_rifs;
8145
8146         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
8147                 return -EIO;
8148         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
8149
8150         mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
8151         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
8152         mlxsw_reg_rgcr_usp_set(rgcr_pl, usp);
8153         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
8154 }
8155
8156 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
8157 {
8158         char rgcr_pl[MLXSW_REG_RGCR_LEN];
8159
8160         mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
8161         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
8162 }
8163
8164 static const struct mlxsw_sp_router_ll_ops mlxsw_sp_router_ll_basic_ops = {
8165         .ralta_write = mlxsw_sp_router_ll_basic_ralta_write,
8166         .ralst_write = mlxsw_sp_router_ll_basic_ralst_write,
8167         .raltb_write = mlxsw_sp_router_ll_basic_raltb_write,
8168 };
8169
8170 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp,
8171                          struct netlink_ext_ack *extack)
8172 {
8173         struct mlxsw_sp_router *router;
8174         int err;
8175
8176         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
8177         if (!router)
8178                 return -ENOMEM;
8179         mutex_init(&router->lock);
8180         mlxsw_sp->router = router;
8181         router->mlxsw_sp = mlxsw_sp;
8182
8183         router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV4] = &mlxsw_sp_router_ll_basic_ops;
8184         router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_router_ll_basic_ops;
8185
8186         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
8187         err = __mlxsw_sp_router_init(mlxsw_sp);
8188         if (err)
8189                 goto err_router_init;
8190
8191         err = mlxsw_sp_rifs_init(mlxsw_sp);
8192         if (err)
8193                 goto err_rifs_init;
8194
8195         err = mlxsw_sp_ipips_init(mlxsw_sp);
8196         if (err)
8197                 goto err_ipips_init;
8198
8199         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
8200                               &mlxsw_sp_nexthop_ht_params);
8201         if (err)
8202                 goto err_nexthop_ht_init;
8203
8204         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
8205                               &mlxsw_sp_nexthop_group_ht_params);
8206         if (err)
8207                 goto err_nexthop_group_ht_init;
8208
8209         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list);
8210         err = mlxsw_sp_lpm_init(mlxsw_sp);
8211         if (err)
8212                 goto err_lpm_init;
8213
8214         err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops);
8215         if (err)
8216                 goto err_mr_init;
8217
8218         err = mlxsw_sp_vrs_init(mlxsw_sp);
8219         if (err)
8220                 goto err_vrs_init;
8221
8222         err = mlxsw_sp_neigh_init(mlxsw_sp);
8223         if (err)
8224                 goto err_neigh_init;
8225
8226         err = mlxsw_sp_mp_hash_init(mlxsw_sp);
8227         if (err)
8228                 goto err_mp_hash_init;
8229
8230         err = mlxsw_sp_dscp_init(mlxsw_sp);
8231         if (err)
8232                 goto err_dscp_init;
8233
8234         INIT_WORK(&router->fib_event_work, mlxsw_sp_router_fib_event_work);
8235         INIT_LIST_HEAD(&router->fib_event_queue);
8236         spin_lock_init(&router->fib_event_queue_lock);
8237
8238         router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event;
8239         err = register_inetaddr_notifier(&router->inetaddr_nb);
8240         if (err)
8241                 goto err_register_inetaddr_notifier;
8242
8243         router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event;
8244         err = register_inet6addr_notifier(&router->inet6addr_nb);
8245         if (err)
8246                 goto err_register_inet6addr_notifier;
8247
8248         mlxsw_sp->router->netevent_nb.notifier_call =
8249                 mlxsw_sp_router_netevent_event;
8250         err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb);
8251         if (err)
8252                 goto err_register_netevent_notifier;
8253
8254         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
8255         err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp),
8256                                     &mlxsw_sp->router->fib_nb,
8257                                     mlxsw_sp_router_fib_dump_flush, extack);
8258         if (err)
8259                 goto err_register_fib_notifier;
8260
8261         return 0;
8262
8263 err_register_fib_notifier:
8264         unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
8265 err_register_netevent_notifier:
8266         unregister_inet6addr_notifier(&router->inet6addr_nb);
8267 err_register_inet6addr_notifier:
8268         unregister_inetaddr_notifier(&router->inetaddr_nb);
8269 err_register_inetaddr_notifier:
8270         mlxsw_core_flush_owq();
8271         WARN_ON(!list_empty(&router->fib_event_queue));
8272 err_dscp_init:
8273 err_mp_hash_init:
8274         mlxsw_sp_neigh_fini(mlxsw_sp);
8275 err_neigh_init:
8276         mlxsw_sp_vrs_fini(mlxsw_sp);
8277 err_vrs_init:
8278         mlxsw_sp_mr_fini(mlxsw_sp);
8279 err_mr_init:
8280         mlxsw_sp_lpm_fini(mlxsw_sp);
8281 err_lpm_init:
8282         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
8283 err_nexthop_group_ht_init:
8284         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
8285 err_nexthop_ht_init:
8286         mlxsw_sp_ipips_fini(mlxsw_sp);
8287 err_ipips_init:
8288         mlxsw_sp_rifs_fini(mlxsw_sp);
8289 err_rifs_init:
8290         __mlxsw_sp_router_fini(mlxsw_sp);
8291 err_router_init:
8292         mutex_destroy(&mlxsw_sp->router->lock);
8293         kfree(mlxsw_sp->router);
8294         return err;
8295 }
8296
8297 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
8298 {
8299         unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp),
8300                                 &mlxsw_sp->router->fib_nb);
8301         unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
8302         unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb);
8303         unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb);
8304         mlxsw_core_flush_owq();
8305         WARN_ON(!list_empty(&mlxsw_sp->router->fib_event_queue));
8306         mlxsw_sp_neigh_fini(mlxsw_sp);
8307         mlxsw_sp_vrs_fini(mlxsw_sp);
8308         mlxsw_sp_mr_fini(mlxsw_sp);
8309         mlxsw_sp_lpm_fini(mlxsw_sp);
8310         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
8311         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
8312         mlxsw_sp_ipips_fini(mlxsw_sp);
8313         mlxsw_sp_rifs_fini(mlxsw_sp);
8314         __mlxsw_sp_router_fini(mlxsw_sp);
8315         mutex_destroy(&mlxsw_sp->router->lock);
8316         kfree(mlxsw_sp->router);
8317 }