2 * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
3 * Copyright (c) 2016-2017 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
5 * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
6 * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
7 * Copyright (c) 2017 Petr Machata <petrm@mellanox.com>
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. Neither the names of the copyright holders nor the names of its
18 * contributors may be used to endorse or promote products derived from
19 * this software without specific prior written permission.
21 * Alternatively, this software may be distributed under the terms of the
22 * GNU General Public License ("GPL") version 2 as published by the Free
23 * Software Foundation.
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
26 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
29 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
38 #include <linux/kernel.h>
39 #include <linux/types.h>
40 #include <linux/rhashtable.h>
41 #include <linux/bitops.h>
42 #include <linux/in6.h>
43 #include <linux/notifier.h>
44 #include <linux/inetdevice.h>
45 #include <linux/netdevice.h>
46 #include <linux/if_bridge.h>
47 #include <linux/socket.h>
48 #include <linux/route.h>
49 #include <net/netevent.h>
50 #include <net/neighbour.h>
52 #include <net/ip_fib.h>
53 #include <net/ip6_fib.h>
54 #include <net/fib_rules.h>
55 #include <net/ip_tunnels.h>
56 #include <net/l3mdev.h>
57 #include <net/addrconf.h>
58 #include <net/ndisc.h>
60 #include <net/fib_notifier.h>
65 #include "spectrum_cnt.h"
66 #include "spectrum_dpipe.h"
67 #include "spectrum_ipip.h"
68 #include "spectrum_router.h"
71 struct mlxsw_sp_lpm_tree;
72 struct mlxsw_sp_rif_ops;
74 struct mlxsw_sp_router {
75 struct mlxsw_sp *mlxsw_sp;
76 struct mlxsw_sp_rif **rifs;
77 struct mlxsw_sp_vr *vrs;
78 struct rhashtable neigh_ht;
79 struct rhashtable nexthop_group_ht;
80 struct rhashtable nexthop_ht;
82 struct mlxsw_sp_lpm_tree *trees;
83 unsigned int tree_count;
86 struct delayed_work dw;
87 unsigned long interval; /* ms */
89 struct delayed_work nexthop_probe_dw;
90 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
91 struct list_head nexthop_neighs_list;
92 struct list_head ipip_list;
94 struct notifier_block fib_nb;
95 const struct mlxsw_sp_rif_ops **rif_ops_arr;
96 const struct mlxsw_sp_ipip_ops **ipip_ops_arr;
100 struct list_head nexthop_list;
101 struct list_head neigh_list;
102 struct net_device *dev;
103 struct mlxsw_sp_fid *fid;
104 unsigned char addr[ETH_ALEN];
108 const struct mlxsw_sp_rif_ops *ops;
109 struct mlxsw_sp *mlxsw_sp;
111 unsigned int counter_ingress;
112 bool counter_ingress_valid;
113 unsigned int counter_egress;
114 bool counter_egress_valid;
117 struct mlxsw_sp_rif_params {
118 struct net_device *dev;
127 struct mlxsw_sp_rif_subport {
128 struct mlxsw_sp_rif common;
137 struct mlxsw_sp_rif_ipip_lb {
138 struct mlxsw_sp_rif common;
139 struct mlxsw_sp_rif_ipip_lb_config lb_config;
140 u16 ul_vr_id; /* Reserved for Spectrum-2. */
143 struct mlxsw_sp_rif_params_ipip_lb {
144 struct mlxsw_sp_rif_params common;
145 struct mlxsw_sp_rif_ipip_lb_config lb_config;
148 struct mlxsw_sp_rif_ops {
149 enum mlxsw_sp_rif_type type;
152 void (*setup)(struct mlxsw_sp_rif *rif,
153 const struct mlxsw_sp_rif_params *params);
154 int (*configure)(struct mlxsw_sp_rif *rif);
155 void (*deconfigure)(struct mlxsw_sp_rif *rif);
156 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
159 static unsigned int *
160 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
161 enum mlxsw_sp_rif_counter_dir dir)
164 case MLXSW_SP_RIF_COUNTER_EGRESS:
165 return &rif->counter_egress;
166 case MLXSW_SP_RIF_COUNTER_INGRESS:
167 return &rif->counter_ingress;
173 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
174 enum mlxsw_sp_rif_counter_dir dir)
177 case MLXSW_SP_RIF_COUNTER_EGRESS:
178 return rif->counter_egress_valid;
179 case MLXSW_SP_RIF_COUNTER_INGRESS:
180 return rif->counter_ingress_valid;
186 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
187 enum mlxsw_sp_rif_counter_dir dir,
191 case MLXSW_SP_RIF_COUNTER_EGRESS:
192 rif->counter_egress_valid = valid;
194 case MLXSW_SP_RIF_COUNTER_INGRESS:
195 rif->counter_ingress_valid = valid;
200 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
201 unsigned int counter_index, bool enable,
202 enum mlxsw_sp_rif_counter_dir dir)
204 char ritr_pl[MLXSW_REG_RITR_LEN];
205 bool is_egress = false;
208 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
210 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
211 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
215 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
217 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
220 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
221 struct mlxsw_sp_rif *rif,
222 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
224 char ricnt_pl[MLXSW_REG_RICNT_LEN];
225 unsigned int *p_counter_index;
229 valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
233 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
234 if (!p_counter_index)
236 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
237 MLXSW_REG_RICNT_OPCODE_NOP);
238 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
241 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
245 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
246 unsigned int counter_index)
248 char ricnt_pl[MLXSW_REG_RICNT_LEN];
250 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
251 MLXSW_REG_RICNT_OPCODE_CLEAR);
252 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
255 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
256 struct mlxsw_sp_rif *rif,
257 enum mlxsw_sp_rif_counter_dir dir)
259 unsigned int *p_counter_index;
262 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
263 if (!p_counter_index)
265 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
270 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
272 goto err_counter_clear;
274 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
275 *p_counter_index, true, dir);
277 goto err_counter_edit;
278 mlxsw_sp_rif_counter_valid_set(rif, dir, true);
283 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
288 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
289 struct mlxsw_sp_rif *rif,
290 enum mlxsw_sp_rif_counter_dir dir)
292 unsigned int *p_counter_index;
294 if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
297 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
298 if (WARN_ON(!p_counter_index))
300 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
301 *p_counter_index, false, dir);
302 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
304 mlxsw_sp_rif_counter_valid_set(rif, dir, false);
307 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
309 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
310 struct devlink *devlink;
312 devlink = priv_to_devlink(mlxsw_sp->core);
313 if (!devlink_dpipe_table_counter_enabled(devlink,
314 MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
316 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
319 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
321 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
323 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
326 static struct mlxsw_sp_rif *
327 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
328 const struct net_device *dev);
330 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
332 struct mlxsw_sp_prefix_usage {
333 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
336 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
337 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
340 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
341 struct mlxsw_sp_prefix_usage *prefix_usage2)
343 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
347 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
349 struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
351 return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
355 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
356 struct mlxsw_sp_prefix_usage *prefix_usage2)
358 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
362 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
363 unsigned char prefix_len)
365 set_bit(prefix_len, prefix_usage->b);
369 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
370 unsigned char prefix_len)
372 clear_bit(prefix_len, prefix_usage->b);
375 struct mlxsw_sp_fib_key {
376 unsigned char addr[sizeof(struct in6_addr)];
377 unsigned char prefix_len;
380 enum mlxsw_sp_fib_entry_type {
381 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
382 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
383 MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
385 /* This is a special case of local delivery, where a packet should be
386 * decapsulated on reception. Note that there is no corresponding ENCAP,
387 * because that's a type of next hop, not of FIB entry. (There can be
388 * several next hops in a REMOTE entry, and some of them may be
389 * encapsulating entries.)
391 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
394 struct mlxsw_sp_nexthop_group;
397 struct mlxsw_sp_fib_node {
398 struct list_head entry_list;
399 struct list_head list;
400 struct rhash_head ht_node;
401 struct mlxsw_sp_fib *fib;
402 struct mlxsw_sp_fib_key key;
405 struct mlxsw_sp_fib_entry_decap {
406 struct mlxsw_sp_ipip_entry *ipip_entry;
410 struct mlxsw_sp_fib_entry {
411 struct list_head list;
412 struct mlxsw_sp_fib_node *fib_node;
413 enum mlxsw_sp_fib_entry_type type;
414 struct list_head nexthop_group_node;
415 struct mlxsw_sp_nexthop_group *nh_group;
416 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
419 struct mlxsw_sp_fib4_entry {
420 struct mlxsw_sp_fib_entry common;
427 struct mlxsw_sp_fib6_entry {
428 struct mlxsw_sp_fib_entry common;
429 struct list_head rt6_list;
433 struct mlxsw_sp_rt6 {
434 struct list_head list;
438 struct mlxsw_sp_lpm_tree {
440 unsigned int ref_count;
441 enum mlxsw_sp_l3proto proto;
442 struct mlxsw_sp_prefix_usage prefix_usage;
445 struct mlxsw_sp_fib {
446 struct rhashtable ht;
447 struct list_head node_list;
448 struct mlxsw_sp_vr *vr;
449 struct mlxsw_sp_lpm_tree *lpm_tree;
450 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
451 struct mlxsw_sp_prefix_usage prefix_usage;
452 enum mlxsw_sp_l3proto proto;
456 u16 id; /* virtual router ID */
457 u32 tb_id; /* kernel fib table id */
458 unsigned int rif_count;
459 struct mlxsw_sp_fib *fib4;
460 struct mlxsw_sp_fib *fib6;
463 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
465 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
466 enum mlxsw_sp_l3proto proto)
468 struct mlxsw_sp_fib *fib;
471 fib = kzalloc(sizeof(*fib), GFP_KERNEL);
473 return ERR_PTR(-ENOMEM);
474 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
476 goto err_rhashtable_init;
477 INIT_LIST_HEAD(&fib->node_list);
487 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
489 WARN_ON(!list_empty(&fib->node_list));
490 WARN_ON(fib->lpm_tree);
491 rhashtable_destroy(&fib->ht);
495 static struct mlxsw_sp_lpm_tree *
496 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
498 static struct mlxsw_sp_lpm_tree *lpm_tree;
501 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
502 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
503 if (lpm_tree->ref_count == 0)
509 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
510 struct mlxsw_sp_lpm_tree *lpm_tree)
512 char ralta_pl[MLXSW_REG_RALTA_LEN];
514 mlxsw_reg_ralta_pack(ralta_pl, true,
515 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
517 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
520 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
521 struct mlxsw_sp_lpm_tree *lpm_tree)
523 char ralta_pl[MLXSW_REG_RALTA_LEN];
525 mlxsw_reg_ralta_pack(ralta_pl, false,
526 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
528 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
532 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
533 struct mlxsw_sp_prefix_usage *prefix_usage,
534 struct mlxsw_sp_lpm_tree *lpm_tree)
536 char ralst_pl[MLXSW_REG_RALST_LEN];
539 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
541 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
544 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
545 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
548 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
549 MLXSW_REG_RALST_BIN_NO_CHILD);
550 last_prefix = prefix;
552 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
555 static struct mlxsw_sp_lpm_tree *
556 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
557 struct mlxsw_sp_prefix_usage *prefix_usage,
558 enum mlxsw_sp_l3proto proto)
560 struct mlxsw_sp_lpm_tree *lpm_tree;
563 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
565 return ERR_PTR(-EBUSY);
566 lpm_tree->proto = proto;
567 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
571 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
574 goto err_left_struct_set;
575 memcpy(&lpm_tree->prefix_usage, prefix_usage,
576 sizeof(lpm_tree->prefix_usage));
580 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
584 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
585 struct mlxsw_sp_lpm_tree *lpm_tree)
587 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
590 static struct mlxsw_sp_lpm_tree *
591 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
592 struct mlxsw_sp_prefix_usage *prefix_usage,
593 enum mlxsw_sp_l3proto proto)
595 struct mlxsw_sp_lpm_tree *lpm_tree;
598 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
599 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
600 if (lpm_tree->ref_count != 0 &&
601 lpm_tree->proto == proto &&
602 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
606 return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
609 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
611 lpm_tree->ref_count++;
614 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
615 struct mlxsw_sp_lpm_tree *lpm_tree)
617 if (--lpm_tree->ref_count == 0)
618 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
621 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
623 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
625 struct mlxsw_sp_lpm_tree *lpm_tree;
629 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
632 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
633 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
634 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
635 sizeof(struct mlxsw_sp_lpm_tree),
637 if (!mlxsw_sp->router->lpm.trees)
640 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
641 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
642 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
648 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
650 kfree(mlxsw_sp->router->lpm.trees);
653 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
655 return !!vr->fib4 || !!vr->fib6;
658 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
660 struct mlxsw_sp_vr *vr;
663 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
664 vr = &mlxsw_sp->router->vrs[i];
665 if (!mlxsw_sp_vr_is_used(vr))
671 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
672 const struct mlxsw_sp_fib *fib, u8 tree_id)
674 char raltb_pl[MLXSW_REG_RALTB_LEN];
676 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
677 (enum mlxsw_reg_ralxx_protocol) fib->proto,
679 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
682 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
683 const struct mlxsw_sp_fib *fib)
685 char raltb_pl[MLXSW_REG_RALTB_LEN];
687 /* Bind to tree 0 which is default */
688 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
689 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
690 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
693 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
695 /* For our purpose, squash main and local table into one */
696 if (tb_id == RT_TABLE_LOCAL)
697 tb_id = RT_TABLE_MAIN;
701 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
704 struct mlxsw_sp_vr *vr;
707 tb_id = mlxsw_sp_fix_tb_id(tb_id);
709 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
710 vr = &mlxsw_sp->router->vrs[i];
711 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
717 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
718 enum mlxsw_sp_l3proto proto)
721 case MLXSW_SP_L3_PROTO_IPV4:
723 case MLXSW_SP_L3_PROTO_IPV6:
729 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
732 struct mlxsw_sp_vr *vr;
735 vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
737 return ERR_PTR(-EBUSY);
738 vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
739 if (IS_ERR(vr->fib4))
740 return ERR_CAST(vr->fib4);
741 vr->fib6 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV6);
742 if (IS_ERR(vr->fib6)) {
743 err = PTR_ERR(vr->fib6);
744 goto err_fib6_create;
750 mlxsw_sp_fib_destroy(vr->fib4);
755 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
757 mlxsw_sp_fib_destroy(vr->fib6);
759 mlxsw_sp_fib_destroy(vr->fib4);
763 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
765 struct mlxsw_sp_vr *vr;
767 tb_id = mlxsw_sp_fix_tb_id(tb_id);
768 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
770 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
774 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
776 if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
777 list_empty(&vr->fib6->node_list))
778 mlxsw_sp_vr_destroy(vr);
782 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
783 enum mlxsw_sp_l3proto proto, u8 tree_id)
785 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
787 if (!mlxsw_sp_vr_is_used(vr))
789 if (fib->lpm_tree && fib->lpm_tree->id == tree_id)
794 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
795 struct mlxsw_sp_fib *fib,
796 struct mlxsw_sp_lpm_tree *new_tree)
798 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
801 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
804 fib->lpm_tree = new_tree;
805 mlxsw_sp_lpm_tree_hold(new_tree);
806 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
810 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
811 struct mlxsw_sp_fib *fib,
812 struct mlxsw_sp_lpm_tree *new_tree)
814 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
815 enum mlxsw_sp_l3proto proto = fib->proto;
816 u8 old_id, new_id = new_tree->id;
817 struct mlxsw_sp_vr *vr;
822 old_id = old_tree->id;
824 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
825 vr = &mlxsw_sp->router->vrs[i];
826 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
828 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
829 mlxsw_sp_vr_fib(vr, proto),
832 goto err_tree_replace;
838 for (i--; i >= 0; i--) {
839 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
841 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
842 mlxsw_sp_vr_fib(vr, proto),
848 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
851 fib->lpm_tree = new_tree;
852 mlxsw_sp_lpm_tree_hold(new_tree);
857 mlxsw_sp_vrs_prefixes(struct mlxsw_sp *mlxsw_sp,
858 enum mlxsw_sp_l3proto proto,
859 struct mlxsw_sp_prefix_usage *req_prefix_usage)
863 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
864 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
865 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
866 unsigned char prefix;
868 if (!mlxsw_sp_vr_is_used(vr))
870 mlxsw_sp_prefix_usage_for_each(prefix, &fib->prefix_usage)
871 mlxsw_sp_prefix_usage_set(req_prefix_usage, prefix);
875 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
877 struct mlxsw_sp_vr *vr;
881 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
884 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
885 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
887 if (!mlxsw_sp->router->vrs)
890 for (i = 0; i < max_vrs; i++) {
891 vr = &mlxsw_sp->router->vrs[i];
898 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
900 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
902 /* At this stage we're guaranteed not to have new incoming
903 * FIB notifications and the work queue is free from FIBs
904 * sitting on top of mlxsw netdevs. However, we can still
905 * have other FIBs queued. Flush the queue before flushing
906 * the device's tables. No need for locks, as we're the only
909 mlxsw_core_flush_owq();
910 mlxsw_sp_router_fib_flush(mlxsw_sp);
911 kfree(mlxsw_sp->router->vrs);
914 static struct net_device *
915 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
917 struct ip_tunnel *tun = netdev_priv(ol_dev);
918 struct net *net = dev_net(ol_dev);
920 return __dev_get_by_index(net, tun->parms.link);
923 static u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
925 struct net_device *d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
928 return l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
930 return l3mdev_fib_table(ol_dev) ? : RT_TABLE_MAIN;
933 static struct mlxsw_sp_rif *
934 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
935 const struct mlxsw_sp_rif_params *params);
937 static struct mlxsw_sp_rif_ipip_lb *
938 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
939 enum mlxsw_sp_ipip_type ipipt,
940 struct net_device *ol_dev)
942 struct mlxsw_sp_rif_params_ipip_lb lb_params;
943 const struct mlxsw_sp_ipip_ops *ipip_ops;
944 struct mlxsw_sp_rif *rif;
946 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
947 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
948 .common.dev = ol_dev,
950 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
953 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common);
955 return ERR_CAST(rif);
956 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
959 static struct mlxsw_sp_ipip_entry *
960 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
961 enum mlxsw_sp_ipip_type ipipt,
962 struct net_device *ol_dev)
964 struct mlxsw_sp_ipip_entry *ipip_entry;
965 struct mlxsw_sp_ipip_entry *ret = NULL;
967 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
969 return ERR_PTR(-ENOMEM);
971 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
973 if (IS_ERR(ipip_entry->ol_lb)) {
974 ret = ERR_CAST(ipip_entry->ol_lb);
975 goto err_ol_ipip_lb_create;
978 ipip_entry->ipipt = ipipt;
979 ipip_entry->ol_dev = ol_dev;
983 err_ol_ipip_lb_create:
989 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp_ipip_entry *ipip_entry)
991 WARN_ON(ipip_entry->ref_count > 0);
992 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
997 mlxsw_sp_ipip_netdev_saddr4(const struct net_device *ol_dev)
999 struct ip_tunnel *tun = netdev_priv(ol_dev);
1001 return tun->parms.iph.saddr;
1004 union mlxsw_sp_l3addr
1005 mlxsw_sp_ipip_netdev_saddr(enum mlxsw_sp_l3proto proto,
1006 const struct net_device *ol_dev)
1009 case MLXSW_SP_L3_PROTO_IPV4:
1010 return (union mlxsw_sp_l3addr) {
1011 .addr4 = mlxsw_sp_ipip_netdev_saddr4(ol_dev),
1013 case MLXSW_SP_L3_PROTO_IPV6:
1018 return (union mlxsw_sp_l3addr) {
1023 __be32 mlxsw_sp_ipip_netdev_daddr4(const struct net_device *ol_dev)
1025 struct ip_tunnel *tun = netdev_priv(ol_dev);
1027 return tun->parms.iph.daddr;
1030 union mlxsw_sp_l3addr
1031 mlxsw_sp_ipip_netdev_daddr(enum mlxsw_sp_l3proto proto,
1032 const struct net_device *ol_dev)
1035 case MLXSW_SP_L3_PROTO_IPV4:
1036 return (union mlxsw_sp_l3addr) {
1037 .addr4 = mlxsw_sp_ipip_netdev_daddr4(ol_dev),
1039 case MLXSW_SP_L3_PROTO_IPV6:
1044 return (union mlxsw_sp_l3addr) {
1049 static bool mlxsw_sp_l3addr_eq(const union mlxsw_sp_l3addr *addr1,
1050 const union mlxsw_sp_l3addr *addr2)
1052 return !memcmp(addr1, addr2, sizeof(*addr1));
1056 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1057 const enum mlxsw_sp_l3proto ul_proto,
1058 union mlxsw_sp_l3addr saddr,
1060 struct mlxsw_sp_ipip_entry *ipip_entry)
1062 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1063 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1064 union mlxsw_sp_l3addr tun_saddr;
1066 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1069 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1070 return tun_ul_tb_id == ul_tb_id &&
1071 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1075 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1076 struct mlxsw_sp_fib_entry *fib_entry,
1077 struct mlxsw_sp_ipip_entry *ipip_entry)
1082 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, 1, &tunnel_index);
1086 ipip_entry->decap_fib_entry = fib_entry;
1087 fib_entry->decap.ipip_entry = ipip_entry;
1088 fib_entry->decap.tunnel_index = tunnel_index;
1092 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1093 struct mlxsw_sp_fib_entry *fib_entry)
1095 /* Unlink this node from the IPIP entry that it's the decap entry of. */
1096 fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1097 fib_entry->decap.ipip_entry = NULL;
1098 mlxsw_sp_kvdl_free(mlxsw_sp, fib_entry->decap.tunnel_index);
1101 static struct mlxsw_sp_fib_node *
1102 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1103 size_t addr_len, unsigned char prefix_len);
1104 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1105 struct mlxsw_sp_fib_entry *fib_entry);
1108 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1109 struct mlxsw_sp_ipip_entry *ipip_entry)
1111 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1113 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1114 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1116 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1120 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1121 struct mlxsw_sp_ipip_entry *ipip_entry,
1122 struct mlxsw_sp_fib_entry *decap_fib_entry)
1124 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1127 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1129 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1130 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1133 /* Given an IPIP entry, find the corresponding decap route. */
1134 static struct mlxsw_sp_fib_entry *
1135 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1136 struct mlxsw_sp_ipip_entry *ipip_entry)
1138 static struct mlxsw_sp_fib_node *fib_node;
1139 const struct mlxsw_sp_ipip_ops *ipip_ops;
1140 struct mlxsw_sp_fib_entry *fib_entry;
1141 unsigned char saddr_prefix_len;
1142 union mlxsw_sp_l3addr saddr;
1143 struct mlxsw_sp_fib *ul_fib;
1144 struct mlxsw_sp_vr *ul_vr;
1150 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1152 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1153 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1157 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1158 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1159 ipip_entry->ol_dev);
1161 switch (ipip_ops->ul_proto) {
1162 case MLXSW_SP_L3_PROTO_IPV4:
1163 saddr4 = be32_to_cpu(saddr.addr4);
1166 saddr_prefix_len = 32;
1168 case MLXSW_SP_L3_PROTO_IPV6:
1173 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1175 if (!fib_node || list_empty(&fib_node->entry_list))
1178 fib_entry = list_first_entry(&fib_node->entry_list,
1179 struct mlxsw_sp_fib_entry, list);
1180 if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1186 static struct mlxsw_sp_ipip_entry *
1187 mlxsw_sp_ipip_entry_get(struct mlxsw_sp *mlxsw_sp,
1188 enum mlxsw_sp_ipip_type ipipt,
1189 struct net_device *ol_dev)
1191 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1192 struct mlxsw_sp_router *router = mlxsw_sp->router;
1193 struct mlxsw_sp_fib_entry *decap_fib_entry;
1194 struct mlxsw_sp_ipip_entry *ipip_entry;
1195 enum mlxsw_sp_l3proto ul_proto;
1196 union mlxsw_sp_l3addr saddr;
1198 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1200 if (ipip_entry->ol_dev == ol_dev)
1203 /* The configuration where several tunnels have the same local
1204 * address in the same underlay table needs special treatment in
1205 * the HW. That is currently not implemented in the driver.
1207 ul_proto = router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1208 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1209 if (mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1210 ul_tb_id, ipip_entry))
1211 return ERR_PTR(-EEXIST);
1214 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1215 if (IS_ERR(ipip_entry))
1218 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1219 if (decap_fib_entry)
1220 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1223 list_add_tail(&ipip_entry->ipip_list_node,
1224 &mlxsw_sp->router->ipip_list);
1227 ++ipip_entry->ref_count;
1232 mlxsw_sp_ipip_entry_put(struct mlxsw_sp *mlxsw_sp,
1233 struct mlxsw_sp_ipip_entry *ipip_entry)
1235 if (--ipip_entry->ref_count == 0) {
1236 list_del(&ipip_entry->ipip_list_node);
1237 if (ipip_entry->decap_fib_entry)
1238 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1239 mlxsw_sp_ipip_entry_destroy(ipip_entry);
1244 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1245 const struct net_device *ul_dev,
1246 enum mlxsw_sp_l3proto ul_proto,
1247 union mlxsw_sp_l3addr ul_dip,
1248 struct mlxsw_sp_ipip_entry *ipip_entry)
1250 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1251 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1252 struct net_device *ipip_ul_dev;
1254 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1257 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev);
1258 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1259 ul_tb_id, ipip_entry) &&
1260 (!ipip_ul_dev || ipip_ul_dev == ul_dev);
1263 /* Given decap parameters, find the corresponding IPIP entry. */
1264 static struct mlxsw_sp_ipip_entry *
1265 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp,
1266 const struct net_device *ul_dev,
1267 enum mlxsw_sp_l3proto ul_proto,
1268 union mlxsw_sp_l3addr ul_dip)
1270 struct mlxsw_sp_ipip_entry *ipip_entry;
1272 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1274 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1282 struct mlxsw_sp_neigh_key {
1283 struct neighbour *n;
1286 struct mlxsw_sp_neigh_entry {
1287 struct list_head rif_list_node;
1288 struct rhash_head ht_node;
1289 struct mlxsw_sp_neigh_key key;
1292 unsigned char ha[ETH_ALEN];
1293 struct list_head nexthop_list; /* list of nexthops using
1296 struct list_head nexthop_neighs_list_node;
1297 unsigned int counter_index;
1301 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
1302 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
1303 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
1304 .key_len = sizeof(struct mlxsw_sp_neigh_key),
1307 struct mlxsw_sp_neigh_entry *
1308 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
1309 struct mlxsw_sp_neigh_entry *neigh_entry)
1312 if (list_empty(&rif->neigh_list))
1315 return list_first_entry(&rif->neigh_list,
1316 typeof(*neigh_entry),
1319 if (neigh_entry->rif_list_node.next == &rif->neigh_list)
1321 return list_next_entry(neigh_entry, rif_list_node);
1324 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
1326 return neigh_entry->key.n->tbl->family;
1330 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
1332 return neigh_entry->ha;
1335 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1337 struct neighbour *n;
1339 n = neigh_entry->key.n;
1340 return ntohl(*((__be32 *) n->primary_key));
1344 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1346 struct neighbour *n;
1348 n = neigh_entry->key.n;
1349 return (struct in6_addr *) &n->primary_key;
1352 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
1353 struct mlxsw_sp_neigh_entry *neigh_entry,
1356 if (!neigh_entry->counter_valid)
1359 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
1363 static struct mlxsw_sp_neigh_entry *
1364 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
1367 struct mlxsw_sp_neigh_entry *neigh_entry;
1369 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
1373 neigh_entry->key.n = n;
1374 neigh_entry->rif = rif;
1375 INIT_LIST_HEAD(&neigh_entry->nexthop_list);
1380 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
1386 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
1387 struct mlxsw_sp_neigh_entry *neigh_entry)
1389 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
1390 &neigh_entry->ht_node,
1391 mlxsw_sp_neigh_ht_params);
1395 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
1396 struct mlxsw_sp_neigh_entry *neigh_entry)
1398 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
1399 &neigh_entry->ht_node,
1400 mlxsw_sp_neigh_ht_params);
1404 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
1405 struct mlxsw_sp_neigh_entry *neigh_entry)
1407 struct devlink *devlink;
1408 const char *table_name;
1410 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
1412 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
1415 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
1422 devlink = priv_to_devlink(mlxsw_sp->core);
1423 return devlink_dpipe_table_counter_enabled(devlink, table_name);
1427 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
1428 struct mlxsw_sp_neigh_entry *neigh_entry)
1430 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
1433 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
1436 neigh_entry->counter_valid = true;
1440 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
1441 struct mlxsw_sp_neigh_entry *neigh_entry)
1443 if (!neigh_entry->counter_valid)
1445 mlxsw_sp_flow_counter_free(mlxsw_sp,
1446 neigh_entry->counter_index);
1447 neigh_entry->counter_valid = false;
1450 static struct mlxsw_sp_neigh_entry *
1451 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1453 struct mlxsw_sp_neigh_entry *neigh_entry;
1454 struct mlxsw_sp_rif *rif;
1457 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
1459 return ERR_PTR(-EINVAL);
1461 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
1463 return ERR_PTR(-ENOMEM);
1465 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
1467 goto err_neigh_entry_insert;
1469 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1470 list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
1474 err_neigh_entry_insert:
1475 mlxsw_sp_neigh_entry_free(neigh_entry);
1476 return ERR_PTR(err);
1480 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1481 struct mlxsw_sp_neigh_entry *neigh_entry)
1483 list_del(&neigh_entry->rif_list_node);
1484 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1485 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
1486 mlxsw_sp_neigh_entry_free(neigh_entry);
1489 static struct mlxsw_sp_neigh_entry *
1490 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1492 struct mlxsw_sp_neigh_key key;
1495 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
1496 &key, mlxsw_sp_neigh_ht_params);
1500 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
1502 unsigned long interval;
1504 #if IS_ENABLED(CONFIG_IPV6)
1505 interval = min_t(unsigned long,
1506 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
1507 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
1509 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
1511 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
1514 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1518 struct net_device *dev;
1519 struct neighbour *n;
1524 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
1526 if (!mlxsw_sp->router->rifs[rif]) {
1527 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1532 dev = mlxsw_sp->router->rifs[rif]->dev;
1533 n = neigh_lookup(&arp_tbl, &dipn, dev);
1535 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
1540 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
1541 neigh_event_send(n, NULL);
1545 #if IS_ENABLED(CONFIG_IPV6)
1546 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1550 struct net_device *dev;
1551 struct neighbour *n;
1552 struct in6_addr dip;
1555 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
1558 if (!mlxsw_sp->router->rifs[rif]) {
1559 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1563 dev = mlxsw_sp->router->rifs[rif]->dev;
1564 n = neigh_lookup(&nd_tbl, &dip, dev);
1566 netdev_err(dev, "Failed to find matching neighbour for IP=%pI6c\n",
1571 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
1572 neigh_event_send(n, NULL);
1576 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1583 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1590 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1592 /* Hardware starts counting at 0, so add 1. */
1595 /* Each record consists of several neighbour entries. */
1596 for (i = 0; i < num_entries; i++) {
1599 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
1600 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
1606 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1610 /* One record contains one entry. */
1611 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
1615 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
1616 char *rauhtd_pl, int rec_index)
1618 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
1619 case MLXSW_REG_RAUHTD_TYPE_IPV4:
1620 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
1623 case MLXSW_REG_RAUHTD_TYPE_IPV6:
1624 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
1630 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1632 u8 num_rec, last_rec_index, num_entries;
1634 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1635 last_rec_index = num_rec - 1;
1637 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1639 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1640 MLXSW_REG_RAUHTD_TYPE_IPV6)
1643 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1645 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1651 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
1653 enum mlxsw_reg_rauhtd_type type)
1658 /* Make sure the neighbour's netdev isn't removed in the
1663 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
1664 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1667 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1670 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1671 for (i = 0; i < num_rec; i++)
1672 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1674 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
1680 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1682 enum mlxsw_reg_rauhtd_type type;
1686 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1690 type = MLXSW_REG_RAUHTD_TYPE_IPV4;
1691 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1695 type = MLXSW_REG_RAUHTD_TYPE_IPV6;
1696 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1702 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1704 struct mlxsw_sp_neigh_entry *neigh_entry;
1706 /* Take RTNL mutex here to prevent lists from changes */
1708 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
1709 nexthop_neighs_list_node)
1710 /* If this neigh have nexthops, make the kernel think this neigh
1711 * is active regardless of the traffic.
1713 neigh_event_send(neigh_entry->key.n, NULL);
1718 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1720 unsigned long interval = mlxsw_sp->router->neighs_update.interval;
1722 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
1723 msecs_to_jiffies(interval));
1726 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1728 struct mlxsw_sp_router *router;
1731 router = container_of(work, struct mlxsw_sp_router,
1732 neighs_update.dw.work);
1733 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
1735 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
1737 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
1739 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
1742 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1744 struct mlxsw_sp_neigh_entry *neigh_entry;
1745 struct mlxsw_sp_router *router;
1747 router = container_of(work, struct mlxsw_sp_router,
1748 nexthop_probe_dw.work);
1749 /* Iterate over nexthop neighbours, find those who are unresolved and
1750 * send arp on them. This solves the chicken-egg problem when
1751 * the nexthop wouldn't get offloaded until the neighbor is resolved
1752 * but it wouldn't get resolved ever in case traffic is flowing in HW
1753 * using different nexthop.
1755 * Take RTNL mutex here to prevent lists from changes.
1758 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
1759 nexthop_neighs_list_node)
1760 if (!neigh_entry->connected)
1761 neigh_event_send(neigh_entry->key.n, NULL);
1764 mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
1765 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1769 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1770 struct mlxsw_sp_neigh_entry *neigh_entry,
1773 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1775 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1776 MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1780 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1781 struct mlxsw_sp_neigh_entry *neigh_entry,
1782 enum mlxsw_reg_rauht_op op)
1784 struct neighbour *n = neigh_entry->key.n;
1785 u32 dip = ntohl(*((__be32 *) n->primary_key));
1786 char rauht_pl[MLXSW_REG_RAUHT_LEN];
1788 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1790 if (neigh_entry->counter_valid)
1791 mlxsw_reg_rauht_pack_counter(rauht_pl,
1792 neigh_entry->counter_index);
1793 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1797 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
1798 struct mlxsw_sp_neigh_entry *neigh_entry,
1799 enum mlxsw_reg_rauht_op op)
1801 struct neighbour *n = neigh_entry->key.n;
1802 char rauht_pl[MLXSW_REG_RAUHT_LEN];
1803 const char *dip = n->primary_key;
1805 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1807 if (neigh_entry->counter_valid)
1808 mlxsw_reg_rauht_pack_counter(rauht_pl,
1809 neigh_entry->counter_index);
1810 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1813 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
1815 struct neighbour *n = neigh_entry->key.n;
1817 /* Packets with a link-local destination address are trapped
1818 * after LPM lookup and never reach the neighbour table, so
1819 * there is no need to program such neighbours to the device.
1821 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
1822 IPV6_ADDR_LINKLOCAL)
1828 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1829 struct mlxsw_sp_neigh_entry *neigh_entry,
1832 if (!adding && !neigh_entry->connected)
1834 neigh_entry->connected = adding;
1835 if (neigh_entry->key.n->tbl->family == AF_INET) {
1836 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1837 mlxsw_sp_rauht_op(adding));
1838 } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
1839 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
1841 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
1842 mlxsw_sp_rauht_op(adding));
1849 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
1850 struct mlxsw_sp_neigh_entry *neigh_entry,
1854 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1856 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1857 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
1860 struct mlxsw_sp_neigh_event_work {
1861 struct work_struct work;
1862 struct mlxsw_sp *mlxsw_sp;
1863 struct neighbour *n;
1866 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1868 struct mlxsw_sp_neigh_event_work *neigh_work =
1869 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1870 struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1871 struct mlxsw_sp_neigh_entry *neigh_entry;
1872 struct neighbour *n = neigh_work->n;
1873 unsigned char ha[ETH_ALEN];
1874 bool entry_connected;
1877 /* If these parameters are changed after we release the lock,
1878 * then we are guaranteed to receive another event letting us
1881 read_lock_bh(&n->lock);
1882 memcpy(ha, n->ha, ETH_ALEN);
1883 nud_state = n->nud_state;
1885 read_unlock_bh(&n->lock);
1888 entry_connected = nud_state & NUD_VALID && !dead;
1889 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1890 if (!entry_connected && !neigh_entry)
1893 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1894 if (IS_ERR(neigh_entry))
1898 memcpy(neigh_entry->ha, ha, ETH_ALEN);
1899 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1900 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
1902 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1903 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1911 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1912 unsigned long event, void *ptr)
1914 struct mlxsw_sp_neigh_event_work *neigh_work;
1915 struct mlxsw_sp_port *mlxsw_sp_port;
1916 struct mlxsw_sp *mlxsw_sp;
1917 unsigned long interval;
1918 struct neigh_parms *p;
1919 struct neighbour *n;
1922 case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1925 /* We don't care about changes in the default table. */
1926 if (!p->dev || (p->tbl->family != AF_INET &&
1927 p->tbl->family != AF_INET6))
1930 /* We are in atomic context and can't take RTNL mutex,
1931 * so use RCU variant to walk the device chain.
1933 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1937 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1938 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1939 mlxsw_sp->router->neighs_update.interval = interval;
1941 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1943 case NETEVENT_NEIGH_UPDATE:
1946 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
1949 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
1953 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1955 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1959 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1960 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1963 /* Take a reference to ensure the neighbour won't be
1964 * destructed until we drop the reference in delayed
1968 mlxsw_core_schedule_work(&neigh_work->work);
1969 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1976 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1980 err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
1981 &mlxsw_sp_neigh_ht_params);
1985 /* Initialize the polling interval according to the default
1988 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1990 /* Create the delayed works for the activity_update */
1991 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
1992 mlxsw_sp_router_neighs_update_work);
1993 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
1994 mlxsw_sp_router_probe_unresolved_nexthops);
1995 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1996 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
2000 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2002 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
2003 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
2004 rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2007 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2008 struct mlxsw_sp_rif *rif)
2010 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2012 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2014 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2015 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2019 enum mlxsw_sp_nexthop_type {
2020 MLXSW_SP_NEXTHOP_TYPE_ETH,
2021 MLXSW_SP_NEXTHOP_TYPE_IPIP,
2024 struct mlxsw_sp_nexthop_key {
2025 struct fib_nh *fib_nh;
2028 struct mlxsw_sp_nexthop {
2029 struct list_head neigh_list_node; /* member of neigh entry list */
2030 struct list_head rif_list_node;
2031 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
2034 struct rhash_head ht_node;
2035 struct mlxsw_sp_nexthop_key key;
2036 unsigned char gw_addr[sizeof(struct in6_addr)];
2038 struct mlxsw_sp_rif *rif;
2039 u8 should_offload:1, /* set indicates this neigh is connected and
2040 * should be put to KVD linear area of this group.
2042 offloaded:1, /* set in case the neigh is actually put into
2043 * KVD linear area of this group.
2045 update:1; /* set indicates that MAC of this neigh should be
2048 enum mlxsw_sp_nexthop_type type;
2050 struct mlxsw_sp_neigh_entry *neigh_entry;
2051 struct mlxsw_sp_ipip_entry *ipip_entry;
2055 struct mlxsw_sp_nexthop_group {
2057 struct rhash_head ht_node;
2058 struct list_head fib_list; /* list of fib entries that use this group */
2059 struct neigh_table *neigh_tbl;
2060 u8 adj_index_valid:1,
2061 gateway:1; /* routes using the group use a gateway */
2065 struct mlxsw_sp_nexthop nexthops[0];
2066 #define nh_rif nexthops[0].rif
2069 static struct fib_info *
2070 mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp)
2072 return nh_grp->priv;
2075 struct mlxsw_sp_nexthop_group_cmp_arg {
2076 enum mlxsw_sp_l3proto proto;
2078 struct fib_info *fi;
2079 struct mlxsw_sp_fib6_entry *fib6_entry;
2084 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
2085 const struct in6_addr *gw, int ifindex)
2089 for (i = 0; i < nh_grp->count; i++) {
2090 const struct mlxsw_sp_nexthop *nh;
2092 nh = &nh_grp->nexthops[i];
2093 if (nh->ifindex == ifindex &&
2094 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
2102 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
2103 const struct mlxsw_sp_fib6_entry *fib6_entry)
2105 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2107 if (nh_grp->count != fib6_entry->nrt6)
2110 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2111 struct in6_addr *gw;
2114 ifindex = mlxsw_sp_rt6->rt->dst.dev->ifindex;
2115 gw = &mlxsw_sp_rt6->rt->rt6i_gateway;
2116 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex))
2124 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
2126 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
2127 const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
2129 switch (cmp_arg->proto) {
2130 case MLXSW_SP_L3_PROTO_IPV4:
2131 return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp);
2132 case MLXSW_SP_L3_PROTO_IPV6:
2133 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
2134 cmp_arg->fib6_entry);
2142 mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp)
2144 return nh_grp->neigh_tbl->family;
2147 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
2149 const struct mlxsw_sp_nexthop_group *nh_grp = data;
2150 const struct mlxsw_sp_nexthop *nh;
2151 struct fib_info *fi;
2155 switch (mlxsw_sp_nexthop_group_type(nh_grp)) {
2157 fi = mlxsw_sp_nexthop4_group_fi(nh_grp);
2158 return jhash(&fi, sizeof(fi), seed);
2160 val = nh_grp->count;
2161 for (i = 0; i < nh_grp->count; i++) {
2162 nh = &nh_grp->nexthops[i];
2165 return jhash(&val, sizeof(val), seed);
2173 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
2175 unsigned int val = fib6_entry->nrt6;
2176 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2177 struct net_device *dev;
2179 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2180 dev = mlxsw_sp_rt6->rt->dst.dev;
2181 val ^= dev->ifindex;
2184 return jhash(&val, sizeof(val), seed);
2188 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
2190 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
2192 switch (cmp_arg->proto) {
2193 case MLXSW_SP_L3_PROTO_IPV4:
2194 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
2195 case MLXSW_SP_L3_PROTO_IPV6:
2196 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
2203 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
2204 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
2205 .hashfn = mlxsw_sp_nexthop_group_hash,
2206 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj,
2207 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp,
2210 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
2211 struct mlxsw_sp_nexthop_group *nh_grp)
2213 if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2217 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
2219 mlxsw_sp_nexthop_group_ht_params);
2222 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
2223 struct mlxsw_sp_nexthop_group *nh_grp)
2225 if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2229 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
2231 mlxsw_sp_nexthop_group_ht_params);
2234 static struct mlxsw_sp_nexthop_group *
2235 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
2236 struct fib_info *fi)
2238 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
2240 cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4;
2242 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
2244 mlxsw_sp_nexthop_group_ht_params);
2247 static struct mlxsw_sp_nexthop_group *
2248 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
2249 struct mlxsw_sp_fib6_entry *fib6_entry)
2251 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
2253 cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6;
2254 cmp_arg.fib6_entry = fib6_entry;
2255 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
2257 mlxsw_sp_nexthop_group_ht_params);
2260 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
2261 .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
2262 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
2263 .key_len = sizeof(struct mlxsw_sp_nexthop_key),
2266 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
2267 struct mlxsw_sp_nexthop *nh)
2269 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
2270 &nh->ht_node, mlxsw_sp_nexthop_ht_params);
2273 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
2274 struct mlxsw_sp_nexthop *nh)
2276 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
2277 mlxsw_sp_nexthop_ht_params);
2280 static struct mlxsw_sp_nexthop *
2281 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
2282 struct mlxsw_sp_nexthop_key key)
2284 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
2285 mlxsw_sp_nexthop_ht_params);
2288 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
2289 const struct mlxsw_sp_fib *fib,
2290 u32 adj_index, u16 ecmp_size,
2294 char raleu_pl[MLXSW_REG_RALEU_LEN];
2296 mlxsw_reg_raleu_pack(raleu_pl,
2297 (enum mlxsw_reg_ralxx_protocol) fib->proto,
2298 fib->vr->id, adj_index, ecmp_size, new_adj_index,
2300 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
2303 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
2304 struct mlxsw_sp_nexthop_group *nh_grp,
2305 u32 old_adj_index, u16 old_ecmp_size)
2307 struct mlxsw_sp_fib_entry *fib_entry;
2308 struct mlxsw_sp_fib *fib = NULL;
2311 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2312 if (fib == fib_entry->fib_node->fib)
2314 fib = fib_entry->fib_node->fib;
2315 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
2326 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
2327 struct mlxsw_sp_nexthop *nh)
2329 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
2330 char ratr_pl[MLXSW_REG_RATR_LEN];
2332 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
2333 true, MLXSW_REG_RATR_TYPE_ETHERNET,
2334 adj_index, neigh_entry->rif);
2335 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
2336 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
2339 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
2341 struct mlxsw_sp_nexthop *nh)
2343 const struct mlxsw_sp_ipip_ops *ipip_ops;
2345 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
2346 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry);
2350 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
2351 struct mlxsw_sp_nexthop_group *nh_grp,
2354 u32 adj_index = nh_grp->adj_index; /* base */
2355 struct mlxsw_sp_nexthop *nh;
2359 for (i = 0; i < nh_grp->count; i++) {
2360 nh = &nh_grp->nexthops[i];
2362 if (!nh->should_offload) {
2367 if (nh->update || reallocate) {
2369 case MLXSW_SP_NEXTHOP_TYPE_ETH:
2370 err = mlxsw_sp_nexthop_mac_update
2371 (mlxsw_sp, adj_index, nh);
2373 case MLXSW_SP_NEXTHOP_TYPE_IPIP:
2374 err = mlxsw_sp_nexthop_ipip_update
2375 (mlxsw_sp, adj_index, nh);
2389 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2390 const struct mlxsw_sp_fib_entry *fib_entry);
2393 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
2394 struct mlxsw_sp_nexthop_group *nh_grp)
2396 struct mlxsw_sp_fib_entry *fib_entry;
2399 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2400 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2403 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2411 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
2412 enum mlxsw_reg_ralue_op op, int err);
2415 mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp)
2417 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
2418 struct mlxsw_sp_fib_entry *fib_entry;
2420 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2421 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2424 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2429 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
2430 struct mlxsw_sp_nexthop_group *nh_grp)
2432 struct mlxsw_sp_nexthop *nh;
2433 bool offload_change = false;
2436 bool old_adj_index_valid;
2442 if (!nh_grp->gateway) {
2443 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2447 for (i = 0; i < nh_grp->count; i++) {
2448 nh = &nh_grp->nexthops[i];
2450 if (nh->should_offload != nh->offloaded) {
2451 offload_change = true;
2452 if (nh->should_offload)
2455 if (nh->should_offload)
2458 if (!offload_change) {
2459 /* Nothing was added or removed, so no need to reallocate. Just
2460 * update MAC on existing adjacency indexes.
2462 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, false);
2464 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2470 /* No neigh of this group is connected so we just set
2471 * the trap and let everthing flow through kernel.
2475 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
2477 /* We ran out of KVD linear space, just set the
2478 * trap and let everything flow through kernel.
2480 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
2483 old_adj_index_valid = nh_grp->adj_index_valid;
2484 old_adj_index = nh_grp->adj_index;
2485 old_ecmp_size = nh_grp->ecmp_size;
2486 nh_grp->adj_index_valid = 1;
2487 nh_grp->adj_index = adj_index;
2488 nh_grp->ecmp_size = ecmp_size;
2489 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, true);
2491 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2495 if (!old_adj_index_valid) {
2496 /* The trap was set for fib entries, so we have to call
2497 * fib entry update to unset it and use adjacency index.
2499 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2501 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
2507 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
2508 old_adj_index, old_ecmp_size);
2509 mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
2511 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
2515 /* Offload state within the group changed, so update the flags. */
2516 mlxsw_sp_nexthop_fib_entries_refresh(nh_grp);
2521 old_adj_index_valid = nh_grp->adj_index_valid;
2522 nh_grp->adj_index_valid = 0;
2523 for (i = 0; i < nh_grp->count; i++) {
2524 nh = &nh_grp->nexthops[i];
2527 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2529 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
2530 if (old_adj_index_valid)
2531 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
2534 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
2538 nh->should_offload = 1;
2539 else if (nh->offloaded)
2540 nh->should_offload = 0;
2545 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2546 struct mlxsw_sp_neigh_entry *neigh_entry,
2549 struct mlxsw_sp_nexthop *nh;
2551 list_for_each_entry(nh, &neigh_entry->nexthop_list,
2553 __mlxsw_sp_nexthop_neigh_update(nh, removing);
2554 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2558 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
2559 struct mlxsw_sp_rif *rif)
2565 list_add(&nh->rif_list_node, &rif->nexthop_list);
2568 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
2573 list_del(&nh->rif_list_node);
2577 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
2578 struct mlxsw_sp_nexthop *nh)
2580 struct mlxsw_sp_neigh_entry *neigh_entry;
2581 struct neighbour *n;
2585 if (!nh->nh_grp->gateway || nh->neigh_entry)
2588 /* Take a reference of neigh here ensuring that neigh would
2589 * not be destructed before the nexthop entry is finished.
2590 * The reference is taken either in neigh_lookup() or
2591 * in neigh_create() in case n is not found.
2593 n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
2595 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
2599 neigh_event_send(n, NULL);
2601 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2603 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2604 if (IS_ERR(neigh_entry)) {
2606 goto err_neigh_entry_create;
2610 /* If that is the first nexthop connected to that neigh, add to
2611 * nexthop_neighs_list
2613 if (list_empty(&neigh_entry->nexthop_list))
2614 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
2615 &mlxsw_sp->router->nexthop_neighs_list);
2617 nh->neigh_entry = neigh_entry;
2618 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
2619 read_lock_bh(&n->lock);
2620 nud_state = n->nud_state;
2622 read_unlock_bh(&n->lock);
2623 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
2627 err_neigh_entry_create:
2632 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
2633 struct mlxsw_sp_nexthop *nh)
2635 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
2636 struct neighbour *n;
2640 n = neigh_entry->key.n;
2642 __mlxsw_sp_nexthop_neigh_update(nh, true);
2643 list_del(&nh->neigh_list_node);
2644 nh->neigh_entry = NULL;
2646 /* If that is the last nexthop connected to that neigh, remove from
2647 * nexthop_neighs_list
2649 if (list_empty(&neigh_entry->nexthop_list))
2650 list_del(&neigh_entry->nexthop_neighs_list_node);
2652 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2653 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2658 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
2659 const struct net_device *dev,
2660 enum mlxsw_sp_ipip_type *p_type)
2662 struct mlxsw_sp_router *router = mlxsw_sp->router;
2663 const struct mlxsw_sp_ipip_ops *ipip_ops;
2664 enum mlxsw_sp_ipip_type ipipt;
2666 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
2667 ipip_ops = router->ipip_ops_arr[ipipt];
2668 if (dev->type == ipip_ops->dev_type) {
2677 static int mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
2678 enum mlxsw_sp_ipip_type ipipt,
2679 struct mlxsw_sp_nexthop *nh,
2680 struct net_device *ol_dev)
2682 if (!nh->nh_grp->gateway || nh->ipip_entry)
2685 nh->ipip_entry = mlxsw_sp_ipip_entry_get(mlxsw_sp, ipipt, ol_dev);
2686 if (IS_ERR(nh->ipip_entry))
2687 return PTR_ERR(nh->ipip_entry);
2689 __mlxsw_sp_nexthop_neigh_update(nh, false);
2693 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
2694 struct mlxsw_sp_nexthop *nh)
2696 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
2701 __mlxsw_sp_nexthop_neigh_update(nh, true);
2702 mlxsw_sp_ipip_entry_put(mlxsw_sp, ipip_entry);
2703 nh->ipip_entry = NULL;
2706 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
2707 const struct fib_nh *fib_nh,
2708 enum mlxsw_sp_ipip_type *p_ipipt)
2710 struct net_device *dev = fib_nh->nh_dev;
2713 fib_nh->nh_parent->fib_type == RTN_UNICAST &&
2714 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
2717 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
2718 struct mlxsw_sp_nexthop *nh)
2721 case MLXSW_SP_NEXTHOP_TYPE_ETH:
2722 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
2723 mlxsw_sp_nexthop_rif_fini(nh);
2725 case MLXSW_SP_NEXTHOP_TYPE_IPIP:
2726 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
2731 static int mlxsw_sp_nexthop4_type_init(struct mlxsw_sp *mlxsw_sp,
2732 struct mlxsw_sp_nexthop *nh,
2733 struct fib_nh *fib_nh)
2735 struct mlxsw_sp_router *router = mlxsw_sp->router;
2736 struct net_device *dev = fib_nh->nh_dev;
2737 enum mlxsw_sp_ipip_type ipipt;
2738 struct mlxsw_sp_rif *rif;
2741 if (mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fib_nh, &ipipt) &&
2742 router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev,
2743 MLXSW_SP_L3_PROTO_IPV4)) {
2744 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
2745 return mlxsw_sp_nexthop_ipip_init(mlxsw_sp, ipipt, nh, dev);
2748 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
2749 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
2753 mlxsw_sp_nexthop_rif_init(nh, rif);
2754 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
2756 goto err_neigh_init;
2761 mlxsw_sp_nexthop_rif_fini(nh);
2765 static void mlxsw_sp_nexthop4_type_fini(struct mlxsw_sp *mlxsw_sp,
2766 struct mlxsw_sp_nexthop *nh)
2768 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
2771 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
2772 struct mlxsw_sp_nexthop_group *nh_grp,
2773 struct mlxsw_sp_nexthop *nh,
2774 struct fib_nh *fib_nh)
2776 struct net_device *dev = fib_nh->nh_dev;
2777 struct in_device *in_dev;
2780 nh->nh_grp = nh_grp;
2781 nh->key.fib_nh = fib_nh;
2782 memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw));
2783 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
2790 in_dev = __in_dev_get_rtnl(dev);
2791 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
2792 fib_nh->nh_flags & RTNH_F_LINKDOWN)
2795 err = mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
2797 goto err_nexthop_neigh_init;
2801 err_nexthop_neigh_init:
2802 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2806 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
2807 struct mlxsw_sp_nexthop *nh)
2809 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
2810 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2813 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
2814 unsigned long event, struct fib_nh *fib_nh)
2816 struct mlxsw_sp_nexthop_key key;
2817 struct mlxsw_sp_nexthop *nh;
2819 if (mlxsw_sp->router->aborted)
2822 key.fib_nh = fib_nh;
2823 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
2824 if (WARN_ON_ONCE(!nh))
2828 case FIB_EVENT_NH_ADD:
2829 mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
2831 case FIB_EVENT_NH_DEL:
2832 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
2836 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2839 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2840 struct mlxsw_sp_rif *rif)
2842 struct mlxsw_sp_nexthop *nh, *tmp;
2844 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
2845 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
2846 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2850 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
2851 const struct fib_info *fi)
2853 return fi->fib_nh->nh_scope == RT_SCOPE_LINK ||
2854 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fi->fib_nh, NULL);
2857 static struct mlxsw_sp_nexthop_group *
2858 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
2860 struct mlxsw_sp_nexthop_group *nh_grp;
2861 struct mlxsw_sp_nexthop *nh;
2862 struct fib_nh *fib_nh;
2867 alloc_size = sizeof(*nh_grp) +
2868 fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
2869 nh_grp = kzalloc(alloc_size, GFP_KERNEL);
2871 return ERR_PTR(-ENOMEM);
2873 INIT_LIST_HEAD(&nh_grp->fib_list);
2874 nh_grp->neigh_tbl = &arp_tbl;
2876 nh_grp->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi);
2877 nh_grp->count = fi->fib_nhs;
2879 for (i = 0; i < nh_grp->count; i++) {
2880 nh = &nh_grp->nexthops[i];
2881 fib_nh = &fi->fib_nh[i];
2882 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
2884 goto err_nexthop4_init;
2886 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
2888 goto err_nexthop_group_insert;
2889 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2892 err_nexthop_group_insert:
2894 for (i--; i >= 0; i--) {
2895 nh = &nh_grp->nexthops[i];
2896 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2900 return ERR_PTR(err);
2904 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
2905 struct mlxsw_sp_nexthop_group *nh_grp)
2907 struct mlxsw_sp_nexthop *nh;
2910 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
2911 for (i = 0; i < nh_grp->count; i++) {
2912 nh = &nh_grp->nexthops[i];
2913 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2915 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2916 WARN_ON_ONCE(nh_grp->adj_index_valid);
2917 fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp));
2921 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
2922 struct mlxsw_sp_fib_entry *fib_entry,
2923 struct fib_info *fi)
2925 struct mlxsw_sp_nexthop_group *nh_grp;
2927 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
2929 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
2931 return PTR_ERR(nh_grp);
2933 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
2934 fib_entry->nh_group = nh_grp;
2938 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
2939 struct mlxsw_sp_fib_entry *fib_entry)
2941 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2943 list_del(&fib_entry->nexthop_group_node);
2944 if (!list_empty(&nh_grp->fib_list))
2946 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
2950 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2952 struct mlxsw_sp_fib4_entry *fib4_entry;
2954 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
2956 return !fib4_entry->tos;
2960 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2962 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
2964 switch (fib_entry->fib_node->fib->proto) {
2965 case MLXSW_SP_L3_PROTO_IPV4:
2966 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
2969 case MLXSW_SP_L3_PROTO_IPV6:
2973 switch (fib_entry->type) {
2974 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2975 return !!nh_group->adj_index_valid;
2976 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2977 return !!nh_group->nh_rif;
2978 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
2985 static struct mlxsw_sp_nexthop *
2986 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
2987 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
2991 for (i = 0; i < nh_grp->count; i++) {
2992 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2993 struct rt6_info *rt = mlxsw_sp_rt6->rt;
2995 if (nh->rif && nh->rif->dev == rt->dst.dev &&
2996 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
3006 mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3008 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3011 if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL ||
3012 fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP) {
3013 nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
3017 for (i = 0; i < nh_grp->count; i++) {
3018 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3021 nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
3023 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
3028 mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3030 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3033 for (i = 0; i < nh_grp->count; i++) {
3034 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3036 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
3041 mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3043 struct mlxsw_sp_fib6_entry *fib6_entry;
3044 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3046 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
3049 if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
3050 list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3051 list)->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
3055 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3056 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3057 struct mlxsw_sp_nexthop *nh;
3059 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3060 if (nh && nh->offloaded)
3061 mlxsw_sp_rt6->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
3063 mlxsw_sp_rt6->rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
3068 mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3070 struct mlxsw_sp_fib6_entry *fib6_entry;
3071 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3073 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
3075 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3076 struct rt6_info *rt = mlxsw_sp_rt6->rt;
3078 rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
3082 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3084 switch (fib_entry->fib_node->fib->proto) {
3085 case MLXSW_SP_L3_PROTO_IPV4:
3086 mlxsw_sp_fib4_entry_offload_set(fib_entry);
3088 case MLXSW_SP_L3_PROTO_IPV6:
3089 mlxsw_sp_fib6_entry_offload_set(fib_entry);
3095 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3097 switch (fib_entry->fib_node->fib->proto) {
3098 case MLXSW_SP_L3_PROTO_IPV4:
3099 mlxsw_sp_fib4_entry_offload_unset(fib_entry);
3101 case MLXSW_SP_L3_PROTO_IPV6:
3102 mlxsw_sp_fib6_entry_offload_unset(fib_entry);
3108 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
3109 enum mlxsw_reg_ralue_op op, int err)
3112 case MLXSW_REG_RALUE_OP_WRITE_DELETE:
3113 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
3114 case MLXSW_REG_RALUE_OP_WRITE_WRITE:
3117 if (mlxsw_sp_fib_entry_should_offload(fib_entry))
3118 mlxsw_sp_fib_entry_offload_set(fib_entry);
3119 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry))
3120 mlxsw_sp_fib_entry_offload_unset(fib_entry);
3128 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
3129 const struct mlxsw_sp_fib_entry *fib_entry,
3130 enum mlxsw_reg_ralue_op op)
3132 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
3133 enum mlxsw_reg_ralxx_protocol proto;
3136 proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
3138 switch (fib->proto) {
3139 case MLXSW_SP_L3_PROTO_IPV4:
3140 p_dip = (u32 *) fib_entry->fib_node->key.addr;
3141 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
3142 fib_entry->fib_node->key.prefix_len,
3145 case MLXSW_SP_L3_PROTO_IPV6:
3146 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
3147 fib_entry->fib_node->key.prefix_len,
3148 fib_entry->fib_node->key.addr);
3153 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
3154 struct mlxsw_sp_fib_entry *fib_entry,
3155 enum mlxsw_reg_ralue_op op)
3157 char ralue_pl[MLXSW_REG_RALUE_LEN];
3158 enum mlxsw_reg_ralue_trap_action trap_action;
3160 u32 adjacency_index = 0;
3163 /* In case the nexthop group adjacency index is valid, use it
3164 * with provided ECMP size. Otherwise, setup trap and pass
3165 * traffic to kernel.
3167 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
3168 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
3169 adjacency_index = fib_entry->nh_group->adj_index;
3170 ecmp_size = fib_entry->nh_group->ecmp_size;
3172 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
3173 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
3176 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3177 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
3178 adjacency_index, ecmp_size);
3179 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3182 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
3183 struct mlxsw_sp_fib_entry *fib_entry,
3184 enum mlxsw_reg_ralue_op op)
3186 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
3187 enum mlxsw_reg_ralue_trap_action trap_action;
3188 char ralue_pl[MLXSW_REG_RALUE_LEN];
3192 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
3193 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
3194 rif_index = rif->rif_index;
3196 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
3197 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
3200 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3201 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
3203 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3206 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
3207 struct mlxsw_sp_fib_entry *fib_entry,
3208 enum mlxsw_reg_ralue_op op)
3210 char ralue_pl[MLXSW_REG_RALUE_LEN];
3212 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3213 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
3214 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3218 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
3219 struct mlxsw_sp_fib_entry *fib_entry,
3220 enum mlxsw_reg_ralue_op op)
3222 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
3223 const struct mlxsw_sp_ipip_ops *ipip_ops;
3225 if (WARN_ON(!ipip_entry))
3228 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
3229 return ipip_ops->fib_entry_op(mlxsw_sp, ipip_entry, op,
3230 fib_entry->decap.tunnel_index);
3233 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
3234 struct mlxsw_sp_fib_entry *fib_entry,
3235 enum mlxsw_reg_ralue_op op)
3237 switch (fib_entry->type) {
3238 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
3239 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
3240 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
3241 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
3242 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
3243 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
3244 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
3245 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp,
3251 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
3252 struct mlxsw_sp_fib_entry *fib_entry,
3253 enum mlxsw_reg_ralue_op op)
3255 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
3257 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
3262 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
3263 struct mlxsw_sp_fib_entry *fib_entry)
3265 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
3266 MLXSW_REG_RALUE_OP_WRITE_WRITE);
3269 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
3270 struct mlxsw_sp_fib_entry *fib_entry)
3272 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
3273 MLXSW_REG_RALUE_OP_WRITE_DELETE);
3277 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
3278 const struct fib_entry_notifier_info *fen_info,
3279 struct mlxsw_sp_fib_entry *fib_entry)
3281 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
3282 struct net_device *dev = fen_info->fi->fib_dev;
3283 struct mlxsw_sp_ipip_entry *ipip_entry;
3284 struct fib_info *fi = fen_info->fi;
3286 switch (fen_info->type) {
3288 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, dev,
3289 MLXSW_SP_L3_PROTO_IPV4, dip);
3291 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
3292 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
3298 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
3300 case RTN_UNREACHABLE: /* fall through */
3301 case RTN_BLACKHOLE: /* fall through */
3303 /* Packets hitting these routes need to be trapped, but
3304 * can do so with a lower priority than packets directed
3305 * at the host, so use action type local instead of trap.
3307 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3310 if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi))
3311 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
3313 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3320 static struct mlxsw_sp_fib4_entry *
3321 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
3322 struct mlxsw_sp_fib_node *fib_node,
3323 const struct fib_entry_notifier_info *fen_info)
3325 struct mlxsw_sp_fib4_entry *fib4_entry;
3326 struct mlxsw_sp_fib_entry *fib_entry;
3329 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
3331 return ERR_PTR(-ENOMEM);
3332 fib_entry = &fib4_entry->common;
3334 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
3336 goto err_fib4_entry_type_set;
3338 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
3340 goto err_nexthop4_group_get;
3342 fib4_entry->prio = fen_info->fi->fib_priority;
3343 fib4_entry->tb_id = fen_info->tb_id;
3344 fib4_entry->type = fen_info->type;
3345 fib4_entry->tos = fen_info->tos;
3347 fib_entry->fib_node = fib_node;
3351 err_nexthop4_group_get:
3352 err_fib4_entry_type_set:
3354 return ERR_PTR(err);
3357 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
3358 struct mlxsw_sp_fib4_entry *fib4_entry)
3360 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
3364 static struct mlxsw_sp_fib4_entry *
3365 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
3366 const struct fib_entry_notifier_info *fen_info)
3368 struct mlxsw_sp_fib4_entry *fib4_entry;
3369 struct mlxsw_sp_fib_node *fib_node;
3370 struct mlxsw_sp_fib *fib;
3371 struct mlxsw_sp_vr *vr;
3373 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
3376 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
3378 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
3379 sizeof(fen_info->dst),
3384 list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
3385 if (fib4_entry->tb_id == fen_info->tb_id &&
3386 fib4_entry->tos == fen_info->tos &&
3387 fib4_entry->type == fen_info->type &&
3388 mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) ==
3397 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
3398 .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
3399 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
3400 .key_len = sizeof(struct mlxsw_sp_fib_key),
3401 .automatic_shrinking = true,
3404 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
3405 struct mlxsw_sp_fib_node *fib_node)
3407 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
3408 mlxsw_sp_fib_ht_params);
3411 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
3412 struct mlxsw_sp_fib_node *fib_node)
3414 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
3415 mlxsw_sp_fib_ht_params);
3418 static struct mlxsw_sp_fib_node *
3419 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
3420 size_t addr_len, unsigned char prefix_len)
3422 struct mlxsw_sp_fib_key key;
3424 memset(&key, 0, sizeof(key));
3425 memcpy(key.addr, addr, addr_len);
3426 key.prefix_len = prefix_len;
3427 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
3430 static struct mlxsw_sp_fib_node *
3431 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
3432 size_t addr_len, unsigned char prefix_len)
3434 struct mlxsw_sp_fib_node *fib_node;
3436 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
3440 INIT_LIST_HEAD(&fib_node->entry_list);
3441 list_add(&fib_node->list, &fib->node_list);
3442 memcpy(fib_node->key.addr, addr, addr_len);
3443 fib_node->key.prefix_len = prefix_len;
3448 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
3450 list_del(&fib_node->list);
3451 WARN_ON(!list_empty(&fib_node->entry_list));
3456 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
3457 const struct mlxsw_sp_fib_entry *fib_entry)
3459 return list_first_entry(&fib_node->entry_list,
3460 struct mlxsw_sp_fib_entry, list) == fib_entry;
3463 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
3464 struct mlxsw_sp_fib *fib,
3465 struct mlxsw_sp_fib_node *fib_node)
3467 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
3468 struct mlxsw_sp_lpm_tree *lpm_tree;
3471 /* Since the tree is shared between all virtual routers we must
3472 * make sure it contains all the required prefix lengths. This
3473 * can be computed by either adding the new prefix length to the
3474 * existing prefix usage of a bound tree, or by aggregating the
3475 * prefix lengths across all virtual routers and adding the new
3479 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
3480 &fib->lpm_tree->prefix_usage);
3482 mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
3483 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
3485 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
3487 if (IS_ERR(lpm_tree))
3488 return PTR_ERR(lpm_tree);
3490 if (fib->lpm_tree && fib->lpm_tree->id == lpm_tree->id)
3493 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
3500 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
3501 struct mlxsw_sp_fib *fib)
3503 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
3504 struct mlxsw_sp_lpm_tree *lpm_tree;
3506 /* Aggregate prefix lengths across all virtual routers to make
3507 * sure we only have used prefix lengths in the LPM tree.
3509 mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
3510 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
3512 if (IS_ERR(lpm_tree))
3514 mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
3517 if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage))
3519 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
3520 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
3521 fib->lpm_tree = NULL;
3524 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
3526 unsigned char prefix_len = fib_node->key.prefix_len;
3527 struct mlxsw_sp_fib *fib = fib_node->fib;
3529 if (fib->prefix_ref_count[prefix_len]++ == 0)
3530 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
3533 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
3535 unsigned char prefix_len = fib_node->key.prefix_len;
3536 struct mlxsw_sp_fib *fib = fib_node->fib;
3538 if (--fib->prefix_ref_count[prefix_len] == 0)
3539 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
3542 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
3543 struct mlxsw_sp_fib_node *fib_node,
3544 struct mlxsw_sp_fib *fib)
3548 err = mlxsw_sp_fib_node_insert(fib, fib_node);
3551 fib_node->fib = fib;
3553 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib, fib_node);
3555 goto err_fib_lpm_tree_link;
3557 mlxsw_sp_fib_node_prefix_inc(fib_node);
3561 err_fib_lpm_tree_link:
3562 fib_node->fib = NULL;
3563 mlxsw_sp_fib_node_remove(fib, fib_node);
3567 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
3568 struct mlxsw_sp_fib_node *fib_node)
3570 struct mlxsw_sp_fib *fib = fib_node->fib;
3572 mlxsw_sp_fib_node_prefix_dec(fib_node);
3573 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib);
3574 fib_node->fib = NULL;
3575 mlxsw_sp_fib_node_remove(fib, fib_node);
3578 static struct mlxsw_sp_fib_node *
3579 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
3580 size_t addr_len, unsigned char prefix_len,
3581 enum mlxsw_sp_l3proto proto)
3583 struct mlxsw_sp_fib_node *fib_node;
3584 struct mlxsw_sp_fib *fib;
3585 struct mlxsw_sp_vr *vr;
3588 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id);
3590 return ERR_CAST(vr);
3591 fib = mlxsw_sp_vr_fib(vr, proto);
3593 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
3597 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
3600 goto err_fib_node_create;
3603 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
3605 goto err_fib_node_init;
3610 mlxsw_sp_fib_node_destroy(fib_node);
3611 err_fib_node_create:
3612 mlxsw_sp_vr_put(vr);
3613 return ERR_PTR(err);
3616 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
3617 struct mlxsw_sp_fib_node *fib_node)
3619 struct mlxsw_sp_vr *vr = fib_node->fib->vr;
3621 if (!list_empty(&fib_node->entry_list))
3623 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
3624 mlxsw_sp_fib_node_destroy(fib_node);
3625 mlxsw_sp_vr_put(vr);
3628 static struct mlxsw_sp_fib4_entry *
3629 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3630 const struct mlxsw_sp_fib4_entry *new4_entry)
3632 struct mlxsw_sp_fib4_entry *fib4_entry;
3634 list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
3635 if (fib4_entry->tb_id > new4_entry->tb_id)
3637 if (fib4_entry->tb_id != new4_entry->tb_id)
3639 if (fib4_entry->tos > new4_entry->tos)
3641 if (fib4_entry->prio >= new4_entry->prio ||
3642 fib4_entry->tos < new4_entry->tos)
3650 mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry,
3651 struct mlxsw_sp_fib4_entry *new4_entry)
3653 struct mlxsw_sp_fib_node *fib_node;
3655 if (WARN_ON(!fib4_entry))
3658 fib_node = fib4_entry->common.fib_node;
3659 list_for_each_entry_from(fib4_entry, &fib_node->entry_list,
3661 if (fib4_entry->tb_id != new4_entry->tb_id ||
3662 fib4_entry->tos != new4_entry->tos ||
3663 fib4_entry->prio != new4_entry->prio)
3667 list_add_tail(&new4_entry->common.list, &fib4_entry->common.list);
3672 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry,
3673 bool replace, bool append)
3675 struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node;
3676 struct mlxsw_sp_fib4_entry *fib4_entry;
3678 fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry);
3681 return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry);
3682 if (replace && WARN_ON(!fib4_entry))
3685 /* Insert new entry before replaced one, so that we can later
3686 * remove the second.
3689 list_add_tail(&new4_entry->common.list,
3690 &fib4_entry->common.list);
3692 struct mlxsw_sp_fib4_entry *last;
3694 list_for_each_entry(last, &fib_node->entry_list, common.list) {
3695 if (new4_entry->tb_id > last->tb_id)
3701 list_add(&new4_entry->common.list,
3702 &fib4_entry->common.list);
3704 list_add(&new4_entry->common.list,
3705 &fib_node->entry_list);
3712 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry)
3714 list_del(&fib4_entry->common.list);
3717 static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp,
3718 struct mlxsw_sp_fib_entry *fib_entry)
3720 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3722 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3725 /* To prevent packet loss, overwrite the previously offloaded
3728 if (!list_is_singular(&fib_node->entry_list)) {
3729 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3730 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3732 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
3735 return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3738 static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp,
3739 struct mlxsw_sp_fib_entry *fib_entry)
3741 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3743 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3746 /* Promote the next entry by overwriting the deleted entry */
3747 if (!list_is_singular(&fib_node->entry_list)) {
3748 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3749 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3751 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
3752 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
3756 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
3759 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
3760 struct mlxsw_sp_fib4_entry *fib4_entry,
3761 bool replace, bool append)
3765 err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append);
3769 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common);
3771 goto err_fib_node_entry_add;
3775 err_fib_node_entry_add:
3776 mlxsw_sp_fib4_node_list_remove(fib4_entry);
3781 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
3782 struct mlxsw_sp_fib4_entry *fib4_entry)
3784 mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common);
3785 mlxsw_sp_fib4_node_list_remove(fib4_entry);
3787 if (fib4_entry->common.type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP)
3788 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, &fib4_entry->common);
3791 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
3792 struct mlxsw_sp_fib4_entry *fib4_entry,
3795 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
3796 struct mlxsw_sp_fib4_entry *replaced;
3801 /* We inserted the new entry before replaced one */
3802 replaced = list_next_entry(fib4_entry, common.list);
3804 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
3805 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
3806 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3810 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
3811 const struct fib_entry_notifier_info *fen_info,
3812 bool replace, bool append)
3814 struct mlxsw_sp_fib4_entry *fib4_entry;
3815 struct mlxsw_sp_fib_node *fib_node;
3818 if (mlxsw_sp->router->aborted)
3821 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
3822 &fen_info->dst, sizeof(fen_info->dst),
3824 MLXSW_SP_L3_PROTO_IPV4);
3825 if (IS_ERR(fib_node)) {
3826 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
3827 return PTR_ERR(fib_node);
3830 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
3831 if (IS_ERR(fib4_entry)) {
3832 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
3833 err = PTR_ERR(fib4_entry);
3834 goto err_fib4_entry_create;
3837 err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace,
3840 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
3841 goto err_fib4_node_entry_link;
3844 mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace);
3848 err_fib4_node_entry_link:
3849 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3850 err_fib4_entry_create:
3851 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3855 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
3856 struct fib_entry_notifier_info *fen_info)
3858 struct mlxsw_sp_fib4_entry *fib4_entry;
3859 struct mlxsw_sp_fib_node *fib_node;
3861 if (mlxsw_sp->router->aborted)
3864 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
3865 if (WARN_ON(!fib4_entry))
3867 fib_node = fib4_entry->common.fib_node;
3869 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
3870 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3871 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3874 static bool mlxsw_sp_fib6_rt_should_ignore(const struct rt6_info *rt)
3876 /* Packets with link-local destination IP arriving to the router
3877 * are trapped to the CPU, so no need to program specific routes
3880 if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LINKLOCAL)
3883 /* Multicast routes aren't supported, so ignore them. Neighbour
3884 * Discovery packets are specifically trapped.
3886 if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_MULTICAST)
3889 /* Cloned routes are irrelevant in the forwarding path. */
3890 if (rt->rt6i_flags & RTF_CACHE)
3896 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct rt6_info *rt)
3898 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3900 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
3902 return ERR_PTR(-ENOMEM);
3904 /* In case of route replace, replaced route is deleted with
3905 * no notification. Take reference to prevent accessing freed
3908 mlxsw_sp_rt6->rt = rt;
3911 return mlxsw_sp_rt6;
3914 #if IS_ENABLED(CONFIG_IPV6)
3915 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3920 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3925 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
3927 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
3928 kfree(mlxsw_sp_rt6);
3931 static bool mlxsw_sp_fib6_rt_can_mp(const struct rt6_info *rt)
3933 /* RTF_CACHE routes are ignored */
3934 return (rt->rt6i_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY;
3937 static struct rt6_info *
3938 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
3940 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3944 static struct mlxsw_sp_fib6_entry *
3945 mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3946 const struct rt6_info *nrt, bool replace)
3948 struct mlxsw_sp_fib6_entry *fib6_entry;
3950 if (!mlxsw_sp_fib6_rt_can_mp(nrt) || replace)
3953 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3954 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3956 /* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same
3959 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
3961 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
3963 if (rt->rt6i_metric < nrt->rt6i_metric)
3965 if (rt->rt6i_metric == nrt->rt6i_metric &&
3966 mlxsw_sp_fib6_rt_can_mp(rt))
3968 if (rt->rt6i_metric > nrt->rt6i_metric)
3975 static struct mlxsw_sp_rt6 *
3976 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
3977 const struct rt6_info *rt)
3979 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3981 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3982 if (mlxsw_sp_rt6->rt == rt)
3983 return mlxsw_sp_rt6;
3989 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
3990 const struct rt6_info *rt,
3991 enum mlxsw_sp_ipip_type *ret)
3993 return rt->dst.dev &&
3994 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->dst.dev, ret);
3997 static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp,
3998 struct mlxsw_sp_nexthop_group *nh_grp,
3999 struct mlxsw_sp_nexthop *nh,
4000 const struct rt6_info *rt)
4002 struct mlxsw_sp_router *router = mlxsw_sp->router;
4003 struct net_device *dev = rt->dst.dev;
4004 enum mlxsw_sp_ipip_type ipipt;
4005 struct mlxsw_sp_rif *rif;
4008 if (mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, &ipipt) &&
4009 router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev,
4010 MLXSW_SP_L3_PROTO_IPV6)) {
4011 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
4012 return mlxsw_sp_nexthop_ipip_init(mlxsw_sp, ipipt, nh, dev);
4015 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
4016 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4019 mlxsw_sp_nexthop_rif_init(nh, rif);
4021 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
4023 goto err_nexthop_neigh_init;
4027 err_nexthop_neigh_init:
4028 mlxsw_sp_nexthop_rif_fini(nh);
4032 static void mlxsw_sp_nexthop6_type_fini(struct mlxsw_sp *mlxsw_sp,
4033 struct mlxsw_sp_nexthop *nh)
4035 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4038 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
4039 struct mlxsw_sp_nexthop_group *nh_grp,
4040 struct mlxsw_sp_nexthop *nh,
4041 const struct rt6_info *rt)
4043 struct net_device *dev = rt->dst.dev;
4045 nh->nh_grp = nh_grp;
4046 memcpy(&nh->gw_addr, &rt->rt6i_gateway, sizeof(nh->gw_addr));
4050 nh->ifindex = dev->ifindex;
4052 return mlxsw_sp_nexthop6_type_init(mlxsw_sp, nh_grp, nh, rt);
4055 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
4056 struct mlxsw_sp_nexthop *nh)
4058 mlxsw_sp_nexthop6_type_fini(mlxsw_sp, nh);
4061 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
4062 const struct rt6_info *rt)
4064 return rt->rt6i_flags & RTF_GATEWAY ||
4065 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
4068 static struct mlxsw_sp_nexthop_group *
4069 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
4070 struct mlxsw_sp_fib6_entry *fib6_entry)
4072 struct mlxsw_sp_nexthop_group *nh_grp;
4073 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4074 struct mlxsw_sp_nexthop *nh;
4079 alloc_size = sizeof(*nh_grp) +
4080 fib6_entry->nrt6 * sizeof(struct mlxsw_sp_nexthop);
4081 nh_grp = kzalloc(alloc_size, GFP_KERNEL);
4083 return ERR_PTR(-ENOMEM);
4084 INIT_LIST_HEAD(&nh_grp->fib_list);
4085 #if IS_ENABLED(CONFIG_IPV6)
4086 nh_grp->neigh_tbl = &nd_tbl;
4088 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
4089 struct mlxsw_sp_rt6, list);
4090 nh_grp->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
4091 nh_grp->count = fib6_entry->nrt6;
4092 for (i = 0; i < nh_grp->count; i++) {
4093 struct rt6_info *rt = mlxsw_sp_rt6->rt;
4095 nh = &nh_grp->nexthops[i];
4096 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
4098 goto err_nexthop6_init;
4099 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
4102 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
4104 goto err_nexthop_group_insert;
4106 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4109 err_nexthop_group_insert:
4111 for (i--; i >= 0; i--) {
4112 nh = &nh_grp->nexthops[i];
4113 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
4116 return ERR_PTR(err);
4120 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
4121 struct mlxsw_sp_nexthop_group *nh_grp)
4123 struct mlxsw_sp_nexthop *nh;
4124 int i = nh_grp->count;
4126 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
4127 for (i--; i >= 0; i--) {
4128 nh = &nh_grp->nexthops[i];
4129 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
4131 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4132 WARN_ON(nh_grp->adj_index_valid);
4136 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
4137 struct mlxsw_sp_fib6_entry *fib6_entry)
4139 struct mlxsw_sp_nexthop_group *nh_grp;
4141 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
4143 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
4145 return PTR_ERR(nh_grp);
4148 list_add_tail(&fib6_entry->common.nexthop_group_node,
4150 fib6_entry->common.nh_group = nh_grp;
4155 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
4156 struct mlxsw_sp_fib_entry *fib_entry)
4158 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
4160 list_del(&fib_entry->nexthop_group_node);
4161 if (!list_empty(&nh_grp->fib_list))
4163 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
4167 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
4168 struct mlxsw_sp_fib6_entry *fib6_entry)
4170 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
4173 fib6_entry->common.nh_group = NULL;
4174 list_del(&fib6_entry->common.nexthop_group_node);
4176 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
4178 goto err_nexthop6_group_get;
4180 /* In case this entry is offloaded, then the adjacency index
4181 * currently associated with it in the device's table is that
4182 * of the old group. Start using the new one instead.
4184 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
4186 goto err_fib_node_entry_add;
4188 if (list_empty(&old_nh_grp->fib_list))
4189 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
4193 err_fib_node_entry_add:
4194 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
4195 err_nexthop6_group_get:
4196 list_add_tail(&fib6_entry->common.nexthop_group_node,
4197 &old_nh_grp->fib_list);
4198 fib6_entry->common.nh_group = old_nh_grp;
4203 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
4204 struct mlxsw_sp_fib6_entry *fib6_entry,
4205 struct rt6_info *rt)
4207 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4210 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
4211 if (IS_ERR(mlxsw_sp_rt6))
4212 return PTR_ERR(mlxsw_sp_rt6);
4214 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
4217 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
4219 goto err_nexthop6_group_update;
4223 err_nexthop6_group_update:
4225 list_del(&mlxsw_sp_rt6->list);
4226 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4231 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
4232 struct mlxsw_sp_fib6_entry *fib6_entry,
4233 struct rt6_info *rt)
4235 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4237 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt);
4238 if (WARN_ON(!mlxsw_sp_rt6))
4242 list_del(&mlxsw_sp_rt6->list);
4243 mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
4244 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4247 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
4248 struct mlxsw_sp_fib_entry *fib_entry,
4249 const struct rt6_info *rt)
4251 /* Packets hitting RTF_REJECT routes need to be discarded by the
4252 * stack. We can rely on their destination device not having a
4253 * RIF (it's the loopback device) and can thus use action type
4254 * local, which will cause them to be trapped with a lower
4255 * priority than packets that need to be locally received.
4257 if (rt->rt6i_flags & (RTF_LOCAL | RTF_ANYCAST))
4258 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
4259 else if (rt->rt6i_flags & RTF_REJECT)
4260 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4261 else if (mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt))
4262 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
4264 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4268 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
4270 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
4272 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
4275 list_del(&mlxsw_sp_rt6->list);
4276 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4280 static struct mlxsw_sp_fib6_entry *
4281 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
4282 struct mlxsw_sp_fib_node *fib_node,
4283 struct rt6_info *rt)
4285 struct mlxsw_sp_fib6_entry *fib6_entry;
4286 struct mlxsw_sp_fib_entry *fib_entry;
4287 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4290 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
4292 return ERR_PTR(-ENOMEM);
4293 fib_entry = &fib6_entry->common;
4295 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
4296 if (IS_ERR(mlxsw_sp_rt6)) {
4297 err = PTR_ERR(mlxsw_sp_rt6);
4298 goto err_rt6_create;
4301 mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, mlxsw_sp_rt6->rt);
4303 INIT_LIST_HEAD(&fib6_entry->rt6_list);
4304 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
4305 fib6_entry->nrt6 = 1;
4306 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
4308 goto err_nexthop6_group_get;
4310 fib_entry->fib_node = fib_node;
4314 err_nexthop6_group_get:
4315 list_del(&mlxsw_sp_rt6->list);
4316 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4319 return ERR_PTR(err);
4322 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
4323 struct mlxsw_sp_fib6_entry *fib6_entry)
4325 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
4326 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
4327 WARN_ON(fib6_entry->nrt6);
4331 static struct mlxsw_sp_fib6_entry *
4332 mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
4333 const struct rt6_info *nrt, bool replace)
4335 struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL;
4337 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
4338 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
4340 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
4342 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
4344 if (replace && rt->rt6i_metric == nrt->rt6i_metric) {
4345 if (mlxsw_sp_fib6_rt_can_mp(rt) ==
4346 mlxsw_sp_fib6_rt_can_mp(nrt))
4348 if (mlxsw_sp_fib6_rt_can_mp(nrt))
4349 fallback = fallback ?: fib6_entry;
4351 if (rt->rt6i_metric > nrt->rt6i_metric)
4352 return fallback ?: fib6_entry;
4359 mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry,
4362 struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node;
4363 struct rt6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry);
4364 struct mlxsw_sp_fib6_entry *fib6_entry;
4366 fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace);
4368 if (replace && WARN_ON(!fib6_entry))
4372 list_add_tail(&new6_entry->common.list,
4373 &fib6_entry->common.list);
4375 struct mlxsw_sp_fib6_entry *last;
4377 list_for_each_entry(last, &fib_node->entry_list, common.list) {
4378 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(last);
4380 if (nrt->rt6i_table->tb6_id > rt->rt6i_table->tb6_id)
4386 list_add(&new6_entry->common.list,
4387 &fib6_entry->common.list);
4389 list_add(&new6_entry->common.list,
4390 &fib_node->entry_list);
4397 mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry)
4399 list_del(&fib6_entry->common.list);
4402 static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp,
4403 struct mlxsw_sp_fib6_entry *fib6_entry,
4408 err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace);
4412 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
4414 goto err_fib_node_entry_add;
4418 err_fib_node_entry_add:
4419 mlxsw_sp_fib6_node_list_remove(fib6_entry);
4424 mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
4425 struct mlxsw_sp_fib6_entry *fib6_entry)
4427 mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common);
4428 mlxsw_sp_fib6_node_list_remove(fib6_entry);
4431 static struct mlxsw_sp_fib6_entry *
4432 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
4433 const struct rt6_info *rt)
4435 struct mlxsw_sp_fib6_entry *fib6_entry;
4436 struct mlxsw_sp_fib_node *fib_node;
4437 struct mlxsw_sp_fib *fib;
4438 struct mlxsw_sp_vr *vr;
4440 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->rt6i_table->tb6_id);
4443 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
4445 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->rt6i_dst.addr,
4446 sizeof(rt->rt6i_dst.addr),
4451 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
4452 struct rt6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
4454 if (rt->rt6i_table->tb6_id == iter_rt->rt6i_table->tb6_id &&
4455 rt->rt6i_metric == iter_rt->rt6i_metric &&
4456 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
4463 static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp,
4464 struct mlxsw_sp_fib6_entry *fib6_entry,
4467 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
4468 struct mlxsw_sp_fib6_entry *replaced;
4473 replaced = list_next_entry(fib6_entry, common.list);
4475 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced);
4476 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced);
4477 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4480 static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
4481 struct rt6_info *rt, bool replace)
4483 struct mlxsw_sp_fib6_entry *fib6_entry;
4484 struct mlxsw_sp_fib_node *fib_node;
4487 if (mlxsw_sp->router->aborted)
4490 if (rt->rt6i_src.plen)
4493 if (mlxsw_sp_fib6_rt_should_ignore(rt))
4496 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->rt6i_table->tb6_id,
4498 sizeof(rt->rt6i_dst.addr),
4500 MLXSW_SP_L3_PROTO_IPV6);
4501 if (IS_ERR(fib_node))
4502 return PTR_ERR(fib_node);
4504 /* Before creating a new entry, try to append route to an existing
4507 fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, replace);
4509 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt);
4511 goto err_fib6_entry_nexthop_add;
4515 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt);
4516 if (IS_ERR(fib6_entry)) {
4517 err = PTR_ERR(fib6_entry);
4518 goto err_fib6_entry_create;
4521 err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace);
4523 goto err_fib6_node_entry_link;
4525 mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace);
4529 err_fib6_node_entry_link:
4530 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4531 err_fib6_entry_create:
4532 err_fib6_entry_nexthop_add:
4533 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4537 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
4538 struct rt6_info *rt)
4540 struct mlxsw_sp_fib6_entry *fib6_entry;
4541 struct mlxsw_sp_fib_node *fib_node;
4543 if (mlxsw_sp->router->aborted)
4546 if (mlxsw_sp_fib6_rt_should_ignore(rt))
4549 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
4550 if (WARN_ON(!fib6_entry))
4553 /* If route is part of a multipath entry, but not the last one
4554 * removed, then only reduce its nexthop group.
4556 if (!list_is_singular(&fib6_entry->rt6_list)) {
4557 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt);
4561 fib_node = fib6_entry->common.fib_node;
4563 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4564 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4565 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4568 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
4569 enum mlxsw_reg_ralxx_protocol proto,
4572 char ralta_pl[MLXSW_REG_RALTA_LEN];
4573 char ralst_pl[MLXSW_REG_RALST_LEN];
4576 mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id);
4577 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
4581 mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id);
4582 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
4586 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
4587 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
4588 char raltb_pl[MLXSW_REG_RALTB_LEN];
4589 char ralue_pl[MLXSW_REG_RALUE_LEN];
4591 mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id);
4592 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
4597 mlxsw_reg_ralue_pack(ralue_pl, proto,
4598 MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0);
4599 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
4600 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
4609 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
4611 enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4;
4614 err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4615 MLXSW_SP_LPM_TREE_MIN);
4619 proto = MLXSW_REG_RALXX_PROTOCOL_IPV6;
4620 return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4621 MLXSW_SP_LPM_TREE_MIN + 1);
4624 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
4625 struct mlxsw_sp_fib_node *fib_node)
4627 struct mlxsw_sp_fib4_entry *fib4_entry, *tmp;
4629 list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list,
4631 bool do_break = &tmp->common.list == &fib_node->entry_list;
4633 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
4634 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
4635 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4636 /* Break when entry list is empty and node was freed.
4637 * Otherwise, we'll access freed memory in the next
4645 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
4646 struct mlxsw_sp_fib_node *fib_node)
4648 struct mlxsw_sp_fib6_entry *fib6_entry, *tmp;
4650 list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list,
4652 bool do_break = &tmp->common.list == &fib_node->entry_list;
4654 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4655 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4656 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4662 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
4663 struct mlxsw_sp_fib_node *fib_node)
4665 switch (fib_node->fib->proto) {
4666 case MLXSW_SP_L3_PROTO_IPV4:
4667 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
4669 case MLXSW_SP_L3_PROTO_IPV6:
4670 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
4675 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
4676 struct mlxsw_sp_vr *vr,
4677 enum mlxsw_sp_l3proto proto)
4679 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
4680 struct mlxsw_sp_fib_node *fib_node, *tmp;
4682 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
4683 bool do_break = &tmp->list == &fib->node_list;
4685 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
4691 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
4695 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
4696 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
4698 if (!mlxsw_sp_vr_is_used(vr))
4700 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
4702 /* If virtual router was only used for IPv4, then it's no
4705 if (!mlxsw_sp_vr_is_used(vr))
4707 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
4711 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
4715 if (mlxsw_sp->router->aborted)
4717 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
4718 mlxsw_sp_router_fib_flush(mlxsw_sp);
4719 mlxsw_sp->router->aborted = true;
4720 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
4722 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
4725 struct mlxsw_sp_fib_event_work {
4726 struct work_struct work;
4728 struct fib6_entry_notifier_info fen6_info;
4729 struct fib_entry_notifier_info fen_info;
4730 struct fib_rule_notifier_info fr_info;
4731 struct fib_nh_notifier_info fnh_info;
4733 struct mlxsw_sp *mlxsw_sp;
4734 unsigned long event;
4737 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
4739 struct mlxsw_sp_fib_event_work *fib_work =
4740 container_of(work, struct mlxsw_sp_fib_event_work, work);
4741 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4742 struct fib_rule *rule;
4743 bool replace, append;
4746 /* Protect internal structures from changes */
4748 switch (fib_work->event) {
4749 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4750 case FIB_EVENT_ENTRY_APPEND: /* fall through */
4751 case FIB_EVENT_ENTRY_ADD:
4752 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4753 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
4754 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
4757 mlxsw_sp_router_fib_abort(mlxsw_sp);
4758 fib_info_put(fib_work->fen_info.fi);
4760 case FIB_EVENT_ENTRY_DEL:
4761 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
4762 fib_info_put(fib_work->fen_info.fi);
4764 case FIB_EVENT_RULE_ADD: /* fall through */
4765 case FIB_EVENT_RULE_DEL:
4766 rule = fib_work->fr_info.rule;
4767 if (!fib4_rule_default(rule) && !rule->l3mdev)
4768 mlxsw_sp_router_fib_abort(mlxsw_sp);
4771 case FIB_EVENT_NH_ADD: /* fall through */
4772 case FIB_EVENT_NH_DEL:
4773 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
4774 fib_work->fnh_info.fib_nh);
4775 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
4782 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
4784 struct mlxsw_sp_fib_event_work *fib_work =
4785 container_of(work, struct mlxsw_sp_fib_event_work, work);
4786 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4787 struct fib_rule *rule;
4792 switch (fib_work->event) {
4793 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4794 case FIB_EVENT_ENTRY_ADD:
4795 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4796 err = mlxsw_sp_router_fib6_add(mlxsw_sp,
4797 fib_work->fen6_info.rt, replace);
4799 mlxsw_sp_router_fib_abort(mlxsw_sp);
4800 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4802 case FIB_EVENT_ENTRY_DEL:
4803 mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt);
4804 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4806 case FIB_EVENT_RULE_ADD: /* fall through */
4807 case FIB_EVENT_RULE_DEL:
4808 rule = fib_work->fr_info.rule;
4809 if (!fib6_rule_default(rule) && !rule->l3mdev)
4810 mlxsw_sp_router_fib_abort(mlxsw_sp);
4818 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
4819 struct fib_notifier_info *info)
4821 switch (fib_work->event) {
4822 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4823 case FIB_EVENT_ENTRY_APPEND: /* fall through */
4824 case FIB_EVENT_ENTRY_ADD: /* fall through */
4825 case FIB_EVENT_ENTRY_DEL:
4826 memcpy(&fib_work->fen_info, info, sizeof(fib_work->fen_info));
4827 /* Take referece on fib_info to prevent it from being
4828 * freed while work is queued. Release it afterwards.
4830 fib_info_hold(fib_work->fen_info.fi);
4832 case FIB_EVENT_RULE_ADD: /* fall through */
4833 case FIB_EVENT_RULE_DEL:
4834 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4835 fib_rule_get(fib_work->fr_info.rule);
4837 case FIB_EVENT_NH_ADD: /* fall through */
4838 case FIB_EVENT_NH_DEL:
4839 memcpy(&fib_work->fnh_info, info, sizeof(fib_work->fnh_info));
4840 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
4845 static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
4846 struct fib_notifier_info *info)
4848 switch (fib_work->event) {
4849 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4850 case FIB_EVENT_ENTRY_ADD: /* fall through */
4851 case FIB_EVENT_ENTRY_DEL:
4852 memcpy(&fib_work->fen6_info, info, sizeof(fib_work->fen6_info));
4853 rt6_hold(fib_work->fen6_info.rt);
4855 case FIB_EVENT_RULE_ADD: /* fall through */
4856 case FIB_EVENT_RULE_DEL:
4857 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4858 fib_rule_get(fib_work->fr_info.rule);
4863 /* Called with rcu_read_lock() */
4864 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
4865 unsigned long event, void *ptr)
4867 struct mlxsw_sp_fib_event_work *fib_work;
4868 struct fib_notifier_info *info = ptr;
4869 struct mlxsw_sp_router *router;
4871 if (!net_eq(info->net, &init_net) ||
4872 (info->family != AF_INET && info->family != AF_INET6))
4875 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
4876 if (WARN_ON(!fib_work))
4879 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
4880 fib_work->mlxsw_sp = router->mlxsw_sp;
4881 fib_work->event = event;
4883 switch (info->family) {
4885 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
4886 mlxsw_sp_router_fib4_event(fib_work, info);
4889 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
4890 mlxsw_sp_router_fib6_event(fib_work, info);
4894 mlxsw_core_schedule_work(&fib_work->work);
4899 static struct mlxsw_sp_rif *
4900 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
4901 const struct net_device *dev)
4905 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
4906 if (mlxsw_sp->router->rifs[i] &&
4907 mlxsw_sp->router->rifs[i]->dev == dev)
4908 return mlxsw_sp->router->rifs[i];
4913 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
4915 char ritr_pl[MLXSW_REG_RITR_LEN];
4918 mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
4919 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4920 if (WARN_ON_ONCE(err))
4923 mlxsw_reg_ritr_enable_set(ritr_pl, false);
4924 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4927 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4928 struct mlxsw_sp_rif *rif)
4930 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
4931 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
4932 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
4936 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
4937 unsigned long event)
4939 struct inet6_dev *inet6_dev;
4940 bool addr_list_empty = true;
4941 struct in_device *idev;
4947 idev = __in_dev_get_rtnl(dev);
4948 if (idev && idev->ifa_list)
4949 addr_list_empty = false;
4951 inet6_dev = __in6_dev_get(dev);
4952 if (addr_list_empty && inet6_dev &&
4953 !list_empty(&inet6_dev->addr_list))
4954 addr_list_empty = false;
4956 if (rif && addr_list_empty &&
4957 !netif_is_l3_slave(rif->dev))
4959 /* It is possible we already removed the RIF ourselves
4960 * if it was assigned to a netdev that is now a bridge
4969 static enum mlxsw_sp_rif_type
4970 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
4971 const struct net_device *dev)
4973 enum mlxsw_sp_fid_type type;
4975 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
4976 return MLXSW_SP_RIF_TYPE_IPIP_LB;
4978 /* Otherwise RIF type is derived from the type of the underlying FID. */
4979 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
4980 type = MLXSW_SP_FID_TYPE_8021Q;
4981 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
4982 type = MLXSW_SP_FID_TYPE_8021Q;
4983 else if (netif_is_bridge_master(dev))
4984 type = MLXSW_SP_FID_TYPE_8021D;
4986 type = MLXSW_SP_FID_TYPE_RFID;
4988 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
4991 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
4995 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
4996 if (!mlxsw_sp->router->rifs[i]) {
5005 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
5007 struct net_device *l3_dev)
5009 struct mlxsw_sp_rif *rif;
5011 rif = kzalloc(rif_size, GFP_KERNEL);
5015 INIT_LIST_HEAD(&rif->nexthop_list);
5016 INIT_LIST_HEAD(&rif->neigh_list);
5017 ether_addr_copy(rif->addr, l3_dev->dev_addr);
5018 rif->mtu = l3_dev->mtu;
5021 rif->rif_index = rif_index;
5026 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
5029 return mlxsw_sp->router->rifs[rif_index];
5032 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
5034 return rif->rif_index;
5037 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
5039 return lb_rif->common.rif_index;
5042 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
5044 return lb_rif->ul_vr_id;
5047 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
5049 return rif->dev->ifindex;
5052 static struct mlxsw_sp_rif *
5053 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
5054 const struct mlxsw_sp_rif_params *params)
5056 u32 tb_id = l3mdev_fib_table(params->dev);
5057 const struct mlxsw_sp_rif_ops *ops;
5058 struct mlxsw_sp_fid *fid = NULL;
5059 enum mlxsw_sp_rif_type type;
5060 struct mlxsw_sp_rif *rif;
5061 struct mlxsw_sp_vr *vr;
5065 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
5066 ops = mlxsw_sp->router->rif_ops_arr[type];
5068 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
5070 return ERR_CAST(vr);
5072 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
5074 goto err_rif_index_alloc;
5076 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
5081 rif->mlxsw_sp = mlxsw_sp;
5085 fid = ops->fid_get(rif);
5094 ops->setup(rif, params);
5096 err = ops->configure(rif);
5100 mlxsw_sp_rif_counters_alloc(rif);
5101 mlxsw_sp->router->rifs[rif_index] = rif;
5108 mlxsw_sp_fid_put(fid);
5112 err_rif_index_alloc:
5113 mlxsw_sp_vr_put(vr);
5114 return ERR_PTR(err);
5117 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
5119 const struct mlxsw_sp_rif_ops *ops = rif->ops;
5120 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5121 struct mlxsw_sp_fid *fid = rif->fid;
5122 struct mlxsw_sp_vr *vr;
5124 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
5125 vr = &mlxsw_sp->router->vrs[rif->vr_id];
5128 mlxsw_sp->router->rifs[rif->rif_index] = NULL;
5129 mlxsw_sp_rif_counters_free(rif);
5130 ops->deconfigure(rif);
5132 /* Loopback RIFs are not associated with a FID. */
5133 mlxsw_sp_fid_put(fid);
5135 mlxsw_sp_vr_put(vr);
5139 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
5140 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
5142 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
5144 params->vid = mlxsw_sp_port_vlan->vid;
5145 params->lag = mlxsw_sp_port->lagged;
5147 params->lag_id = mlxsw_sp_port->lag_id;
5149 params->system_port = mlxsw_sp_port->local_port;
5153 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
5154 struct net_device *l3_dev)
5156 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
5157 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
5158 u16 vid = mlxsw_sp_port_vlan->vid;
5159 struct mlxsw_sp_rif *rif;
5160 struct mlxsw_sp_fid *fid;
5163 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5165 struct mlxsw_sp_rif_params params = {
5169 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan);
5170 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms);
5172 return PTR_ERR(rif);
5175 /* FID was already created, just take a reference */
5176 fid = rif->ops->fid_get(rif);
5177 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
5179 goto err_fid_port_vid_map;
5181 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
5183 goto err_port_vid_learning_set;
5185 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
5186 BR_STATE_FORWARDING);
5188 goto err_port_vid_stp_set;
5190 mlxsw_sp_port_vlan->fid = fid;
5194 err_port_vid_stp_set:
5195 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
5196 err_port_vid_learning_set:
5197 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
5198 err_fid_port_vid_map:
5199 mlxsw_sp_fid_put(fid);
5204 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
5206 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
5207 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
5208 u16 vid = mlxsw_sp_port_vlan->vid;
5210 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
5213 mlxsw_sp_port_vlan->fid = NULL;
5214 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
5215 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
5216 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
5217 /* If router port holds the last reference on the rFID, then the
5218 * associated Sub-port RIF will be destroyed.
5220 mlxsw_sp_fid_put(fid);
5223 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
5224 struct net_device *port_dev,
5225 unsigned long event, u16 vid)
5227 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
5228 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
5230 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
5231 if (WARN_ON(!mlxsw_sp_port_vlan))
5236 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
5239 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
5246 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
5247 unsigned long event)
5249 if (netif_is_bridge_port(port_dev) ||
5250 netif_is_lag_port(port_dev) ||
5251 netif_is_ovs_port(port_dev))
5254 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
5257 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
5258 struct net_device *lag_dev,
5259 unsigned long event, u16 vid)
5261 struct net_device *port_dev;
5262 struct list_head *iter;
5265 netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
5266 if (mlxsw_sp_port_dev_check(port_dev)) {
5267 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
5278 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
5279 unsigned long event)
5281 if (netif_is_bridge_port(lag_dev))
5284 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
5287 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
5288 unsigned long event)
5290 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
5291 struct mlxsw_sp_rif_params params = {
5294 struct mlxsw_sp_rif *rif;
5298 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms);
5300 return PTR_ERR(rif);
5303 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5304 mlxsw_sp_rif_destroy(rif);
5311 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
5312 unsigned long event)
5314 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
5315 u16 vid = vlan_dev_vlan_id(vlan_dev);
5317 if (netif_is_bridge_port(vlan_dev))
5320 if (mlxsw_sp_port_dev_check(real_dev))
5321 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
5323 else if (netif_is_lag_master(real_dev))
5324 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
5326 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
5327 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
5332 static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
5333 unsigned long event)
5335 if (mlxsw_sp_port_dev_check(dev))
5336 return mlxsw_sp_inetaddr_port_event(dev, event);
5337 else if (netif_is_lag_master(dev))
5338 return mlxsw_sp_inetaddr_lag_event(dev, event);
5339 else if (netif_is_bridge_master(dev))
5340 return mlxsw_sp_inetaddr_bridge_event(dev, event);
5341 else if (is_vlan_dev(dev))
5342 return mlxsw_sp_inetaddr_vlan_event(dev, event);
5347 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
5348 unsigned long event, void *ptr)
5350 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
5351 struct net_device *dev = ifa->ifa_dev->dev;
5352 struct mlxsw_sp *mlxsw_sp;
5353 struct mlxsw_sp_rif *rif;
5356 mlxsw_sp = mlxsw_sp_lower_get(dev);
5360 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5361 if (!mlxsw_sp_rif_should_config(rif, dev, event))
5364 err = __mlxsw_sp_inetaddr_event(dev, event);
5366 return notifier_from_errno(err);
5369 struct mlxsw_sp_inet6addr_event_work {
5370 struct work_struct work;
5371 struct net_device *dev;
5372 unsigned long event;
5375 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
5377 struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
5378 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
5379 struct net_device *dev = inet6addr_work->dev;
5380 unsigned long event = inet6addr_work->event;
5381 struct mlxsw_sp *mlxsw_sp;
5382 struct mlxsw_sp_rif *rif;
5385 mlxsw_sp = mlxsw_sp_lower_get(dev);
5389 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5390 if (!mlxsw_sp_rif_should_config(rif, dev, event))
5393 __mlxsw_sp_inetaddr_event(dev, event);
5397 kfree(inet6addr_work);
5400 /* Called with rcu_read_lock() */
5401 int mlxsw_sp_inet6addr_event(struct notifier_block *unused,
5402 unsigned long event, void *ptr)
5404 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
5405 struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
5406 struct net_device *dev = if6->idev->dev;
5408 if (!mlxsw_sp_port_dev_lower_find_rcu(dev))
5411 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
5412 if (!inet6addr_work)
5415 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
5416 inet6addr_work->dev = dev;
5417 inet6addr_work->event = event;
5419 mlxsw_core_schedule_work(&inet6addr_work->work);
5424 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
5425 const char *mac, int mtu)
5427 char ritr_pl[MLXSW_REG_RITR_LEN];
5430 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
5431 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5435 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
5436 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
5437 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
5438 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5441 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
5443 struct mlxsw_sp *mlxsw_sp;
5444 struct mlxsw_sp_rif *rif;
5448 mlxsw_sp = mlxsw_sp_lower_get(dev);
5452 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5455 fid_index = mlxsw_sp_fid_index(rif->fid);
5457 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
5461 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
5466 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
5468 goto err_rif_fdb_op;
5470 ether_addr_copy(rif->addr, dev->dev_addr);
5471 rif->mtu = dev->mtu;
5473 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
5478 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
5480 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
5484 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
5485 struct net_device *l3_dev)
5487 struct mlxsw_sp_rif *rif;
5489 /* If netdev is already associated with a RIF, then we need to
5490 * destroy it and create a new one with the new virtual router ID.
5492 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5494 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
5496 return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
5499 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
5500 struct net_device *l3_dev)
5502 struct mlxsw_sp_rif *rif;
5504 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5507 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
5510 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
5511 struct netdev_notifier_changeupper_info *info)
5513 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
5520 case NETDEV_PRECHANGEUPPER:
5522 case NETDEV_CHANGEUPPER:
5524 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
5526 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
5533 static struct mlxsw_sp_rif_subport *
5534 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
5536 return container_of(rif, struct mlxsw_sp_rif_subport, common);
5539 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
5540 const struct mlxsw_sp_rif_params *params)
5542 struct mlxsw_sp_rif_subport *rif_subport;
5544 rif_subport = mlxsw_sp_rif_subport_rif(rif);
5545 rif_subport->vid = params->vid;
5546 rif_subport->lag = params->lag;
5548 rif_subport->lag_id = params->lag_id;
5550 rif_subport->system_port = params->system_port;
5553 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
5555 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5556 struct mlxsw_sp_rif_subport *rif_subport;
5557 char ritr_pl[MLXSW_REG_RITR_LEN];
5559 rif_subport = mlxsw_sp_rif_subport_rif(rif);
5560 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
5561 rif->rif_index, rif->vr_id, rif->dev->mtu);
5562 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
5563 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
5564 rif_subport->lag ? rif_subport->lag_id :
5565 rif_subport->system_port,
5568 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5571 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
5575 err = mlxsw_sp_rif_subport_op(rif, true);
5579 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5580 mlxsw_sp_fid_index(rif->fid), true);
5582 goto err_rif_fdb_op;
5584 mlxsw_sp_fid_rif_set(rif->fid, rif);
5588 mlxsw_sp_rif_subport_op(rif, false);
5592 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
5594 struct mlxsw_sp_fid *fid = rif->fid;
5596 mlxsw_sp_fid_rif_set(fid, NULL);
5597 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5598 mlxsw_sp_fid_index(fid), false);
5599 mlxsw_sp_rif_subport_op(rif, false);
5602 static struct mlxsw_sp_fid *
5603 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
5605 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
5608 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
5609 .type = MLXSW_SP_RIF_TYPE_SUBPORT,
5610 .rif_size = sizeof(struct mlxsw_sp_rif_subport),
5611 .setup = mlxsw_sp_rif_subport_setup,
5612 .configure = mlxsw_sp_rif_subport_configure,
5613 .deconfigure = mlxsw_sp_rif_subport_deconfigure,
5614 .fid_get = mlxsw_sp_rif_subport_fid_get,
5617 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
5618 enum mlxsw_reg_ritr_if_type type,
5619 u16 vid_fid, bool enable)
5621 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5622 char ritr_pl[MLXSW_REG_RITR_LEN];
5624 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
5626 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
5627 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
5629 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5632 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
5634 return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
5637 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
5639 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5640 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5643 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
5647 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5648 mlxsw_sp_router_port(mlxsw_sp), true);
5650 goto err_fid_mc_flood_set;
5652 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5653 mlxsw_sp_router_port(mlxsw_sp), true);
5655 goto err_fid_bc_flood_set;
5657 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5658 mlxsw_sp_fid_index(rif->fid), true);
5660 goto err_rif_fdb_op;
5662 mlxsw_sp_fid_rif_set(rif->fid, rif);
5666 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5667 mlxsw_sp_router_port(mlxsw_sp), false);
5668 err_fid_bc_flood_set:
5669 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5670 mlxsw_sp_router_port(mlxsw_sp), false);
5671 err_fid_mc_flood_set:
5672 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5676 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
5678 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5679 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5680 struct mlxsw_sp_fid *fid = rif->fid;
5682 mlxsw_sp_fid_rif_set(fid, NULL);
5683 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5684 mlxsw_sp_fid_index(fid), false);
5685 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5686 mlxsw_sp_router_port(mlxsw_sp), false);
5687 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5688 mlxsw_sp_router_port(mlxsw_sp), false);
5689 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5692 static struct mlxsw_sp_fid *
5693 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
5695 u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
5697 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
5700 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
5701 .type = MLXSW_SP_RIF_TYPE_VLAN,
5702 .rif_size = sizeof(struct mlxsw_sp_rif),
5703 .configure = mlxsw_sp_rif_vlan_configure,
5704 .deconfigure = mlxsw_sp_rif_vlan_deconfigure,
5705 .fid_get = mlxsw_sp_rif_vlan_fid_get,
5708 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
5710 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5711 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5714 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
5719 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5720 mlxsw_sp_router_port(mlxsw_sp), true);
5722 goto err_fid_mc_flood_set;
5724 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5725 mlxsw_sp_router_port(mlxsw_sp), true);
5727 goto err_fid_bc_flood_set;
5729 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5730 mlxsw_sp_fid_index(rif->fid), true);
5732 goto err_rif_fdb_op;
5734 mlxsw_sp_fid_rif_set(rif->fid, rif);
5738 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5739 mlxsw_sp_router_port(mlxsw_sp), false);
5740 err_fid_bc_flood_set:
5741 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5742 mlxsw_sp_router_port(mlxsw_sp), false);
5743 err_fid_mc_flood_set:
5744 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5748 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
5750 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5751 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5752 struct mlxsw_sp_fid *fid = rif->fid;
5754 mlxsw_sp_fid_rif_set(fid, NULL);
5755 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5756 mlxsw_sp_fid_index(fid), false);
5757 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5758 mlxsw_sp_router_port(mlxsw_sp), false);
5759 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5760 mlxsw_sp_router_port(mlxsw_sp), false);
5761 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5764 static struct mlxsw_sp_fid *
5765 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
5767 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
5770 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
5771 .type = MLXSW_SP_RIF_TYPE_FID,
5772 .rif_size = sizeof(struct mlxsw_sp_rif),
5773 .configure = mlxsw_sp_rif_fid_configure,
5774 .deconfigure = mlxsw_sp_rif_fid_deconfigure,
5775 .fid_get = mlxsw_sp_rif_fid_fid_get,
5778 static struct mlxsw_sp_rif_ipip_lb *
5779 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
5781 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
5785 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
5786 const struct mlxsw_sp_rif_params *params)
5788 struct mlxsw_sp_rif_params_ipip_lb *params_lb;
5789 struct mlxsw_sp_rif_ipip_lb *rif_lb;
5791 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
5793 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
5794 rif_lb->lb_config = params_lb->lb_config;
5798 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif,
5799 struct mlxsw_sp_vr *ul_vr, bool enable)
5801 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
5802 struct mlxsw_sp_rif *rif = &lb_rif->common;
5803 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5804 char ritr_pl[MLXSW_REG_RITR_LEN];
5807 switch (lb_cf.ul_protocol) {
5808 case MLXSW_SP_L3_PROTO_IPV4:
5809 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
5810 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
5811 rif->rif_index, rif->vr_id, rif->dev->mtu);
5812 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
5813 MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
5814 ul_vr->id, saddr4, lb_cf.okey);
5817 case MLXSW_SP_L3_PROTO_IPV6:
5818 return -EAFNOSUPPORT;
5821 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5825 mlxsw_sp_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
5827 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5828 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
5829 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5830 struct mlxsw_sp_vr *ul_vr;
5833 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id);
5835 return PTR_ERR(ul_vr);
5837 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, true);
5839 goto err_loopback_op;
5841 lb_rif->ul_vr_id = ul_vr->id;
5846 mlxsw_sp_vr_put(ul_vr);
5850 static void mlxsw_sp_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
5852 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5853 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5854 struct mlxsw_sp_vr *ul_vr;
5856 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
5857 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, false);
5860 mlxsw_sp_vr_put(ul_vr);
5863 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_ipip_lb_ops = {
5864 .type = MLXSW_SP_RIF_TYPE_IPIP_LB,
5865 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb),
5866 .setup = mlxsw_sp_rif_ipip_lb_setup,
5867 .configure = mlxsw_sp_rif_ipip_lb_configure,
5868 .deconfigure = mlxsw_sp_rif_ipip_lb_deconfigure,
5871 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
5872 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops,
5873 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_ops,
5874 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops,
5875 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp_rif_ipip_lb_ops,
5878 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
5880 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5882 mlxsw_sp->router->rifs = kcalloc(max_rifs,
5883 sizeof(struct mlxsw_sp_rif *),
5885 if (!mlxsw_sp->router->rifs)
5888 mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
5893 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
5897 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
5898 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
5900 kfree(mlxsw_sp->router->rifs);
5903 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
5905 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
5906 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
5910 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
5912 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
5915 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
5917 struct mlxsw_sp_router *router;
5919 /* Flush pending FIB notifications and then flush the device's
5920 * table before requesting another dump. The FIB notification
5921 * block is unregistered, so no need to take RTNL.
5923 mlxsw_core_flush_owq();
5924 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
5925 mlxsw_sp_router_fib_flush(router->mlxsw_sp);
5928 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5930 char rgcr_pl[MLXSW_REG_RGCR_LEN];
5934 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
5936 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5938 mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
5939 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
5940 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
5946 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
5948 char rgcr_pl[MLXSW_REG_RGCR_LEN];
5950 mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
5951 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
5954 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5956 struct mlxsw_sp_router *router;
5959 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
5962 mlxsw_sp->router = router;
5963 router->mlxsw_sp = mlxsw_sp;
5965 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
5966 err = __mlxsw_sp_router_init(mlxsw_sp);
5968 goto err_router_init;
5970 err = mlxsw_sp_rifs_init(mlxsw_sp);
5974 err = mlxsw_sp_ipips_init(mlxsw_sp);
5976 goto err_ipips_init;
5978 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
5979 &mlxsw_sp_nexthop_ht_params);
5981 goto err_nexthop_ht_init;
5983 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
5984 &mlxsw_sp_nexthop_group_ht_params);
5986 goto err_nexthop_group_ht_init;
5988 err = mlxsw_sp_lpm_init(mlxsw_sp);
5992 err = mlxsw_sp_vrs_init(mlxsw_sp);
5996 err = mlxsw_sp_neigh_init(mlxsw_sp);
5998 goto err_neigh_init;
6000 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
6001 err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
6002 mlxsw_sp_router_fib_dump_flush);
6004 goto err_register_fib_notifier;
6008 err_register_fib_notifier:
6009 mlxsw_sp_neigh_fini(mlxsw_sp);
6011 mlxsw_sp_vrs_fini(mlxsw_sp);
6013 mlxsw_sp_lpm_fini(mlxsw_sp);
6015 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
6016 err_nexthop_group_ht_init:
6017 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
6018 err_nexthop_ht_init:
6019 mlxsw_sp_ipips_fini(mlxsw_sp);
6021 mlxsw_sp_rifs_fini(mlxsw_sp);
6023 __mlxsw_sp_router_fini(mlxsw_sp);
6025 kfree(mlxsw_sp->router);
6029 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
6031 unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
6032 mlxsw_sp_neigh_fini(mlxsw_sp);
6033 mlxsw_sp_vrs_fini(mlxsw_sp);
6034 mlxsw_sp_lpm_fini(mlxsw_sp);
6035 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
6036 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
6037 mlxsw_sp_ipips_fini(mlxsw_sp);
6038 mlxsw_sp_rifs_fini(mlxsw_sp);
6039 __mlxsw_sp_router_fini(mlxsw_sp);
6040 kfree(mlxsw_sp->router);