2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/netdevice.h>
34 #include <linux/mlx5/driver.h>
35 #include <linux/mlx5/vport.h>
36 #include "mlx5_core.h"
41 /* General purpose, use for short periods of time.
42 * Beware of lock dependencies (preferably, no locks should be acquired
45 static DEFINE_MUTEX(lag_mutex);
47 static int mlx5_cmd_create_lag(struct mlx5_core_dev *dev, u8 remap_port1,
50 u32 in[MLX5_ST_SZ_DW(create_lag_in)] = {};
51 void *lag_ctx = MLX5_ADDR_OF(create_lag_in, in, ctx);
53 MLX5_SET(create_lag_in, in, opcode, MLX5_CMD_OP_CREATE_LAG);
55 MLX5_SET(lagc, lag_ctx, tx_remap_affinity_1, remap_port1);
56 MLX5_SET(lagc, lag_ctx, tx_remap_affinity_2, remap_port2);
58 return mlx5_cmd_exec_in(dev, create_lag, in);
61 static int mlx5_cmd_modify_lag(struct mlx5_core_dev *dev, u8 remap_port1,
64 u32 in[MLX5_ST_SZ_DW(modify_lag_in)] = {};
65 void *lag_ctx = MLX5_ADDR_OF(modify_lag_in, in, ctx);
67 MLX5_SET(modify_lag_in, in, opcode, MLX5_CMD_OP_MODIFY_LAG);
68 MLX5_SET(modify_lag_in, in, field_select, 0x1);
70 MLX5_SET(lagc, lag_ctx, tx_remap_affinity_1, remap_port1);
71 MLX5_SET(lagc, lag_ctx, tx_remap_affinity_2, remap_port2);
73 return mlx5_cmd_exec_in(dev, modify_lag, in);
76 int mlx5_cmd_create_vport_lag(struct mlx5_core_dev *dev)
78 u32 in[MLX5_ST_SZ_DW(create_vport_lag_in)] = {};
80 MLX5_SET(create_vport_lag_in, in, opcode, MLX5_CMD_OP_CREATE_VPORT_LAG);
82 return mlx5_cmd_exec_in(dev, create_vport_lag, in);
84 EXPORT_SYMBOL(mlx5_cmd_create_vport_lag);
86 int mlx5_cmd_destroy_vport_lag(struct mlx5_core_dev *dev)
88 u32 in[MLX5_ST_SZ_DW(destroy_vport_lag_in)] = {};
90 MLX5_SET(destroy_vport_lag_in, in, opcode, MLX5_CMD_OP_DESTROY_VPORT_LAG);
92 return mlx5_cmd_exec_in(dev, destroy_vport_lag, in);
94 EXPORT_SYMBOL(mlx5_cmd_destroy_vport_lag);
96 int mlx5_lag_dev_get_netdev_idx(struct mlx5_lag *ldev,
97 struct net_device *ndev)
101 for (i = 0; i < MLX5_MAX_PORTS; i++)
102 if (ldev->pf[i].netdev == ndev)
108 static bool __mlx5_lag_is_roce(struct mlx5_lag *ldev)
110 return !!(ldev->flags & MLX5_LAG_FLAG_ROCE);
113 static bool __mlx5_lag_is_sriov(struct mlx5_lag *ldev)
115 return !!(ldev->flags & MLX5_LAG_FLAG_SRIOV);
118 static void mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker,
119 u8 *port1, u8 *port2)
123 if (!tracker->netdev_state[MLX5_LAG_P1].tx_enabled ||
124 !tracker->netdev_state[MLX5_LAG_P1].link_up) {
129 if (!tracker->netdev_state[MLX5_LAG_P2].tx_enabled ||
130 !tracker->netdev_state[MLX5_LAG_P2].link_up)
134 void mlx5_modify_lag(struct mlx5_lag *ldev,
135 struct lag_tracker *tracker)
137 struct mlx5_core_dev *dev0 = ldev->pf[MLX5_LAG_P1].dev;
138 u8 v2p_port1, v2p_port2;
141 mlx5_infer_tx_affinity_mapping(tracker, &v2p_port1,
144 if (v2p_port1 != ldev->v2p_map[MLX5_LAG_P1] ||
145 v2p_port2 != ldev->v2p_map[MLX5_LAG_P2]) {
146 ldev->v2p_map[MLX5_LAG_P1] = v2p_port1;
147 ldev->v2p_map[MLX5_LAG_P2] = v2p_port2;
149 mlx5_core_info(dev0, "modify lag map port 1:%d port 2:%d",
150 ldev->v2p_map[MLX5_LAG_P1],
151 ldev->v2p_map[MLX5_LAG_P2]);
153 err = mlx5_cmd_modify_lag(dev0, v2p_port1, v2p_port2);
156 "Failed to modify LAG (%d)\n",
161 static int mlx5_create_lag(struct mlx5_lag *ldev,
162 struct lag_tracker *tracker)
164 struct mlx5_core_dev *dev0 = ldev->pf[MLX5_LAG_P1].dev;
167 mlx5_infer_tx_affinity_mapping(tracker, &ldev->v2p_map[MLX5_LAG_P1],
168 &ldev->v2p_map[MLX5_LAG_P2]);
170 mlx5_core_info(dev0, "lag map port 1:%d port 2:%d",
171 ldev->v2p_map[MLX5_LAG_P1], ldev->v2p_map[MLX5_LAG_P2]);
173 err = mlx5_cmd_create_lag(dev0, ldev->v2p_map[MLX5_LAG_P1],
174 ldev->v2p_map[MLX5_LAG_P2]);
177 "Failed to create LAG (%d)\n",
182 int mlx5_activate_lag(struct mlx5_lag *ldev,
183 struct lag_tracker *tracker,
186 bool roce_lag = !!(flags & MLX5_LAG_FLAG_ROCE);
187 struct mlx5_core_dev *dev0 = ldev->pf[MLX5_LAG_P1].dev;
190 err = mlx5_create_lag(ldev, tracker);
194 "Failed to activate RoCE LAG\n");
197 "Failed to activate VF LAG\n"
198 "Make sure all VFs are unbound prior to VF LAG activation or deactivation\n");
203 ldev->flags |= flags;
207 static int mlx5_deactivate_lag(struct mlx5_lag *ldev)
209 struct mlx5_core_dev *dev0 = ldev->pf[MLX5_LAG_P1].dev;
210 u32 in[MLX5_ST_SZ_DW(destroy_lag_in)] = {};
211 bool roce_lag = __mlx5_lag_is_roce(ldev);
214 ldev->flags &= ~MLX5_LAG_MODE_FLAGS;
216 MLX5_SET(destroy_lag_in, in, opcode, MLX5_CMD_OP_DESTROY_LAG);
217 err = mlx5_cmd_exec_in(dev0, destroy_lag, in);
221 "Failed to deactivate RoCE LAG; driver restart required\n");
224 "Failed to deactivate VF LAG; driver restart required\n"
225 "Make sure all VFs are unbound prior to VF LAG activation or deactivation\n");
232 static bool mlx5_lag_check_prereq(struct mlx5_lag *ldev)
234 if (!ldev->pf[MLX5_LAG_P1].dev || !ldev->pf[MLX5_LAG_P2].dev)
237 #ifdef CONFIG_MLX5_ESWITCH
238 return mlx5_esw_lag_prereq(ldev->pf[MLX5_LAG_P1].dev,
239 ldev->pf[MLX5_LAG_P2].dev);
241 return (!mlx5_sriov_is_enabled(ldev->pf[MLX5_LAG_P1].dev) &&
242 !mlx5_sriov_is_enabled(ldev->pf[MLX5_LAG_P2].dev));
246 static void mlx5_lag_add_ib_devices(struct mlx5_lag *ldev)
250 for (i = 0; i < MLX5_MAX_PORTS; i++)
252 mlx5_add_dev_by_protocol(ldev->pf[i].dev,
253 MLX5_INTERFACE_PROTOCOL_IB);
256 static void mlx5_lag_remove_ib_devices(struct mlx5_lag *ldev)
260 for (i = 0; i < MLX5_MAX_PORTS; i++)
262 mlx5_remove_dev_by_protocol(ldev->pf[i].dev,
263 MLX5_INTERFACE_PROTOCOL_IB);
266 static void mlx5_do_bond(struct mlx5_lag *ldev)
268 struct mlx5_core_dev *dev0 = ldev->pf[MLX5_LAG_P1].dev;
269 struct mlx5_core_dev *dev1 = ldev->pf[MLX5_LAG_P2].dev;
270 struct lag_tracker tracker;
271 bool do_bond, roce_lag;
277 mutex_lock(&lag_mutex);
278 tracker = ldev->tracker;
279 mutex_unlock(&lag_mutex);
281 do_bond = tracker.is_bonded && mlx5_lag_check_prereq(ldev);
283 if (do_bond && !__mlx5_lag_is_active(ldev)) {
284 roce_lag = !mlx5_sriov_is_enabled(dev0) &&
285 !mlx5_sriov_is_enabled(dev1);
287 #ifdef CONFIG_MLX5_ESWITCH
288 roce_lag &= dev0->priv.eswitch->mode == MLX5_ESWITCH_NONE &&
289 dev1->priv.eswitch->mode == MLX5_ESWITCH_NONE;
293 mlx5_lag_remove_ib_devices(ldev);
295 err = mlx5_activate_lag(ldev, &tracker,
296 roce_lag ? MLX5_LAG_FLAG_ROCE :
297 MLX5_LAG_FLAG_SRIOV);
300 mlx5_lag_add_ib_devices(ldev);
306 mlx5_add_dev_by_protocol(dev0, MLX5_INTERFACE_PROTOCOL_IB);
307 mlx5_nic_vport_enable_roce(dev1);
309 } else if (do_bond && __mlx5_lag_is_active(ldev)) {
310 mlx5_modify_lag(ldev, &tracker);
311 } else if (!do_bond && __mlx5_lag_is_active(ldev)) {
312 roce_lag = __mlx5_lag_is_roce(ldev);
315 mlx5_remove_dev_by_protocol(dev0, MLX5_INTERFACE_PROTOCOL_IB);
316 mlx5_nic_vport_disable_roce(dev1);
319 err = mlx5_deactivate_lag(ldev);
324 mlx5_lag_add_ib_devices(ldev);
328 static void mlx5_queue_bond_work(struct mlx5_lag *ldev, unsigned long delay)
330 queue_delayed_work(ldev->wq, &ldev->bond_work, delay);
333 static void mlx5_do_bond_work(struct work_struct *work)
335 struct delayed_work *delayed_work = to_delayed_work(work);
336 struct mlx5_lag *ldev = container_of(delayed_work, struct mlx5_lag,
340 status = mlx5_dev_list_trylock();
343 mlx5_queue_bond_work(ldev, HZ);
348 mlx5_dev_list_unlock();
351 static int mlx5_handle_changeupper_event(struct mlx5_lag *ldev,
352 struct lag_tracker *tracker,
353 struct net_device *ndev,
354 struct netdev_notifier_changeupper_info *info)
356 struct net_device *upper = info->upper_dev, *ndev_tmp;
357 struct netdev_lag_upper_info *lag_upper_info = NULL;
363 if (!netif_is_lag_master(upper))
367 lag_upper_info = info->upper_info;
369 /* The event may still be of interest if the slave does not belong to
370 * us, but is enslaved to a master which has one or more of our netdevs
371 * as slaves (e.g., if a new slave is added to a master that bonds two
372 * of our netdevs, we should unbond).
375 for_each_netdev_in_bond_rcu(upper, ndev_tmp) {
376 idx = mlx5_lag_dev_get_netdev_idx(ldev, ndev_tmp);
378 bond_status |= (1 << idx);
384 /* None of this lagdev's netdevs are slaves of this master. */
385 if (!(bond_status & 0x3))
389 tracker->tx_type = lag_upper_info->tx_type;
391 /* Determine bonding status:
392 * A device is considered bonded if both its physical ports are slaves
393 * of the same lag master, and only them.
394 * Lag mode must be activebackup or hash.
396 is_bonded = (num_slaves == MLX5_MAX_PORTS) &&
397 (bond_status == 0x3) &&
398 ((tracker->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) ||
399 (tracker->tx_type == NETDEV_LAG_TX_TYPE_HASH));
401 if (tracker->is_bonded != is_bonded) {
402 tracker->is_bonded = is_bonded;
409 static int mlx5_handle_changelowerstate_event(struct mlx5_lag *ldev,
410 struct lag_tracker *tracker,
411 struct net_device *ndev,
412 struct netdev_notifier_changelowerstate_info *info)
414 struct netdev_lag_lower_state_info *lag_lower_info;
417 if (!netif_is_lag_port(ndev))
420 idx = mlx5_lag_dev_get_netdev_idx(ldev, ndev);
424 /* This information is used to determine virtual to physical
427 lag_lower_info = info->lower_state_info;
431 tracker->netdev_state[idx] = *lag_lower_info;
436 static int mlx5_lag_netdev_event(struct notifier_block *this,
437 unsigned long event, void *ptr)
439 struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
440 struct lag_tracker tracker;
441 struct mlx5_lag *ldev;
444 if ((event != NETDEV_CHANGEUPPER) && (event != NETDEV_CHANGELOWERSTATE))
447 ldev = container_of(this, struct mlx5_lag, nb);
448 tracker = ldev->tracker;
451 case NETDEV_CHANGEUPPER:
452 changed = mlx5_handle_changeupper_event(ldev, &tracker, ndev,
455 case NETDEV_CHANGELOWERSTATE:
456 changed = mlx5_handle_changelowerstate_event(ldev, &tracker,
461 mutex_lock(&lag_mutex);
462 ldev->tracker = tracker;
463 mutex_unlock(&lag_mutex);
466 mlx5_queue_bond_work(ldev, 0);
471 static struct mlx5_lag *mlx5_lag_dev_alloc(void)
473 struct mlx5_lag *ldev;
475 ldev = kzalloc(sizeof(*ldev), GFP_KERNEL);
479 ldev->wq = create_singlethread_workqueue("mlx5_lag");
485 INIT_DELAYED_WORK(&ldev->bond_work, mlx5_do_bond_work);
490 static void mlx5_lag_dev_free(struct mlx5_lag *ldev)
492 destroy_workqueue(ldev->wq);
496 static void mlx5_lag_dev_add_pf(struct mlx5_lag *ldev,
497 struct mlx5_core_dev *dev,
498 struct net_device *netdev)
500 unsigned int fn = PCI_FUNC(dev->pdev->devfn);
502 if (fn >= MLX5_MAX_PORTS)
505 mutex_lock(&lag_mutex);
506 ldev->pf[fn].dev = dev;
507 ldev->pf[fn].netdev = netdev;
508 ldev->tracker.netdev_state[fn].link_up = 0;
509 ldev->tracker.netdev_state[fn].tx_enabled = 0;
511 dev->priv.lag = ldev;
513 mutex_unlock(&lag_mutex);
516 static void mlx5_lag_dev_remove_pf(struct mlx5_lag *ldev,
517 struct mlx5_core_dev *dev)
521 for (i = 0; i < MLX5_MAX_PORTS; i++)
522 if (ldev->pf[i].dev == dev)
525 if (i == MLX5_MAX_PORTS)
528 mutex_lock(&lag_mutex);
529 memset(&ldev->pf[i], 0, sizeof(*ldev->pf));
531 dev->priv.lag = NULL;
532 mutex_unlock(&lag_mutex);
535 /* Must be called with intf_mutex held */
536 void mlx5_lag_add(struct mlx5_core_dev *dev, struct net_device *netdev)
538 struct mlx5_lag *ldev = NULL;
539 struct mlx5_core_dev *tmp_dev;
542 if (!MLX5_CAP_GEN(dev, vport_group_manager) ||
543 !MLX5_CAP_GEN(dev, lag_master) ||
544 (MLX5_CAP_GEN(dev, num_lag_ports) != MLX5_MAX_PORTS))
547 tmp_dev = mlx5_get_next_phys_dev(dev);
549 ldev = tmp_dev->priv.lag;
552 ldev = mlx5_lag_dev_alloc();
554 mlx5_core_err(dev, "Failed to alloc lag dev\n");
559 mlx5_lag_dev_add_pf(ldev, dev, netdev);
561 if (!ldev->nb.notifier_call) {
562 ldev->nb.notifier_call = mlx5_lag_netdev_event;
563 if (register_netdevice_notifier_net(&init_net, &ldev->nb)) {
564 ldev->nb.notifier_call = NULL;
565 mlx5_core_err(dev, "Failed to register LAG netdev notifier\n");
569 err = mlx5_lag_mp_init(ldev);
571 mlx5_core_err(dev, "Failed to init multipath lag err=%d\n",
575 /* Must be called with intf_mutex held */
576 void mlx5_lag_remove(struct mlx5_core_dev *dev)
578 struct mlx5_lag *ldev;
581 ldev = mlx5_lag_dev_get(dev);
585 if (__mlx5_lag_is_active(ldev))
586 mlx5_deactivate_lag(ldev);
588 mlx5_lag_dev_remove_pf(ldev, dev);
590 for (i = 0; i < MLX5_MAX_PORTS; i++)
594 if (i == MLX5_MAX_PORTS) {
595 if (ldev->nb.notifier_call) {
596 unregister_netdevice_notifier_net(&init_net, &ldev->nb);
597 ldev->nb.notifier_call = NULL;
599 mlx5_lag_mp_cleanup(ldev);
600 cancel_delayed_work_sync(&ldev->bond_work);
601 mlx5_lag_dev_free(ldev);
605 bool mlx5_lag_is_roce(struct mlx5_core_dev *dev)
607 struct mlx5_lag *ldev;
610 mutex_lock(&lag_mutex);
611 ldev = mlx5_lag_dev_get(dev);
612 res = ldev && __mlx5_lag_is_roce(ldev);
613 mutex_unlock(&lag_mutex);
617 EXPORT_SYMBOL(mlx5_lag_is_roce);
619 bool mlx5_lag_is_active(struct mlx5_core_dev *dev)
621 struct mlx5_lag *ldev;
624 mutex_lock(&lag_mutex);
625 ldev = mlx5_lag_dev_get(dev);
626 res = ldev && __mlx5_lag_is_active(ldev);
627 mutex_unlock(&lag_mutex);
631 EXPORT_SYMBOL(mlx5_lag_is_active);
633 bool mlx5_lag_is_sriov(struct mlx5_core_dev *dev)
635 struct mlx5_lag *ldev;
638 mutex_lock(&lag_mutex);
639 ldev = mlx5_lag_dev_get(dev);
640 res = ldev && __mlx5_lag_is_sriov(ldev);
641 mutex_unlock(&lag_mutex);
645 EXPORT_SYMBOL(mlx5_lag_is_sriov);
647 void mlx5_lag_update(struct mlx5_core_dev *dev)
649 struct mlx5_lag *ldev;
651 mlx5_dev_list_lock();
652 ldev = mlx5_lag_dev_get(dev);
659 mlx5_dev_list_unlock();
662 struct net_device *mlx5_lag_get_roce_netdev(struct mlx5_core_dev *dev)
664 struct net_device *ndev = NULL;
665 struct mlx5_lag *ldev;
667 mutex_lock(&lag_mutex);
668 ldev = mlx5_lag_dev_get(dev);
670 if (!(ldev && __mlx5_lag_is_roce(ldev)))
673 if (ldev->tracker.tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
674 ndev = ldev->tracker.netdev_state[MLX5_LAG_P1].tx_enabled ?
675 ldev->pf[MLX5_LAG_P1].netdev :
676 ldev->pf[MLX5_LAG_P2].netdev;
678 ndev = ldev->pf[MLX5_LAG_P1].netdev;
684 mutex_unlock(&lag_mutex);
688 EXPORT_SYMBOL(mlx5_lag_get_roce_netdev);
690 bool mlx5_lag_intf_add(struct mlx5_interface *intf, struct mlx5_priv *priv)
692 struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev,
694 struct mlx5_lag *ldev;
696 if (intf->protocol != MLX5_INTERFACE_PROTOCOL_IB)
699 ldev = mlx5_lag_dev_get(dev);
700 if (!ldev || !__mlx5_lag_is_roce(ldev) ||
701 ldev->pf[MLX5_LAG_P1].dev == dev)
704 /* If bonded, we do not add an IB device for PF1. */
708 int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev,
713 int outlen = MLX5_ST_SZ_BYTES(query_cong_statistics_out);
714 struct mlx5_core_dev *mdev[MLX5_MAX_PORTS];
715 struct mlx5_lag *ldev;
720 out = kvzalloc(outlen, GFP_KERNEL);
724 memset(values, 0, sizeof(*values) * num_counters);
726 mutex_lock(&lag_mutex);
727 ldev = mlx5_lag_dev_get(dev);
728 if (ldev && __mlx5_lag_is_roce(ldev)) {
729 num_ports = MLX5_MAX_PORTS;
730 mdev[MLX5_LAG_P1] = ldev->pf[MLX5_LAG_P1].dev;
731 mdev[MLX5_LAG_P2] = ldev->pf[MLX5_LAG_P2].dev;
734 mdev[MLX5_LAG_P1] = dev;
737 for (i = 0; i < num_ports; ++i) {
738 u32 in[MLX5_ST_SZ_DW(query_cong_statistics_in)] = {};
740 MLX5_SET(query_cong_statistics_in, in, opcode,
741 MLX5_CMD_OP_QUERY_CONG_STATISTICS);
742 ret = mlx5_cmd_exec_inout(mdev[i], query_cong_statistics, in,
747 for (j = 0; j < num_counters; ++j)
748 values[j] += be64_to_cpup((__be64 *)(out + offsets[j]));
752 mutex_unlock(&lag_mutex);
756 EXPORT_SYMBOL(mlx5_lag_query_cong_counters);