drivers/net/ethernet/mellanox/mlx5/core/en_rep.c

   1 /*
   2  * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <generated/utsrelease.h>
  34 #include <linux/mlx5/fs.h>
  35 #include <net/switchdev.h>
  36 #include <net/pkt_cls.h>
  37 #include <net/act_api.h>
  38 #include <net/netevent.h>
  39 #include <net/arp.h>
  40 #include <net/devlink.h>
  41 #include <net/ipv6_stubs.h>
  42
  43 #include "eswitch.h"
  44 #include "eswitch_offloads_chains.h"
  45 #include "en.h"
  46 #include "en_rep.h"
  47 #include "en_tc.h"
  48 #include "en/tc_tun.h"
  49 #include "fs_core.h"
  50 #include "lib/port_tun.h"
  51 #include "lib/mlx5.h"
  52 #define CREATE_TRACE_POINTS
  53 #include "diag/en_rep_tracepoint.h"
  54
  55 #define MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE \
  56         max(0x7, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE)
  57 #define MLX5E_REP_PARAMS_DEF_NUM_CHANNELS 1
  58
  59 static const char mlx5e_rep_driver_name[] = "mlx5e_rep";
  60
  61 struct mlx5e_rep_indr_block_priv {
  62         struct net_device *netdev;
  63         struct mlx5e_rep_priv *rpriv;
  64
  65         struct list_head list;
  66 };
  67
  68 static void mlx5e_rep_indr_unregister_block(struct mlx5e_rep_priv *rpriv,
  69                                             struct net_device *netdev);
  70
  71 static void mlx5e_rep_get_drvinfo(struct net_device *dev,
  72                                   struct ethtool_drvinfo *drvinfo)
  73 {
  74         struct mlx5e_priv *priv = netdev_priv(dev);
  75         struct mlx5_core_dev *mdev = priv->mdev;
  76
  77         strlcpy(drvinfo->driver, mlx5e_rep_driver_name,
  78                 sizeof(drvinfo->driver));
  79         strlcpy(drvinfo->version, UTS_RELEASE, sizeof(drvinfo->version));
  80         snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
  81                  "%d.%d.%04d (%.16s)",
  82                  fw_rev_maj(mdev), fw_rev_min(mdev),
  83                  fw_rev_sub(mdev), mdev->board_id);
  84 }
  85
  86 static void mlx5e_uplink_rep_get_drvinfo(struct net_device *dev,
  87                                          struct ethtool_drvinfo *drvinfo)
  88 {
  89         struct mlx5e_priv *priv = netdev_priv(dev);
  90
  91         mlx5e_rep_get_drvinfo(dev, drvinfo);
  92         strlcpy(drvinfo->bus_info, pci_name(priv->mdev->pdev),
  93                 sizeof(drvinfo->bus_info));
  94 }
  95
  96 static const struct counter_desc sw_rep_stats_desc[] = {
  97         { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_packets) },
  98         { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_bytes) },
  99         { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_packets) },
 100         { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_bytes) },
 101 };
 102
 103 struct vport_stats {
 104         u64 vport_rx_packets;
 105         u64 vport_tx_packets;
 106         u64 vport_rx_bytes;
 107         u64 vport_tx_bytes;
 108 };
 109
 110 static const struct counter_desc vport_rep_stats_desc[] = {
 111         { MLX5E_DECLARE_STAT(struct vport_stats, vport_rx_packets) },
 112         { MLX5E_DECLARE_STAT(struct vport_stats, vport_rx_bytes) },
 113         { MLX5E_DECLARE_STAT(struct vport_stats, vport_tx_packets) },
 114         { MLX5E_DECLARE_STAT(struct vport_stats, vport_tx_bytes) },
 115 };
 116
 117 #define NUM_VPORT_REP_SW_COUNTERS ARRAY_SIZE(sw_rep_stats_desc)
 118 #define NUM_VPORT_REP_HW_COUNTERS ARRAY_SIZE(vport_rep_stats_desc)
 119
 120 static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(sw_rep)
 121 {
 122         return NUM_VPORT_REP_SW_COUNTERS;
 123 }
 124
 125 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(sw_rep)
 126 {
 127         int i;
 128
 129         for (i = 0; i < NUM_VPORT_REP_SW_COUNTERS; i++)
 130                 strcpy(data + (idx++) * ETH_GSTRING_LEN,
 131                        sw_rep_stats_desc[i].format);
 132         return idx;
 133 }
 134
 135 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(sw_rep)
 136 {
 137         int i;
 138
 139         for (i = 0; i < NUM_VPORT_REP_SW_COUNTERS; i++)
 140                 data[idx++] = MLX5E_READ_CTR64_CPU(&priv->stats.sw,
 141                                                    sw_rep_stats_desc, i);
 142         return idx;
 143 }
 144
 145 static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(sw_rep)
 146 {
 147         struct mlx5e_sw_stats *s = &priv->stats.sw;
 148         struct rtnl_link_stats64 stats64 = {};
 149
 150         memset(s, 0, sizeof(*s));
 151         mlx5e_fold_sw_stats64(priv, &stats64);
 152
 153         s->rx_packets = stats64.rx_packets;
 154         s->rx_bytes   = stats64.rx_bytes;
 155         s->tx_packets = stats64.tx_packets;
 156         s->tx_bytes   = stats64.tx_bytes;
 157         s->tx_queue_dropped = stats64.tx_dropped;
 158 }
 159
 160 static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(vport_rep)
 161 {
 162         return NUM_VPORT_REP_HW_COUNTERS;
 163 }
 164
 165 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(vport_rep)
 166 {
 167         int i;
 168
 169         for (i = 0; i < NUM_VPORT_REP_HW_COUNTERS; i++)
 170                 strcpy(data + (idx++) * ETH_GSTRING_LEN, vport_rep_stats_desc[i].format);
 171         return idx;
 172 }
 173
 174 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(vport_rep)
 175 {
 176         int i;
 177
 178         for (i = 0; i < NUM_VPORT_REP_HW_COUNTERS; i++)
 179                 data[idx++] = MLX5E_READ_CTR64_CPU(&priv->stats.vf_vport,
 180                                                    vport_rep_stats_desc, i);
 181         return idx;
 182 }
 183
 184 static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(vport_rep)
 185 {
 186         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 187         struct mlx5e_rep_priv *rpriv = priv->ppriv;
 188         struct mlx5_eswitch_rep *rep = rpriv->rep;
 189         struct rtnl_link_stats64 *vport_stats;
 190         struct ifla_vf_stats vf_stats;
 191         int err;
 192
 193         err = mlx5_eswitch_get_vport_stats(esw, rep->vport, &vf_stats);
 194         if (err) {
 195                 pr_warn("vport %d error %d reading stats\n", rep->vport, err);
 196                 return;
 197         }
 198
 199         vport_stats = &priv->stats.vf_vport;
 200         /* flip tx/rx as we are reporting the counters for the switch vport */
 201         vport_stats->rx_packets = vf_stats.tx_packets;
 202         vport_stats->rx_bytes   = vf_stats.tx_bytes;
 203         vport_stats->tx_packets = vf_stats.rx_packets;
 204         vport_stats->tx_bytes   = vf_stats.rx_bytes;
 205 }
 206
 207 static void mlx5e_rep_get_strings(struct net_device *dev,
 208                                   u32 stringset, uint8_t *data)
 209 {
 210         struct mlx5e_priv *priv = netdev_priv(dev);
 211
 212         switch (stringset) {
 213         case ETH_SS_STATS:
 214                 mlx5e_stats_fill_strings(priv, data);
 215                 break;
 216         }
 217 }
 218
 219 static void mlx5e_rep_get_ethtool_stats(struct net_device *dev,
 220                                         struct ethtool_stats *stats, u64 *data)
 221 {
 222         struct mlx5e_priv *priv = netdev_priv(dev);
 223
 224         mlx5e_ethtool_get_ethtool_stats(priv, stats, data);
 225 }
 226
 227 static int mlx5e_rep_get_sset_count(struct net_device *dev, int sset)
 228 {
 229         struct mlx5e_priv *priv = netdev_priv(dev);
 230
 231         switch (sset) {
 232         case ETH_SS_STATS:
 233                 return mlx5e_stats_total_num(priv);
 234         default:
 235                 return -EOPNOTSUPP;
 236         }
 237 }
 238
 239 static void mlx5e_rep_get_ringparam(struct net_device *dev,
 240                                 struct ethtool_ringparam *param)
 241 {
 242         struct mlx5e_priv *priv = netdev_priv(dev);
 243
 244         mlx5e_ethtool_get_ringparam(priv, param);
 245 }
 246
 247 static int mlx5e_rep_set_ringparam(struct net_device *dev,
 248                                struct ethtool_ringparam *param)
 249 {
 250         struct mlx5e_priv *priv = netdev_priv(dev);
 251
 252         return mlx5e_ethtool_set_ringparam(priv, param);
 253 }
 254
 255 static int mlx5e_replace_rep_vport_rx_rule(struct mlx5e_priv *priv,
 256                                            struct mlx5_flow_destination *dest)
 257 {
 258         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 259         struct mlx5e_rep_priv *rpriv = priv->ppriv;
 260         struct mlx5_eswitch_rep *rep = rpriv->rep;
 261         struct mlx5_flow_handle *flow_rule;
 262
 263         flow_rule = mlx5_eswitch_create_vport_rx_rule(esw,
 264                                                       rep->vport,
 265                                                       dest);
 266         if (IS_ERR(flow_rule))
 267                 return PTR_ERR(flow_rule);
 268
 269         mlx5_del_flow_rules(rpriv->vport_rx_rule);
 270         rpriv->vport_rx_rule = flow_rule;
 271         return 0;
 272 }
 273
 274 static void mlx5e_rep_get_channels(struct net_device *dev,
 275                                    struct ethtool_channels *ch)
 276 {
 277         struct mlx5e_priv *priv = netdev_priv(dev);
 278
 279         mlx5e_ethtool_get_channels(priv, ch);
 280 }
 281
 282 static int mlx5e_rep_set_channels(struct net_device *dev,
 283                                   struct ethtool_channels *ch)
 284 {
 285         struct mlx5e_priv *priv = netdev_priv(dev);
 286         u16 curr_channels_amount = priv->channels.params.num_channels;
 287         u32 new_channels_amount = ch->combined_count;
 288         struct mlx5_flow_destination new_dest;
 289         int err = 0;
 290
 291         err = mlx5e_ethtool_set_channels(priv, ch);
 292         if (err)
 293                 return err;
 294
 295         if (curr_channels_amount == 1 && new_channels_amount > 1) {
 296                 new_dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 297                 new_dest.ft = priv->fs.ttc.ft.t;
 298         } else if (new_channels_amount == 1 && curr_channels_amount > 1) {
 299                 new_dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 300                 new_dest.tir_num = priv->direct_tir[0].tirn;
 301         } else {
 302                 return 0;
 303         }
 304
 305         err = mlx5e_replace_rep_vport_rx_rule(priv, &new_dest);
 306         if (err) {
 307                 netdev_warn(priv->netdev, "Failed to update vport rx rule, when going from (%d) channels to (%d) channels\n",
 308                             curr_channels_amount, new_channels_amount);
 309                 return err;
 310         }
 311
 312         return 0;
 313 }
 314
 315 static int mlx5e_rep_get_coalesce(struct net_device *netdev,
 316                                   struct ethtool_coalesce *coal)
 317 {
 318         struct mlx5e_priv *priv = netdev_priv(netdev);
 319
 320         return mlx5e_ethtool_get_coalesce(priv, coal);
 321 }
 322
 323 static int mlx5e_rep_set_coalesce(struct net_device *netdev,
 324                                   struct ethtool_coalesce *coal)
 325 {
 326         struct mlx5e_priv *priv = netdev_priv(netdev);
 327
 328         return mlx5e_ethtool_set_coalesce(priv, coal);
 329 }
 330
 331 static u32 mlx5e_rep_get_rxfh_key_size(struct net_device *netdev)
 332 {
 333         struct mlx5e_priv *priv = netdev_priv(netdev);
 334
 335         return mlx5e_ethtool_get_rxfh_key_size(priv);
 336 }
 337
 338 static u32 mlx5e_rep_get_rxfh_indir_size(struct net_device *netdev)
 339 {
 340         struct mlx5e_priv *priv = netdev_priv(netdev);
 341
 342         return mlx5e_ethtool_get_rxfh_indir_size(priv);
 343 }
 344
 345 static void mlx5e_uplink_rep_get_pauseparam(struct net_device *netdev,
 346                                             struct ethtool_pauseparam *pauseparam)
 347 {
 348         struct mlx5e_priv *priv = netdev_priv(netdev);
 349
 350         mlx5e_ethtool_get_pauseparam(priv, pauseparam);
 351 }
 352
 353 static int mlx5e_uplink_rep_set_pauseparam(struct net_device *netdev,
 354                                            struct ethtool_pauseparam *pauseparam)
 355 {
 356         struct mlx5e_priv *priv = netdev_priv(netdev);
 357
 358         return mlx5e_ethtool_set_pauseparam(priv, pauseparam);
 359 }
 360
 361 static int mlx5e_uplink_rep_get_link_ksettings(struct net_device *netdev,
 362                                                struct ethtool_link_ksettings *link_ksettings)
 363 {
 364         struct mlx5e_priv *priv = netdev_priv(netdev);
 365
 366         return mlx5e_ethtool_get_link_ksettings(priv, link_ksettings);
 367 }
 368
 369 static int mlx5e_uplink_rep_set_link_ksettings(struct net_device *netdev,
 370                                                const struct ethtool_link_ksettings *link_ksettings)
 371 {
 372         struct mlx5e_priv *priv = netdev_priv(netdev);
 373
 374         return mlx5e_ethtool_set_link_ksettings(priv, link_ksettings);
 375 }
 376
 377 static const struct ethtool_ops mlx5e_rep_ethtool_ops = {
 378         .get_drvinfo       = mlx5e_rep_get_drvinfo,
 379         .get_link          = ethtool_op_get_link,
 380         .get_strings       = mlx5e_rep_get_strings,
 381         .get_sset_count    = mlx5e_rep_get_sset_count,
 382         .get_ethtool_stats = mlx5e_rep_get_ethtool_stats,
 383         .get_ringparam     = mlx5e_rep_get_ringparam,
 384         .set_ringparam     = mlx5e_rep_set_ringparam,
 385         .get_channels      = mlx5e_rep_get_channels,
 386         .set_channels      = mlx5e_rep_set_channels,
 387         .get_coalesce      = mlx5e_rep_get_coalesce,
 388         .set_coalesce      = mlx5e_rep_set_coalesce,
 389         .get_rxfh_key_size   = mlx5e_rep_get_rxfh_key_size,
 390         .get_rxfh_indir_size = mlx5e_rep_get_rxfh_indir_size,
 391 };
 392
 393 static const struct ethtool_ops mlx5e_uplink_rep_ethtool_ops = {
 394         .get_drvinfo       = mlx5e_uplink_rep_get_drvinfo,
 395         .get_link          = ethtool_op_get_link,
 396         .get_strings       = mlx5e_rep_get_strings,
 397         .get_sset_count    = mlx5e_rep_get_sset_count,
 398         .get_ethtool_stats = mlx5e_rep_get_ethtool_stats,
 399         .get_ringparam     = mlx5e_rep_get_ringparam,
 400         .set_ringparam     = mlx5e_rep_set_ringparam,
 401         .get_channels      = mlx5e_rep_get_channels,
 402         .set_channels      = mlx5e_rep_set_channels,
 403         .get_coalesce      = mlx5e_rep_get_coalesce,
 404         .set_coalesce      = mlx5e_rep_set_coalesce,
 405         .get_link_ksettings = mlx5e_uplink_rep_get_link_ksettings,
 406         .set_link_ksettings = mlx5e_uplink_rep_set_link_ksettings,
 407         .get_rxfh_key_size   = mlx5e_rep_get_rxfh_key_size,
 408         .get_rxfh_indir_size = mlx5e_rep_get_rxfh_indir_size,
 409         .get_pauseparam    = mlx5e_uplink_rep_get_pauseparam,
 410         .set_pauseparam    = mlx5e_uplink_rep_set_pauseparam,
 411 };
 412
 413 static void mlx5e_rep_get_port_parent_id(struct net_device *dev,
 414                                          struct netdev_phys_item_id *ppid)
 415 {
 416         struct mlx5e_priv *priv;
 417         u64 parent_id;
 418
 419         priv = netdev_priv(dev);
 420
 421         parent_id = mlx5_query_nic_system_image_guid(priv->mdev);
 422         ppid->id_len = sizeof(parent_id);
 423         memcpy(ppid->id, &parent_id, sizeof(parent_id));
 424 }
 425
 426 static void mlx5e_sqs2vport_stop(struct mlx5_eswitch *esw,
 427                                  struct mlx5_eswitch_rep *rep)
 428 {
 429         struct mlx5e_rep_sq *rep_sq, *tmp;
 430         struct mlx5e_rep_priv *rpriv;
 431
 432         if (esw->mode != MLX5_ESWITCH_OFFLOADS)
 433                 return;
 434
 435         rpriv = mlx5e_rep_to_rep_priv(rep);
 436         list_for_each_entry_safe(rep_sq, tmp, &rpriv->vport_sqs_list, list) {
 437                 mlx5_eswitch_del_send_to_vport_rule(rep_sq->send_to_vport_rule);
 438                 list_del(&rep_sq->list);
 439                 kfree(rep_sq);
 440         }
 441 }
 442
 443 static int mlx5e_sqs2vport_start(struct mlx5_eswitch *esw,
 444                                  struct mlx5_eswitch_rep *rep,
 445                                  u32 *sqns_array, int sqns_num)
 446 {
 447         struct mlx5_flow_handle *flow_rule;
 448         struct mlx5e_rep_priv *rpriv;
 449         struct mlx5e_rep_sq *rep_sq;
 450         int err;
 451         int i;
 452
 453         if (esw->mode != MLX5_ESWITCH_OFFLOADS)
 454                 return 0;
 455
 456         rpriv = mlx5e_rep_to_rep_priv(rep);
 457         for (i = 0; i < sqns_num; i++) {
 458                 rep_sq = kzalloc(sizeof(*rep_sq), GFP_KERNEL);
 459                 if (!rep_sq) {
 460                         err = -ENOMEM;
 461                         goto out_err;
 462                 }
 463
 464                 /* Add re-inject rule to the PF/representor sqs */
 465                 flow_rule = mlx5_eswitch_add_send_to_vport_rule(esw,
 466                                                                 rep->vport,
 467                                                                 sqns_array[i]);
 468                 if (IS_ERR(flow_rule)) {
 469                         err = PTR_ERR(flow_rule);
 470                         kfree(rep_sq);
 471                         goto out_err;
 472                 }
 473                 rep_sq->send_to_vport_rule = flow_rule;
 474                 list_add(&rep_sq->list, &rpriv->vport_sqs_list);
 475         }
 476         return 0;
 477
 478 out_err:
 479         mlx5e_sqs2vport_stop(esw, rep);
 480         return err;
 481 }
 482
 483 int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv)
 484 {
 485         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 486         struct mlx5e_rep_priv *rpriv = priv->ppriv;
 487         struct mlx5_eswitch_rep *rep = rpriv->rep;
 488         struct mlx5e_channel *c;
 489         int n, tc, num_sqs = 0;
 490         int err = -ENOMEM;
 491         u32 *sqs;
 492
 493         sqs = kcalloc(priv->channels.num * priv->channels.params.num_tc, sizeof(*sqs), GFP_KERNEL);
 494         if (!sqs)
 495                 goto out;
 496
 497         for (n = 0; n < priv->channels.num; n++) {
 498                 c = priv->channels.c[n];
 499                 for (tc = 0; tc < c->num_tc; tc++)
 500                         sqs[num_sqs++] = c->sq[tc].sqn;
 501         }
 502
 503         err = mlx5e_sqs2vport_start(esw, rep, sqs, num_sqs);
 504         kfree(sqs);
 505
 506 out:
 507         if (err)
 508                 netdev_warn(priv->netdev, "Failed to add SQs FWD rules %d\n", err);
 509         return err;
 510 }
 511
 512 void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv)
 513 {
 514         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 515         struct mlx5e_rep_priv *rpriv = priv->ppriv;
 516         struct mlx5_eswitch_rep *rep = rpriv->rep;
 517
 518         mlx5e_sqs2vport_stop(esw, rep);
 519 }
 520
 521 static unsigned long mlx5e_rep_ipv6_interval(void)
 522 {
 523         if (IS_ENABLED(CONFIG_IPV6) && ipv6_stub->nd_tbl)
 524                 return NEIGH_VAR(&ipv6_stub->nd_tbl->parms, DELAY_PROBE_TIME);
 525
 526         return ~0UL;
 527 }
 528
 529 static void mlx5e_rep_neigh_update_init_interval(struct mlx5e_rep_priv *rpriv)
 530 {
 531         unsigned long ipv4_interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
 532         unsigned long ipv6_interval = mlx5e_rep_ipv6_interval();
 533         struct net_device *netdev = rpriv->netdev;
 534         struct mlx5e_priv *priv = netdev_priv(netdev);
 535
 536         rpriv->neigh_update.min_interval = min_t(unsigned long, ipv6_interval, ipv4_interval);
 537         mlx5_fc_update_sampling_interval(priv->mdev, rpriv->neigh_update.min_interval);
 538 }
 539
 540 void mlx5e_rep_queue_neigh_stats_work(struct mlx5e_priv *priv)
 541 {
 542         struct mlx5e_rep_priv *rpriv = priv->ppriv;
 543         struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update;
 544
 545         mlx5_fc_queue_stats_work(priv->mdev,
 546                                  &neigh_update->neigh_stats_work,
 547                                  neigh_update->min_interval);
 548 }
 549
 550 static bool mlx5e_rep_neigh_entry_hold(struct mlx5e_neigh_hash_entry *nhe)
 551 {
 552         return refcount_inc_not_zero(&nhe->refcnt);
 553 }
 554
 555 static void mlx5e_rep_neigh_entry_remove(struct mlx5e_neigh_hash_entry *nhe);
 556
 557 static void mlx5e_rep_neigh_entry_release(struct mlx5e_neigh_hash_entry *nhe)
 558 {
 559         if (refcount_dec_and_test(&nhe->refcnt)) {
 560                 mlx5e_rep_neigh_entry_remove(nhe);
 561                 kfree_rcu(nhe, rcu);
 562         }
 563 }
 564
 565 static struct mlx5e_neigh_hash_entry *
 566 mlx5e_get_next_nhe(struct mlx5e_rep_priv *rpriv,
 567                    struct mlx5e_neigh_hash_entry *nhe)
 568 {
 569         struct mlx5e_neigh_hash_entry *next = NULL;
 570
 571         rcu_read_lock();
 572
 573         for (next = nhe ?
 574                      list_next_or_null_rcu(&rpriv->neigh_update.neigh_list,
 575                                            &nhe->neigh_list,
 576                                            struct mlx5e_neigh_hash_entry,
 577                                            neigh_list) :
 578                      list_first_or_null_rcu(&rpriv->neigh_update.neigh_list,
 579                                             struct mlx5e_neigh_hash_entry,
 580                                             neigh_list);
 581              next;
 582              next = list_next_or_null_rcu(&rpriv->neigh_update.neigh_list,
 583                                           &next->neigh_list,
 584                                           struct mlx5e_neigh_hash_entry,
 585                                           neigh_list))
 586                 if (mlx5e_rep_neigh_entry_hold(next))
 587                         break;
 588
 589         rcu_read_unlock();
 590
 591         if (nhe)
 592                 mlx5e_rep_neigh_entry_release(nhe);
 593
 594         return next;
 595 }
 596
 597 static void mlx5e_rep_neigh_stats_work(struct work_struct *work)
 598 {
 599         struct mlx5e_rep_priv *rpriv = container_of(work, struct mlx5e_rep_priv,
 600                                                     neigh_update.neigh_stats_work.work);
 601         struct net_device *netdev = rpriv->netdev;
 602         struct mlx5e_priv *priv = netdev_priv(netdev);
 603         struct mlx5e_neigh_hash_entry *nhe = NULL;
 604
 605         rtnl_lock();
 606         if (!list_empty(&rpriv->neigh_update.neigh_list))
 607                 mlx5e_rep_queue_neigh_stats_work(priv);
 608
 609         while ((nhe = mlx5e_get_next_nhe(rpriv, nhe)) != NULL)
 610                 mlx5e_tc_update_neigh_used_value(nhe);
 611
 612         rtnl_unlock();
 613 }
 614
 615 static void mlx5e_rep_update_flows(struct mlx5e_priv *priv,
 616                                    struct mlx5e_encap_entry *e,
 617                                    bool neigh_connected,
 618                                    unsigned char ha[ETH_ALEN])
 619 {
 620         struct ethhdr *eth = (struct ethhdr *)e->encap_header;
 621         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 622         bool encap_connected;
 623         LIST_HEAD(flow_list);
 624
 625         ASSERT_RTNL();
 626
 627         /* wait for encap to be fully initialized */
 628         wait_for_completion(&e->res_ready);
 629
 630         mutex_lock(&esw->offloads.encap_tbl_lock);
 631         encap_connected = !!(e->flags & MLX5_ENCAP_ENTRY_VALID);
 632         if (e->compl_result < 0 || (encap_connected == neigh_connected &&
 633                                     ether_addr_equal(e->h_dest, ha)))
 634                 goto unlock;
 635
 636         mlx5e_take_all_encap_flows(e, &flow_list);
 637
 638         if ((e->flags & MLX5_ENCAP_ENTRY_VALID) &&
 639             (!neigh_connected || !ether_addr_equal(e->h_dest, ha)))
 640                 mlx5e_tc_encap_flows_del(priv, e, &flow_list);
 641
 642         if (neigh_connected && !(e->flags & MLX5_ENCAP_ENTRY_VALID)) {
 643                 ether_addr_copy(e->h_dest, ha);
 644                 ether_addr_copy(eth->h_dest, ha);
 645                 /* Update the encap source mac, in case that we delete
 646                  * the flows when encap source mac changed.
 647                  */
 648                 ether_addr_copy(eth->h_source, e->route_dev->dev_addr);
 649
 650                 mlx5e_tc_encap_flows_add(priv, e, &flow_list);
 651         }
 652 unlock:
 653         mutex_unlock(&esw->offloads.encap_tbl_lock);
 654         mlx5e_put_encap_flow_list(priv, &flow_list);
 655 }
 656
 657 static void mlx5e_rep_neigh_update(struct work_struct *work)
 658 {
 659         struct mlx5e_neigh_hash_entry *nhe =
 660                 container_of(work, struct mlx5e_neigh_hash_entry, neigh_update_work);
 661         struct neighbour *n = nhe->n;
 662         struct mlx5e_encap_entry *e;
 663         unsigned char ha[ETH_ALEN];
 664         struct mlx5e_priv *priv;
 665         bool neigh_connected;
 666         u8 nud_state, dead;
 667
 668         rtnl_lock();
 669
 670         /* If these parameters are changed after we release the lock,
 671          * we'll receive another event letting us know about it.
 672          * We use this lock to avoid inconsistency between the neigh validity
 673          * and it's hw address.
 674          */
 675         read_lock_bh(&n->lock);
 676         memcpy(ha, n->ha, ETH_ALEN);
 677         nud_state = n->nud_state;
 678         dead = n->dead;
 679         read_unlock_bh(&n->lock);
 680
 681         neigh_connected = (nud_state & NUD_VALID) && !dead;
 682
 683         trace_mlx5e_rep_neigh_update(nhe, ha, neigh_connected);
 684
 685         list_for_each_entry(e, &nhe->encap_list, encap_list) {
 686                 if (!mlx5e_encap_take(e))
 687                         continue;
 688
 689                 priv = netdev_priv(e->out_dev);
 690                 mlx5e_rep_update_flows(priv, e, neigh_connected, ha);
 691                 mlx5e_encap_put(priv, e);
 692         }
 693         mlx5e_rep_neigh_entry_release(nhe);
 694         rtnl_unlock();
 695         neigh_release(n);
 696 }
 697
 698 static struct mlx5e_rep_indr_block_priv *
 699 mlx5e_rep_indr_block_priv_lookup(struct mlx5e_rep_priv *rpriv,
 700                                  struct net_device *netdev)
 701 {
 702         struct mlx5e_rep_indr_block_priv *cb_priv;
 703
 704         /* All callback list access should be protected by RTNL. */
 705         ASSERT_RTNL();
 706
 707         list_for_each_entry(cb_priv,
 708                             &rpriv->uplink_priv.tc_indr_block_priv_list,
 709                             list)
 710                 if (cb_priv->netdev == netdev)
 711                         return cb_priv;
 712
 713         return NULL;
 714 }
 715
 716 static void mlx5e_rep_indr_clean_block_privs(struct mlx5e_rep_priv *rpriv)
 717 {
 718         struct mlx5e_rep_indr_block_priv *cb_priv, *temp;
 719         struct list_head *head = &rpriv->uplink_priv.tc_indr_block_priv_list;
 720
 721         list_for_each_entry_safe(cb_priv, temp, head, list) {
 722                 mlx5e_rep_indr_unregister_block(rpriv, cb_priv->netdev);
 723                 kfree(cb_priv);
 724         }
 725 }
 726
 727 static int
 728 mlx5e_rep_indr_offload(struct net_device *netdev,
 729                        struct flow_cls_offload *flower,
 730                        struct mlx5e_rep_indr_block_priv *indr_priv)
 731 {
 732         unsigned long flags = MLX5_TC_FLAG(EGRESS) | MLX5_TC_FLAG(ESW_OFFLOAD);
 733         struct mlx5e_priv *priv = netdev_priv(indr_priv->rpriv->netdev);
 734         int err = 0;
 735
 736         switch (flower->command) {
 737         case FLOW_CLS_REPLACE:
 738                 err = mlx5e_configure_flower(netdev, priv, flower, flags);
 739                 break;
 740         case FLOW_CLS_DESTROY:
 741                 err = mlx5e_delete_flower(netdev, priv, flower, flags);
 742                 break;
 743         case FLOW_CLS_STATS:
 744                 err = mlx5e_stats_flower(netdev, priv, flower, flags);
 745                 break;
 746         default:
 747                 err = -EOPNOTSUPP;
 748         }
 749
 750         return err;
 751 }
 752
 753 static int mlx5e_rep_indr_setup_block_cb(enum tc_setup_type type,
 754                                          void *type_data, void *indr_priv)
 755 {
 756         struct mlx5e_rep_indr_block_priv *priv = indr_priv;
 757
 758         switch (type) {
 759         case TC_SETUP_CLSFLOWER:
 760                 return mlx5e_rep_indr_offload(priv->netdev, type_data, priv);
 761         default:
 762                 return -EOPNOTSUPP;
 763         }
 764 }
 765
 766 static void mlx5e_rep_indr_tc_block_unbind(void *cb_priv)
 767 {
 768         struct mlx5e_rep_indr_block_priv *indr_priv = cb_priv;
 769
 770         list_del(&indr_priv->list);
 771         kfree(indr_priv);
 772 }
 773
 774 static LIST_HEAD(mlx5e_block_cb_list);
 775
 776 static int
 777 mlx5e_rep_indr_setup_tc_block(struct net_device *netdev,
 778                               struct mlx5e_rep_priv *rpriv,
 779                               struct flow_block_offload *f)
 780 {
 781         struct mlx5e_rep_indr_block_priv *indr_priv;
 782         struct flow_block_cb *block_cb;
 783
 784         if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
 785                 return -EOPNOTSUPP;
 786
 787         f->unlocked_driver_cb = true;
 788         f->driver_block_list = &mlx5e_block_cb_list;
 789
 790         switch (f->command) {
 791         case FLOW_BLOCK_BIND:
 792                 indr_priv = mlx5e_rep_indr_block_priv_lookup(rpriv, netdev);
 793                 if (indr_priv)
 794                         return -EEXIST;
 795
 796                 indr_priv = kmalloc(sizeof(*indr_priv), GFP_KERNEL);
 797                 if (!indr_priv)
 798                         return -ENOMEM;
 799
 800                 indr_priv->netdev = netdev;
 801                 indr_priv->rpriv = rpriv;
 802                 list_add(&indr_priv->list,
 803                          &rpriv->uplink_priv.tc_indr_block_priv_list);
 804
 805                 block_cb = flow_block_cb_alloc(mlx5e_rep_indr_setup_block_cb,
 806                                                indr_priv, indr_priv,
 807                                                mlx5e_rep_indr_tc_block_unbind);
 808                 if (IS_ERR(block_cb)) {
 809                         list_del(&indr_priv->list);
 810                         kfree(indr_priv);
 811                         return PTR_ERR(block_cb);
 812                 }
 813                 flow_block_cb_add(block_cb, f);
 814                 list_add_tail(&block_cb->driver_list, &mlx5e_block_cb_list);
 815
 816                 return 0;
 817         case FLOW_BLOCK_UNBIND:
 818                 indr_priv = mlx5e_rep_indr_block_priv_lookup(rpriv, netdev);
 819                 if (!indr_priv)
 820                         return -ENOENT;
 821
 822                 block_cb = flow_block_cb_lookup(f->block,
 823                                                 mlx5e_rep_indr_setup_block_cb,
 824                                                 indr_priv);
 825                 if (!block_cb)
 826                         return -ENOENT;
 827
 828                 flow_block_cb_remove(block_cb, f);
 829                 list_del(&block_cb->driver_list);
 830                 return 0;
 831         default:
 832                 return -EOPNOTSUPP;
 833         }
 834         return 0;
 835 }
 836
 837 static
 838 int mlx5e_rep_indr_setup_tc_cb(struct net_device *netdev, void *cb_priv,
 839                                enum tc_setup_type type, void *type_data)
 840 {
 841         switch (type) {
 842         case TC_SETUP_BLOCK:
 843                 return mlx5e_rep_indr_setup_tc_block(netdev, cb_priv,
 844                                                       type_data);
 845         default:
 846                 return -EOPNOTSUPP;
 847         }
 848 }
 849
 850 static int mlx5e_rep_indr_register_block(struct mlx5e_rep_priv *rpriv,
 851                                          struct net_device *netdev)
 852 {
 853         int err;
 854
 855         err = __flow_indr_block_cb_register(netdev, rpriv,
 856                                             mlx5e_rep_indr_setup_tc_cb,
 857                                             rpriv);
 858         if (err) {
 859                 struct mlx5e_priv *priv = netdev_priv(rpriv->netdev);
 860
 861                 mlx5_core_err(priv->mdev, "Failed to register remote block notifier for %s err=%d\n",
 862                               netdev_name(netdev), err);
 863         }
 864         return err;
 865 }
 866
 867 static void mlx5e_rep_indr_unregister_block(struct mlx5e_rep_priv *rpriv,
 868                                             struct net_device *netdev)
 869 {
 870         __flow_indr_block_cb_unregister(netdev, mlx5e_rep_indr_setup_tc_cb,
 871                                         rpriv);
 872 }
 873
 874 static int mlx5e_nic_rep_netdevice_event(struct notifier_block *nb,
 875                                          unsigned long event, void *ptr)
 876 {
 877         struct mlx5e_rep_priv *rpriv = container_of(nb, struct mlx5e_rep_priv,
 878                                                      uplink_priv.netdevice_nb);
 879         struct mlx5e_priv *priv = netdev_priv(rpriv->netdev);
 880         struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
 881
 882         if (!mlx5e_tc_tun_device_to_offload(priv, netdev) &&
 883             !(is_vlan_dev(netdev) && vlan_dev_real_dev(netdev) == rpriv->netdev))
 884                 return NOTIFY_OK;
 885
 886         switch (event) {
 887         case NETDEV_REGISTER:
 888                 mlx5e_rep_indr_register_block(rpriv, netdev);
 889                 break;
 890         case NETDEV_UNREGISTER:
 891                 mlx5e_rep_indr_unregister_block(rpriv, netdev);
 892                 break;
 893         }
 894         return NOTIFY_OK;
 895 }
 896
 897 static void
 898 mlx5e_rep_queue_neigh_update_work(struct mlx5e_priv *priv,
 899                                   struct mlx5e_neigh_hash_entry *nhe,
 900                                   struct neighbour *n)
 901 {
 902         /* Take a reference to ensure the neighbour and mlx5 encap
 903          * entry won't be destructed until we drop the reference in
 904          * delayed work.
 905          */
 906         neigh_hold(n);
 907
 908         /* This assignment is valid as long as the the neigh reference
 909          * is taken
 910          */
 911         nhe->n = n;
 912
 913         if (!queue_work(priv->wq, &nhe->neigh_update_work)) {
 914                 mlx5e_rep_neigh_entry_release(nhe);
 915                 neigh_release(n);
 916         }
 917 }
 918
 919 static struct mlx5e_neigh_hash_entry *
 920 mlx5e_rep_neigh_entry_lookup(struct mlx5e_priv *priv,
 921                              struct mlx5e_neigh *m_neigh);
 922
 923 static int mlx5e_rep_netevent_event(struct notifier_block *nb,
 924                                     unsigned long event, void *ptr)
 925 {
 926         struct mlx5e_rep_priv *rpriv = container_of(nb, struct mlx5e_rep_priv,
 927                                                     neigh_update.netevent_nb);
 928         struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update;
 929         struct net_device *netdev = rpriv->netdev;
 930         struct mlx5e_priv *priv = netdev_priv(netdev);
 931         struct mlx5e_neigh_hash_entry *nhe = NULL;
 932         struct mlx5e_neigh m_neigh = {};
 933         struct neigh_parms *p;
 934         struct neighbour *n;
 935         bool found = false;
 936
 937         switch (event) {
 938         case NETEVENT_NEIGH_UPDATE:
 939                 n = ptr;
 940 #if IS_ENABLED(CONFIG_IPV6)
 941                 if (n->tbl != ipv6_stub->nd_tbl && n->tbl != &arp_tbl)
 942 #else
 943                 if (n->tbl != &arp_tbl)
 944 #endif
 945                         return NOTIFY_DONE;
 946
 947                 m_neigh.dev = n->dev;
 948                 m_neigh.family = n->ops->family;
 949                 memcpy(&m_neigh.dst_ip, n->primary_key, n->tbl->key_len);
 950
 951                 rcu_read_lock();
 952                 nhe = mlx5e_rep_neigh_entry_lookup(priv, &m_neigh);
 953                 rcu_read_unlock();
 954                 if (!nhe)
 955                         return NOTIFY_DONE;
 956
 957                 mlx5e_rep_queue_neigh_update_work(priv, nhe, n);
 958                 break;
 959
 960         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
 961                 p = ptr;
 962
 963                 /* We check the device is present since we don't care about
 964                  * changes in the default table, we only care about changes
 965                  * done per device delay prob time parameter.
 966                  */
 967 #if IS_ENABLED(CONFIG_IPV6)
 968                 if (!p->dev || (p->tbl != ipv6_stub->nd_tbl && p->tbl != &arp_tbl))
 969 #else
 970                 if (!p->dev || p->tbl != &arp_tbl)
 971 #endif
 972                         return NOTIFY_DONE;
 973
 974                 rcu_read_lock();
 975                 list_for_each_entry_rcu(nhe, &neigh_update->neigh_list,
 976                                         neigh_list) {
 977                         if (p->dev == nhe->m_neigh.dev) {
 978                                 found = true;
 979                                 break;
 980                         }
 981                 }
 982                 rcu_read_unlock();
 983                 if (!found)
 984                         return NOTIFY_DONE;
 985
 986                 neigh_update->min_interval = min_t(unsigned long,
 987                                                    NEIGH_VAR(p, DELAY_PROBE_TIME),
 988                                                    neigh_update->min_interval);
 989                 mlx5_fc_update_sampling_interval(priv->mdev,
 990                                                  neigh_update->min_interval);
 991                 break;
 992         }
 993         return NOTIFY_DONE;
 994 }
 995
 996 static const struct rhashtable_params mlx5e_neigh_ht_params = {
 997         .head_offset = offsetof(struct mlx5e_neigh_hash_entry, rhash_node),
 998         .key_offset = offsetof(struct mlx5e_neigh_hash_entry, m_neigh),
 999         .key_len = sizeof(struct mlx5e_neigh),
1000         .automatic_shrinking = true,
1001 };
1002
1003 static int mlx5e_rep_neigh_init(struct mlx5e_rep_priv *rpriv)
1004 {
1005         struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update;
1006         int err;
1007
1008         err = rhashtable_init(&neigh_update->neigh_ht, &mlx5e_neigh_ht_params);
1009         if (err)
1010                 return err;
1011
1012         INIT_LIST_HEAD(&neigh_update->neigh_list);
1013         mutex_init(&neigh_update->encap_lock);
1014         INIT_DELAYED_WORK(&neigh_update->neigh_stats_work,
1015                           mlx5e_rep_neigh_stats_work);
1016         mlx5e_rep_neigh_update_init_interval(rpriv);
1017
1018         rpriv->neigh_update.netevent_nb.notifier_call = mlx5e_rep_netevent_event;
1019         err = register_netevent_notifier(&rpriv->neigh_update.netevent_nb);
1020         if (err)
1021                 goto out_err;
1022         return 0;
1023
1024 out_err:
1025         rhashtable_destroy(&neigh_update->neigh_ht);
1026         return err;
1027 }
1028
1029 static void mlx5e_rep_neigh_cleanup(struct mlx5e_rep_priv *rpriv)
1030 {
1031         struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update;
1032         struct mlx5e_priv *priv = netdev_priv(rpriv->netdev);
1033
1034         unregister_netevent_notifier(&neigh_update->netevent_nb);
1035
1036         flush_workqueue(priv->wq); /* flush neigh update works */
1037
1038         cancel_delayed_work_sync(&rpriv->neigh_update.neigh_stats_work);
1039
1040         mutex_destroy(&neigh_update->encap_lock);
1041         rhashtable_destroy(&neigh_update->neigh_ht);
1042 }
1043
1044 static int mlx5e_rep_neigh_entry_insert(struct mlx5e_priv *priv,
1045                                         struct mlx5e_neigh_hash_entry *nhe)
1046 {
1047         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1048         int err;
1049
1050         err = rhashtable_insert_fast(&rpriv->neigh_update.neigh_ht,
1051                                      &nhe->rhash_node,
1052                                      mlx5e_neigh_ht_params);
1053         if (err)
1054                 return err;
1055
1056         list_add_rcu(&nhe->neigh_list, &rpriv->neigh_update.neigh_list);
1057
1058         return err;
1059 }
1060
1061 static void mlx5e_rep_neigh_entry_remove(struct mlx5e_neigh_hash_entry *nhe)
1062 {
1063         struct mlx5e_rep_priv *rpriv = nhe->priv->ppriv;
1064
1065         mutex_lock(&rpriv->neigh_update.encap_lock);
1066
1067         list_del_rcu(&nhe->neigh_list);
1068
1069         rhashtable_remove_fast(&rpriv->neigh_update.neigh_ht,
1070                                &nhe->rhash_node,
1071                                mlx5e_neigh_ht_params);
1072         mutex_unlock(&rpriv->neigh_update.encap_lock);
1073 }
1074
1075 /* This function must only be called under the representor's encap_lock or
1076  * inside rcu read lock section.
1077  */
1078 static struct mlx5e_neigh_hash_entry *
1079 mlx5e_rep_neigh_entry_lookup(struct mlx5e_priv *priv,
1080                              struct mlx5e_neigh *m_neigh)
1081 {
1082         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1083         struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update;
1084         struct mlx5e_neigh_hash_entry *nhe;
1085
1086         nhe = rhashtable_lookup_fast(&neigh_update->neigh_ht, m_neigh,
1087                                      mlx5e_neigh_ht_params);
1088         return nhe && mlx5e_rep_neigh_entry_hold(nhe) ? nhe : NULL;
1089 }
1090
1091 static int mlx5e_rep_neigh_entry_create(struct mlx5e_priv *priv,
1092                                         struct mlx5e_encap_entry *e,
1093                                         struct mlx5e_neigh_hash_entry **nhe)
1094 {
1095         int err;
1096
1097         *nhe = kzalloc(sizeof(**nhe), GFP_KERNEL);
1098         if (!*nhe)
1099                 return -ENOMEM;
1100
1101         (*nhe)->priv = priv;
1102         memcpy(&(*nhe)->m_neigh, &e->m_neigh, sizeof(e->m_neigh));
1103         INIT_WORK(&(*nhe)->neigh_update_work, mlx5e_rep_neigh_update);
1104         spin_lock_init(&(*nhe)->encap_list_lock);
1105         INIT_LIST_HEAD(&(*nhe)->encap_list);
1106         refcount_set(&(*nhe)->refcnt, 1);
1107
1108         err = mlx5e_rep_neigh_entry_insert(priv, *nhe);
1109         if (err)
1110                 goto out_free;
1111         return 0;
1112
1113 out_free:
1114         kfree(*nhe);
1115         return err;
1116 }
1117
1118 int mlx5e_rep_encap_entry_attach(struct mlx5e_priv *priv,
1119                                  struct mlx5e_encap_entry *e)
1120 {
1121         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1122         struct mlx5_rep_uplink_priv *uplink_priv = &rpriv->uplink_priv;
1123         struct mlx5_tun_entropy *tun_entropy = &uplink_priv->tun_entropy;
1124         struct mlx5e_neigh_hash_entry *nhe;
1125         int err;
1126
1127         err = mlx5_tun_entropy_refcount_inc(tun_entropy, e->reformat_type);
1128         if (err)
1129                 return err;
1130
1131         mutex_lock(&rpriv->neigh_update.encap_lock);
1132         nhe = mlx5e_rep_neigh_entry_lookup(priv, &e->m_neigh);
1133         if (!nhe) {
1134                 err = mlx5e_rep_neigh_entry_create(priv, e, &nhe);
1135                 if (err) {
1136                         mutex_unlock(&rpriv->neigh_update.encap_lock);
1137                         mlx5_tun_entropy_refcount_dec(tun_entropy,
1138                                                       e->reformat_type);
1139                         return err;
1140                 }
1141         }
1142
1143         e->nhe = nhe;
1144         spin_lock(&nhe->encap_list_lock);
1145         list_add_rcu(&e->encap_list, &nhe->encap_list);
1146         spin_unlock(&nhe->encap_list_lock);
1147
1148         mutex_unlock(&rpriv->neigh_update.encap_lock);
1149
1150         return 0;
1151 }
1152
1153 void mlx5e_rep_encap_entry_detach(struct mlx5e_priv *priv,
1154                                   struct mlx5e_encap_entry *e)
1155 {
1156         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1157         struct mlx5_rep_uplink_priv *uplink_priv = &rpriv->uplink_priv;
1158         struct mlx5_tun_entropy *tun_entropy = &uplink_priv->tun_entropy;
1159
1160         if (!e->nhe)
1161                 return;
1162
1163         spin_lock(&e->nhe->encap_list_lock);
1164         list_del_rcu(&e->encap_list);
1165         spin_unlock(&e->nhe->encap_list_lock);
1166
1167         mlx5e_rep_neigh_entry_release(e->nhe);
1168         e->nhe = NULL;
1169         mlx5_tun_entropy_refcount_dec(tun_entropy, e->reformat_type);
1170 }
1171
1172 static int mlx5e_rep_open(struct net_device *dev)
1173 {
1174         struct mlx5e_priv *priv = netdev_priv(dev);
1175         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1176         struct mlx5_eswitch_rep *rep = rpriv->rep;
1177         int err;
1178
1179         mutex_lock(&priv->state_lock);
1180         err = mlx5e_open_locked(dev);
1181         if (err)
1182                 goto unlock;
1183
1184         if (!mlx5_modify_vport_admin_state(priv->mdev,
1185                                            MLX5_VPORT_STATE_OP_MOD_ESW_VPORT,
1186                                            rep->vport, 1,
1187                                            MLX5_VPORT_ADMIN_STATE_UP))
1188                 netif_carrier_on(dev);
1189
1190 unlock:
1191         mutex_unlock(&priv->state_lock);
1192         return err;
1193 }
1194
1195 static int mlx5e_rep_close(struct net_device *dev)
1196 {
1197         struct mlx5e_priv *priv = netdev_priv(dev);
1198         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1199         struct mlx5_eswitch_rep *rep = rpriv->rep;
1200         int ret;
1201
1202         mutex_lock(&priv->state_lock);
1203         mlx5_modify_vport_admin_state(priv->mdev,
1204                                       MLX5_VPORT_STATE_OP_MOD_ESW_VPORT,
1205                                       rep->vport, 1,
1206                                       MLX5_VPORT_ADMIN_STATE_DOWN);
1207         ret = mlx5e_close_locked(dev);
1208         mutex_unlock(&priv->state_lock);
1209         return ret;
1210 }
1211
1212 static int
1213 mlx5e_rep_setup_tc_cls_flower(struct mlx5e_priv *priv,
1214                               struct flow_cls_offload *cls_flower, int flags)
1215 {
1216         switch (cls_flower->command) {
1217         case FLOW_CLS_REPLACE:
1218                 return mlx5e_configure_flower(priv->netdev, priv, cls_flower,
1219                                               flags);
1220         case FLOW_CLS_DESTROY:
1221                 return mlx5e_delete_flower(priv->netdev, priv, cls_flower,
1222                                            flags);
1223         case FLOW_CLS_STATS:
1224                 return mlx5e_stats_flower(priv->netdev, priv, cls_flower,
1225                                           flags);
1226         default:
1227                 return -EOPNOTSUPP;
1228         }
1229 }
1230
1231 static
1232 int mlx5e_rep_setup_tc_cls_matchall(struct mlx5e_priv *priv,
1233                                     struct tc_cls_matchall_offload *ma)
1234 {
1235         switch (ma->command) {
1236         case TC_CLSMATCHALL_REPLACE:
1237                 return mlx5e_tc_configure_matchall(priv, ma);
1238         case TC_CLSMATCHALL_DESTROY:
1239                 return mlx5e_tc_delete_matchall(priv, ma);
1240         case TC_CLSMATCHALL_STATS:
1241                 mlx5e_tc_stats_matchall(priv, ma);
1242                 return 0;
1243         default:
1244                 return -EOPNOTSUPP;
1245         }
1246 }
1247
1248 static int mlx5e_rep_setup_tc_cb(enum tc_setup_type type, void *type_data,
1249                                  void *cb_priv)
1250 {
1251         unsigned long flags = MLX5_TC_FLAG(INGRESS) | MLX5_TC_FLAG(ESW_OFFLOAD);
1252         struct mlx5e_priv *priv = cb_priv;
1253
1254         switch (type) {
1255         case TC_SETUP_CLSFLOWER:
1256                 return mlx5e_rep_setup_tc_cls_flower(priv, type_data, flags);
1257         case TC_SETUP_CLSMATCHALL:
1258                 return mlx5e_rep_setup_tc_cls_matchall(priv, type_data);
1259         default:
1260                 return -EOPNOTSUPP;
1261         }
1262 }
1263
1264 static int mlx5e_rep_setup_ft_cb(enum tc_setup_type type, void *type_data,
1265                                  void *cb_priv)
1266 {
1267         struct flow_cls_offload tmp, *f = type_data;
1268         struct mlx5e_priv *priv = cb_priv;
1269         struct mlx5_eswitch *esw;
1270         unsigned long flags;
1271         int err;
1272
1273         flags = MLX5_TC_FLAG(INGRESS) |
1274                 MLX5_TC_FLAG(ESW_OFFLOAD) |
1275                 MLX5_TC_FLAG(FT_OFFLOAD);
1276         esw = priv->mdev->priv.eswitch;
1277
1278         switch (type) {
1279         case TC_SETUP_CLSFLOWER:
1280                 memcpy(&tmp, f, sizeof(*f));
1281
1282                 if (!mlx5_esw_chains_prios_supported(esw) ||
1283                     tmp.common.chain_index)
1284                         return -EOPNOTSUPP;
1285
1286                 /* Re-use tc offload path by moving the ft flow to the
1287                  * reserved ft chain.
1288                  *
1289                  * FT offload can use prio range [0, INT_MAX], so we normalize
1290                  * it to range [1, mlx5_esw_chains_get_prio_range(esw)]
1291                  * as with tc, where prio 0 isn't supported.
1292                  *
1293                  * We only support chain 0 of FT offload.
1294                  */
1295                 if (tmp.common.prio >= mlx5_esw_chains_get_prio_range(esw))
1296                         return -EOPNOTSUPP;
1297                 if (tmp.common.chain_index != 0)
1298                         return -EOPNOTSUPP;
1299
1300                 tmp.common.chain_index = mlx5_esw_chains_get_ft_chain(esw);
1301                 tmp.common.prio++;
1302                 err = mlx5e_rep_setup_tc_cls_flower(priv, &tmp, flags);
1303                 memcpy(&f->stats, &tmp.stats, sizeof(f->stats));
1304                 return err;
1305         default:
1306                 return -EOPNOTSUPP;
1307         }
1308 }
1309
1310 static LIST_HEAD(mlx5e_rep_block_tc_cb_list);
1311 static LIST_HEAD(mlx5e_rep_block_ft_cb_list);
1312 static int mlx5e_rep_setup_tc(struct net_device *dev, enum tc_setup_type type,
1313                               void *type_data)
1314 {
1315         struct mlx5e_priv *priv = netdev_priv(dev);
1316         struct flow_block_offload *f = type_data;
1317
1318         f->unlocked_driver_cb = true;
1319
1320         switch (type) {
1321         case TC_SETUP_BLOCK:
1322                 return flow_block_cb_setup_simple(type_data,
1323                                                   &mlx5e_rep_block_tc_cb_list,
1324                                                   mlx5e_rep_setup_tc_cb,
1325                                                   priv, priv, true);
1326         case TC_SETUP_FT:
1327                 return flow_block_cb_setup_simple(type_data,
1328                                                   &mlx5e_rep_block_ft_cb_list,
1329                                                   mlx5e_rep_setup_ft_cb,
1330                                                   priv, priv, true);
1331         default:
1332                 return -EOPNOTSUPP;
1333         }
1334 }
1335
1336 bool mlx5e_is_uplink_rep(struct mlx5e_priv *priv)
1337 {
1338         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1339         struct mlx5_eswitch_rep *rep;
1340
1341         if (!MLX5_ESWITCH_MANAGER(priv->mdev))
1342                 return false;
1343
1344         if (!rpriv) /* non vport rep mlx5e instances don't use this field */
1345                 return false;
1346
1347         rep = rpriv->rep;
1348         return (rep->vport == MLX5_VPORT_UPLINK);
1349 }
1350
1351 static bool mlx5e_rep_has_offload_stats(const struct net_device *dev, int attr_id)
1352 {
1353         switch (attr_id) {
1354         case IFLA_OFFLOAD_XSTATS_CPU_HIT:
1355                         return true;
1356         }
1357
1358         return false;
1359 }
1360
1361 static int
1362 mlx5e_get_sw_stats64(const struct net_device *dev,
1363                      struct rtnl_link_stats64 *stats)
1364 {
1365         struct mlx5e_priv *priv = netdev_priv(dev);
1366
1367         mlx5e_fold_sw_stats64(priv, stats);
1368         return 0;
1369 }
1370
1371 static int mlx5e_rep_get_offload_stats(int attr_id, const struct net_device *dev,
1372                                        void *sp)
1373 {
1374         switch (attr_id) {
1375         case IFLA_OFFLOAD_XSTATS_CPU_HIT:
1376                 return mlx5e_get_sw_stats64(dev, sp);
1377         }
1378
1379         return -EINVAL;
1380 }
1381
1382 static void
1383 mlx5e_rep_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
1384 {
1385         struct mlx5e_priv *priv = netdev_priv(dev);
1386
1387         /* update HW stats in background for next time */
1388         mlx5e_queue_update_stats(priv);
1389         memcpy(stats, &priv->stats.vf_vport, sizeof(*stats));
1390 }
1391
1392 static int mlx5e_rep_change_mtu(struct net_device *netdev, int new_mtu)
1393 {
1394         return mlx5e_change_mtu(netdev, new_mtu, NULL);
1395 }
1396
1397 static int mlx5e_uplink_rep_change_mtu(struct net_device *netdev, int new_mtu)
1398 {
1399         return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu);
1400 }
1401
1402 static int mlx5e_uplink_rep_set_mac(struct net_device *netdev, void *addr)
1403 {
1404         struct sockaddr *saddr = addr;
1405
1406         if (!is_valid_ether_addr(saddr->sa_data))
1407                 return -EADDRNOTAVAIL;
1408
1409         ether_addr_copy(netdev->dev_addr, saddr->sa_data);
1410         return 0;
1411 }
1412
1413 static int mlx5e_uplink_rep_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
1414                                         __be16 vlan_proto)
1415 {
1416         netdev_warn_once(dev, "legacy vf vlan setting isn't supported in switchdev mode\n");
1417
1418         if (vlan != 0)
1419                 return -EOPNOTSUPP;
1420
1421         /* allow setting 0-vid for compatibility with libvirt */
1422         return 0;
1423 }
1424
1425 static struct devlink_port *mlx5e_get_devlink_port(struct net_device *dev)
1426 {
1427         struct mlx5e_priv *priv = netdev_priv(dev);
1428         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1429
1430         return &rpriv->dl_port;
1431 }
1432
1433 static const struct net_device_ops mlx5e_netdev_ops_rep = {
1434         .ndo_open                = mlx5e_rep_open,
1435         .ndo_stop                = mlx5e_rep_close,
1436         .ndo_start_xmit          = mlx5e_xmit,
1437         .ndo_setup_tc            = mlx5e_rep_setup_tc,
1438         .ndo_get_devlink_port = mlx5e_get_devlink_port,
1439         .ndo_get_stats64         = mlx5e_rep_get_stats,
1440         .ndo_has_offload_stats   = mlx5e_rep_has_offload_stats,
1441         .ndo_get_offload_stats   = mlx5e_rep_get_offload_stats,
1442         .ndo_change_mtu          = mlx5e_rep_change_mtu,
1443 };
1444
1445 static const struct net_device_ops mlx5e_netdev_ops_uplink_rep = {
1446         .ndo_open                = mlx5e_open,
1447         .ndo_stop                = mlx5e_close,
1448         .ndo_start_xmit          = mlx5e_xmit,
1449         .ndo_set_mac_address     = mlx5e_uplink_rep_set_mac,
1450         .ndo_setup_tc            = mlx5e_rep_setup_tc,
1451         .ndo_get_devlink_port = mlx5e_get_devlink_port,
1452         .ndo_get_stats64         = mlx5e_get_stats,
1453         .ndo_has_offload_stats   = mlx5e_rep_has_offload_stats,
1454         .ndo_get_offload_stats   = mlx5e_rep_get_offload_stats,
1455         .ndo_change_mtu          = mlx5e_uplink_rep_change_mtu,
1456         .ndo_udp_tunnel_add      = mlx5e_add_vxlan_port,
1457         .ndo_udp_tunnel_del      = mlx5e_del_vxlan_port,
1458         .ndo_features_check      = mlx5e_features_check,
1459         .ndo_set_vf_mac          = mlx5e_set_vf_mac,
1460         .ndo_set_vf_rate         = mlx5e_set_vf_rate,
1461         .ndo_get_vf_config       = mlx5e_get_vf_config,
1462         .ndo_get_vf_stats        = mlx5e_get_vf_stats,
1463         .ndo_set_vf_vlan         = mlx5e_uplink_rep_set_vf_vlan,
1464         .ndo_set_features        = mlx5e_set_features,
1465 };
1466
1467 bool mlx5e_eswitch_rep(struct net_device *netdev)
1468 {
1469         if (netdev->netdev_ops == &mlx5e_netdev_ops_rep ||
1470             netdev->netdev_ops == &mlx5e_netdev_ops_uplink_rep)
1471                 return true;
1472
1473         return false;
1474 }
1475
1476 static void mlx5e_build_rep_params(struct net_device *netdev)
1477 {
1478         struct mlx5e_priv *priv = netdev_priv(netdev);
1479         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1480         struct mlx5_eswitch_rep *rep = rpriv->rep;
1481         struct mlx5_core_dev *mdev = priv->mdev;
1482         struct mlx5e_params *params;
1483
1484         u8 cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
1485                                          MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
1486                                          MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
1487
1488         params = &priv->channels.params;
1489         params->hard_mtu    = MLX5E_ETH_HARD_MTU;
1490         params->sw_mtu      = netdev->mtu;
1491
1492         /* SQ */
1493         if (rep->vport == MLX5_VPORT_UPLINK)
1494                 params->log_sq_size = MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
1495         else
1496                 params->log_sq_size = MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE;
1497
1498         /* RQ */
1499         mlx5e_build_rq_params(mdev, params);
1500
1501         /* CQ moderation params */
1502         params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
1503         mlx5e_set_rx_cq_mode_params(params, cq_period_mode);
1504
1505         params->num_tc                = 1;
1506         params->tunneled_offload_en = false;
1507
1508         mlx5_query_min_inline(mdev, &params->tx_min_inline_mode);
1509
1510         /* RSS */
1511         mlx5e_build_rss_params(&priv->rss_params, params->num_channels);
1512 }
1513
1514 static void mlx5e_build_rep_netdev(struct net_device *netdev)
1515 {
1516         struct mlx5e_priv *priv = netdev_priv(netdev);
1517         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1518         struct mlx5_eswitch_rep *rep = rpriv->rep;
1519         struct mlx5_core_dev *mdev = priv->mdev;
1520
1521         if (rep->vport == MLX5_VPORT_UPLINK) {
1522                 SET_NETDEV_DEV(netdev, mdev->device);
1523                 netdev->netdev_ops = &mlx5e_netdev_ops_uplink_rep;
1524                 /* we want a persistent mac for the uplink rep */
1525                 mlx5_query_mac_address(mdev, netdev->dev_addr);
1526                 netdev->ethtool_ops = &mlx5e_uplink_rep_ethtool_ops;
1527 #ifdef CONFIG_MLX5_CORE_EN_DCB
1528                 if (MLX5_CAP_GEN(mdev, qos))
1529                         netdev->dcbnl_ops = &mlx5e_dcbnl_ops;
1530 #endif
1531         } else {
1532                 netdev->netdev_ops = &mlx5e_netdev_ops_rep;
1533                 eth_hw_addr_random(netdev);
1534                 netdev->ethtool_ops = &mlx5e_rep_ethtool_ops;
1535         }
1536
1537         netdev->watchdog_timeo    = 15 * HZ;
1538
1539         netdev->features       |= NETIF_F_NETNS_LOCAL;
1540
1541         netdev->hw_features    |= NETIF_F_HW_TC;
1542         netdev->hw_features    |= NETIF_F_SG;
1543         netdev->hw_features    |= NETIF_F_IP_CSUM;
1544         netdev->hw_features    |= NETIF_F_IPV6_CSUM;
1545         netdev->hw_features    |= NETIF_F_GRO;
1546         netdev->hw_features    |= NETIF_F_TSO;
1547         netdev->hw_features    |= NETIF_F_TSO6;
1548         netdev->hw_features    |= NETIF_F_RXCSUM;
1549
1550         if (rep->vport == MLX5_VPORT_UPLINK)
1551                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
1552         else
1553                 netdev->features |= NETIF_F_VLAN_CHALLENGED;
1554
1555         netdev->features |= netdev->hw_features;
1556 }
1557
1558 static int mlx5e_init_rep(struct mlx5_core_dev *mdev,
1559                           struct net_device *netdev,
1560                           const struct mlx5e_profile *profile,
1561                           void *ppriv)
1562 {
1563         struct mlx5e_priv *priv = netdev_priv(netdev);
1564         int err;
1565
1566         err = mlx5e_netdev_init(netdev, priv, mdev, profile, ppriv);
1567         if (err)
1568                 return err;
1569
1570         priv->channels.params.num_channels = MLX5E_REP_PARAMS_DEF_NUM_CHANNELS;
1571
1572         mlx5e_build_rep_params(netdev);
1573         mlx5e_build_rep_netdev(netdev);
1574
1575         mlx5e_timestamp_init(priv);
1576
1577         return 0;
1578 }
1579
1580 static void mlx5e_cleanup_rep(struct mlx5e_priv *priv)
1581 {
1582         mlx5e_netdev_cleanup(priv->netdev, priv);
1583 }
1584
1585 static int mlx5e_create_rep_ttc_table(struct mlx5e_priv *priv)
1586 {
1587         struct ttc_params ttc_params = {};
1588         int tt, err;
1589
1590         priv->fs.ns = mlx5_get_flow_namespace(priv->mdev,
1591                                               MLX5_FLOW_NAMESPACE_KERNEL);
1592
1593         /* The inner_ttc in the ttc params is intentionally not set */
1594         ttc_params.any_tt_tirn = priv->direct_tir[0].tirn;
1595         mlx5e_set_ttc_ft_params(&ttc_params);
1596         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
1597                 ttc_params.indir_tirn[tt] = priv->indir_tir[tt].tirn;
1598
1599         err = mlx5e_create_ttc_table(priv, &ttc_params, &priv->fs.ttc);
1600         if (err) {
1601                 netdev_err(priv->netdev, "Failed to create rep ttc table, err=%d\n", err);
1602                 return err;
1603         }
1604         return 0;
1605 }
1606
1607 static int mlx5e_create_rep_vport_rx_rule(struct mlx5e_priv *priv)
1608 {
1609         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1610         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1611         struct mlx5_eswitch_rep *rep = rpriv->rep;
1612         struct mlx5_flow_handle *flow_rule;
1613         struct mlx5_flow_destination dest;
1614
1615         dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
1616         dest.tir_num = priv->direct_tir[0].tirn;
1617         flow_rule = mlx5_eswitch_create_vport_rx_rule(esw,
1618                                                       rep->vport,
1619                                                       &dest);
1620         if (IS_ERR(flow_rule))
1621                 return PTR_ERR(flow_rule);
1622         rpriv->vport_rx_rule = flow_rule;
1623         return 0;
1624 }
1625
1626 static int mlx5e_init_rep_rx(struct mlx5e_priv *priv)
1627 {
1628         struct mlx5_core_dev *mdev = priv->mdev;
1629         int err;
1630
1631         mlx5e_init_l2_addr(priv);
1632
1633         err = mlx5e_open_drop_rq(priv, &priv->drop_rq);
1634         if (err) {
1635                 mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
1636                 return err;
1637         }
1638
1639         err = mlx5e_create_indirect_rqt(priv);
1640         if (err)
1641                 goto err_close_drop_rq;
1642
1643         err = mlx5e_create_direct_rqts(priv, priv->direct_tir);
1644         if (err)
1645                 goto err_destroy_indirect_rqts;
1646
1647         err = mlx5e_create_indirect_tirs(priv, false);
1648         if (err)
1649                 goto err_destroy_direct_rqts;
1650
1651         err = mlx5e_create_direct_tirs(priv, priv->direct_tir);
1652         if (err)
1653                 goto err_destroy_indirect_tirs;
1654
1655         err = mlx5e_create_rep_ttc_table(priv);
1656         if (err)
1657                 goto err_destroy_direct_tirs;
1658
1659         err = mlx5e_create_rep_vport_rx_rule(priv);
1660         if (err)
1661                 goto err_destroy_ttc_table;
1662
1663         return 0;
1664
1665 err_destroy_ttc_table:
1666         mlx5e_destroy_ttc_table(priv, &priv->fs.ttc);
1667 err_destroy_direct_tirs:
1668         mlx5e_destroy_direct_tirs(priv, priv->direct_tir);
1669 err_destroy_indirect_tirs:
1670         mlx5e_destroy_indirect_tirs(priv, false);
1671 err_destroy_direct_rqts:
1672         mlx5e_destroy_direct_rqts(priv, priv->direct_tir);
1673 err_destroy_indirect_rqts:
1674         mlx5e_destroy_rqt(priv, &priv->indir_rqt);
1675 err_close_drop_rq:
1676         mlx5e_close_drop_rq(&priv->drop_rq);
1677         return err;
1678 }
1679
1680 static void mlx5e_cleanup_rep_rx(struct mlx5e_priv *priv)
1681 {
1682         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1683
1684         mlx5_del_flow_rules(rpriv->vport_rx_rule);
1685         mlx5e_destroy_ttc_table(priv, &priv->fs.ttc);
1686         mlx5e_destroy_direct_tirs(priv, priv->direct_tir);
1687         mlx5e_destroy_indirect_tirs(priv, false);
1688         mlx5e_destroy_direct_rqts(priv, priv->direct_tir);
1689         mlx5e_destroy_rqt(priv, &priv->indir_rqt);
1690         mlx5e_close_drop_rq(&priv->drop_rq);
1691 }
1692
1693 static int mlx5e_init_ul_rep_rx(struct mlx5e_priv *priv)
1694 {
1695         int err = mlx5e_init_rep_rx(priv);
1696
1697         if (err)
1698                 return err;
1699
1700         mlx5e_create_q_counters(priv);
1701         return 0;
1702 }
1703
1704 static void mlx5e_cleanup_ul_rep_rx(struct mlx5e_priv *priv)
1705 {
1706         mlx5e_destroy_q_counters(priv);
1707         mlx5e_cleanup_rep_rx(priv);
1708 }
1709
1710 static int mlx5e_init_uplink_rep_tx(struct mlx5e_rep_priv *rpriv)
1711 {
1712         struct mlx5_rep_uplink_priv *uplink_priv;
1713         struct net_device *netdev;
1714         struct mlx5e_priv *priv;
1715         int err;
1716
1717         netdev = rpriv->netdev;
1718         priv = netdev_priv(netdev);
1719         uplink_priv = &rpriv->uplink_priv;
1720
1721         mutex_init(&uplink_priv->unready_flows_lock);
1722         INIT_LIST_HEAD(&uplink_priv->unready_flows);
1723
1724         /* init shared tc flow table */
1725         err = mlx5e_tc_esw_init(&uplink_priv->tc_ht);
1726         if (err)
1727                 return err;
1728
1729         mlx5_init_port_tun_entropy(&uplink_priv->tun_entropy, priv->mdev);
1730
1731         /* init indirect block notifications */
1732         INIT_LIST_HEAD(&uplink_priv->tc_indr_block_priv_list);
1733         uplink_priv->netdevice_nb.notifier_call = mlx5e_nic_rep_netdevice_event;
1734         err = register_netdevice_notifier_dev_net(rpriv->netdev,
1735                                                   &uplink_priv->netdevice_nb,
1736                                                   &uplink_priv->netdevice_nn);
1737         if (err) {
1738                 mlx5_core_err(priv->mdev, "Failed to register netdev notifier\n");
1739                 goto tc_esw_cleanup;
1740         }
1741
1742         return 0;
1743
1744 tc_esw_cleanup:
1745         mlx5e_tc_esw_cleanup(&uplink_priv->tc_ht);
1746         return err;
1747 }
1748
1749 static int mlx5e_init_rep_tx(struct mlx5e_priv *priv)
1750 {
1751         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1752         int err;
1753
1754         err = mlx5e_create_tises(priv);
1755         if (err) {
1756                 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err);
1757                 return err;
1758         }
1759
1760         if (rpriv->rep->vport == MLX5_VPORT_UPLINK) {
1761                 err = mlx5e_init_uplink_rep_tx(rpriv);
1762                 if (err)
1763                         goto destroy_tises;
1764         }
1765
1766         return 0;
1767
1768 destroy_tises:
1769         mlx5e_destroy_tises(priv);
1770         return err;
1771 }
1772
1773 static void mlx5e_cleanup_uplink_rep_tx(struct mlx5e_rep_priv *rpriv)
1774 {
1775         struct mlx5_rep_uplink_priv *uplink_priv = &rpriv->uplink_priv;
1776
1777         /* clean indirect TC block notifications */
1778         unregister_netdevice_notifier_dev_net(rpriv->netdev,
1779                                               &uplink_priv->netdevice_nb,
1780                                               &uplink_priv->netdevice_nn);
1781         mlx5e_rep_indr_clean_block_privs(rpriv);
1782
1783         /* delete shared tc flow table */
1784         mlx5e_tc_esw_cleanup(&rpriv->uplink_priv.tc_ht);
1785         mutex_destroy(&rpriv->uplink_priv.unready_flows_lock);
1786 }
1787
1788 static void mlx5e_cleanup_rep_tx(struct mlx5e_priv *priv)
1789 {
1790         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1791
1792         mlx5e_destroy_tises(priv);
1793
1794         if (rpriv->rep->vport == MLX5_VPORT_UPLINK)
1795                 mlx5e_cleanup_uplink_rep_tx(rpriv);
1796 }
1797
1798 static void mlx5e_rep_enable(struct mlx5e_priv *priv)
1799 {
1800         mlx5e_set_netdev_mtu_boundaries(priv);
1801 }
1802
1803 static int mlx5e_update_rep_rx(struct mlx5e_priv *priv)
1804 {
1805         return 0;
1806 }
1807
1808 static int uplink_rep_async_event(struct notifier_block *nb, unsigned long event, void *data)
1809 {
1810         struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb);
1811
1812         if (event == MLX5_EVENT_TYPE_PORT_CHANGE) {
1813                 struct mlx5_eqe *eqe = data;
1814
1815                 switch (eqe->sub_type) {
1816                 case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
1817                 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
1818                         queue_work(priv->wq, &priv->update_carrier_work);
1819                         break;
1820                 default:
1821                         return NOTIFY_DONE;
1822                 }
1823
1824                 return NOTIFY_OK;
1825         }
1826
1827         if (event == MLX5_DEV_EVENT_PORT_AFFINITY) {
1828                 struct mlx5e_rep_priv *rpriv = priv->ppriv;
1829
1830                 queue_work(priv->wq, &rpriv->uplink_priv.reoffload_flows_work);
1831
1832                 return NOTIFY_OK;
1833         }
1834
1835         return NOTIFY_DONE;
1836 }
1837
1838 static void mlx5e_uplink_rep_enable(struct mlx5e_priv *priv)
1839 {
1840         struct net_device *netdev = priv->netdev;
1841         struct mlx5_core_dev *mdev = priv->mdev;
1842         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1843         u16 max_mtu;
1844
1845         netdev->min_mtu = ETH_MIN_MTU;
1846         mlx5_query_port_max_mtu(priv->mdev, &max_mtu, 1);
1847         netdev->max_mtu = MLX5E_HW2SW_MTU(&priv->channels.params, max_mtu);
1848         mlx5e_set_dev_port_mtu(priv);
1849
1850         INIT_WORK(&rpriv->uplink_priv.reoffload_flows_work,
1851                   mlx5e_tc_reoffload_flows_work);
1852
1853         mlx5_lag_add(mdev, netdev);
1854         priv->events_nb.notifier_call = uplink_rep_async_event;
1855         mlx5_notifier_register(mdev, &priv->events_nb);
1856 #ifdef CONFIG_MLX5_CORE_EN_DCB
1857         mlx5e_dcbnl_initialize(priv);
1858         mlx5e_dcbnl_init_app(priv);
1859 #endif
1860 }
1861
1862 static void mlx5e_uplink_rep_disable(struct mlx5e_priv *priv)
1863 {
1864         struct net_device *netdev = priv->netdev;
1865         struct mlx5_core_dev *mdev = priv->mdev;
1866         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1867
1868 #ifdef CONFIG_MLX5_CORE_EN_DCB
1869         mlx5e_dcbnl_delete_app(priv);
1870 #endif
1871         mlx5_notifier_unregister(mdev, &priv->events_nb);
1872         cancel_work_sync(&rpriv->uplink_priv.reoffload_flows_work);
1873         mlx5_lag_remove(mdev, netdev);
1874 }
1875
1876 static MLX5E_DEFINE_STATS_GRP(sw_rep, 0);
1877 static MLX5E_DEFINE_STATS_GRP(vport_rep, MLX5E_NDO_UPDATE_STATS);
1878
1879 /* The stats groups order is opposite to the update_stats() order calls */
1880 static mlx5e_stats_grp_t mlx5e_rep_stats_grps[] = {
1881         &MLX5E_STATS_GRP(sw_rep),
1882         &MLX5E_STATS_GRP(vport_rep),
1883 };
1884
1885 static unsigned int mlx5e_rep_stats_grps_num(struct mlx5e_priv *priv)
1886 {
1887         return ARRAY_SIZE(mlx5e_rep_stats_grps);
1888 }
1889
1890 /* The stats groups order is opposite to the update_stats() order calls */
1891 static mlx5e_stats_grp_t mlx5e_ul_rep_stats_grps[] = {
1892         &MLX5E_STATS_GRP(sw),
1893         &MLX5E_STATS_GRP(qcnt),
1894         &MLX5E_STATS_GRP(vnic_env),
1895         &MLX5E_STATS_GRP(vport),
1896         &MLX5E_STATS_GRP(802_3),
1897         &MLX5E_STATS_GRP(2863),
1898         &MLX5E_STATS_GRP(2819),
1899         &MLX5E_STATS_GRP(phy),
1900         &MLX5E_STATS_GRP(eth_ext),
1901         &MLX5E_STATS_GRP(pcie),
1902         &MLX5E_STATS_GRP(per_prio),
1903         &MLX5E_STATS_GRP(pme),
1904         &MLX5E_STATS_GRP(channels),
1905         &MLX5E_STATS_GRP(per_port_buff_congest),
1906 };
1907
1908 static unsigned int mlx5e_ul_rep_stats_grps_num(struct mlx5e_priv *priv)
1909 {
1910         return ARRAY_SIZE(mlx5e_ul_rep_stats_grps);
1911 }
1912
1913 static const struct mlx5e_profile mlx5e_rep_profile = {
1914         .init                   = mlx5e_init_rep,
1915         .cleanup                = mlx5e_cleanup_rep,
1916         .init_rx                = mlx5e_init_rep_rx,
1917         .cleanup_rx             = mlx5e_cleanup_rep_rx,
1918         .init_tx                = mlx5e_init_rep_tx,
1919         .cleanup_tx             = mlx5e_cleanup_rep_tx,
1920         .enable                 = mlx5e_rep_enable,
1921         .update_rx              = mlx5e_update_rep_rx,
1922         .update_stats           = mlx5e_update_ndo_stats,
1923         .rx_handlers.handle_rx_cqe       = mlx5e_handle_rx_cqe_rep,
1924         .rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq,
1925         .max_tc                 = 1,
1926         .rq_groups              = MLX5E_NUM_RQ_GROUPS(REGULAR),
1927         .stats_grps             = mlx5e_rep_stats_grps,
1928         .stats_grps_num         = mlx5e_rep_stats_grps_num,
1929 };
1930
1931 static const struct mlx5e_profile mlx5e_uplink_rep_profile = {
1932         .init                   = mlx5e_init_rep,
1933         .cleanup                = mlx5e_cleanup_rep,
1934         .init_rx                = mlx5e_init_ul_rep_rx,
1935         .cleanup_rx             = mlx5e_cleanup_ul_rep_rx,
1936         .init_tx                = mlx5e_init_rep_tx,
1937         .cleanup_tx             = mlx5e_cleanup_rep_tx,
1938         .enable                 = mlx5e_uplink_rep_enable,
1939         .disable                = mlx5e_uplink_rep_disable,
1940         .update_rx              = mlx5e_update_rep_rx,
1941         .update_stats           = mlx5e_update_ndo_stats,
1942         .update_carrier         = mlx5e_update_carrier,
1943         .rx_handlers.handle_rx_cqe       = mlx5e_handle_rx_cqe_rep,
1944         .rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq,
1945         .max_tc                 = MLX5E_MAX_NUM_TC,
1946         .rq_groups              = MLX5E_NUM_RQ_GROUPS(REGULAR),
1947         .stats_grps             = mlx5e_ul_rep_stats_grps,
1948         .stats_grps_num         = mlx5e_ul_rep_stats_grps_num,
1949 };
1950
1951 static bool
1952 is_devlink_port_supported(const struct mlx5_core_dev *dev,
1953                           const struct mlx5e_rep_priv *rpriv)
1954 {
1955         return rpriv->rep->vport == MLX5_VPORT_UPLINK ||
1956                rpriv->rep->vport == MLX5_VPORT_PF ||
1957                mlx5_eswitch_is_vf_vport(dev->priv.eswitch, rpriv->rep->vport);
1958 }
1959
1960 static unsigned int
1961 vport_to_devlink_port_index(const struct mlx5_core_dev *dev, u16 vport_num)
1962 {
1963         return (MLX5_CAP_GEN(dev, vhca_id) << 16) | vport_num;
1964 }
1965
1966 static int register_devlink_port(struct mlx5_core_dev *dev,
1967                                  struct mlx5e_rep_priv *rpriv)
1968 {
1969         struct devlink *devlink = priv_to_devlink(dev);
1970         struct mlx5_eswitch_rep *rep = rpriv->rep;
1971         struct netdev_phys_item_id ppid = {};
1972         unsigned int dl_port_index = 0;
1973
1974         if (!is_devlink_port_supported(dev, rpriv))
1975                 return 0;
1976
1977         mlx5e_rep_get_port_parent_id(rpriv->netdev, &ppid);
1978
1979         if (rep->vport == MLX5_VPORT_UPLINK) {
1980                 devlink_port_attrs_set(&rpriv->dl_port,
1981                                        DEVLINK_PORT_FLAVOUR_PHYSICAL,
1982                                        PCI_FUNC(dev->pdev->devfn), false, 0,
1983                                        &ppid.id[0], ppid.id_len);
1984                 dl_port_index = vport_to_devlink_port_index(dev, rep->vport);
1985         } else if (rep->vport == MLX5_VPORT_PF) {
1986                 devlink_port_attrs_pci_pf_set(&rpriv->dl_port,
1987                                               &ppid.id[0], ppid.id_len,
1988                                               dev->pdev->devfn);
1989                 dl_port_index = rep->vport;
1990         } else if (mlx5_eswitch_is_vf_vport(dev->priv.eswitch,
1991                                             rpriv->rep->vport)) {
1992                 devlink_port_attrs_pci_vf_set(&rpriv->dl_port,
1993                                               &ppid.id[0], ppid.id_len,
1994                                               dev->pdev->devfn,
1995                                               rep->vport - 1);
1996                 dl_port_index = vport_to_devlink_port_index(dev, rep->vport);
1997         }
1998
1999         return devlink_port_register(devlink, &rpriv->dl_port, dl_port_index);
2000 }
2001
2002 static void unregister_devlink_port(struct mlx5_core_dev *dev,
2003                                     struct mlx5e_rep_priv *rpriv)
2004 {
2005         if (is_devlink_port_supported(dev, rpriv))
2006                 devlink_port_unregister(&rpriv->dl_port);
2007 }
2008
2009 /* e-Switch vport representors */
2010 static int
2011 mlx5e_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
2012 {
2013         const struct mlx5e_profile *profile;
2014         struct mlx5e_rep_priv *rpriv;
2015         struct net_device *netdev;
2016         int nch, err;
2017
2018         rpriv = kzalloc(sizeof(*rpriv), GFP_KERNEL);
2019         if (!rpriv)
2020                 return -ENOMEM;
2021
2022         /* rpriv->rep to be looked up when profile->init() is called */
2023         rpriv->rep = rep;
2024
2025         nch = mlx5e_get_max_num_channels(dev);
2026         profile = (rep->vport == MLX5_VPORT_UPLINK) ?
2027                   &mlx5e_uplink_rep_profile : &mlx5e_rep_profile;
2028         netdev = mlx5e_create_netdev(dev, profile, nch, rpriv);
2029         if (!netdev) {
2030                 pr_warn("Failed to create representor netdev for vport %d\n",
2031                         rep->vport);
2032                 kfree(rpriv);
2033                 return -EINVAL;
2034         }
2035
2036         dev_net_set(netdev, mlx5_core_net(dev));
2037         rpriv->netdev = netdev;
2038         rep->rep_data[REP_ETH].priv = rpriv;
2039         INIT_LIST_HEAD(&rpriv->vport_sqs_list);
2040
2041         if (rep->vport == MLX5_VPORT_UPLINK) {
2042                 err = mlx5e_create_mdev_resources(dev);
2043                 if (err)
2044                         goto err_destroy_netdev;
2045         }
2046
2047         err = mlx5e_attach_netdev(netdev_priv(netdev));
2048         if (err) {
2049                 pr_warn("Failed to attach representor netdev for vport %d\n",
2050                         rep->vport);
2051                 goto err_destroy_mdev_resources;
2052         }
2053
2054         err = mlx5e_rep_neigh_init(rpriv);
2055         if (err) {
2056                 pr_warn("Failed to initialized neighbours handling for vport %d\n",
2057                         rep->vport);
2058                 goto err_detach_netdev;
2059         }
2060
2061         err = register_devlink_port(dev, rpriv);
2062         if (err) {
2063                 esw_warn(dev, "Failed to register devlink port %d\n",
2064                          rep->vport);
2065                 goto err_neigh_cleanup;
2066         }
2067
2068         err = register_netdev(netdev);
2069         if (err) {
2070                 pr_warn("Failed to register representor netdev for vport %d\n",
2071                         rep->vport);
2072                 goto err_devlink_cleanup;
2073         }
2074
2075         if (is_devlink_port_supported(dev, rpriv))
2076                 devlink_port_type_eth_set(&rpriv->dl_port, netdev);
2077         return 0;
2078
2079 err_devlink_cleanup:
2080         unregister_devlink_port(dev, rpriv);
2081
2082 err_neigh_cleanup:
2083         mlx5e_rep_neigh_cleanup(rpriv);
2084
2085 err_detach_netdev:
2086         mlx5e_detach_netdev(netdev_priv(netdev));
2087
2088 err_destroy_mdev_resources:
2089         if (rep->vport == MLX5_VPORT_UPLINK)
2090                 mlx5e_destroy_mdev_resources(dev);
2091
2092 err_destroy_netdev:
2093         mlx5e_destroy_netdev(netdev_priv(netdev));
2094         kfree(rpriv);
2095         return err;
2096 }
2097
2098 static void
2099 mlx5e_vport_rep_unload(struct mlx5_eswitch_rep *rep)
2100 {
2101         struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep);
2102         struct net_device *netdev = rpriv->netdev;
2103         struct mlx5e_priv *priv = netdev_priv(netdev);
2104         struct mlx5_core_dev *dev = priv->mdev;
2105         void *ppriv = priv->ppriv;
2106
2107         if (is_devlink_port_supported(dev, rpriv))
2108                 devlink_port_type_clear(&rpriv->dl_port);
2109         unregister_netdev(netdev);
2110         unregister_devlink_port(dev, rpriv);
2111         mlx5e_rep_neigh_cleanup(rpriv);
2112         mlx5e_detach_netdev(priv);
2113         if (rep->vport == MLX5_VPORT_UPLINK)
2114                 mlx5e_destroy_mdev_resources(priv->mdev);
2115         mlx5e_destroy_netdev(priv);
2116         kfree(ppriv); /* mlx5e_rep_priv */
2117 }
2118
2119 static void *mlx5e_vport_rep_get_proto_dev(struct mlx5_eswitch_rep *rep)
2120 {
2121         struct mlx5e_rep_priv *rpriv;
2122
2123         rpriv = mlx5e_rep_to_rep_priv(rep);
2124
2125         return rpriv->netdev;
2126 }
2127
2128 static const struct mlx5_eswitch_rep_ops rep_ops = {
2129         .load = mlx5e_vport_rep_load,
2130         .unload = mlx5e_vport_rep_unload,
2131         .get_proto_dev = mlx5e_vport_rep_get_proto_dev
2132 };
2133
2134 void mlx5e_rep_register_vport_reps(struct mlx5_core_dev *mdev)
2135 {
2136         struct mlx5_eswitch *esw = mdev->priv.eswitch;
2137
2138         mlx5_eswitch_register_vport_reps(esw, &rep_ops, REP_ETH);
2139 }
2140
2141 void mlx5e_rep_unregister_vport_reps(struct mlx5_core_dev *mdev)
2142 {
2143         struct mlx5_eswitch *esw = mdev->priv.eswitch;
2144
2145         mlx5_eswitch_unregister_vport_reps(esw, REP_ETH);
2146 }