drivers/infiniband/hw/mlx5/main.c

   1 /*
   2  * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/debugfs.h>
  34 #include <linux/highmem.h>
  35 #include <linux/module.h>
  36 #include <linux/init.h>
  37 #include <linux/errno.h>
  38 #include <linux/pci.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/slab.h>
  41 #include <linux/bitmap.h>
  42 #if defined(CONFIG_X86)
  43 #include <asm/pat.h>
  44 #endif
  45 #include <linux/sched.h>
  46 #include <linux/sched/mm.h>
  47 #include <linux/sched/task.h>
  48 #include <linux/delay.h>
  49 #include <rdma/ib_user_verbs.h>
  50 #include <rdma/ib_addr.h>
  51 #include <rdma/ib_cache.h>
  52 #include <linux/mlx5/port.h>
  53 #include <linux/mlx5/vport.h>
  54 #include <linux/mlx5/fs.h>
  55 #include <linux/mlx5/eswitch.h>
  56 #include <linux/list.h>
  57 #include <rdma/ib_smi.h>
  58 #include <rdma/ib_umem.h>
  59 #include <linux/in.h>
  60 #include <linux/etherdevice.h>
  61 #include "mlx5_ib.h"
  62 #include "ib_rep.h"
  63 #include "cmd.h"
  64 #include "srq.h"
  65 #include <linux/mlx5/fs_helpers.h>
  66 #include <linux/mlx5/accel.h>
  67 #include <rdma/uverbs_std_types.h>
  68 #include <rdma/mlx5_user_ioctl_verbs.h>
  69 #include <rdma/mlx5_user_ioctl_cmds.h>
  70 #include <rdma/ib_umem_odp.h>
  71
  72 #define UVERBS_MODULE_NAME mlx5_ib
  73 #include <rdma/uverbs_named_ioctl.h>
  74
  75 #define DRIVER_NAME "mlx5_ib"
  76 #define DRIVER_VERSION "5.0-0"
  77
  78 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
  79 MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");
  80 MODULE_LICENSE("Dual BSD/GPL");
  81
  82 static char mlx5_version[] =
  83         DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v"
  84         DRIVER_VERSION "\n";
  85
  86 struct mlx5_ib_event_work {
  87         struct work_struct      work;
  88         union {
  89                 struct mlx5_ib_dev            *dev;
  90                 struct mlx5_ib_multiport_info *mpi;
  91         };
  92         bool                    is_slave;
  93         unsigned int            event;
  94         void                    *param;
  95 };
  96
  97 enum {
  98         MLX5_ATOMIC_SIZE_QP_8BYTES = 1 << 3,
  99 };
 100
 101 static struct workqueue_struct *mlx5_ib_event_wq;
 102 static LIST_HEAD(mlx5_ib_unaffiliated_port_list);
 103 static LIST_HEAD(mlx5_ib_dev_list);
 104 /*
 105  * This mutex should be held when accessing either of the above lists
 106  */
 107 static DEFINE_MUTEX(mlx5_ib_multiport_mutex);
 108
 109 /* We can't use an array for xlt_emergency_page because dma_map_single
 110  * doesn't work on kernel modules memory
 111  */
 112 static unsigned long xlt_emergency_page;
 113 static struct mutex xlt_emergency_page_mutex;
 114
 115 struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)
 116 {
 117         struct mlx5_ib_dev *dev;
 118
 119         mutex_lock(&mlx5_ib_multiport_mutex);
 120         dev = mpi->ibdev;
 121         mutex_unlock(&mlx5_ib_multiport_mutex);
 122         return dev;
 123 }
 124
 125 static enum rdma_link_layer
 126 mlx5_port_type_cap_to_rdma_ll(int port_type_cap)
 127 {
 128         switch (port_type_cap) {
 129         case MLX5_CAP_PORT_TYPE_IB:
 130                 return IB_LINK_LAYER_INFINIBAND;
 131         case MLX5_CAP_PORT_TYPE_ETH:
 132                 return IB_LINK_LAYER_ETHERNET;
 133         default:
 134                 return IB_LINK_LAYER_UNSPECIFIED;
 135         }
 136 }
 137
 138 static enum rdma_link_layer
 139 mlx5_ib_port_link_layer(struct ib_device *device, u8 port_num)
 140 {
 141         struct mlx5_ib_dev *dev = to_mdev(device);
 142         int port_type_cap = MLX5_CAP_GEN(dev->mdev, port_type);
 143
 144         return mlx5_port_type_cap_to_rdma_ll(port_type_cap);
 145 }
 146
 147 static int get_port_state(struct ib_device *ibdev,
 148                           u8 port_num,
 149                           enum ib_port_state *state)
 150 {
 151         struct ib_port_attr attr;
 152         int ret;
 153
 154         memset(&attr, 0, sizeof(attr));
 155         ret = ibdev->ops.query_port(ibdev, port_num, &attr);
 156         if (!ret)
 157                 *state = attr.state;
 158         return ret;
 159 }
 160
 161 static struct mlx5_roce *mlx5_get_rep_roce(struct mlx5_ib_dev *dev,
 162                                            struct net_device *ndev,
 163                                            u8 *port_num)
 164 {
 165         struct mlx5_eswitch *esw = dev->mdev->priv.eswitch;
 166         struct net_device *rep_ndev;
 167         struct mlx5_ib_port *port;
 168         int i;
 169
 170         for (i = 0; i < dev->num_ports; i++) {
 171                 port  = &dev->port[i];
 172                 if (!port->rep)
 173                         continue;
 174
 175                 read_lock(&port->roce.netdev_lock);
 176                 rep_ndev = mlx5_ib_get_rep_netdev(esw,
 177                                                   port->rep->vport);
 178                 if (rep_ndev == ndev) {
 179                         read_unlock(&port->roce.netdev_lock);
 180                         *port_num = i + 1;
 181                         return &port->roce;
 182                 }
 183                 read_unlock(&port->roce.netdev_lock);
 184         }
 185
 186         return NULL;
 187 }
 188
 189 static int mlx5_netdev_event(struct notifier_block *this,
 190                              unsigned long event, void *ptr)
 191 {
 192         struct mlx5_roce *roce = container_of(this, struct mlx5_roce, nb);
 193         struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
 194         u8 port_num = roce->native_port_num;
 195         struct mlx5_core_dev *mdev;
 196         struct mlx5_ib_dev *ibdev;
 197
 198         ibdev = roce->dev;
 199         mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
 200         if (!mdev)
 201                 return NOTIFY_DONE;
 202
 203         switch (event) {
 204         case NETDEV_REGISTER:
 205                 /* Should already be registered during the load */
 206                 if (ibdev->is_rep)
 207                         break;
 208                 write_lock(&roce->netdev_lock);
 209                 if (ndev->dev.parent == mdev->device)
 210                         roce->netdev = ndev;
 211                 write_unlock(&roce->netdev_lock);
 212                 break;
 213
 214         case NETDEV_UNREGISTER:
 215                 /* In case of reps, ib device goes away before the netdevs */
 216                 write_lock(&roce->netdev_lock);
 217                 if (roce->netdev == ndev)
 218                         roce->netdev = NULL;
 219                 write_unlock(&roce->netdev_lock);
 220                 break;
 221
 222         case NETDEV_CHANGE:
 223         case NETDEV_UP:
 224         case NETDEV_DOWN: {
 225                 struct net_device *lag_ndev = mlx5_lag_get_roce_netdev(mdev);
 226                 struct net_device *upper = NULL;
 227
 228                 if (lag_ndev) {
 229                         upper = netdev_master_upper_dev_get(lag_ndev);
 230                         dev_put(lag_ndev);
 231                 }
 232
 233                 if (ibdev->is_rep)
 234                         roce = mlx5_get_rep_roce(ibdev, ndev, &port_num);
 235                 if (!roce)
 236                         return NOTIFY_DONE;
 237                 if ((upper == ndev || (!upper && ndev == roce->netdev))
 238                     && ibdev->ib_active) {
 239                         struct ib_event ibev = { };
 240                         enum ib_port_state port_state;
 241
 242                         if (get_port_state(&ibdev->ib_dev, port_num,
 243                                            &port_state))
 244                                 goto done;
 245
 246                         if (roce->last_port_state == port_state)
 247                                 goto done;
 248
 249                         roce->last_port_state = port_state;
 250                         ibev.device = &ibdev->ib_dev;
 251                         if (port_state == IB_PORT_DOWN)
 252                                 ibev.event = IB_EVENT_PORT_ERR;
 253                         else if (port_state == IB_PORT_ACTIVE)
 254                                 ibev.event = IB_EVENT_PORT_ACTIVE;
 255                         else
 256                                 goto done;
 257
 258                         ibev.element.port_num = port_num;
 259                         ib_dispatch_event(&ibev);
 260                 }
 261                 break;
 262         }
 263
 264         default:
 265                 break;
 266         }
 267 done:
 268         mlx5_ib_put_native_port_mdev(ibdev, port_num);
 269         return NOTIFY_DONE;
 270 }
 271
 272 static struct net_device *mlx5_ib_get_netdev(struct ib_device *device,
 273                                              u8 port_num)
 274 {
 275         struct mlx5_ib_dev *ibdev = to_mdev(device);
 276         struct net_device *ndev;
 277         struct mlx5_core_dev *mdev;
 278
 279         mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
 280         if (!mdev)
 281                 return NULL;
 282
 283         ndev = mlx5_lag_get_roce_netdev(mdev);
 284         if (ndev)
 285                 goto out;
 286
 287         /* Ensure ndev does not disappear before we invoke dev_hold()
 288          */
 289         read_lock(&ibdev->port[port_num - 1].roce.netdev_lock);
 290         ndev = ibdev->port[port_num - 1].roce.netdev;
 291         if (ndev)
 292                 dev_hold(ndev);
 293         read_unlock(&ibdev->port[port_num - 1].roce.netdev_lock);
 294
 295 out:
 296         mlx5_ib_put_native_port_mdev(ibdev, port_num);
 297         return ndev;
 298 }
 299
 300 struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev,
 301                                                    u8 ib_port_num,
 302                                                    u8 *native_port_num)
 303 {
 304         enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
 305                                                           ib_port_num);
 306         struct mlx5_core_dev *mdev = NULL;
 307         struct mlx5_ib_multiport_info *mpi;
 308         struct mlx5_ib_port *port;
 309
 310         if (!mlx5_core_mp_enabled(ibdev->mdev) ||
 311             ll != IB_LINK_LAYER_ETHERNET) {
 312                 if (native_port_num)
 313                         *native_port_num = ib_port_num;
 314                 return ibdev->mdev;
 315         }
 316
 317         if (native_port_num)
 318                 *native_port_num = 1;
 319
 320         port = &ibdev->port[ib_port_num - 1];
 321         if (!port)
 322                 return NULL;
 323
 324         spin_lock(&port->mp.mpi_lock);
 325         mpi = ibdev->port[ib_port_num - 1].mp.mpi;
 326         if (mpi && !mpi->unaffiliate) {
 327                 mdev = mpi->mdev;
 328                 /* If it's the master no need to refcount, it'll exist
 329                  * as long as the ib_dev exists.
 330                  */
 331                 if (!mpi->is_master)
 332                         mpi->mdev_refcnt++;
 333         }
 334         spin_unlock(&port->mp.mpi_lock);
 335
 336         return mdev;
 337 }
 338
 339 void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *ibdev, u8 port_num)
 340 {
 341         enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
 342                                                           port_num);
 343         struct mlx5_ib_multiport_info *mpi;
 344         struct mlx5_ib_port *port;
 345
 346         if (!mlx5_core_mp_enabled(ibdev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
 347                 return;
 348
 349         port = &ibdev->port[port_num - 1];
 350
 351         spin_lock(&port->mp.mpi_lock);
 352         mpi = ibdev->port[port_num - 1].mp.mpi;
 353         if (mpi->is_master)
 354                 goto out;
 355
 356         mpi->mdev_refcnt--;
 357         if (mpi->unaffiliate)
 358                 complete(&mpi->unref_comp);
 359 out:
 360         spin_unlock(&port->mp.mpi_lock);
 361 }
 362
 363 static int translate_eth_legacy_proto_oper(u32 eth_proto_oper, u8 *active_speed,
 364                                            u8 *active_width)
 365 {
 366         switch (eth_proto_oper) {
 367         case MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII):
 368         case MLX5E_PROT_MASK(MLX5E_1000BASE_KX):
 369         case MLX5E_PROT_MASK(MLX5E_100BASE_TX):
 370         case MLX5E_PROT_MASK(MLX5E_1000BASE_T):
 371                 *active_width = IB_WIDTH_1X;
 372                 *active_speed = IB_SPEED_SDR;
 373                 break;
 374         case MLX5E_PROT_MASK(MLX5E_10GBASE_T):
 375         case MLX5E_PROT_MASK(MLX5E_10GBASE_CX4):
 376         case MLX5E_PROT_MASK(MLX5E_10GBASE_KX4):
 377         case MLX5E_PROT_MASK(MLX5E_10GBASE_KR):
 378         case MLX5E_PROT_MASK(MLX5E_10GBASE_CR):
 379         case MLX5E_PROT_MASK(MLX5E_10GBASE_SR):
 380         case MLX5E_PROT_MASK(MLX5E_10GBASE_ER):
 381                 *active_width = IB_WIDTH_1X;
 382                 *active_speed = IB_SPEED_QDR;
 383                 break;
 384         case MLX5E_PROT_MASK(MLX5E_25GBASE_CR):
 385         case MLX5E_PROT_MASK(MLX5E_25GBASE_KR):
 386         case MLX5E_PROT_MASK(MLX5E_25GBASE_SR):
 387                 *active_width = IB_WIDTH_1X;
 388                 *active_speed = IB_SPEED_EDR;
 389                 break;
 390         case MLX5E_PROT_MASK(MLX5E_40GBASE_CR4):
 391         case MLX5E_PROT_MASK(MLX5E_40GBASE_KR4):
 392         case MLX5E_PROT_MASK(MLX5E_40GBASE_SR4):
 393         case MLX5E_PROT_MASK(MLX5E_40GBASE_LR4):
 394                 *active_width = IB_WIDTH_4X;
 395                 *active_speed = IB_SPEED_QDR;
 396                 break;
 397         case MLX5E_PROT_MASK(MLX5E_50GBASE_CR2):
 398         case MLX5E_PROT_MASK(MLX5E_50GBASE_KR2):
 399         case MLX5E_PROT_MASK(MLX5E_50GBASE_SR2):
 400                 *active_width = IB_WIDTH_1X;
 401                 *active_speed = IB_SPEED_HDR;
 402                 break;
 403         case MLX5E_PROT_MASK(MLX5E_56GBASE_R4):
 404                 *active_width = IB_WIDTH_4X;
 405                 *active_speed = IB_SPEED_FDR;
 406                 break;
 407         case MLX5E_PROT_MASK(MLX5E_100GBASE_CR4):
 408         case MLX5E_PROT_MASK(MLX5E_100GBASE_SR4):
 409         case MLX5E_PROT_MASK(MLX5E_100GBASE_KR4):
 410         case MLX5E_PROT_MASK(MLX5E_100GBASE_LR4):
 411                 *active_width = IB_WIDTH_4X;
 412                 *active_speed = IB_SPEED_EDR;
 413                 break;
 414         default:
 415                 return -EINVAL;
 416         }
 417
 418         return 0;
 419 }
 420
 421 static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u8 *active_speed,
 422                                         u8 *active_width)
 423 {
 424         switch (eth_proto_oper) {
 425         case MLX5E_PROT_MASK(MLX5E_SGMII_100M):
 426         case MLX5E_PROT_MASK(MLX5E_1000BASE_X_SGMII):
 427                 *active_width = IB_WIDTH_1X;
 428                 *active_speed = IB_SPEED_SDR;
 429                 break;
 430         case MLX5E_PROT_MASK(MLX5E_5GBASE_R):
 431                 *active_width = IB_WIDTH_1X;
 432                 *active_speed = IB_SPEED_DDR;
 433                 break;
 434         case MLX5E_PROT_MASK(MLX5E_10GBASE_XFI_XAUI_1):
 435                 *active_width = IB_WIDTH_1X;
 436                 *active_speed = IB_SPEED_QDR;
 437                 break;
 438         case MLX5E_PROT_MASK(MLX5E_40GBASE_XLAUI_4_XLPPI_4):
 439                 *active_width = IB_WIDTH_4X;
 440                 *active_speed = IB_SPEED_QDR;
 441                 break;
 442         case MLX5E_PROT_MASK(MLX5E_25GAUI_1_25GBASE_CR_KR):
 443                 *active_width = IB_WIDTH_1X;
 444                 *active_speed = IB_SPEED_EDR;
 445                 break;
 446         case MLX5E_PROT_MASK(MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2):
 447                 *active_width = IB_WIDTH_2X;
 448                 *active_speed = IB_SPEED_EDR;
 449                 break;
 450         case MLX5E_PROT_MASK(MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR):
 451                 *active_width = IB_WIDTH_1X;
 452                 *active_speed = IB_SPEED_HDR;
 453                 break;
 454         case MLX5E_PROT_MASK(MLX5E_CAUI_4_100GBASE_CR4_KR4):
 455                 *active_width = IB_WIDTH_4X;
 456                 *active_speed = IB_SPEED_EDR;
 457                 break;
 458         case MLX5E_PROT_MASK(MLX5E_100GAUI_2_100GBASE_CR2_KR2):
 459                 *active_width = IB_WIDTH_2X;
 460                 *active_speed = IB_SPEED_HDR;
 461                 break;
 462         case MLX5E_PROT_MASK(MLX5E_200GAUI_4_200GBASE_CR4_KR4):
 463                 *active_width = IB_WIDTH_4X;
 464                 *active_speed = IB_SPEED_HDR;
 465                 break;
 466         default:
 467                 return -EINVAL;
 468         }
 469
 470         return 0;
 471 }
 472
 473 static int translate_eth_proto_oper(u32 eth_proto_oper, u8 *active_speed,
 474                                     u8 *active_width, bool ext)
 475 {
 476         return ext ?
 477                 translate_eth_ext_proto_oper(eth_proto_oper, active_speed,
 478                                              active_width) :
 479                 translate_eth_legacy_proto_oper(eth_proto_oper, active_speed,
 480                                                 active_width);
 481 }
 482
 483 static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
 484                                 struct ib_port_attr *props)
 485 {
 486         struct mlx5_ib_dev *dev = to_mdev(device);
 487         u32 out[MLX5_ST_SZ_DW(ptys_reg)] = {0};
 488         struct mlx5_core_dev *mdev;
 489         struct net_device *ndev, *upper;
 490         enum ib_mtu ndev_ib_mtu;
 491         bool put_mdev = true;
 492         u16 qkey_viol_cntr;
 493         u32 eth_prot_oper;
 494         u8 mdev_port_num;
 495         bool ext;
 496         int err;
 497
 498         mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
 499         if (!mdev) {
 500                 /* This means the port isn't affiliated yet. Get the
 501                  * info for the master port instead.
 502                  */
 503                 put_mdev = false;
 504                 mdev = dev->mdev;
 505                 mdev_port_num = 1;
 506                 port_num = 1;
 507         }
 508
 509         /* Possible bad flows are checked before filling out props so in case
 510          * of an error it will still be zeroed out.
 511          * Use native port in case of reps
 512          */
 513         if (dev->is_rep)
 514                 err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
 515                                            1);
 516         else
 517                 err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
 518                                            mdev_port_num);
 519         if (err)
 520                 goto out;
 521         ext = MLX5_CAP_PCAM_FEATURE(dev->mdev, ptys_extended_ethernet);
 522         eth_prot_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext, eth_proto_oper);
 523
 524         props->active_width     = IB_WIDTH_4X;
 525         props->active_speed     = IB_SPEED_QDR;
 526
 527         translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
 528                                  &props->active_width, ext);
 529
 530         props->port_cap_flags |= IB_PORT_CM_SUP;
 531         props->ip_gids = true;
 532
 533         props->gid_tbl_len      = MLX5_CAP_ROCE(dev->mdev,
 534                                                 roce_address_table_size);
 535         props->max_mtu          = IB_MTU_4096;
 536         props->max_msg_sz       = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
 537         props->pkey_tbl_len     = 1;
 538         props->state            = IB_PORT_DOWN;
 539         props->phys_state       = IB_PORT_PHYS_STATE_DISABLED;
 540
 541         mlx5_query_nic_vport_qkey_viol_cntr(mdev, &qkey_viol_cntr);
 542         props->qkey_viol_cntr = qkey_viol_cntr;
 543
 544         /* If this is a stub query for an unaffiliated port stop here */
 545         if (!put_mdev)
 546                 goto out;
 547
 548         ndev = mlx5_ib_get_netdev(device, port_num);
 549         if (!ndev)
 550                 goto out;
 551
 552         if (dev->lag_active) {
 553                 rcu_read_lock();
 554                 upper = netdev_master_upper_dev_get_rcu(ndev);
 555                 if (upper) {
 556                         dev_put(ndev);
 557                         ndev = upper;
 558                         dev_hold(ndev);
 559                 }
 560                 rcu_read_unlock();
 561         }
 562
 563         if (netif_running(ndev) && netif_carrier_ok(ndev)) {
 564                 props->state      = IB_PORT_ACTIVE;
 565                 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 566         }
 567
 568         ndev_ib_mtu = iboe_get_mtu(ndev->mtu);
 569
 570         dev_put(ndev);
 571
 572         props->active_mtu       = min(props->max_mtu, ndev_ib_mtu);
 573 out:
 574         if (put_mdev)
 575                 mlx5_ib_put_native_port_mdev(dev, port_num);
 576         return err;
 577 }
 578
 579 static int set_roce_addr(struct mlx5_ib_dev *dev, u8 port_num,
 580                          unsigned int index, const union ib_gid *gid,
 581                          const struct ib_gid_attr *attr)
 582 {
 583         enum ib_gid_type gid_type = IB_GID_TYPE_IB;
 584         u16 vlan_id = 0xffff;
 585         u8 roce_version = 0;
 586         u8 roce_l3_type = 0;
 587         u8 mac[ETH_ALEN];
 588         int ret;
 589
 590         if (gid) {
 591                 gid_type = attr->gid_type;
 592                 ret = rdma_read_gid_l2_fields(attr, &vlan_id, &mac[0]);
 593                 if (ret)
 594                         return ret;
 595         }
 596
 597         switch (gid_type) {
 598         case IB_GID_TYPE_IB:
 599                 roce_version = MLX5_ROCE_VERSION_1;
 600                 break;
 601         case IB_GID_TYPE_ROCE_UDP_ENCAP:
 602                 roce_version = MLX5_ROCE_VERSION_2;
 603                 if (ipv6_addr_v4mapped((void *)gid))
 604                         roce_l3_type = MLX5_ROCE_L3_TYPE_IPV4;
 605                 else
 606                         roce_l3_type = MLX5_ROCE_L3_TYPE_IPV6;
 607                 break;
 608
 609         default:
 610                 mlx5_ib_warn(dev, "Unexpected GID type %u\n", gid_type);
 611         }
 612
 613         return mlx5_core_roce_gid_set(dev->mdev, index, roce_version,
 614                                       roce_l3_type, gid->raw, mac,
 615                                       vlan_id < VLAN_CFI_MASK, vlan_id,
 616                                       port_num);
 617 }
 618
 619 static int mlx5_ib_add_gid(const struct ib_gid_attr *attr,
 620                            __always_unused void **context)
 621 {
 622         return set_roce_addr(to_mdev(attr->device), attr->port_num,
 623                              attr->index, &attr->gid, attr);
 624 }
 625
 626 static int mlx5_ib_del_gid(const struct ib_gid_attr *attr,
 627                            __always_unused void **context)
 628 {
 629         return set_roce_addr(to_mdev(attr->device), attr->port_num,
 630                              attr->index, NULL, NULL);
 631 }
 632
 633 __be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev,
 634                                const struct ib_gid_attr *attr)
 635 {
 636         if (attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)
 637                 return 0;
 638
 639         return cpu_to_be16(MLX5_CAP_ROCE(dev->mdev, r_roce_min_src_udp_port));
 640 }
 641
 642 static int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev)
 643 {
 644         if (MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_IB)
 645                 return !MLX5_CAP_GEN(dev->mdev, ib_virt);
 646         return 0;
 647 }
 648
 649 enum {
 650         MLX5_VPORT_ACCESS_METHOD_MAD,
 651         MLX5_VPORT_ACCESS_METHOD_HCA,
 652         MLX5_VPORT_ACCESS_METHOD_NIC,
 653 };
 654
 655 static int mlx5_get_vport_access_method(struct ib_device *ibdev)
 656 {
 657         if (mlx5_use_mad_ifc(to_mdev(ibdev)))
 658                 return MLX5_VPORT_ACCESS_METHOD_MAD;
 659
 660         if (mlx5_ib_port_link_layer(ibdev, 1) ==
 661             IB_LINK_LAYER_ETHERNET)
 662                 return MLX5_VPORT_ACCESS_METHOD_NIC;
 663
 664         return MLX5_VPORT_ACCESS_METHOD_HCA;
 665 }
 666
 667 static void get_atomic_caps(struct mlx5_ib_dev *dev,
 668                             u8 atomic_size_qp,
 669                             struct ib_device_attr *props)
 670 {
 671         u8 tmp;
 672         u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
 673         u8 atomic_req_8B_endianness_mode =
 674                 MLX5_CAP_ATOMIC(dev->mdev, atomic_req_8B_endianness_mode);
 675
 676         /* Check if HW supports 8 bytes standard atomic operations and capable
 677          * of host endianness respond
 678          */
 679         tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD;
 680         if (((atomic_operations & tmp) == tmp) &&
 681             (atomic_size_qp & MLX5_ATOMIC_SIZE_QP_8BYTES) &&
 682             (atomic_req_8B_endianness_mode)) {
 683                 props->atomic_cap = IB_ATOMIC_HCA;
 684         } else {
 685                 props->atomic_cap = IB_ATOMIC_NONE;
 686         }
 687 }
 688
 689 static void get_atomic_caps_qp(struct mlx5_ib_dev *dev,
 690                                struct ib_device_attr *props)
 691 {
 692         u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
 693
 694         get_atomic_caps(dev, atomic_size_qp, props);
 695 }
 696
 697 static int mlx5_query_system_image_guid(struct ib_device *ibdev,
 698                                         __be64 *sys_image_guid)
 699 {
 700         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 701         struct mlx5_core_dev *mdev = dev->mdev;
 702         u64 tmp;
 703         int err;
 704
 705         switch (mlx5_get_vport_access_method(ibdev)) {
 706         case MLX5_VPORT_ACCESS_METHOD_MAD:
 707                 return mlx5_query_mad_ifc_system_image_guid(ibdev,
 708                                                             sys_image_guid);
 709
 710         case MLX5_VPORT_ACCESS_METHOD_HCA:
 711                 err = mlx5_query_hca_vport_system_image_guid(mdev, &tmp);
 712                 break;
 713
 714         case MLX5_VPORT_ACCESS_METHOD_NIC:
 715                 err = mlx5_query_nic_vport_system_image_guid(mdev, &tmp);
 716                 break;
 717
 718         default:
 719                 return -EINVAL;
 720         }
 721
 722         if (!err)
 723                 *sys_image_guid = cpu_to_be64(tmp);
 724
 725         return err;
 726
 727 }
 728
 729 static int mlx5_query_max_pkeys(struct ib_device *ibdev,
 730                                 u16 *max_pkeys)
 731 {
 732         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 733         struct mlx5_core_dev *mdev = dev->mdev;
 734
 735         switch (mlx5_get_vport_access_method(ibdev)) {
 736         case MLX5_VPORT_ACCESS_METHOD_MAD:
 737                 return mlx5_query_mad_ifc_max_pkeys(ibdev, max_pkeys);
 738
 739         case MLX5_VPORT_ACCESS_METHOD_HCA:
 740         case MLX5_VPORT_ACCESS_METHOD_NIC:
 741                 *max_pkeys = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev,
 742                                                 pkey_table_size));
 743                 return 0;
 744
 745         default:
 746                 return -EINVAL;
 747         }
 748 }
 749
 750 static int mlx5_query_vendor_id(struct ib_device *ibdev,
 751                                 u32 *vendor_id)
 752 {
 753         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 754
 755         switch (mlx5_get_vport_access_method(ibdev)) {
 756         case MLX5_VPORT_ACCESS_METHOD_MAD:
 757                 return mlx5_query_mad_ifc_vendor_id(ibdev, vendor_id);
 758
 759         case MLX5_VPORT_ACCESS_METHOD_HCA:
 760         case MLX5_VPORT_ACCESS_METHOD_NIC:
 761                 return mlx5_core_query_vendor_id(dev->mdev, vendor_id);
 762
 763         default:
 764                 return -EINVAL;
 765         }
 766 }
 767
 768 static int mlx5_query_node_guid(struct mlx5_ib_dev *dev,
 769                                 __be64 *node_guid)
 770 {
 771         u64 tmp;
 772         int err;
 773
 774         switch (mlx5_get_vport_access_method(&dev->ib_dev)) {
 775         case MLX5_VPORT_ACCESS_METHOD_MAD:
 776                 return mlx5_query_mad_ifc_node_guid(dev, node_guid);
 777
 778         case MLX5_VPORT_ACCESS_METHOD_HCA:
 779                 err = mlx5_query_hca_vport_node_guid(dev->mdev, &tmp);
 780                 break;
 781
 782         case MLX5_VPORT_ACCESS_METHOD_NIC:
 783                 err = mlx5_query_nic_vport_node_guid(dev->mdev, &tmp);
 784                 break;
 785
 786         default:
 787                 return -EINVAL;
 788         }
 789
 790         if (!err)
 791                 *node_guid = cpu_to_be64(tmp);
 792
 793         return err;
 794 }
 795
 796 struct mlx5_reg_node_desc {
 797         u8      desc[IB_DEVICE_NODE_DESC_MAX];
 798 };
 799
 800 static int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc)
 801 {
 802         struct mlx5_reg_node_desc in;
 803
 804         if (mlx5_use_mad_ifc(dev))
 805                 return mlx5_query_mad_ifc_node_desc(dev, node_desc);
 806
 807         memset(&in, 0, sizeof(in));
 808
 809         return mlx5_core_access_reg(dev->mdev, &in, sizeof(in), node_desc,
 810                                     sizeof(struct mlx5_reg_node_desc),
 811                                     MLX5_REG_NODE_DESC, 0, 0);
 812 }
 813
 814 static int mlx5_ib_query_device(struct ib_device *ibdev,
 815                                 struct ib_device_attr *props,
 816                                 struct ib_udata *uhw)
 817 {
 818         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 819         struct mlx5_core_dev *mdev = dev->mdev;
 820         int err = -ENOMEM;
 821         int max_sq_desc;
 822         int max_rq_sg;
 823         int max_sq_sg;
 824         u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
 825         bool raw_support = !mlx5_core_mp_enabled(mdev);
 826         struct mlx5_ib_query_device_resp resp = {};
 827         size_t resp_len;
 828         u64 max_tso;
 829
 830         resp_len = sizeof(resp.comp_mask) + sizeof(resp.response_length);
 831         if (uhw->outlen && uhw->outlen < resp_len)
 832                 return -EINVAL;
 833
 834         resp.response_length = resp_len;
 835
 836         if (uhw->inlen && !ib_is_udata_cleared(uhw, 0, uhw->inlen))
 837                 return -EINVAL;
 838
 839         memset(props, 0, sizeof(*props));
 840         err = mlx5_query_system_image_guid(ibdev,
 841                                            &props->sys_image_guid);
 842         if (err)
 843                 return err;
 844
 845         err = mlx5_query_max_pkeys(ibdev, &props->max_pkeys);
 846         if (err)
 847                 return err;
 848
 849         err = mlx5_query_vendor_id(ibdev, &props->vendor_id);
 850         if (err)
 851                 return err;
 852
 853         props->fw_ver = ((u64)fw_rev_maj(dev->mdev) << 32) |
 854                 (fw_rev_min(dev->mdev) << 16) |
 855                 fw_rev_sub(dev->mdev);
 856         props->device_cap_flags    = IB_DEVICE_CHANGE_PHY_PORT |
 857                 IB_DEVICE_PORT_ACTIVE_EVENT             |
 858                 IB_DEVICE_SYS_IMAGE_GUID                |
 859                 IB_DEVICE_RC_RNR_NAK_GEN;
 860
 861         if (MLX5_CAP_GEN(mdev, pkv))
 862                 props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
 863         if (MLX5_CAP_GEN(mdev, qkv))
 864                 props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
 865         if (MLX5_CAP_GEN(mdev, apm))
 866                 props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
 867         if (MLX5_CAP_GEN(mdev, xrc))
 868                 props->device_cap_flags |= IB_DEVICE_XRC;
 869         if (MLX5_CAP_GEN(mdev, imaicl)) {
 870                 props->device_cap_flags |= IB_DEVICE_MEM_WINDOW |
 871                                            IB_DEVICE_MEM_WINDOW_TYPE_2B;
 872                 props->max_mw = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
 873                 /* We support 'Gappy' memory registration too */
 874                 props->device_cap_flags |= IB_DEVICE_SG_GAPS_REG;
 875         }
 876         props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
 877         if (MLX5_CAP_GEN(mdev, sho)) {
 878                 props->device_cap_flags |= IB_DEVICE_INTEGRITY_HANDOVER;
 879                 /* At this stage no support for signature handover */
 880                 props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |
 881                                       IB_PROT_T10DIF_TYPE_2 |
 882                                       IB_PROT_T10DIF_TYPE_3;
 883                 props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
 884                                        IB_GUARD_T10DIF_CSUM;
 885         }
 886         if (MLX5_CAP_GEN(mdev, block_lb_mc))
 887                 props->device_cap_flags |= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
 888
 889         if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) && raw_support) {
 890                 if (MLX5_CAP_ETH(mdev, csum_cap)) {
 891                         /* Legacy bit to support old userspace libraries */
 892                         props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
 893                         props->raw_packet_caps |= IB_RAW_PACKET_CAP_IP_CSUM;
 894                 }
 895
 896                 if (MLX5_CAP_ETH(dev->mdev, vlan_cap))
 897                         props->raw_packet_caps |=
 898                                 IB_RAW_PACKET_CAP_CVLAN_STRIPPING;
 899
 900                 if (field_avail(typeof(resp), tso_caps, uhw->outlen)) {
 901                         max_tso = MLX5_CAP_ETH(mdev, max_lso_cap);
 902                         if (max_tso) {
 903                                 resp.tso_caps.max_tso = 1 << max_tso;
 904                                 resp.tso_caps.supported_qpts |=
 905                                         1 << IB_QPT_RAW_PACKET;
 906                                 resp.response_length += sizeof(resp.tso_caps);
 907                         }
 908                 }
 909
 910                 if (field_avail(typeof(resp), rss_caps, uhw->outlen)) {
 911                         resp.rss_caps.rx_hash_function =
 912                                                 MLX5_RX_HASH_FUNC_TOEPLITZ;
 913                         resp.rss_caps.rx_hash_fields_mask =
 914                                                 MLX5_RX_HASH_SRC_IPV4 |
 915                                                 MLX5_RX_HASH_DST_IPV4 |
 916                                                 MLX5_RX_HASH_SRC_IPV6 |
 917                                                 MLX5_RX_HASH_DST_IPV6 |
 918                                                 MLX5_RX_HASH_SRC_PORT_TCP |
 919                                                 MLX5_RX_HASH_DST_PORT_TCP |
 920                                                 MLX5_RX_HASH_SRC_PORT_UDP |
 921                                                 MLX5_RX_HASH_DST_PORT_UDP |
 922                                                 MLX5_RX_HASH_INNER;
 923                         if (mlx5_accel_ipsec_device_caps(dev->mdev) &
 924                             MLX5_ACCEL_IPSEC_CAP_DEVICE)
 925                                 resp.rss_caps.rx_hash_fields_mask |=
 926                                         MLX5_RX_HASH_IPSEC_SPI;
 927                         resp.response_length += sizeof(resp.rss_caps);
 928                 }
 929         } else {
 930                 if (field_avail(typeof(resp), tso_caps, uhw->outlen))
 931                         resp.response_length += sizeof(resp.tso_caps);
 932                 if (field_avail(typeof(resp), rss_caps, uhw->outlen))
 933                         resp.response_length += sizeof(resp.rss_caps);
 934         }
 935
 936         if (MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
 937                 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
 938                 props->device_cap_flags |= IB_DEVICE_UD_TSO;
 939         }
 940
 941         if (MLX5_CAP_GEN(dev->mdev, rq_delay_drop) &&
 942             MLX5_CAP_GEN(dev->mdev, general_notification_event) &&
 943             raw_support)
 944                 props->raw_packet_caps |= IB_RAW_PACKET_CAP_DELAY_DROP;
 945
 946         if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
 947             MLX5_CAP_IPOIB_ENHANCED(mdev, csum_cap))
 948                 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
 949
 950         if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
 951             MLX5_CAP_ETH(dev->mdev, scatter_fcs) &&
 952             raw_support) {
 953                 /* Legacy bit to support old userspace libraries */
 954                 props->device_cap_flags |= IB_DEVICE_RAW_SCATTER_FCS;
 955                 props->raw_packet_caps |= IB_RAW_PACKET_CAP_SCATTER_FCS;
 956         }
 957
 958         if (MLX5_CAP_DEV_MEM(mdev, memic)) {
 959                 props->max_dm_size =
 960                         MLX5_CAP_DEV_MEM(mdev, max_memic_size);
 961         }
 962
 963         if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS))
 964                 props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING;
 965
 966         if (MLX5_CAP_GEN(mdev, end_pad))
 967                 props->device_cap_flags |= IB_DEVICE_PCI_WRITE_END_PADDING;
 968
 969         props->vendor_part_id      = mdev->pdev->device;
 970         props->hw_ver              = mdev->pdev->revision;
 971
 972         props->max_mr_size         = ~0ull;
 973         props->page_size_cap       = ~(min_page_size - 1);
 974         props->max_qp              = 1 << MLX5_CAP_GEN(mdev, log_max_qp);
 975         props->max_qp_wr           = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
 976         max_rq_sg =  MLX5_CAP_GEN(mdev, max_wqe_sz_rq) /
 977                      sizeof(struct mlx5_wqe_data_seg);
 978         max_sq_desc = min_t(int, MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512);
 979         max_sq_sg = (max_sq_desc - sizeof(struct mlx5_wqe_ctrl_seg) -
 980                      sizeof(struct mlx5_wqe_raddr_seg)) /
 981                 sizeof(struct mlx5_wqe_data_seg);
 982         props->max_send_sge = max_sq_sg;
 983         props->max_recv_sge = max_rq_sg;
 984         props->max_sge_rd          = MLX5_MAX_SGE_RD;
 985         props->max_cq              = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
 986         props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;
 987         props->max_mr              = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
 988         props->max_pd              = 1 << MLX5_CAP_GEN(mdev, log_max_pd);
 989         props->max_qp_rd_atom      = 1 << MLX5_CAP_GEN(mdev, log_max_ra_req_qp);
 990         props->max_qp_init_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_res_qp);
 991         props->max_srq             = 1 << MLX5_CAP_GEN(mdev, log_max_srq);
 992         props->max_srq_wr = (1 << MLX5_CAP_GEN(mdev, log_max_srq_sz)) - 1;
 993         props->local_ca_ack_delay  = MLX5_CAP_GEN(mdev, local_ca_ack_delay);
 994         props->max_res_rd_atom     = props->max_qp_rd_atom * props->max_qp;
 995         props->max_srq_sge         = max_rq_sg - 1;
 996         props->max_fast_reg_page_list_len =
 997                 1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size);
 998         props->max_pi_fast_reg_page_list_len =
 999                 props->max_fast_reg_page_list_len / 2;
1000         props->max_sgl_rd =
1001                 MLX5_CAP_GEN(mdev, max_sgl_for_optimized_performance);
1002         get_atomic_caps_qp(dev, props);
1003         props->masked_atomic_cap   = IB_ATOMIC_NONE;
1004         props->max_mcast_grp       = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
1005         props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
1006         props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
1007                                            props->max_mcast_grp;
1008         props->max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */
1009         props->max_ah = INT_MAX;
1010         props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);
1011         props->timestamp_mask = 0x7FFFFFFFFFFFFFFFULL;
1012
1013         if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
1014                 if (dev->odp_caps.general_caps & IB_ODP_SUPPORT)
1015                         props->device_cap_flags |= IB_DEVICE_ON_DEMAND_PAGING;
1016                 props->odp_caps = dev->odp_caps;
1017         }
1018
1019         if (MLX5_CAP_GEN(mdev, cd))
1020                 props->device_cap_flags |= IB_DEVICE_CROSS_CHANNEL;
1021
1022         if (mlx5_core_is_vf(mdev))
1023                 props->device_cap_flags |= IB_DEVICE_VIRTUAL_FUNCTION;
1024
1025         if (mlx5_ib_port_link_layer(ibdev, 1) ==
1026             IB_LINK_LAYER_ETHERNET && raw_support) {
1027                 props->rss_caps.max_rwq_indirection_tables =
1028                         1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt);
1029                 props->rss_caps.max_rwq_indirection_table_size =
1030                         1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt_size);
1031                 props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET;
1032                 props->max_wq_type_rq =
1033                         1 << MLX5_CAP_GEN(dev->mdev, log_max_rq);
1034         }
1035
1036         if (MLX5_CAP_GEN(mdev, tag_matching)) {
1037                 props->tm_caps.max_num_tags =
1038                         (1 << MLX5_CAP_GEN(mdev, log_tag_matching_list_sz)) - 1;
1039                 props->tm_caps.max_ops =
1040                         1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
1041                 props->tm_caps.max_sge = MLX5_TM_MAX_SGE;
1042         }
1043
1044         if (MLX5_CAP_GEN(mdev, tag_matching) &&
1045             MLX5_CAP_GEN(mdev, rndv_offload_rc)) {
1046                 props->tm_caps.flags = IB_TM_CAP_RNDV_RC;
1047                 props->tm_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE;
1048         }
1049
1050         if (MLX5_CAP_GEN(dev->mdev, cq_moderation)) {
1051                 props->cq_caps.max_cq_moderation_count =
1052                                                 MLX5_MAX_CQ_COUNT;
1053                 props->cq_caps.max_cq_moderation_period =
1054                                                 MLX5_MAX_CQ_PERIOD;
1055         }
1056
1057         if (field_avail(typeof(resp), cqe_comp_caps, uhw->outlen)) {
1058                 resp.response_length += sizeof(resp.cqe_comp_caps);
1059
1060                 if (MLX5_CAP_GEN(dev->mdev, cqe_compression)) {
1061                         resp.cqe_comp_caps.max_num =
1062                                 MLX5_CAP_GEN(dev->mdev,
1063                                              cqe_compression_max_num);
1064
1065                         resp.cqe_comp_caps.supported_format =
1066                                 MLX5_IB_CQE_RES_FORMAT_HASH |
1067                                 MLX5_IB_CQE_RES_FORMAT_CSUM;
1068
1069                         if (MLX5_CAP_GEN(dev->mdev, mini_cqe_resp_stride_index))
1070                                 resp.cqe_comp_caps.supported_format |=
1071                                         MLX5_IB_CQE_RES_FORMAT_CSUM_STRIDX;
1072                 }
1073         }
1074
1075         if (field_avail(typeof(resp), packet_pacing_caps, uhw->outlen) &&
1076             raw_support) {
1077                 if (MLX5_CAP_QOS(mdev, packet_pacing) &&
1078                     MLX5_CAP_GEN(mdev, qos)) {
1079                         resp.packet_pacing_caps.qp_rate_limit_max =
1080                                 MLX5_CAP_QOS(mdev, packet_pacing_max_rate);
1081                         resp.packet_pacing_caps.qp_rate_limit_min =
1082                                 MLX5_CAP_QOS(mdev, packet_pacing_min_rate);
1083                         resp.packet_pacing_caps.supported_qpts |=
1084                                 1 << IB_QPT_RAW_PACKET;
1085                         if (MLX5_CAP_QOS(mdev, packet_pacing_burst_bound) &&
1086                             MLX5_CAP_QOS(mdev, packet_pacing_typical_size))
1087                                 resp.packet_pacing_caps.cap_flags |=
1088                                         MLX5_IB_PP_SUPPORT_BURST;
1089                 }
1090                 resp.response_length += sizeof(resp.packet_pacing_caps);
1091         }
1092
1093         if (field_avail(typeof(resp), mlx5_ib_support_multi_pkt_send_wqes,
1094                         uhw->outlen)) {
1095                 if (MLX5_CAP_ETH(mdev, multi_pkt_send_wqe))
1096                         resp.mlx5_ib_support_multi_pkt_send_wqes =
1097                                 MLX5_IB_ALLOW_MPW;
1098
1099                 if (MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe))
1100                         resp.mlx5_ib_support_multi_pkt_send_wqes |=
1101                                 MLX5_IB_SUPPORT_EMPW;
1102
1103                 resp.response_length +=
1104                         sizeof(resp.mlx5_ib_support_multi_pkt_send_wqes);
1105         }
1106
1107         if (field_avail(typeof(resp), flags, uhw->outlen)) {
1108                 resp.response_length += sizeof(resp.flags);
1109
1110                 if (MLX5_CAP_GEN(mdev, cqe_compression_128))
1111                         resp.flags |=
1112                                 MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_COMP;
1113
1114                 if (MLX5_CAP_GEN(mdev, cqe_128_always))
1115                         resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_PAD;
1116                 if (MLX5_CAP_GEN(mdev, qp_packet_based))
1117                         resp.flags |=
1118                                 MLX5_IB_QUERY_DEV_RESP_PACKET_BASED_CREDIT_MODE;
1119
1120                 resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_SCAT2CQE_DCT;
1121         }
1122
1123         if (field_avail(typeof(resp), sw_parsing_caps,
1124                         uhw->outlen)) {
1125                 resp.response_length += sizeof(resp.sw_parsing_caps);
1126                 if (MLX5_CAP_ETH(mdev, swp)) {
1127                         resp.sw_parsing_caps.sw_parsing_offloads |=
1128                                 MLX5_IB_SW_PARSING;
1129
1130                         if (MLX5_CAP_ETH(mdev, swp_csum))
1131                                 resp.sw_parsing_caps.sw_parsing_offloads |=
1132                                         MLX5_IB_SW_PARSING_CSUM;
1133
1134                         if (MLX5_CAP_ETH(mdev, swp_lso))
1135                                 resp.sw_parsing_caps.sw_parsing_offloads |=
1136                                         MLX5_IB_SW_PARSING_LSO;
1137
1138                         if (resp.sw_parsing_caps.sw_parsing_offloads)
1139                                 resp.sw_parsing_caps.supported_qpts =
1140                                         BIT(IB_QPT_RAW_PACKET);
1141                 }
1142         }
1143
1144         if (field_avail(typeof(resp), striding_rq_caps, uhw->outlen) &&
1145             raw_support) {
1146                 resp.response_length += sizeof(resp.striding_rq_caps);
1147                 if (MLX5_CAP_GEN(mdev, striding_rq)) {
1148                         resp.striding_rq_caps.min_single_stride_log_num_of_bytes =
1149                                 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES;
1150                         resp.striding_rq_caps.max_single_stride_log_num_of_bytes =
1151                                 MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES;
1152                         if (MLX5_CAP_GEN(dev->mdev, ext_stride_num_range))
1153                                 resp.striding_rq_caps
1154                                         .min_single_wqe_log_num_of_strides =
1155                                         MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
1156                         else
1157                                 resp.striding_rq_caps
1158                                         .min_single_wqe_log_num_of_strides =
1159                                         MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
1160                         resp.striding_rq_caps.max_single_wqe_log_num_of_strides =
1161                                 MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES;
1162                         resp.striding_rq_caps.supported_qpts =
1163                                 BIT(IB_QPT_RAW_PACKET);
1164                 }
1165         }
1166
1167         if (field_avail(typeof(resp), tunnel_offloads_caps,
1168                         uhw->outlen)) {
1169                 resp.response_length += sizeof(resp.tunnel_offloads_caps);
1170                 if (MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan))
1171                         resp.tunnel_offloads_caps |=
1172                                 MLX5_IB_TUNNELED_OFFLOADS_VXLAN;
1173                 if (MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx))
1174                         resp.tunnel_offloads_caps |=
1175                                 MLX5_IB_TUNNELED_OFFLOADS_GENEVE;
1176                 if (MLX5_CAP_ETH(mdev, tunnel_stateless_gre))
1177                         resp.tunnel_offloads_caps |=
1178                                 MLX5_IB_TUNNELED_OFFLOADS_GRE;
1179                 if (MLX5_CAP_GEN(mdev, flex_parser_protocols) &
1180                     MLX5_FLEX_PROTO_CW_MPLS_GRE)
1181                         resp.tunnel_offloads_caps |=
1182                                 MLX5_IB_TUNNELED_OFFLOADS_MPLS_GRE;
1183                 if (MLX5_CAP_GEN(mdev, flex_parser_protocols) &
1184                     MLX5_FLEX_PROTO_CW_MPLS_UDP)
1185                         resp.tunnel_offloads_caps |=
1186                                 MLX5_IB_TUNNELED_OFFLOADS_MPLS_UDP;
1187         }
1188
1189         if (uhw->outlen) {
1190                 err = ib_copy_to_udata(uhw, &resp, resp.response_length);
1191
1192                 if (err)
1193                         return err;
1194         }
1195
1196         return 0;
1197 }
1198
1199 enum mlx5_ib_width {
1200         MLX5_IB_WIDTH_1X        = 1 << 0,
1201         MLX5_IB_WIDTH_2X        = 1 << 1,
1202         MLX5_IB_WIDTH_4X        = 1 << 2,
1203         MLX5_IB_WIDTH_8X        = 1 << 3,
1204         MLX5_IB_WIDTH_12X       = 1 << 4
1205 };
1206
1207 static void translate_active_width(struct ib_device *ibdev, u8 active_width,
1208                                   u8 *ib_width)
1209 {
1210         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1211
1212         if (active_width & MLX5_IB_WIDTH_1X)
1213                 *ib_width = IB_WIDTH_1X;
1214         else if (active_width & MLX5_IB_WIDTH_2X)
1215                 *ib_width = IB_WIDTH_2X;
1216         else if (active_width & MLX5_IB_WIDTH_4X)
1217                 *ib_width = IB_WIDTH_4X;
1218         else if (active_width & MLX5_IB_WIDTH_8X)
1219                 *ib_width = IB_WIDTH_8X;
1220         else if (active_width & MLX5_IB_WIDTH_12X)
1221                 *ib_width = IB_WIDTH_12X;
1222         else {
1223                 mlx5_ib_dbg(dev, "Invalid active_width %d, setting width to default value: 4x\n",
1224                             (int)active_width);
1225                 *ib_width = IB_WIDTH_4X;
1226         }
1227
1228         return;
1229 }
1230
1231 static int mlx5_mtu_to_ib_mtu(int mtu)
1232 {
1233         switch (mtu) {
1234         case 256: return 1;
1235         case 512: return 2;
1236         case 1024: return 3;
1237         case 2048: return 4;
1238         case 4096: return 5;
1239         default:
1240                 pr_warn("invalid mtu\n");
1241                 return -1;
1242         }
1243 }
1244
1245 enum ib_max_vl_num {
1246         __IB_MAX_VL_0           = 1,
1247         __IB_MAX_VL_0_1         = 2,
1248         __IB_MAX_VL_0_3         = 3,
1249         __IB_MAX_VL_0_7         = 4,
1250         __IB_MAX_VL_0_14        = 5,
1251 };
1252
1253 enum mlx5_vl_hw_cap {
1254         MLX5_VL_HW_0    = 1,
1255         MLX5_VL_HW_0_1  = 2,
1256         MLX5_VL_HW_0_2  = 3,
1257         MLX5_VL_HW_0_3  = 4,
1258         MLX5_VL_HW_0_4  = 5,
1259         MLX5_VL_HW_0_5  = 6,
1260         MLX5_VL_HW_0_6  = 7,
1261         MLX5_VL_HW_0_7  = 8,
1262         MLX5_VL_HW_0_14 = 15
1263 };
1264
1265 static int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap,
1266                                 u8 *max_vl_num)
1267 {
1268         switch (vl_hw_cap) {
1269         case MLX5_VL_HW_0:
1270                 *max_vl_num = __IB_MAX_VL_0;
1271                 break;
1272         case MLX5_VL_HW_0_1:
1273                 *max_vl_num = __IB_MAX_VL_0_1;
1274                 break;
1275         case MLX5_VL_HW_0_3:
1276                 *max_vl_num = __IB_MAX_VL_0_3;
1277                 break;
1278         case MLX5_VL_HW_0_7:
1279                 *max_vl_num = __IB_MAX_VL_0_7;
1280                 break;
1281         case MLX5_VL_HW_0_14:
1282                 *max_vl_num = __IB_MAX_VL_0_14;
1283                 break;
1284
1285         default:
1286                 return -EINVAL;
1287         }
1288
1289         return 0;
1290 }
1291
1292 static int mlx5_query_hca_port(struct ib_device *ibdev, u8 port,
1293                                struct ib_port_attr *props)
1294 {
1295         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1296         struct mlx5_core_dev *mdev = dev->mdev;
1297         struct mlx5_hca_vport_context *rep;
1298         u16 max_mtu;
1299         u16 oper_mtu;
1300         int err;
1301         u8 ib_link_width_oper;
1302         u8 vl_hw_cap;
1303
1304         rep = kzalloc(sizeof(*rep), GFP_KERNEL);
1305         if (!rep) {
1306                 err = -ENOMEM;
1307                 goto out;
1308         }
1309
1310         /* props being zeroed by the caller, avoid zeroing it here */
1311
1312         err = mlx5_query_hca_vport_context(mdev, 0, port, 0, rep);
1313         if (err)
1314                 goto out;
1315
1316         props->lid              = rep->lid;
1317         props->lmc              = rep->lmc;
1318         props->sm_lid           = rep->sm_lid;
1319         props->sm_sl            = rep->sm_sl;
1320         props->state            = rep->vport_state;
1321         props->phys_state       = rep->port_physical_state;
1322         props->port_cap_flags   = rep->cap_mask1;
1323         props->gid_tbl_len      = mlx5_get_gid_table_len(MLX5_CAP_GEN(mdev, gid_table_size));
1324         props->max_msg_sz       = 1 << MLX5_CAP_GEN(mdev, log_max_msg);
1325         props->pkey_tbl_len     = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev, pkey_table_size));
1326         props->bad_pkey_cntr    = rep->pkey_violation_counter;
1327         props->qkey_viol_cntr   = rep->qkey_violation_counter;
1328         props->subnet_timeout   = rep->subnet_timeout;
1329         props->init_type_reply  = rep->init_type_reply;
1330
1331         if (props->port_cap_flags & IB_PORT_CAP_MASK2_SUP)
1332                 props->port_cap_flags2 = rep->cap_mask2;
1333
1334         err = mlx5_query_port_link_width_oper(mdev, &ib_link_width_oper, port);
1335         if (err)
1336                 goto out;
1337
1338         translate_active_width(ibdev, ib_link_width_oper, &props->active_width);
1339
1340         err = mlx5_query_port_ib_proto_oper(mdev, &props->active_speed, port);
1341         if (err)
1342                 goto out;
1343
1344         mlx5_query_port_max_mtu(mdev, &max_mtu, port);
1345
1346         props->max_mtu = mlx5_mtu_to_ib_mtu(max_mtu);
1347
1348         mlx5_query_port_oper_mtu(mdev, &oper_mtu, port);
1349
1350         props->active_mtu = mlx5_mtu_to_ib_mtu(oper_mtu);
1351
1352         err = mlx5_query_port_vl_hw_cap(mdev, &vl_hw_cap, port);
1353         if (err)
1354                 goto out;
1355
1356         err = translate_max_vl_num(ibdev, vl_hw_cap,
1357                                    &props->max_vl_num);
1358 out:
1359         kfree(rep);
1360         return err;
1361 }
1362
1363 int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
1364                        struct ib_port_attr *props)
1365 {
1366         unsigned int count;
1367         int ret;
1368
1369         switch (mlx5_get_vport_access_method(ibdev)) {
1370         case MLX5_VPORT_ACCESS_METHOD_MAD:
1371                 ret = mlx5_query_mad_ifc_port(ibdev, port, props);
1372                 break;
1373
1374         case MLX5_VPORT_ACCESS_METHOD_HCA:
1375                 ret = mlx5_query_hca_port(ibdev, port, props);
1376                 break;
1377
1378         case MLX5_VPORT_ACCESS_METHOD_NIC:
1379                 ret = mlx5_query_port_roce(ibdev, port, props);
1380                 break;
1381
1382         default:
1383                 ret = -EINVAL;
1384         }
1385
1386         if (!ret && props) {
1387                 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1388                 struct mlx5_core_dev *mdev;
1389                 bool put_mdev = true;
1390
1391                 mdev = mlx5_ib_get_native_port_mdev(dev, port, NULL);
1392                 if (!mdev) {
1393                         /* If the port isn't affiliated yet query the master.
1394                          * The master and slave will have the same values.
1395                          */
1396                         mdev = dev->mdev;
1397                         port = 1;
1398                         put_mdev = false;
1399                 }
1400                 count = mlx5_core_reserved_gids_count(mdev);
1401                 if (put_mdev)
1402                         mlx5_ib_put_native_port_mdev(dev, port);
1403                 props->gid_tbl_len -= count;
1404         }
1405         return ret;
1406 }
1407
1408 static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u8 port,
1409                                   struct ib_port_attr *props)
1410 {
1411         int ret;
1412
1413         /* Only link layer == ethernet is valid for representors
1414          * and we always use port 1
1415          */
1416         ret = mlx5_query_port_roce(ibdev, port, props);
1417         if (ret || !props)
1418                 return ret;
1419
1420         /* We don't support GIDS */
1421         props->gid_tbl_len = 0;
1422
1423         return ret;
1424 }
1425
1426 static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
1427                              union ib_gid *gid)
1428 {
1429         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1430         struct mlx5_core_dev *mdev = dev->mdev;
1431
1432         switch (mlx5_get_vport_access_method(ibdev)) {
1433         case MLX5_VPORT_ACCESS_METHOD_MAD:
1434                 return mlx5_query_mad_ifc_gids(ibdev, port, index, gid);
1435
1436         case MLX5_VPORT_ACCESS_METHOD_HCA:
1437                 return mlx5_query_hca_vport_gid(mdev, 0, port, 0, index, gid);
1438
1439         default:
1440                 return -EINVAL;
1441         }
1442
1443 }
1444
1445 static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u8 port,
1446                                    u16 index, u16 *pkey)
1447 {
1448         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1449         struct mlx5_core_dev *mdev;
1450         bool put_mdev = true;
1451         u8 mdev_port_num;
1452         int err;
1453
1454         mdev = mlx5_ib_get_native_port_mdev(dev, port, &mdev_port_num);
1455         if (!mdev) {
1456                 /* The port isn't affiliated yet, get the PKey from the master
1457                  * port. For RoCE the PKey tables will be the same.
1458                  */
1459                 put_mdev = false;
1460                 mdev = dev->mdev;
1461                 mdev_port_num = 1;
1462         }
1463
1464         err = mlx5_query_hca_vport_pkey(mdev, 0, mdev_port_num, 0,
1465                                         index, pkey);
1466         if (put_mdev)
1467                 mlx5_ib_put_native_port_mdev(dev, port);
1468
1469         return err;
1470 }
1471
1472 static int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
1473                               u16 *pkey)
1474 {
1475         switch (mlx5_get_vport_access_method(ibdev)) {
1476         case MLX5_VPORT_ACCESS_METHOD_MAD:
1477                 return mlx5_query_mad_ifc_pkey(ibdev, port, index, pkey);
1478
1479         case MLX5_VPORT_ACCESS_METHOD_HCA:
1480         case MLX5_VPORT_ACCESS_METHOD_NIC:
1481                 return mlx5_query_hca_nic_pkey(ibdev, port, index, pkey);
1482         default:
1483                 return -EINVAL;
1484         }
1485 }
1486
1487 static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
1488                                  struct ib_device_modify *props)
1489 {
1490         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1491         struct mlx5_reg_node_desc in;
1492         struct mlx5_reg_node_desc out;
1493         int err;
1494
1495         if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
1496                 return -EOPNOTSUPP;
1497
1498         if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
1499                 return 0;
1500
1501         /*
1502          * If possible, pass node desc to FW, so it can generate
1503          * a 144 trap.  If cmd fails, just ignore.
1504          */
1505         memcpy(&in, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1506         err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out,
1507                                    sizeof(out), MLX5_REG_NODE_DESC, 0, 1);
1508         if (err)
1509                 return err;
1510
1511         memcpy(ibdev->node_desc, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1512
1513         return err;
1514 }
1515
1516 static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u8 port_num, u32 mask,
1517                                 u32 value)
1518 {
1519         struct mlx5_hca_vport_context ctx = {};
1520         struct mlx5_core_dev *mdev;
1521         u8 mdev_port_num;
1522         int err;
1523
1524         mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
1525         if (!mdev)
1526                 return -ENODEV;
1527
1528         err = mlx5_query_hca_vport_context(mdev, 0, mdev_port_num, 0, &ctx);
1529         if (err)
1530                 goto out;
1531
1532         if (~ctx.cap_mask1_perm & mask) {
1533                 mlx5_ib_warn(dev, "trying to change bitmask 0x%X but change supported 0x%X\n",
1534                              mask, ctx.cap_mask1_perm);
1535                 err = -EINVAL;
1536                 goto out;
1537         }
1538
1539         ctx.cap_mask1 = value;
1540         ctx.cap_mask1_perm = mask;
1541         err = mlx5_core_modify_hca_vport_context(mdev, 0, mdev_port_num,
1542                                                  0, &ctx);
1543
1544 out:
1545         mlx5_ib_put_native_port_mdev(dev, port_num);
1546
1547         return err;
1548 }
1549
1550 static int mlx5_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,
1551                                struct ib_port_modify *props)
1552 {
1553         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1554         struct ib_port_attr attr;
1555         u32 tmp;
1556         int err;
1557         u32 change_mask;
1558         u32 value;
1559         bool is_ib = (mlx5_ib_port_link_layer(ibdev, port) ==
1560                       IB_LINK_LAYER_INFINIBAND);
1561
1562         /* CM layer calls ib_modify_port() regardless of the link layer. For
1563          * Ethernet ports, qkey violation and Port capabilities are meaningless.
1564          */
1565         if (!is_ib)
1566                 return 0;
1567
1568         if (MLX5_CAP_GEN(dev->mdev, ib_virt) && is_ib) {
1569                 change_mask = props->clr_port_cap_mask | props->set_port_cap_mask;
1570                 value = ~props->clr_port_cap_mask | props->set_port_cap_mask;
1571                 return set_port_caps_atomic(dev, port, change_mask, value);
1572         }
1573
1574         mutex_lock(&dev->cap_mask_mutex);
1575
1576         err = ib_query_port(ibdev, port, &attr);
1577         if (err)
1578                 goto out;
1579
1580         tmp = (attr.port_cap_flags | props->set_port_cap_mask) &
1581                 ~props->clr_port_cap_mask;
1582
1583         err = mlx5_set_port_caps(dev->mdev, port, tmp);
1584
1585 out:
1586         mutex_unlock(&dev->cap_mask_mutex);
1587         return err;
1588 }
1589
1590 static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps)
1591 {
1592         mlx5_ib_dbg(dev, "MLX5_LIB_CAP_4K_UAR = %s\n",
1593                     caps & MLX5_LIB_CAP_4K_UAR ? "y" : "n");
1594 }
1595
1596 static u16 calc_dynamic_bfregs(int uars_per_sys_page)
1597 {
1598         /* Large page with non 4k uar support might limit the dynamic size */
1599         if (uars_per_sys_page == 1  && PAGE_SIZE > 4096)
1600                 return MLX5_MIN_DYN_BFREGS;
1601
1602         return MLX5_MAX_DYN_BFREGS;
1603 }
1604
1605 static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
1606                              struct mlx5_ib_alloc_ucontext_req_v2 *req,
1607                              struct mlx5_bfreg_info *bfregi)
1608 {
1609         int uars_per_sys_page;
1610         int bfregs_per_sys_page;
1611         int ref_bfregs = req->total_num_bfregs;
1612
1613         if (req->total_num_bfregs == 0)
1614                 return -EINVAL;
1615
1616         BUILD_BUG_ON(MLX5_MAX_BFREGS % MLX5_NON_FP_BFREGS_IN_PAGE);
1617         BUILD_BUG_ON(MLX5_MAX_BFREGS < MLX5_NON_FP_BFREGS_IN_PAGE);
1618
1619         if (req->total_num_bfregs > MLX5_MAX_BFREGS)
1620                 return -ENOMEM;
1621
1622         uars_per_sys_page = get_uars_per_sys_page(dev, lib_uar_4k);
1623         bfregs_per_sys_page = uars_per_sys_page * MLX5_NON_FP_BFREGS_PER_UAR;
1624         /* This holds the required static allocation asked by the user */
1625         req->total_num_bfregs = ALIGN(req->total_num_bfregs, bfregs_per_sys_page);
1626         if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
1627                 return -EINVAL;
1628
1629         bfregi->num_static_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
1630         bfregi->num_dyn_bfregs = ALIGN(calc_dynamic_bfregs(uars_per_sys_page), bfregs_per_sys_page);
1631         bfregi->total_num_bfregs = req->total_num_bfregs + bfregi->num_dyn_bfregs;
1632         bfregi->num_sys_pages = bfregi->total_num_bfregs / bfregs_per_sys_page;
1633
1634         mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, allocated %d, total bfregs %d, using %d sys pages\n",
1635                     MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
1636                     lib_uar_4k ? "yes" : "no", ref_bfregs,
1637                     req->total_num_bfregs, bfregi->total_num_bfregs,
1638                     bfregi->num_sys_pages);
1639
1640         return 0;
1641 }
1642
1643 static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1644 {
1645         struct mlx5_bfreg_info *bfregi;
1646         int err;
1647         int i;
1648
1649         bfregi = &context->bfregi;
1650         for (i = 0; i < bfregi->num_static_sys_pages; i++) {
1651                 err = mlx5_cmd_alloc_uar(dev->mdev, &bfregi->sys_pages[i]);
1652                 if (err)
1653                         goto error;
1654
1655                 mlx5_ib_dbg(dev, "allocated uar %d\n", bfregi->sys_pages[i]);
1656         }
1657
1658         for (i = bfregi->num_static_sys_pages; i < bfregi->num_sys_pages; i++)
1659                 bfregi->sys_pages[i] = MLX5_IB_INVALID_UAR_INDEX;
1660
1661         return 0;
1662
1663 error:
1664         for (--i; i >= 0; i--)
1665                 if (mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]))
1666                         mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1667
1668         return err;
1669 }
1670
1671 static void deallocate_uars(struct mlx5_ib_dev *dev,
1672                             struct mlx5_ib_ucontext *context)
1673 {
1674         struct mlx5_bfreg_info *bfregi;
1675         int i;
1676
1677         bfregi = &context->bfregi;
1678         for (i = 0; i < bfregi->num_sys_pages; i++)
1679                 if (i < bfregi->num_static_sys_pages ||
1680                     bfregi->sys_pages[i] != MLX5_IB_INVALID_UAR_INDEX)
1681                         mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]);
1682 }
1683
1684 int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
1685 {
1686         int err = 0;
1687
1688         mutex_lock(&dev->lb.mutex);
1689         if (td)
1690                 dev->lb.user_td++;
1691         if (qp)
1692                 dev->lb.qps++;
1693
1694         if (dev->lb.user_td == 2 ||
1695             dev->lb.qps == 1) {
1696                 if (!dev->lb.enabled) {
1697                         err = mlx5_nic_vport_update_local_lb(dev->mdev, true);
1698                         dev->lb.enabled = true;
1699                 }
1700         }
1701
1702         mutex_unlock(&dev->lb.mutex);
1703
1704         return err;
1705 }
1706
1707 void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
1708 {
1709         mutex_lock(&dev->lb.mutex);
1710         if (td)
1711                 dev->lb.user_td--;
1712         if (qp)
1713                 dev->lb.qps--;
1714
1715         if (dev->lb.user_td == 1 &&
1716             dev->lb.qps == 0) {
1717                 if (dev->lb.enabled) {
1718                         mlx5_nic_vport_update_local_lb(dev->mdev, false);
1719                         dev->lb.enabled = false;
1720                 }
1721         }
1722
1723         mutex_unlock(&dev->lb.mutex);
1724 }
1725
1726 static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn,
1727                                           u16 uid)
1728 {
1729         int err;
1730
1731         if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1732                 return 0;
1733
1734         err = mlx5_cmd_alloc_transport_domain(dev->mdev, tdn, uid);
1735         if (err)
1736                 return err;
1737
1738         if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1739             (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1740              !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1741                 return err;
1742
1743         return mlx5_ib_enable_lb(dev, true, false);
1744 }
1745
1746 static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn,
1747                                              u16 uid)
1748 {
1749         if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1750                 return;
1751
1752         mlx5_cmd_dealloc_transport_domain(dev->mdev, tdn, uid);
1753
1754         if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1755             (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1756              !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1757                 return;
1758
1759         mlx5_ib_disable_lb(dev, true, false);
1760 }
1761
1762 static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
1763                                   struct ib_udata *udata)
1764 {
1765         struct ib_device *ibdev = uctx->device;
1766         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1767         struct mlx5_ib_alloc_ucontext_req_v2 req = {};
1768         struct mlx5_ib_alloc_ucontext_resp resp = {};
1769         struct mlx5_core_dev *mdev = dev->mdev;
1770         struct mlx5_ib_ucontext *context = to_mucontext(uctx);
1771         struct mlx5_bfreg_info *bfregi;
1772         int ver;
1773         int err;
1774         size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
1775                                      max_cqe_version);
1776         u32 dump_fill_mkey;
1777         bool lib_uar_4k;
1778
1779         if (!dev->ib_active)
1780                 return -EAGAIN;
1781
1782         if (udata->inlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
1783                 ver = 0;
1784         else if (udata->inlen >= min_req_v2)
1785                 ver = 2;
1786         else
1787                 return -EINVAL;
1788
1789         err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
1790         if (err)
1791                 return err;
1792
1793         if (req.flags & ~MLX5_IB_ALLOC_UCTX_DEVX)
1794                 return -EOPNOTSUPP;
1795
1796         if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
1797                 return -EOPNOTSUPP;
1798
1799         req.total_num_bfregs = ALIGN(req.total_num_bfregs,
1800                                     MLX5_NON_FP_BFREGS_PER_UAR);
1801         if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
1802                 return -EINVAL;
1803
1804         resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
1805         if (dev->wc_support)
1806                 resp.bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size);
1807         resp.cache_line_size = cache_line_size();
1808         resp.max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
1809         resp.max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
1810         resp.max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1811         resp.max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1812         resp.max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
1813         resp.cqe_version = min_t(__u8,
1814                                  (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
1815                                  req.max_cqe_version);
1816         resp.log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1817                                 MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;
1818         resp.num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1819                                         MLX5_CAP_GEN(dev->mdev, num_of_uars_per_page) : 1;
1820         resp.response_length = min(offsetof(typeof(resp), response_length) +
1821                                    sizeof(resp.response_length), udata->outlen);
1822
1823         if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE) {
1824                 if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_EGRESS))
1825                         resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM;
1826                 if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_REQUIRED_METADATA)
1827                         resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_REQ_METADATA;
1828                 if (MLX5_CAP_FLOWTABLE(dev->mdev, flow_table_properties_nic_receive.ft_field_support.outer_esp_spi))
1829                         resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_SPI_STEERING;
1830                 if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN)
1831                         resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_TX_IV_IS_ESN;
1832                 /* MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_FULL_OFFLOAD is currently always 0 */
1833         }
1834
1835         lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
1836         bfregi = &context->bfregi;
1837
1838         /* updates req->total_num_bfregs */
1839         err = calc_total_bfregs(dev, lib_uar_4k, &req, bfregi);
1840         if (err)
1841                 goto out_ctx;
1842
1843         mutex_init(&bfregi->lock);
1844         bfregi->lib_uar_4k = lib_uar_4k;
1845         bfregi->count = kcalloc(bfregi->total_num_bfregs, sizeof(*bfregi->count),
1846                                 GFP_KERNEL);
1847         if (!bfregi->count) {
1848                 err = -ENOMEM;
1849                 goto out_ctx;
1850         }
1851
1852         bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
1853                                     sizeof(*bfregi->sys_pages),
1854                                     GFP_KERNEL);
1855         if (!bfregi->sys_pages) {
1856                 err = -ENOMEM;
1857                 goto out_count;
1858         }
1859
1860         err = allocate_uars(dev, context);
1861         if (err)
1862                 goto out_sys_pages;
1863
1864         if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX) {
1865                 err = mlx5_ib_devx_create(dev, true);
1866                 if (err < 0)
1867                         goto out_uars;
1868                 context->devx_uid = err;
1869         }
1870
1871         err = mlx5_ib_alloc_transport_domain(dev, &context->tdn,
1872                                              context->devx_uid);
1873         if (err)
1874                 goto out_devx;
1875
1876         if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
1877                 err = mlx5_cmd_dump_fill_mkey(dev->mdev, &dump_fill_mkey);
1878                 if (err)
1879                         goto out_mdev;
1880         }
1881
1882         INIT_LIST_HEAD(&context->db_page_list);
1883         mutex_init(&context->db_page_mutex);
1884
1885         resp.tot_bfregs = req.total_num_bfregs;
1886         resp.num_ports = dev->num_ports;
1887
1888         if (field_avail(typeof(resp), cqe_version, udata->outlen))
1889                 resp.response_length += sizeof(resp.cqe_version);
1890
1891         if (field_avail(typeof(resp), cmds_supp_uhw, udata->outlen)) {
1892                 resp.cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
1893                                       MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
1894                 resp.response_length += sizeof(resp.cmds_supp_uhw);
1895         }
1896
1897         if (field_avail(typeof(resp), eth_min_inline, udata->outlen)) {
1898                 if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
1899                         mlx5_query_min_inline(dev->mdev, &resp.eth_min_inline);
1900                         resp.eth_min_inline++;
1901                 }
1902                 resp.response_length += sizeof(resp.eth_min_inline);
1903         }
1904
1905         if (field_avail(typeof(resp), clock_info_versions, udata->outlen)) {
1906                 if (mdev->clock_info)
1907                         resp.clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);
1908                 resp.response_length += sizeof(resp.clock_info_versions);
1909         }
1910
1911         /*
1912          * We don't want to expose information from the PCI bar that is located
1913          * after 4096 bytes, so if the arch only supports larger pages, let's
1914          * pretend we don't support reading the HCA's core clock. This is also
1915          * forced by mmap function.
1916          */
1917         if (field_avail(typeof(resp), hca_core_clock_offset, udata->outlen)) {
1918                 if (PAGE_SIZE <= 4096) {
1919                         resp.comp_mask |=
1920                                 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
1921                         resp.hca_core_clock_offset =
1922                                 offsetof(struct mlx5_init_seg, internal_timer_h) % PAGE_SIZE;
1923                 }
1924                 resp.response_length += sizeof(resp.hca_core_clock_offset);
1925         }
1926
1927         if (field_avail(typeof(resp), log_uar_size, udata->outlen))
1928                 resp.response_length += sizeof(resp.log_uar_size);
1929
1930         if (field_avail(typeof(resp), num_uars_per_page, udata->outlen))
1931                 resp.response_length += sizeof(resp.num_uars_per_page);
1932
1933         if (field_avail(typeof(resp), num_dyn_bfregs, udata->outlen)) {
1934                 resp.num_dyn_bfregs = bfregi->num_dyn_bfregs;
1935                 resp.response_length += sizeof(resp.num_dyn_bfregs);
1936         }
1937
1938         if (field_avail(typeof(resp), dump_fill_mkey, udata->outlen)) {
1939                 if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
1940                         resp.dump_fill_mkey = dump_fill_mkey;
1941                         resp.comp_mask |=
1942                                 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY;
1943                 }
1944                 resp.response_length += sizeof(resp.dump_fill_mkey);
1945         }
1946
1947         err = ib_copy_to_udata(udata, &resp, resp.response_length);
1948         if (err)
1949                 goto out_mdev;
1950
1951         bfregi->ver = ver;
1952         bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
1953         context->cqe_version = resp.cqe_version;
1954         context->lib_caps = req.lib_caps;
1955         print_lib_caps(dev, context->lib_caps);
1956
1957         if (dev->lag_active) {
1958                 u8 port = mlx5_core_native_port_num(dev->mdev) - 1;
1959
1960                 atomic_set(&context->tx_port_affinity,
1961                            atomic_add_return(
1962                                    1, &dev->port[port].roce.tx_port_affinity));
1963         }
1964
1965         return 0;
1966
1967 out_mdev:
1968         mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
1969 out_devx:
1970         if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX)
1971                 mlx5_ib_devx_destroy(dev, context->devx_uid);
1972
1973 out_uars:
1974         deallocate_uars(dev, context);
1975
1976 out_sys_pages:
1977         kfree(bfregi->sys_pages);
1978
1979 out_count:
1980         kfree(bfregi->count);
1981
1982 out_ctx:
1983         return err;
1984 }
1985
1986 static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
1987 {
1988         struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
1989         struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
1990         struct mlx5_bfreg_info *bfregi;
1991
1992         bfregi = &context->bfregi;
1993         mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
1994
1995         if (context->devx_uid)
1996                 mlx5_ib_devx_destroy(dev, context->devx_uid);
1997
1998         deallocate_uars(dev, context);
1999         kfree(bfregi->sys_pages);
2000         kfree(bfregi->count);
2001 }
2002
2003 static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
2004                                  int uar_idx)
2005 {
2006         int fw_uars_per_page;
2007
2008         fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? MLX5_UARS_IN_PAGE : 1;
2009
2010         return (dev->mdev->bar_addr >> PAGE_SHIFT) + uar_idx / fw_uars_per_page;
2011 }
2012
2013 static int get_command(unsigned long offset)
2014 {
2015         return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
2016 }
2017
2018 static int get_arg(unsigned long offset)
2019 {
2020         return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
2021 }
2022
2023 static int get_index(unsigned long offset)
2024 {
2025         return get_arg(offset);
2026 }
2027
2028 /* Index resides in an extra byte to enable larger values than 255 */
2029 static int get_extended_index(unsigned long offset)
2030 {
2031         return get_arg(offset) | ((offset >> 16) & 0xff) << 8;
2032 }
2033
2034
2035 static void mlx5_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
2036 {
2037 }
2038
2039 static inline char *mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)
2040 {
2041         switch (cmd) {
2042         case MLX5_IB_MMAP_WC_PAGE:
2043                 return "WC";
2044         case MLX5_IB_MMAP_REGULAR_PAGE:
2045                 return "best effort WC";
2046         case MLX5_IB_MMAP_NC_PAGE:
2047                 return "NC";
2048         case MLX5_IB_MMAP_DEVICE_MEM:
2049                 return "Device Memory";
2050         default:
2051                 return NULL;
2052         }
2053 }
2054
2055 static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,
2056                                         struct vm_area_struct *vma,
2057                                         struct mlx5_ib_ucontext *context)
2058 {
2059         if ((vma->vm_end - vma->vm_start != PAGE_SIZE) ||
2060             !(vma->vm_flags & VM_SHARED))
2061                 return -EINVAL;
2062
2063         if (get_index(vma->vm_pgoff) != MLX5_IB_CLOCK_INFO_V1)
2064                 return -EOPNOTSUPP;
2065
2066         if (vma->vm_flags & (VM_WRITE | VM_EXEC))
2067                 return -EPERM;
2068         vma->vm_flags &= ~VM_MAYWRITE;
2069
2070         if (!dev->mdev->clock_info)
2071                 return -EOPNOTSUPP;
2072
2073         return vm_insert_page(vma, vma->vm_start,
2074                               virt_to_page(dev->mdev->clock_info));
2075 }
2076
2077 static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
2078                     struct vm_area_struct *vma,
2079                     struct mlx5_ib_ucontext *context)
2080 {
2081         struct mlx5_bfreg_info *bfregi = &context->bfregi;
2082         int err;
2083         unsigned long idx;
2084         phys_addr_t pfn;
2085         pgprot_t prot;
2086         u32 bfreg_dyn_idx = 0;
2087         u32 uar_index;
2088         int dyn_uar = (cmd == MLX5_IB_MMAP_ALLOC_WC);
2089         int max_valid_idx = dyn_uar ? bfregi->num_sys_pages :
2090                                 bfregi->num_static_sys_pages;
2091
2092         if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2093                 return -EINVAL;
2094
2095         if (dyn_uar)
2096                 idx = get_extended_index(vma->vm_pgoff) + bfregi->num_static_sys_pages;
2097         else
2098                 idx = get_index(vma->vm_pgoff);
2099
2100         if (idx >= max_valid_idx) {
2101                 mlx5_ib_warn(dev, "invalid uar index %lu, max=%d\n",
2102                              idx, max_valid_idx);
2103                 return -EINVAL;
2104         }
2105
2106         switch (cmd) {
2107         case MLX5_IB_MMAP_WC_PAGE:
2108         case MLX5_IB_MMAP_ALLOC_WC:
2109 /* Some architectures don't support WC memory */
2110 #if defined(CONFIG_X86)
2111                 if (!pat_enabled())
2112                         return -EPERM;
2113 #elif !(defined(CONFIG_PPC) || (defined(CONFIG_ARM) && defined(CONFIG_MMU)))
2114                         return -EPERM;
2115 #endif
2116         /* fall through */
2117         case MLX5_IB_MMAP_REGULAR_PAGE:
2118                 /* For MLX5_IB_MMAP_REGULAR_PAGE do the best effort to get WC */
2119                 prot = pgprot_writecombine(vma->vm_page_prot);
2120                 break;
2121         case MLX5_IB_MMAP_NC_PAGE:
2122                 prot = pgprot_noncached(vma->vm_page_prot);
2123                 break;
2124         default:
2125                 return -EINVAL;
2126         }
2127
2128         if (dyn_uar) {
2129                 int uars_per_page;
2130
2131                 uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
2132                 bfreg_dyn_idx = idx * (uars_per_page * MLX5_NON_FP_BFREGS_PER_UAR);
2133                 if (bfreg_dyn_idx >= bfregi->total_num_bfregs) {
2134                         mlx5_ib_warn(dev, "invalid bfreg_dyn_idx %u, max=%u\n",
2135                                      bfreg_dyn_idx, bfregi->total_num_bfregs);
2136                         return -EINVAL;
2137                 }
2138
2139                 mutex_lock(&bfregi->lock);
2140                 /* Fail if uar already allocated, first bfreg index of each
2141                  * page holds its count.
2142                  */
2143                 if (bfregi->count[bfreg_dyn_idx]) {
2144                         mlx5_ib_warn(dev, "wrong offset, idx %lu is busy, bfregn=%u\n", idx, bfreg_dyn_idx);
2145                         mutex_unlock(&bfregi->lock);
2146                         return -EINVAL;
2147                 }
2148
2149                 bfregi->count[bfreg_dyn_idx]++;
2150                 mutex_unlock(&bfregi->lock);
2151
2152                 err = mlx5_cmd_alloc_uar(dev->mdev, &uar_index);
2153                 if (err) {
2154                         mlx5_ib_warn(dev, "UAR alloc failed\n");
2155                         goto free_bfreg;
2156                 }
2157         } else {
2158                 uar_index = bfregi->sys_pages[idx];
2159         }
2160
2161         pfn = uar_index2pfn(dev, uar_index);
2162         mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
2163
2164         err = rdma_user_mmap_io(&context->ibucontext, vma, pfn, PAGE_SIZE,
2165                                 prot, NULL);
2166         if (err) {
2167                 mlx5_ib_err(dev,
2168                             "rdma_user_mmap_io failed with error=%d, mmap_cmd=%s\n",
2169                             err, mmap_cmd2str(cmd));
2170                 goto err;
2171         }
2172
2173         if (dyn_uar)
2174                 bfregi->sys_pages[idx] = uar_index;
2175         return 0;
2176
2177 err:
2178         if (!dyn_uar)
2179                 return err;
2180
2181         mlx5_cmd_free_uar(dev->mdev, idx);
2182
2183 free_bfreg:
2184         mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx);
2185
2186         return err;
2187 }
2188
2189 static int dm_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
2190 {
2191         struct mlx5_ib_ucontext *mctx = to_mucontext(context);
2192         struct mlx5_ib_dev *dev = to_mdev(context->device);
2193         u16 page_idx = get_extended_index(vma->vm_pgoff);
2194         size_t map_size = vma->vm_end - vma->vm_start;
2195         u32 npages = map_size >> PAGE_SHIFT;
2196         phys_addr_t pfn;
2197
2198         if (find_next_zero_bit(mctx->dm_pages, page_idx + npages, page_idx) !=
2199             page_idx + npages)
2200                 return -EINVAL;
2201
2202         pfn = ((dev->mdev->bar_addr +
2203               MLX5_CAP64_DEV_MEM(dev->mdev, memic_bar_start_addr)) >>
2204               PAGE_SHIFT) +
2205               page_idx;
2206         return rdma_user_mmap_io(context, vma, pfn, map_size,
2207                                  pgprot_writecombine(vma->vm_page_prot),
2208                                  NULL);
2209 }
2210
2211 static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
2212 {
2213         struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
2214         struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
2215         unsigned long command;
2216         phys_addr_t pfn;
2217
2218         command = get_command(vma->vm_pgoff);
2219         switch (command) {
2220         case MLX5_IB_MMAP_WC_PAGE:
2221         case MLX5_IB_MMAP_NC_PAGE:
2222         case MLX5_IB_MMAP_REGULAR_PAGE:
2223         case MLX5_IB_MMAP_ALLOC_WC:
2224                 return uar_mmap(dev, command, vma, context);
2225
2226         case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
2227                 return -ENOSYS;
2228
2229         case MLX5_IB_MMAP_CORE_CLOCK:
2230                 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2231                         return -EINVAL;
2232
2233                 if (vma->vm_flags & VM_WRITE)
2234                         return -EPERM;
2235                 vma->vm_flags &= ~VM_MAYWRITE;
2236
2237                 /* Don't expose to user-space information it shouldn't have */
2238                 if (PAGE_SIZE > 4096)
2239                         return -EOPNOTSUPP;
2240
2241                 pfn = (dev->mdev->iseg_base +
2242                        offsetof(struct mlx5_init_seg, internal_timer_h)) >>
2243                         PAGE_SHIFT;
2244                 return rdma_user_mmap_io(&context->ibucontext, vma, pfn,
2245                                          PAGE_SIZE,
2246                                          pgprot_noncached(vma->vm_page_prot),
2247                                          NULL);
2248         case MLX5_IB_MMAP_CLOCK_INFO:
2249                 return mlx5_ib_mmap_clock_info_page(dev, vma, context);
2250
2251         case MLX5_IB_MMAP_DEVICE_MEM:
2252                 return dm_mmap(ibcontext, vma);
2253
2254         default:
2255                 return -EINVAL;
2256         }
2257
2258         return 0;
2259 }
2260
2261 static inline int check_dm_type_support(struct mlx5_ib_dev *dev,
2262                                         u32 type)
2263 {
2264         switch (type) {
2265         case MLX5_IB_UAPI_DM_TYPE_MEMIC:
2266                 if (!MLX5_CAP_DEV_MEM(dev->mdev, memic))
2267                         return -EOPNOTSUPP;
2268                 break;
2269         case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
2270         case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
2271                 if (!capable(CAP_SYS_RAWIO) ||
2272                     !capable(CAP_NET_RAW))
2273                         return -EPERM;
2274
2275                 if (!(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) ||
2276                       MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, sw_owner)))
2277                         return -EOPNOTSUPP;
2278                 break;
2279         }
2280
2281         return 0;
2282 }
2283
2284 static int handle_alloc_dm_memic(struct ib_ucontext *ctx,
2285                                  struct mlx5_ib_dm *dm,
2286                                  struct ib_dm_alloc_attr *attr,
2287                                  struct uverbs_attr_bundle *attrs)
2288 {
2289         struct mlx5_dm *dm_db = &to_mdev(ctx->device)->dm;
2290         u64 start_offset;
2291         u32 page_idx;
2292         int err;
2293
2294         dm->size = roundup(attr->length, MLX5_MEMIC_BASE_SIZE);
2295
2296         err = mlx5_cmd_alloc_memic(dm_db, &dm->dev_addr,
2297                                    dm->size, attr->alignment);
2298         if (err)
2299                 return err;
2300
2301         page_idx = (dm->dev_addr - pci_resource_start(dm_db->dev->pdev, 0) -
2302                     MLX5_CAP64_DEV_MEM(dm_db->dev, memic_bar_start_addr)) >>
2303                     PAGE_SHIFT;
2304
2305         err = uverbs_copy_to(attrs,
2306                              MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
2307                              &page_idx, sizeof(page_idx));
2308         if (err)
2309                 goto err_dealloc;
2310
2311         start_offset = dm->dev_addr & ~PAGE_MASK;
2312         err = uverbs_copy_to(attrs,
2313                              MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
2314                              &start_offset, sizeof(start_offset));
2315         if (err)
2316                 goto err_dealloc;
2317
2318         bitmap_set(to_mucontext(ctx)->dm_pages, page_idx,
2319                    DIV_ROUND_UP(dm->size, PAGE_SIZE));
2320
2321         return 0;
2322
2323 err_dealloc:
2324         mlx5_cmd_dealloc_memic(dm_db, dm->dev_addr, dm->size);
2325
2326         return err;
2327 }
2328
2329 static int handle_alloc_dm_sw_icm(struct ib_ucontext *ctx,
2330                                   struct mlx5_ib_dm *dm,
2331                                   struct ib_dm_alloc_attr *attr,
2332                                   struct uverbs_attr_bundle *attrs,
2333                                   int type)
2334 {
2335         struct mlx5_core_dev *dev = to_mdev(ctx->device)->mdev;
2336         u64 act_size;
2337         int err;
2338
2339         /* Allocation size must a multiple of the basic block size
2340          * and a power of 2.
2341          */
2342         act_size = round_up(attr->length, MLX5_SW_ICM_BLOCK_SIZE(dev));
2343         act_size = roundup_pow_of_two(act_size);
2344
2345         dm->size = act_size;
2346         err = mlx5_dm_sw_icm_alloc(dev, type, act_size,
2347                                    to_mucontext(ctx)->devx_uid, &dm->dev_addr,
2348                                    &dm->icm_dm.obj_id);
2349         if (err)
2350                 return err;
2351
2352         err = uverbs_copy_to(attrs,
2353                              MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
2354                              &dm->dev_addr, sizeof(dm->dev_addr));
2355         if (err)
2356                 mlx5_dm_sw_icm_dealloc(dev, type, dm->size,
2357                                        to_mucontext(ctx)->devx_uid, dm->dev_addr,
2358                                        dm->icm_dm.obj_id);
2359
2360         return err;
2361 }
2362
2363 struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,
2364                                struct ib_ucontext *context,
2365                                struct ib_dm_alloc_attr *attr,
2366                                struct uverbs_attr_bundle *attrs)
2367 {
2368         struct mlx5_ib_dm *dm;
2369         enum mlx5_ib_uapi_dm_type type;
2370         int err;
2371
2372         err = uverbs_get_const_default(&type, attrs,
2373                                        MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
2374                                        MLX5_IB_UAPI_DM_TYPE_MEMIC);
2375         if (err)
2376                 return ERR_PTR(err);
2377
2378         mlx5_ib_dbg(to_mdev(ibdev), "alloc_dm req: dm_type=%d user_length=0x%llx log_alignment=%d\n",
2379                     type, attr->length, attr->alignment);
2380
2381         err = check_dm_type_support(to_mdev(ibdev), type);
2382         if (err)
2383                 return ERR_PTR(err);
2384
2385         dm = kzalloc(sizeof(*dm), GFP_KERNEL);
2386         if (!dm)
2387                 return ERR_PTR(-ENOMEM);
2388
2389         dm->type = type;
2390
2391         switch (type) {
2392         case MLX5_IB_UAPI_DM_TYPE_MEMIC:
2393                 err = handle_alloc_dm_memic(context, dm,
2394                                             attr,
2395                                             attrs);
2396                 break;
2397         case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
2398                 err = handle_alloc_dm_sw_icm(context, dm,
2399                                              attr, attrs,
2400                                              MLX5_SW_ICM_TYPE_STEERING);
2401                 break;
2402         case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
2403                 err = handle_alloc_dm_sw_icm(context, dm,
2404                                              attr, attrs,
2405                                              MLX5_SW_ICM_TYPE_HEADER_MODIFY);
2406                 break;
2407         default:
2408                 err = -EOPNOTSUPP;
2409         }
2410
2411         if (err)
2412                 goto err_free;
2413
2414         return &dm->ibdm;
2415
2416 err_free:
2417         kfree(dm);
2418         return ERR_PTR(err);
2419 }
2420
2421 int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, struct uverbs_attr_bundle *attrs)
2422 {
2423         struct mlx5_ib_ucontext *ctx = rdma_udata_to_drv_context(
2424                 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2425         struct mlx5_core_dev *dev = to_mdev(ibdm->device)->mdev;
2426         struct mlx5_dm *dm_db = &to_mdev(ibdm->device)->dm;
2427         struct mlx5_ib_dm *dm = to_mdm(ibdm);
2428         u32 page_idx;
2429         int ret;
2430
2431         switch (dm->type) {
2432         case MLX5_IB_UAPI_DM_TYPE_MEMIC:
2433                 ret = mlx5_cmd_dealloc_memic(dm_db, dm->dev_addr, dm->size);
2434                 if (ret)
2435                         return ret;
2436
2437                 page_idx = (dm->dev_addr - pci_resource_start(dev->pdev, 0) -
2438                             MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr)) >>
2439                             PAGE_SHIFT;
2440                 bitmap_clear(ctx->dm_pages, page_idx,
2441                              DIV_ROUND_UP(dm->size, PAGE_SIZE));
2442                 break;
2443         case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
2444                 ret = mlx5_dm_sw_icm_dealloc(dev, MLX5_SW_ICM_TYPE_STEERING,
2445                                              dm->size, ctx->devx_uid, dm->dev_addr,
2446                                              dm->icm_dm.obj_id);
2447                 if (ret)
2448                         return ret;
2449                 break;
2450         case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
2451                 ret = mlx5_dm_sw_icm_dealloc(dev, MLX5_SW_ICM_TYPE_HEADER_MODIFY,
2452                                              dm->size, ctx->devx_uid, dm->dev_addr,
2453                                              dm->icm_dm.obj_id);
2454                 if (ret)
2455                         return ret;
2456                 break;
2457         default:
2458                 return -EOPNOTSUPP;
2459         }
2460
2461         kfree(dm);
2462
2463         return 0;
2464 }
2465
2466 static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
2467 {
2468         struct mlx5_ib_pd *pd = to_mpd(ibpd);
2469         struct ib_device *ibdev = ibpd->device;
2470         struct mlx5_ib_alloc_pd_resp resp;
2471         int err;
2472         u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};
2473         u32 in[MLX5_ST_SZ_DW(alloc_pd_in)]   = {};
2474         u16 uid = 0;
2475         struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
2476                 udata, struct mlx5_ib_ucontext, ibucontext);
2477
2478         uid = context ? context->devx_uid : 0;
2479         MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
2480         MLX5_SET(alloc_pd_in, in, uid, uid);
2481         err = mlx5_cmd_exec(to_mdev(ibdev)->mdev, in, sizeof(in),
2482                             out, sizeof(out));
2483         if (err)
2484                 return err;
2485
2486         pd->pdn = MLX5_GET(alloc_pd_out, out, pd);
2487         pd->uid = uid;
2488         if (udata) {
2489                 resp.pdn = pd->pdn;
2490                 if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
2491                         mlx5_cmd_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn, uid);
2492                         return -EFAULT;
2493                 }
2494         }
2495
2496         return 0;
2497 }
2498
2499 static void mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
2500 {
2501         struct mlx5_ib_dev *mdev = to_mdev(pd->device);
2502         struct mlx5_ib_pd *mpd = to_mpd(pd);
2503
2504         mlx5_cmd_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid);
2505 }
2506
2507 enum {
2508         MATCH_CRITERIA_ENABLE_OUTER_BIT,
2509         MATCH_CRITERIA_ENABLE_MISC_BIT,
2510         MATCH_CRITERIA_ENABLE_INNER_BIT,
2511         MATCH_CRITERIA_ENABLE_MISC2_BIT
2512 };
2513
2514 #define HEADER_IS_ZERO(match_criteria, headers)                            \
2515         !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
2516                     0, MLX5_FLD_SZ_BYTES(fte_match_param, headers)))       \
2517
2518 static u8 get_match_criteria_enable(u32 *match_criteria)
2519 {
2520         u8 match_criteria_enable;
2521
2522         match_criteria_enable =
2523                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
2524                 MATCH_CRITERIA_ENABLE_OUTER_BIT;
2525         match_criteria_enable |=
2526                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
2527                 MATCH_CRITERIA_ENABLE_MISC_BIT;
2528         match_criteria_enable |=
2529                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
2530                 MATCH_CRITERIA_ENABLE_INNER_BIT;
2531         match_criteria_enable |=
2532                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
2533                 MATCH_CRITERIA_ENABLE_MISC2_BIT;
2534
2535         return match_criteria_enable;
2536 }
2537
2538 static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)
2539 {
2540         u8 entry_mask;
2541         u8 entry_val;
2542         int err = 0;
2543
2544         if (!mask)
2545                 goto out;
2546
2547         entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c,
2548                               ip_protocol);
2549         entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v,
2550                              ip_protocol);
2551         if (!entry_mask) {
2552                 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);
2553                 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
2554                 goto out;
2555         }
2556         /* Don't override existing ip protocol */
2557         if (mask != entry_mask || val != entry_val)
2558                 err = -EINVAL;
2559 out:
2560         return err;
2561 }
2562
2563 static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val,
2564                            bool inner)
2565 {
2566         if (inner) {
2567                 MLX5_SET(fte_match_set_misc,
2568                          misc_c, inner_ipv6_flow_label, mask);
2569                 MLX5_SET(fte_match_set_misc,
2570                          misc_v, inner_ipv6_flow_label, val);
2571         } else {
2572                 MLX5_SET(fte_match_set_misc,
2573                          misc_c, outer_ipv6_flow_label, mask);
2574                 MLX5_SET(fte_match_set_misc,
2575                          misc_v, outer_ipv6_flow_label, val);
2576         }
2577 }
2578
2579 static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
2580 {
2581         MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);
2582         MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);
2583         MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2);
2584         MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);
2585 }
2586
2587 static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)
2588 {
2589         if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) &&
2590             !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL))
2591                 return -EOPNOTSUPP;
2592
2593         if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) &&
2594             !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP))
2595                 return -EOPNOTSUPP;
2596
2597         if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) &&
2598             !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS))
2599                 return -EOPNOTSUPP;
2600
2601         if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) &&
2602             !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL))
2603                 return -EOPNOTSUPP;
2604
2605         return 0;
2606 }
2607
2608 #define LAST_ETH_FIELD vlan_tag
2609 #define LAST_IB_FIELD sl
2610 #define LAST_IPV4_FIELD tos
2611 #define LAST_IPV6_FIELD traffic_class
2612 #define LAST_TCP_UDP_FIELD src_port
2613 #define LAST_TUNNEL_FIELD tunnel_id
2614 #define LAST_FLOW_TAG_FIELD tag_id
2615 #define LAST_DROP_FIELD size
2616 #define LAST_COUNTERS_FIELD counters
2617
2618 /* Field is the last supported field */
2619 #define FIELDS_NOT_SUPPORTED(filter, field)\
2620         memchr_inv((void *)&filter.field  +\
2621                    sizeof(filter.field), 0,\
2622                    sizeof(filter) -\
2623                    offsetof(typeof(filter), field) -\
2624                    sizeof(filter.field))
2625
2626 int parse_flow_flow_action(struct mlx5_ib_flow_action *maction,
2627                            bool is_egress,
2628                            struct mlx5_flow_act *action)
2629 {
2630
2631         switch (maction->ib_action.type) {
2632         case IB_FLOW_ACTION_ESP:
2633                 if (action->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |
2634                                       MLX5_FLOW_CONTEXT_ACTION_DECRYPT))
2635                         return -EINVAL;
2636                 /* Currently only AES_GCM keymat is supported by the driver */
2637                 action->esp_id = (uintptr_t)maction->esp_aes_gcm.ctx;
2638                 action->action |= is_egress ?
2639                         MLX5_FLOW_CONTEXT_ACTION_ENCRYPT :
2640                         MLX5_FLOW_CONTEXT_ACTION_DECRYPT;
2641                 return 0;
2642         case IB_FLOW_ACTION_UNSPECIFIED:
2643                 if (maction->flow_action_raw.sub_type ==
2644                     MLX5_IB_FLOW_ACTION_MODIFY_HEADER) {
2645                         if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
2646                                 return -EINVAL;
2647                         action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
2648                         action->modify_hdr =
2649                                 maction->flow_action_raw.modify_hdr;
2650                         return 0;
2651                 }
2652                 if (maction->flow_action_raw.sub_type ==
2653                     MLX5_IB_FLOW_ACTION_DECAP) {
2654                         if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP)
2655                                 return -EINVAL;
2656                         action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
2657                         return 0;
2658                 }
2659                 if (maction->flow_action_raw.sub_type ==
2660                     MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) {
2661                         if (action->action &
2662                             MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT)
2663                                 return -EINVAL;
2664                         action->action |=
2665                                 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
2666                         action->pkt_reformat =
2667                                 maction->flow_action_raw.pkt_reformat;
2668                         return 0;
2669                 }
2670                 /* fall through */
2671         default:
2672                 return -EOPNOTSUPP;
2673         }
2674 }
2675
2676 static int parse_flow_attr(struct mlx5_core_dev *mdev,
2677                            struct mlx5_flow_spec *spec,
2678                            const union ib_flow_spec *ib_spec,
2679                            const struct ib_flow_attr *flow_attr,
2680                            struct mlx5_flow_act *action, u32 prev_type)
2681 {
2682         struct mlx5_flow_context *flow_context = &spec->flow_context;
2683         u32 *match_c = spec->match_criteria;
2684         u32 *match_v = spec->match_value;
2685         void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
2686                                            misc_parameters);
2687         void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
2688                                            misc_parameters);
2689         void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c,
2690                                             misc_parameters_2);
2691         void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v,
2692                                             misc_parameters_2);
2693         void *headers_c;
2694         void *headers_v;
2695         int match_ipv;
2696         int ret;
2697
2698         if (ib_spec->type & IB_FLOW_SPEC_INNER) {
2699                 headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
2700                                          inner_headers);
2701                 headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
2702                                          inner_headers);
2703                 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2704                                         ft_field_support.inner_ip_version);
2705         } else {
2706                 headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
2707                                          outer_headers);
2708                 headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
2709                                          outer_headers);
2710                 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2711                                         ft_field_support.outer_ip_version);
2712         }
2713
2714         switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
2715         case IB_FLOW_SPEC_ETH:
2716                 if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
2717                         return -EOPNOTSUPP;
2718
2719                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2720                                              dmac_47_16),
2721                                 ib_spec->eth.mask.dst_mac);
2722                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2723                                              dmac_47_16),
2724                                 ib_spec->eth.val.dst_mac);
2725
2726                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2727                                              smac_47_16),
2728                                 ib_spec->eth.mask.src_mac);
2729                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2730                                              smac_47_16),
2731                                 ib_spec->eth.val.src_mac);
2732
2733                 if (ib_spec->eth.mask.vlan_tag) {
2734                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2735                                  cvlan_tag, 1);
2736                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2737                                  cvlan_tag, 1);
2738
2739                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2740                                  first_vid, ntohs(ib_spec->eth.mask.vlan_tag));
2741                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2742                                  first_vid, ntohs(ib_spec->eth.val.vlan_tag));
2743
2744                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2745                                  first_cfi,
2746                                  ntohs(ib_spec->eth.mask.vlan_tag) >> 12);
2747                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2748                                  first_cfi,
2749                                  ntohs(ib_spec->eth.val.vlan_tag) >> 12);
2750
2751                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2752                                  first_prio,
2753                                  ntohs(ib_spec->eth.mask.vlan_tag) >> 13);
2754                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2755                                  first_prio,
2756                                  ntohs(ib_spec->eth.val.vlan_tag) >> 13);
2757                 }
2758                 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2759                          ethertype, ntohs(ib_spec->eth.mask.ether_type));
2760                 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2761                          ethertype, ntohs(ib_spec->eth.val.ether_type));
2762                 break;
2763         case IB_FLOW_SPEC_IPV4:
2764                 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
2765                         return -EOPNOTSUPP;
2766
2767                 if (match_ipv) {
2768                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2769                                  ip_version, 0xf);
2770                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2771                                  ip_version, MLX5_FS_IPV4_VERSION);
2772                 } else {
2773                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2774                                  ethertype, 0xffff);
2775                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2776                                  ethertype, ETH_P_IP);
2777                 }
2778
2779                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2780                                     src_ipv4_src_ipv6.ipv4_layout.ipv4),
2781                        &ib_spec->ipv4.mask.src_ip,
2782                        sizeof(ib_spec->ipv4.mask.src_ip));
2783                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2784                                     src_ipv4_src_ipv6.ipv4_layout.ipv4),
2785                        &ib_spec->ipv4.val.src_ip,
2786                        sizeof(ib_spec->ipv4.val.src_ip));
2787                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2788                                     dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
2789                        &ib_spec->ipv4.mask.dst_ip,
2790                        sizeof(ib_spec->ipv4.mask.dst_ip));
2791                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2792                                     dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
2793                        &ib_spec->ipv4.val.dst_ip,
2794                        sizeof(ib_spec->ipv4.val.dst_ip));
2795
2796                 set_tos(headers_c, headers_v,
2797                         ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos);
2798
2799                 if (set_proto(headers_c, headers_v,
2800                               ib_spec->ipv4.mask.proto,
2801                               ib_spec->ipv4.val.proto))
2802                         return -EINVAL;
2803                 break;
2804         case IB_FLOW_SPEC_IPV6:
2805                 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
2806                         return -EOPNOTSUPP;
2807
2808                 if (match_ipv) {
2809                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2810                                  ip_version, 0xf);
2811                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2812                                  ip_version, MLX5_FS_IPV6_VERSION);
2813                 } else {
2814                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2815                                  ethertype, 0xffff);
2816                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2817                                  ethertype, ETH_P_IPV6);
2818                 }
2819
2820                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2821                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
2822                        &ib_spec->ipv6.mask.src_ip,
2823                        sizeof(ib_spec->ipv6.mask.src_ip));
2824                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2825                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
2826                        &ib_spec->ipv6.val.src_ip,
2827                        sizeof(ib_spec->ipv6.val.src_ip));
2828                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2829                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
2830                        &ib_spec->ipv6.mask.dst_ip,
2831                        sizeof(ib_spec->ipv6.mask.dst_ip));
2832                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2833                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
2834                        &ib_spec->ipv6.val.dst_ip,
2835                        sizeof(ib_spec->ipv6.val.dst_ip));
2836
2837                 set_tos(headers_c, headers_v,
2838                         ib_spec->ipv6.mask.traffic_class,
2839                         ib_spec->ipv6.val.traffic_class);
2840
2841                 if (set_proto(headers_c, headers_v,
2842                               ib_spec->ipv6.mask.next_hdr,
2843                               ib_spec->ipv6.val.next_hdr))
2844                         return -EINVAL;
2845
2846                 set_flow_label(misc_params_c, misc_params_v,
2847                                ntohl(ib_spec->ipv6.mask.flow_label),
2848                                ntohl(ib_spec->ipv6.val.flow_label),
2849                                ib_spec->type & IB_FLOW_SPEC_INNER);
2850                 break;
2851         case IB_FLOW_SPEC_ESP:
2852                 if (ib_spec->esp.mask.seq)
2853                         return -EOPNOTSUPP;
2854
2855                 MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi,
2856                          ntohl(ib_spec->esp.mask.spi));
2857                 MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi,
2858                          ntohl(ib_spec->esp.val.spi));
2859                 break;
2860         case IB_FLOW_SPEC_TCP:
2861                 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
2862                                          LAST_TCP_UDP_FIELD))
2863                         return -EOPNOTSUPP;
2864
2865                 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP))
2866                         return -EINVAL;
2867
2868                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,
2869                          ntohs(ib_spec->tcp_udp.mask.src_port));
2870                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
2871                          ntohs(ib_spec->tcp_udp.val.src_port));
2872
2873                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,
2874                          ntohs(ib_spec->tcp_udp.mask.dst_port));
2875                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
2876                          ntohs(ib_spec->tcp_udp.val.dst_port));
2877                 break;
2878         case IB_FLOW_SPEC_UDP:
2879                 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
2880                                          LAST_TCP_UDP_FIELD))
2881                         return -EOPNOTSUPP;
2882
2883                 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP))
2884                         return -EINVAL;
2885
2886                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
2887                          ntohs(ib_spec->tcp_udp.mask.src_port));
2888                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
2889                          ntohs(ib_spec->tcp_udp.val.src_port));
2890
2891                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
2892                          ntohs(ib_spec->tcp_udp.mask.dst_port));
2893                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
2894                          ntohs(ib_spec->tcp_udp.val.dst_port));
2895                 break;
2896         case IB_FLOW_SPEC_GRE:
2897                 if (ib_spec->gre.mask.c_ks_res0_ver)
2898                         return -EOPNOTSUPP;
2899
2900                 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE))
2901                         return -EINVAL;
2902
2903                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
2904                          0xff);
2905                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
2906                          IPPROTO_GRE);
2907
2908                 MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol,
2909                          ntohs(ib_spec->gre.mask.protocol));
2910                 MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol,
2911                          ntohs(ib_spec->gre.val.protocol));
2912
2913                 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,
2914                                     gre_key.nvgre.hi),
2915                        &ib_spec->gre.mask.key,
2916                        sizeof(ib_spec->gre.mask.key));
2917                 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v,
2918                                     gre_key.nvgre.hi),
2919                        &ib_spec->gre.val.key,
2920                        sizeof(ib_spec->gre.val.key));
2921                 break;
2922         case IB_FLOW_SPEC_MPLS:
2923                 switch (prev_type) {
2924                 case IB_FLOW_SPEC_UDP:
2925                         if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2926                                                    ft_field_support.outer_first_mpls_over_udp),
2927                                                    &ib_spec->mpls.mask.tag))
2928                                 return -EOPNOTSUPP;
2929
2930                         memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
2931                                             outer_first_mpls_over_udp),
2932                                &ib_spec->mpls.val.tag,
2933                                sizeof(ib_spec->mpls.val.tag));
2934                         memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
2935                                             outer_first_mpls_over_udp),
2936                                &ib_spec->mpls.mask.tag,
2937                                sizeof(ib_spec->mpls.mask.tag));
2938                         break;
2939                 case IB_FLOW_SPEC_GRE:
2940                         if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2941                                                    ft_field_support.outer_first_mpls_over_gre),
2942                                                    &ib_spec->mpls.mask.tag))
2943                                 return -EOPNOTSUPP;
2944
2945                         memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
2946                                             outer_first_mpls_over_gre),
2947                                &ib_spec->mpls.val.tag,
2948                                sizeof(ib_spec->mpls.val.tag));
2949                         memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
2950                                             outer_first_mpls_over_gre),
2951                                &ib_spec->mpls.mask.tag,
2952                                sizeof(ib_spec->mpls.mask.tag));
2953                         break;
2954                 default:
2955                         if (ib_spec->type & IB_FLOW_SPEC_INNER) {
2956                                 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2957                                                            ft_field_support.inner_first_mpls),
2958                                                            &ib_spec->mpls.mask.tag))
2959                                         return -EOPNOTSUPP;
2960
2961                                 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
2962                                                     inner_first_mpls),
2963                                        &ib_spec->mpls.val.tag,
2964                                        sizeof(ib_spec->mpls.val.tag));
2965                                 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
2966                                                     inner_first_mpls),
2967                                        &ib_spec->mpls.mask.tag,
2968                                        sizeof(ib_spec->mpls.mask.tag));
2969                         } else {
2970                                 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2971                                                            ft_field_support.outer_first_mpls),
2972                                                            &ib_spec->mpls.mask.tag))
2973                                         return -EOPNOTSUPP;
2974
2975                                 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
2976                                                     outer_first_mpls),
2977                                        &ib_spec->mpls.val.tag,
2978                                        sizeof(ib_spec->mpls.val.tag));
2979                                 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
2980                                                     outer_first_mpls),
2981                                        &ib_spec->mpls.mask.tag,
2982                                        sizeof(ib_spec->mpls.mask.tag));
2983                         }
2984                 }
2985                 break;
2986         case IB_FLOW_SPEC_VXLAN_TUNNEL:
2987                 if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask,
2988                                          LAST_TUNNEL_FIELD))
2989                         return -EOPNOTSUPP;
2990
2991                 MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,
2992                          ntohl(ib_spec->tunnel.mask.tunnel_id));
2993                 MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,
2994                          ntohl(ib_spec->tunnel.val.tunnel_id));
2995                 break;
2996         case IB_FLOW_SPEC_ACTION_TAG:
2997                 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag,
2998                                          LAST_FLOW_TAG_FIELD))
2999                         return -EOPNOTSUPP;
3000                 if (ib_spec->flow_tag.tag_id >= BIT(24))
3001                         return -EINVAL;
3002
3003                 flow_context->flow_tag = ib_spec->flow_tag.tag_id;
3004                 flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
3005                 break;
3006         case IB_FLOW_SPEC_ACTION_DROP:
3007                 if (FIELDS_NOT_SUPPORTED(ib_spec->drop,
3008                                          LAST_DROP_FIELD))
3009                         return -EOPNOTSUPP;
3010                 action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
3011                 break;
3012         case IB_FLOW_SPEC_ACTION_HANDLE:
3013                 ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act),
3014                         flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action);
3015                 if (ret)
3016                         return ret;
3017                 break;
3018         case IB_FLOW_SPEC_ACTION_COUNT:
3019                 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count,
3020                                          LAST_COUNTERS_FIELD))
3021                         return -EOPNOTSUPP;
3022
3023                 /* for now support only one counters spec per flow */
3024                 if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
3025                         return -EINVAL;
3026
3027                 action->counters = ib_spec->flow_count.counters;
3028                 action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
3029                 break;
3030         default:
3031                 return -EINVAL;
3032         }
3033
3034         return 0;
3035 }
3036
3037 /* If a flow could catch both multicast and unicast packets,
3038  * it won't fall into the multicast flow steering table and this rule
3039  * could steal other multicast packets.
3040  */
3041 static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)
3042 {
3043         union ib_flow_spec *flow_spec;
3044
3045         if (ib_attr->type != IB_FLOW_ATTR_NORMAL ||
3046             ib_attr->num_of_specs < 1)
3047                 return false;
3048
3049         flow_spec = (union ib_flow_spec *)(ib_attr + 1);
3050         if (flow_spec->type == IB_FLOW_SPEC_IPV4) {
3051                 struct ib_flow_spec_ipv4 *ipv4_spec;
3052
3053                 ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;
3054                 if (ipv4_is_multicast(ipv4_spec->val.dst_ip))
3055                         return true;
3056
3057                 return false;
3058         }
3059
3060         if (flow_spec->type == IB_FLOW_SPEC_ETH) {
3061                 struct ib_flow_spec_eth *eth_spec;
3062
3063                 eth_spec = (struct ib_flow_spec_eth *)flow_spec;
3064                 return is_multicast_ether_addr(eth_spec->mask.dst_mac) &&
3065                        is_multicast_ether_addr(eth_spec->val.dst_mac);
3066         }
3067
3068         return false;
3069 }
3070
3071 enum valid_spec {
3072         VALID_SPEC_INVALID,
3073         VALID_SPEC_VALID,
3074         VALID_SPEC_NA,
3075 };
3076
3077 static enum valid_spec
3078 is_valid_esp_aes_gcm(struct mlx5_core_dev *mdev,
3079                      const struct mlx5_flow_spec *spec,
3080                      const struct mlx5_flow_act *flow_act,
3081                      bool egress)
3082 {
3083         const u32 *match_c = spec->match_criteria;
3084         bool is_crypto =
3085                 (flow_act->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |
3086                                      MLX5_FLOW_CONTEXT_ACTION_DECRYPT));
3087         bool is_ipsec = mlx5_fs_is_ipsec_flow(match_c);
3088         bool is_drop = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_DROP;
3089
3090         /*
3091          * Currently only crypto is supported in egress, when regular egress
3092          * rules would be supported, always return VALID_SPEC_NA.
3093          */
3094         if (!is_crypto)
3095                 return VALID_SPEC_NA;
3096
3097         return is_crypto && is_ipsec &&
3098                 (!egress || (!is_drop &&
3099                              !(spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG))) ?
3100                 VALID_SPEC_VALID : VALID_SPEC_INVALID;
3101 }
3102
3103 static bool is_valid_spec(struct mlx5_core_dev *mdev,
3104                           const struct mlx5_flow_spec *spec,
3105                           const struct mlx5_flow_act *flow_act,
3106                           bool egress)
3107 {
3108         /* We curretly only support ipsec egress flow */
3109         return is_valid_esp_aes_gcm(mdev, spec, flow_act, egress) != VALID_SPEC_INVALID;
3110 }
3111
3112 static bool is_valid_ethertype(struct mlx5_core_dev *mdev,
3113                                const struct ib_flow_attr *flow_attr,
3114                                bool check_inner)
3115 {
3116         union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1);
3117         int match_ipv = check_inner ?
3118                         MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
3119                                         ft_field_support.inner_ip_version) :
3120                         MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
3121                                         ft_field_support.outer_ip_version);
3122         int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0;
3123         bool ipv4_spec_valid, ipv6_spec_valid;
3124         unsigned int ip_spec_type = 0;
3125         bool has_ethertype = false;
3126         unsigned int spec_index;
3127         bool mask_valid = true;
3128         u16 eth_type = 0;
3129         bool type_valid;
3130
3131         /* Validate that ethertype is correct */
3132         for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
3133                 if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) &&
3134                     ib_spec->eth.mask.ether_type) {
3135                         mask_valid = (ib_spec->eth.mask.ether_type ==
3136                                       htons(0xffff));
3137                         has_ethertype = true;
3138                         eth_type = ntohs(ib_spec->eth.val.ether_type);
3139                 } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) ||
3140                            (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) {
3141                         ip_spec_type = ib_spec->type;
3142                 }
3143                 ib_spec = (void *)ib_spec + ib_spec->size;
3144         }
3145
3146         type_valid = (!has_ethertype) || (!ip_spec_type);
3147         if (!type_valid && mask_valid) {
3148                 ipv4_spec_valid = (eth_type == ETH_P_IP) &&
3149                         (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit));
3150                 ipv6_spec_valid = (eth_type == ETH_P_IPV6) &&
3151                         (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit));
3152
3153                 type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) ||
3154                              (((eth_type == ETH_P_MPLS_UC) ||
3155                                (eth_type == ETH_P_MPLS_MC)) && match_ipv);
3156         }
3157
3158         return type_valid;
3159 }
3160
3161 static bool is_valid_attr(struct mlx5_core_dev *mdev,
3162                           const struct ib_flow_attr *flow_attr)
3163 {
3164         return is_valid_ethertype(mdev, flow_attr, false) &&
3165                is_valid_ethertype(mdev, flow_attr, true);
3166 }
3167
3168 static void put_flow_table(struct mlx5_ib_dev *dev,
3169                            struct mlx5_ib_flow_prio *prio, bool ft_added)
3170 {
3171         prio->refcount -= !!ft_added;
3172         if (!prio->refcount) {
3173                 mlx5_destroy_flow_table(prio->flow_table);
3174                 prio->flow_table = NULL;
3175         }
3176 }
3177
3178 static void counters_clear_description(struct ib_counters *counters)
3179 {
3180         struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
3181
3182         mutex_lock(&mcounters->mcntrs_mutex);
3183         kfree(mcounters->counters_data);
3184         mcounters->counters_data = NULL;
3185         mcounters->cntrs_max_index = 0;
3186         mutex_unlock(&mcounters->mcntrs_mutex);
3187 }
3188
3189 static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
3190 {
3191         struct mlx5_ib_flow_handler *handler = container_of(flow_id,
3192                                                           struct mlx5_ib_flow_handler,
3193                                                           ibflow);
3194         struct mlx5_ib_flow_handler *iter, *tmp;
3195         struct mlx5_ib_dev *dev = handler->dev;
3196
3197         mutex_lock(&dev->flow_db->lock);
3198
3199         list_for_each_entry_safe(iter, tmp, &handler->list, list) {
3200                 mlx5_del_flow_rules(iter->rule);
3201                 put_flow_table(dev, iter->prio, true);
3202                 list_del(&iter->list);
3203                 kfree(iter);
3204         }
3205
3206         mlx5_del_flow_rules(handler->rule);
3207         put_flow_table(dev, handler->prio, true);
3208         if (handler->ibcounters &&
3209             atomic_read(&handler->ibcounters->usecnt) == 1)
3210                 counters_clear_description(handler->ibcounters);
3211
3212         mutex_unlock(&dev->flow_db->lock);
3213         if (handler->flow_matcher)
3214                 atomic_dec(&handler->flow_matcher->usecnt);
3215         kfree(handler);
3216
3217         return 0;
3218 }
3219
3220 static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
3221 {
3222         priority *= 2;
3223         if (!dont_trap)
3224                 priority++;
3225         return priority;
3226 }
3227
3228 enum flow_table_type {
3229         MLX5_IB_FT_RX,
3230         MLX5_IB_FT_TX
3231 };
3232
3233 #define MLX5_FS_MAX_TYPES        6
3234 #define MLX5_FS_MAX_ENTRIES      BIT(16)
3235
3236 static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns,
3237                                            struct mlx5_ib_flow_prio *prio,
3238                                            int priority,
3239                                            int num_entries, int num_groups,
3240                                            u32 flags)
3241 {
3242         struct mlx5_flow_table *ft;
3243
3244         ft = mlx5_create_auto_grouped_flow_table(ns, priority,
3245                                                  num_entries,
3246                                                  num_groups,
3247                                                  0, flags);
3248         if (IS_ERR(ft))
3249                 return ERR_CAST(ft);
3250
3251         prio->flow_table = ft;
3252         prio->refcount = 0;
3253         return prio;
3254 }
3255
3256 static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
3257                                                 struct ib_flow_attr *flow_attr,
3258                                                 enum flow_table_type ft_type)
3259 {
3260         bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
3261         struct mlx5_flow_namespace *ns = NULL;
3262         struct mlx5_ib_flow_prio *prio;
3263         struct mlx5_flow_table *ft;
3264         int max_table_size;
3265         int num_entries;
3266         int num_groups;
3267         bool esw_encap;
3268         u32 flags = 0;
3269         int priority;
3270
3271         max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
3272                                                        log_max_ft_size));
3273         esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
3274                 DEVLINK_ESWITCH_ENCAP_MODE_NONE;
3275         if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
3276                 enum mlx5_flow_namespace_type fn_type;
3277
3278                 if (flow_is_multicast_only(flow_attr) &&
3279                     !dont_trap)
3280                         priority = MLX5_IB_FLOW_MCAST_PRIO;
3281                 else
3282                         priority = ib_prio_to_core_prio(flow_attr->priority,
3283                                                         dont_trap);
3284                 if (ft_type == MLX5_IB_FT_RX) {
3285                         fn_type = MLX5_FLOW_NAMESPACE_BYPASS;
3286                         prio = &dev->flow_db->prios[priority];
3287                         if (!dev->is_rep && !esw_encap &&
3288                             MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap))
3289                                 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
3290                         if (!dev->is_rep && !esw_encap &&
3291                             MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
3292                                         reformat_l3_tunnel_to_l2))
3293                                 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
3294                 } else {
3295                         max_table_size =
3296                                 BIT(MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
3297                                                               log_max_ft_size));
3298                         fn_type = MLX5_FLOW_NAMESPACE_EGRESS;
3299                         prio = &dev->flow_db->egress_prios[priority];
3300                         if (!dev->is_rep && !esw_encap &&
3301                             MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat))
3302                                 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
3303                 }
3304                 ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
3305                 num_entries = MLX5_FS_MAX_ENTRIES;
3306                 num_groups = MLX5_FS_MAX_TYPES;
3307         } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
3308                    flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
3309                 ns = mlx5_get_flow_namespace(dev->mdev,
3310                                              MLX5_FLOW_NAMESPACE_LEFTOVERS);
3311                 build_leftovers_ft_param(&priority,
3312                                          &num_entries,
3313                                          &num_groups);
3314                 prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];
3315         } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
3316                 if (!MLX5_CAP_FLOWTABLE(dev->mdev,
3317                                         allow_sniffer_and_nic_rx_shared_tir))
3318                         return ERR_PTR(-ENOTSUPP);
3319
3320                 ns = mlx5_get_flow_namespace(dev->mdev, ft_type == MLX5_IB_FT_RX ?
3321                                              MLX5_FLOW_NAMESPACE_SNIFFER_RX :
3322                                              MLX5_FLOW_NAMESPACE_SNIFFER_TX);
3323
3324                 prio = &dev->flow_db->sniffer[ft_type];
3325                 priority = 0;
3326                 num_entries = 1;
3327                 num_groups = 1;
3328         }
3329
3330         if (!ns)
3331                 return ERR_PTR(-ENOTSUPP);
3332
3333         max_table_size = min_t(int, num_entries, max_table_size);
3334
3335         ft = prio->flow_table;
3336         if (!ft)
3337                 return _get_prio(ns, prio, priority, max_table_size, num_groups,
3338                                  flags);
3339
3340         return prio;
3341 }
3342
3343 static void set_underlay_qp(struct mlx5_ib_dev *dev,
3344                             struct mlx5_flow_spec *spec,
3345                             u32 underlay_qpn)
3346 {
3347         void *misc_params_c = MLX5_ADDR_OF(fte_match_param,
3348                                            spec->match_criteria,
3349                                            misc_parameters);
3350         void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
3351                                            misc_parameters);
3352
3353         if (underlay_qpn &&
3354             MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
3355                                       ft_field_support.bth_dst_qp)) {
3356                 MLX5_SET(fte_match_set_misc,
3357                          misc_params_v, bth_dst_qp, underlay_qpn);
3358                 MLX5_SET(fte_match_set_misc,
3359                          misc_params_c, bth_dst_qp, 0xffffff);
3360         }
3361 }
3362
3363 static int read_flow_counters(struct ib_device *ibdev,
3364                               struct mlx5_read_counters_attr *read_attr)
3365 {
3366         struct mlx5_fc *fc = read_attr->hw_cntrs_hndl;
3367         struct mlx5_ib_dev *dev = to_mdev(ibdev);
3368
3369         return mlx5_fc_query(dev->mdev, fc,
3370                              &read_attr->out[IB_COUNTER_PACKETS],
3371                              &read_attr->out[IB_COUNTER_BYTES]);
3372 }
3373
3374 /* flow counters currently expose two counters packets and bytes */
3375 #define FLOW_COUNTERS_NUM 2
3376 static int counters_set_description(struct ib_counters *counters,
3377                                     enum mlx5_ib_counters_type counters_type,
3378                                     struct mlx5_ib_flow_counters_desc *desc_data,
3379                                     u32 ncounters)
3380 {
3381         struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
3382         u32 cntrs_max_index = 0;
3383         int i;
3384
3385         if (counters_type != MLX5_IB_COUNTERS_FLOW)
3386                 return -EINVAL;
3387
3388         /* init the fields for the object */
3389         mcounters->type = counters_type;
3390         mcounters->read_counters = read_flow_counters;
3391         mcounters->counters_num = FLOW_COUNTERS_NUM;
3392         mcounters->ncounters = ncounters;
3393         /* each counter entry have both description and index pair */
3394         for (i = 0; i < ncounters; i++) {
3395                 if (desc_data[i].description > IB_COUNTER_BYTES)
3396                         return -EINVAL;
3397
3398                 if (cntrs_max_index <= desc_data[i].index)
3399                         cntrs_max_index = desc_data[i].index + 1;
3400         }
3401
3402         mutex_lock(&mcounters->mcntrs_mutex);
3403         mcounters->counters_data = desc_data;
3404         mcounters->cntrs_max_index = cntrs_max_index;
3405         mutex_unlock(&mcounters->mcntrs_mutex);
3406
3407         return 0;
3408 }
3409
3410 #define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2))
3411 static int flow_counters_set_data(struct ib_counters *ibcounters,
3412                                   struct mlx5_ib_create_flow *ucmd)
3413 {
3414         struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters);
3415         struct mlx5_ib_flow_counters_data *cntrs_data = NULL;
3416         struct mlx5_ib_flow_counters_desc *desc_data = NULL;
3417         bool hw_hndl = false;
3418         int ret = 0;
3419
3420         if (ucmd && ucmd->ncounters_data != 0) {
3421                 cntrs_data = ucmd->data;
3422                 if (cntrs_data->ncounters > MAX_COUNTERS_NUM)
3423                         return -EINVAL;
3424
3425                 desc_data = kcalloc(cntrs_data->ncounters,
3426                                     sizeof(*desc_data),
3427                                     GFP_KERNEL);
3428                 if (!desc_data)
3429                         return  -ENOMEM;
3430
3431                 if (copy_from_user(desc_data,
3432                                    u64_to_user_ptr(cntrs_data->counters_data),
3433                                    sizeof(*desc_data) * cntrs_data->ncounters)) {
3434                         ret = -EFAULT;
3435                         goto free;
3436                 }
3437         }
3438
3439         if (!mcounters->hw_cntrs_hndl) {
3440                 mcounters->hw_cntrs_hndl = mlx5_fc_create(
3441                         to_mdev(ibcounters->device)->mdev, false);
3442                 if (IS_ERR(mcounters->hw_cntrs_hndl)) {
3443                         ret = PTR_ERR(mcounters->hw_cntrs_hndl);
3444                         goto free;
3445                 }
3446                 hw_hndl = true;
3447         }
3448
3449         if (desc_data) {
3450                 /* counters already bound to at least one flow */
3451                 if (mcounters->cntrs_max_index) {
3452                         ret = -EINVAL;
3453                         goto free_hndl;
3454                 }
3455
3456                 ret = counters_set_description(ibcounters,
3457                                                MLX5_IB_COUNTERS_FLOW,
3458                                                desc_data,
3459                                                cntrs_data->ncounters);
3460                 if (ret)
3461                         goto free_hndl;
3462
3463         } else if (!mcounters->cntrs_max_index) {
3464                 /* counters not bound yet, must have udata passed */
3465                 ret = -EINVAL;
3466                 goto free_hndl;
3467         }
3468
3469         return 0;
3470
3471 free_hndl:
3472         if (hw_hndl) {
3473                 mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev,
3474                                 mcounters->hw_cntrs_hndl);
3475                 mcounters->hw_cntrs_hndl = NULL;
3476         }
3477 free:
3478         kfree(desc_data);
3479         return ret;
3480 }
3481
3482 static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev,
3483                                          struct mlx5_flow_spec *spec,
3484                                          struct mlx5_eswitch_rep *rep)
3485 {
3486         struct mlx5_eswitch *esw = dev->mdev->priv.eswitch;
3487         void *misc;
3488
3489         if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
3490                 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
3491                                     misc_parameters_2);
3492
3493                 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
3494                          mlx5_eswitch_get_vport_metadata_for_match(esw,
3495                                                                    rep->vport));
3496                 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
3497                                     misc_parameters_2);
3498
3499                 MLX5_SET_TO_ONES(fte_match_set_misc2, misc, metadata_reg_c_0);
3500         } else {
3501                 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
3502                                     misc_parameters);
3503
3504                 MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport);
3505
3506                 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
3507                                     misc_parameters);
3508
3509                 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
3510         }
3511 }
3512
3513 static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
3514                                                       struct mlx5_ib_flow_prio *ft_prio,
3515                                                       const struct ib_flow_attr *flow_attr,
3516                                                       struct mlx5_flow_destination *dst,
3517                                                       u32 underlay_qpn,
3518                                                       struct mlx5_ib_create_flow *ucmd)
3519 {
3520         struct mlx5_flow_table  *ft = ft_prio->flow_table;
3521         struct mlx5_ib_flow_handler *handler;
3522         struct mlx5_flow_act flow_act = {};
3523         struct mlx5_flow_spec *spec;
3524         struct mlx5_flow_destination dest_arr[2] = {};
3525         struct mlx5_flow_destination *rule_dst = dest_arr;
3526         const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
3527         unsigned int spec_index;
3528         u32 prev_type = 0;
3529         int err = 0;
3530         int dest_num = 0;
3531         bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
3532
3533         if (!is_valid_attr(dev->mdev, flow_attr))
3534                 return ERR_PTR(-EINVAL);
3535
3536         if (dev->is_rep && is_egress)
3537                 return ERR_PTR(-EINVAL);
3538
3539         spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
3540         handler = kzalloc(sizeof(*handler), GFP_KERNEL);
3541         if (!handler || !spec) {
3542                 err = -ENOMEM;
3543                 goto free;
3544         }
3545
3546         INIT_LIST_HEAD(&handler->list);
3547
3548         for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
3549                 err = parse_flow_attr(dev->mdev, spec,
3550                                       ib_flow, flow_attr, &flow_act,
3551                                       prev_type);
3552                 if (err < 0)
3553                         goto free;
3554
3555                 prev_type = ((union ib_flow_spec *)ib_flow)->type;
3556                 ib_flow += ((union ib_flow_spec *)ib_flow)->size;
3557         }
3558
3559         if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) {
3560                 memcpy(&dest_arr[0], dst, sizeof(*dst));
3561                 dest_num++;
3562         }
3563
3564         if (!flow_is_multicast_only(flow_attr))
3565                 set_underlay_qp(dev, spec, underlay_qpn);
3566
3567         if (dev->is_rep) {
3568                 struct mlx5_eswitch_rep *rep;
3569
3570                 rep = dev->port[flow_attr->port - 1].rep;
3571                 if (!rep) {
3572                         err = -EINVAL;
3573                         goto free;
3574                 }
3575
3576                 mlx5_ib_set_rule_source_port(dev, spec, rep);
3577         }
3578
3579         spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
3580
3581         if (is_egress &&
3582             !is_valid_spec(dev->mdev, spec, &flow_act, is_egress)) {
3583                 err = -EINVAL;
3584                 goto free;
3585         }
3586
3587         if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
3588                 struct mlx5_ib_mcounters *mcounters;
3589
3590                 err = flow_counters_set_data(flow_act.counters, ucmd);
3591                 if (err)
3592                         goto free;
3593
3594                 mcounters = to_mcounters(flow_act.counters);
3595                 handler->ibcounters = flow_act.counters;
3596                 dest_arr[dest_num].type =
3597                         MLX5_FLOW_DESTINATION_TYPE_COUNTER;
3598                 dest_arr[dest_num].counter_id =
3599                         mlx5_fc_id(mcounters->hw_cntrs_hndl);
3600                 dest_num++;
3601         }
3602
3603         if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) {
3604                 if (!dest_num)
3605                         rule_dst = NULL;
3606         } else {
3607                 if (is_egress)
3608                         flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
3609                 else
3610                         flow_act.action |=
3611                                 dest_num ?  MLX5_FLOW_CONTEXT_ACTION_FWD_DEST :
3612                                         MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
3613         }
3614
3615         if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG)  &&
3616             (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
3617              flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
3618                 mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",
3619                              spec->flow_context.flow_tag, flow_attr->type);
3620                 err = -EINVAL;
3621                 goto free;
3622         }
3623         handler->rule = mlx5_add_flow_rules(ft, spec,
3624                                             &flow_act,
3625                                             rule_dst, dest_num);
3626
3627         if (IS_ERR(handler->rule)) {
3628                 err = PTR_ERR(handler->rule);
3629                 goto free;
3630         }
3631
3632         ft_prio->refcount++;
3633         handler->prio = ft_prio;
3634         handler->dev = dev;
3635
3636         ft_prio->flow_table = ft;
3637 free:
3638         if (err && handler) {
3639                 if (handler->ibcounters &&
3640                     atomic_read(&handler->ibcounters->usecnt) == 1)
3641                         counters_clear_description(handler->ibcounters);
3642                 kfree(handler);
3643         }
3644         kvfree(spec);
3645         return err ? ERR_PTR(err) : handler;
3646 }
3647
3648 static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
3649                                                      struct mlx5_ib_flow_prio *ft_prio,
3650                                                      const struct ib_flow_attr *flow_attr,
3651                                                      struct mlx5_flow_destination *dst)
3652 {
3653         return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL);
3654 }
3655
3656 static struct mlx5_ib_flow_handler *create_dont_trap_rule(struct mlx5_ib_dev *dev,
3657                                                           struct mlx5_ib_flow_prio *ft_prio,
3658                                                           struct ib_flow_attr *flow_attr,
3659                                                           struct mlx5_flow_destination *dst)
3660 {
3661         struct mlx5_ib_flow_handler *handler_dst = NULL;
3662         struct mlx5_ib_flow_handler *handler = NULL;
3663
3664         handler = create_flow_rule(dev, ft_prio, flow_attr, NULL);
3665         if (!IS_ERR(handler)) {
3666                 handler_dst = create_flow_rule(dev, ft_prio,
3667                                                flow_attr, dst);
3668                 if (IS_ERR(handler_dst)) {
3669                         mlx5_del_flow_rules(handler->rule);
3670                         ft_prio->refcount--;
3671                         kfree(handler);
3672                         handler = handler_dst;
3673                 } else {
3674                         list_add(&handler_dst->list, &handler->list);
3675                 }
3676         }
3677
3678         return handler;
3679 }
3680 enum {
3681         LEFTOVERS_MC,
3682         LEFTOVERS_UC,
3683 };
3684
3685 static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev,
3686                                                           struct mlx5_ib_flow_prio *ft_prio,
3687                                                           struct ib_flow_attr *flow_attr,
3688                                                           struct mlx5_flow_destination *dst)
3689 {
3690         struct mlx5_ib_flow_handler *handler_ucast = NULL;
3691         struct mlx5_ib_flow_handler *handler = NULL;
3692
3693         static struct {
3694                 struct ib_flow_attr     flow_attr;
3695                 struct ib_flow_spec_eth eth_flow;
3696         } leftovers_specs[] = {
3697                 [LEFTOVERS_MC] = {
3698                         .flow_attr = {
3699                                 .num_of_specs = 1,
3700                                 .size = sizeof(leftovers_specs[0])
3701                         },
3702                         .eth_flow = {
3703                                 .type = IB_FLOW_SPEC_ETH,
3704                                 .size = sizeof(struct ib_flow_spec_eth),
3705                                 .mask = {.dst_mac = {0x1} },
3706                                 .val =  {.dst_mac = {0x1} }
3707                         }
3708                 },
3709                 [LEFTOVERS_UC] = {
3710                         .flow_attr = {
3711                                 .num_of_specs = 1,
3712                                 .size = sizeof(leftovers_specs[0])
3713                         },
3714                         .eth_flow = {
3715                                 .type = IB_FLOW_SPEC_ETH,
3716                                 .size = sizeof(struct ib_flow_spec_eth),
3717                                 .mask = {.dst_mac = {0x1} },
3718                                 .val = {.dst_mac = {} }
3719                         }
3720                 }
3721         };
3722
3723         handler = create_flow_rule(dev, ft_prio,
3724                                    &leftovers_specs[LEFTOVERS_MC].flow_attr,
3725                                    dst);
3726         if (!IS_ERR(handler) &&
3727             flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
3728                 handler_ucast = create_flow_rule(dev, ft_prio,
3729                                                  &leftovers_specs[LEFTOVERS_UC].flow_attr,
3730                                                  dst);
3731                 if (IS_ERR(handler_ucast)) {
3732                         mlx5_del_flow_rules(handler->rule);
3733                         ft_prio->refcount--;
3734                         kfree(handler);
3735                         handler = handler_ucast;
3736                 } else {
3737                         list_add(&handler_ucast->list, &handler->list);
3738                 }
3739         }
3740
3741         return handler;
3742 }
3743
3744 static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,
3745                                                         struct mlx5_ib_flow_prio *ft_rx,
3746                                                         struct mlx5_ib_flow_prio *ft_tx,
3747                                                         struct mlx5_flow_destination *dst)
3748 {
3749         struct mlx5_ib_flow_handler *handler_rx;
3750         struct mlx5_ib_flow_handler *handler_tx;
3751         int err;
3752         static const struct ib_flow_attr flow_attr  = {
3753                 .num_of_specs = 0,
3754                 .size = sizeof(flow_attr)
3755         };
3756
3757         handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst);
3758         if (IS_ERR(handler_rx)) {
3759                 err = PTR_ERR(handler_rx);
3760                 goto err;
3761         }
3762
3763         handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst);
3764         if (IS_ERR(handler_tx)) {
3765                 err = PTR_ERR(handler_tx);
3766                 goto err_tx;
3767         }
3768
3769         list_add(&handler_tx->list, &handler_rx->list);
3770
3771         return handler_rx;
3772
3773 err_tx:
3774         mlx5_del_flow_rules(handler_rx->rule);
3775         ft_rx->refcount--;
3776         kfree(handler_rx);
3777 err:
3778         return ERR_PTR(err);
3779 }
3780
3781 static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
3782                                            struct ib_flow_attr *flow_attr,
3783                                            int domain,
3784                                            struct ib_udata *udata)
3785 {
3786         struct mlx5_ib_dev *dev = to_mdev(qp->device);
3787         struct mlx5_ib_qp *mqp = to_mqp(qp);
3788         struct mlx5_ib_flow_handler *handler = NULL;
3789         struct mlx5_flow_destination *dst = NULL;
3790         struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
3791         struct mlx5_ib_flow_prio *ft_prio;
3792         bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
3793         struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr;
3794         size_t min_ucmd_sz, required_ucmd_sz;
3795         int err;
3796         int underlay_qpn;
3797
3798         if (udata && udata->inlen) {
3799                 min_ucmd_sz = offsetof(typeof(ucmd_hdr), reserved) +
3800                                 sizeof(ucmd_hdr.reserved);
3801                 if (udata->inlen < min_ucmd_sz)
3802                         return ERR_PTR(-EOPNOTSUPP);
3803
3804                 err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz);
3805                 if (err)
3806                         return ERR_PTR(err);
3807
3808                 /* currently supports only one counters data */
3809                 if (ucmd_hdr.ncounters_data > 1)
3810                         return ERR_PTR(-EINVAL);
3811
3812                 required_ucmd_sz = min_ucmd_sz +
3813                         sizeof(struct mlx5_ib_flow_counters_data) *
3814                         ucmd_hdr.ncounters_data;
3815                 if (udata->inlen > required_ucmd_sz &&
3816                     !ib_is_udata_cleared(udata, required_ucmd_sz,
3817                                          udata->inlen - required_ucmd_sz))
3818                         return ERR_PTR(-EOPNOTSUPP);
3819
3820                 ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL);
3821                 if (!ucmd)
3822                         return ERR_PTR(-ENOMEM);
3823
3824                 err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz);
3825                 if (err)
3826                         goto free_ucmd;
3827         }
3828
3829         if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) {
3830                 err = -ENOMEM;
3831                 goto free_ucmd;
3832         }
3833
3834         if (domain != IB_FLOW_DOMAIN_USER ||
3835             flow_attr->port > dev->num_ports ||
3836             (flow_attr->flags & ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP |
3837                                   IB_FLOW_ATTR_FLAGS_EGRESS))) {
3838                 err = -EINVAL;
3839                 goto free_ucmd;
3840         }
3841
3842         if (is_egress &&
3843             (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
3844              flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
3845                 err = -EINVAL;
3846                 goto free_ucmd;
3847         }
3848
3849         dst = kzalloc(sizeof(*dst), GFP_KERNEL);
3850         if (!dst) {
3851                 err = -ENOMEM;
3852                 goto free_ucmd;
3853         }
3854
3855         mutex_lock(&dev->flow_db->lock);
3856
3857         ft_prio = get_flow_table(dev, flow_attr,
3858                                  is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX);
3859         if (IS_ERR(ft_prio)) {
3860                 err = PTR_ERR(ft_prio);
3861                 goto unlock;
3862         }
3863         if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
3864                 ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);
3865                 if (IS_ERR(ft_prio_tx)) {
3866                         err = PTR_ERR(ft_prio_tx);
3867                         ft_prio_tx = NULL;
3868                         goto destroy_ft;
3869                 }
3870         }
3871
3872         if (is_egress) {
3873                 dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT;
3874         } else {
3875                 dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
3876                 if (mqp->flags & MLX5_IB_QP_RSS)
3877                         dst->tir_num = mqp->rss_qp.tirn;
3878                 else
3879                         dst->tir_num = mqp->raw_packet_qp.rq.tirn;
3880         }
3881
3882         if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
3883                 if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP)  {
3884                         handler = create_dont_trap_rule(dev, ft_prio,
3885                                                         flow_attr, dst);
3886                 } else {
3887                         underlay_qpn = (mqp->flags & MLX5_IB_QP_UNDERLAY) ?
3888                                         mqp->underlay_qpn : 0;
3889                         handler = _create_flow_rule(dev, ft_prio, flow_attr,
3890                                                     dst, underlay_qpn, ucmd);
3891                 }
3892         } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
3893                    flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
3894                 handler = create_leftovers_rule(dev, ft_prio, flow_attr,
3895                                                 dst);
3896         } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
3897                 handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);
3898         } else {
3899                 err = -EINVAL;
3900                 goto destroy_ft;
3901         }
3902
3903         if (IS_ERR(handler)) {
3904                 err = PTR_ERR(handler);
3905                 handler = NULL;
3906                 goto destroy_ft;
3907         }
3908
3909         mutex_unlock(&dev->flow_db->lock);
3910         kfree(dst);
3911         kfree(ucmd);
3912
3913         return &handler->ibflow;
3914
3915 destroy_ft:
3916         put_flow_table(dev, ft_prio, false);
3917         if (ft_prio_tx)
3918                 put_flow_table(dev, ft_prio_tx, false);
3919 unlock:
3920         mutex_unlock(&dev->flow_db->lock);
3921         kfree(dst);
3922 free_ucmd:
3923         kfree(ucmd);
3924         return ERR_PTR(err);
3925 }
3926
3927 static struct mlx5_ib_flow_prio *
3928 _get_flow_table(struct mlx5_ib_dev *dev,
3929                 struct mlx5_ib_flow_matcher *fs_matcher,
3930                 bool mcast)
3931 {
3932         struct mlx5_flow_namespace *ns = NULL;
3933         struct mlx5_ib_flow_prio *prio = NULL;
3934         int max_table_size = 0;
3935         bool esw_encap;
3936         u32 flags = 0;
3937         int priority;
3938
3939         if (mcast)
3940                 priority = MLX5_IB_FLOW_MCAST_PRIO;
3941         else
3942                 priority = ib_prio_to_core_prio(fs_matcher->priority, false);
3943
3944         esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
3945                 DEVLINK_ESWITCH_ENCAP_MODE_NONE;
3946         if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
3947                 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
3948                                         log_max_ft_size));
3949                 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap)
3950                         flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
3951                 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
3952                                               reformat_l3_tunnel_to_l2) &&
3953                     !esw_encap)
3954                         flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
3955         } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) {
3956                 max_table_size = BIT(
3957                         MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size));
3958                 if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && !esw_encap)
3959                         flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
3960         } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) {
3961                 max_table_size = BIT(
3962                         MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size));
3963                 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap)
3964                         flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
3965                 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, reformat_l3_tunnel_to_l2) &&
3966                     esw_encap)
3967                         flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
3968                 priority = FDB_BYPASS_PATH;
3969         } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) {
3970                 max_table_size =
3971                         BIT(MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
3972                                                        log_max_ft_size));
3973                 priority = fs_matcher->priority;
3974         }
3975
3976         max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES);
3977
3978         ns = mlx5_get_flow_namespace(dev->mdev, fs_matcher->ns_type);
3979         if (!ns)
3980                 return ERR_PTR(-ENOTSUPP);
3981
3982         if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS)
3983                 prio = &dev->flow_db->prios[priority];
3984         else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS)
3985                 prio = &dev->flow_db->egress_prios[priority];
3986         else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB)
3987                 prio = &dev->flow_db->fdb;
3988         else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX)
3989                 prio = &dev->flow_db->rdma_rx[priority];
3990
3991         if (!prio)
3992                 return ERR_PTR(-EINVAL);
3993
3994         if (prio->flow_table)
3995                 return prio;
3996
3997         return _get_prio(ns, prio, priority, max_table_size,
3998                          MLX5_FS_MAX_TYPES, flags);
3999 }
4000
4001 static struct mlx5_ib_flow_handler *
4002 _create_raw_flow_rule(struct mlx5_ib_dev *dev,
4003                       struct mlx5_ib_flow_prio *ft_prio,
4004                       struct mlx5_flow_destination *dst,
4005                       struct mlx5_ib_flow_matcher  *fs_matcher,
4006                       struct mlx5_flow_context *flow_context,
4007                       struct mlx5_flow_act *flow_act,
4008                       void *cmd_in, int inlen,
4009                       int dst_num)
4010 {
4011         struct mlx5_ib_flow_handler *handler;
4012         struct mlx5_flow_spec *spec;
4013         struct mlx5_flow_table *ft = ft_prio->flow_table;
4014         int err = 0;
4015
4016         spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
4017         handler = kzalloc(sizeof(*handler), GFP_KERNEL);
4018         if (!handler || !spec) {
4019                 err = -ENOMEM;
4020                 goto free;
4021         }
4022
4023         INIT_LIST_HEAD(&handler->list);
4024
4025         memcpy(spec->match_value, cmd_in, inlen);
4026         memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params,
4027                fs_matcher->mask_len);
4028         spec->match_criteria_enable = fs_matcher->match_criteria_enable;
4029         spec->flow_context = *flow_context;
4030
4031         handler->rule = mlx5_add_flow_rules(ft, spec,
4032                                             flow_act, dst, dst_num);
4033
4034         if (IS_ERR(handler->rule)) {
4035                 err = PTR_ERR(handler->rule);
4036                 goto free;
4037         }
4038
4039         ft_prio->refcount++;
4040         handler->prio = ft_prio;
4041         handler->dev = dev;
4042         ft_prio->flow_table = ft;
4043
4044 free:
4045         if (err)
4046                 kfree(handler);
4047         kvfree(spec);
4048         return err ? ERR_PTR(err) : handler;
4049 }
4050
4051 static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher,
4052                                 void *match_v)
4053 {
4054         void *match_c;
4055         void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4;
4056         void *dmac, *dmac_mask;
4057         void *ipv4, *ipv4_mask;
4058
4059         if (!(fs_matcher->match_criteria_enable &
4060               (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT)))
4061                 return false;
4062
4063         match_c = fs_matcher->matcher_mask.match_params;
4064         match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v,
4065                                            outer_headers);
4066         match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c,
4067                                            outer_headers);
4068
4069         dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
4070                             dmac_47_16);
4071         dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
4072                                  dmac_47_16);
4073
4074         if (is_multicast_ether_addr(dmac) &&
4075             is_multicast_ether_addr(dmac_mask))
4076                 return true;
4077
4078         ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
4079                             dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
4080
4081         ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
4082                                  dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
4083
4084         if (ipv4_is_multicast(*(__be32 *)(ipv4)) &&
4085             ipv4_is_multicast(*(__be32 *)(ipv4_mask)))
4086                 return true;
4087
4088         return false;
4089 }
4090
4091 struct mlx5_ib_flow_handler *
4092 mlx5_ib_raw_fs_rule_add(struct mlx5_ib_dev *dev,
4093                         struct mlx5_ib_flow_matcher *fs_matcher,
4094                         struct mlx5_flow_context *flow_context,
4095                         struct mlx5_flow_act *flow_act,
4096                         u32 counter_id,
4097                         void *cmd_in, int inlen, int dest_id,
4098                         int dest_type)
4099 {
4100         struct mlx5_flow_destination *dst;
4101         struct mlx5_ib_flow_prio *ft_prio;
4102         struct mlx5_ib_flow_handler *handler;
4103         int dst_num = 0;
4104         bool mcast;
4105         int err;
4106
4107         if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL)
4108                 return ERR_PTR(-EOPNOTSUPP);
4109
4110         if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO)
4111                 return ERR_PTR(-ENOMEM);
4112
4113         dst = kcalloc(2, sizeof(*dst), GFP_KERNEL);
4114         if (!dst)
4115                 return ERR_PTR(-ENOMEM);
4116
4117         mcast = raw_fs_is_multicast(fs_matcher, cmd_in);
4118         mutex_lock(&dev->flow_db->lock);
4119
4120         ft_prio = _get_flow_table(dev, fs_matcher, mcast);
4121         if (IS_ERR(ft_prio)) {
4122                 err = PTR_ERR(ft_prio);
4123                 goto unlock;
4124         }
4125
4126         if (dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR) {
4127                 dst[dst_num].type = dest_type;
4128                 dst[dst_num].tir_num = dest_id;
4129                 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
4130         } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) {
4131                 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM;
4132                 dst[dst_num].ft_num = dest_id;
4133                 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
4134         } else {
4135                 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_PORT;
4136                 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
4137         }
4138
4139         dst_num++;
4140
4141         if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
4142                 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
4143                 dst[dst_num].counter_id = counter_id;
4144                 dst_num++;
4145         }
4146
4147         handler = _create_raw_flow_rule(dev, ft_prio, dst, fs_matcher,
4148                                         flow_context, flow_act,
4149                                         cmd_in, inlen, dst_num);
4150
4151         if (IS_ERR(handler)) {
4152                 err = PTR_ERR(handler);
4153                 goto destroy_ft;
4154         }
4155
4156         mutex_unlock(&dev->flow_db->lock);
4157         atomic_inc(&fs_matcher->usecnt);
4158         handler->flow_matcher = fs_matcher;
4159
4160         kfree(dst);
4161
4162         return handler;
4163
4164 destroy_ft:
4165         put_flow_table(dev, ft_prio, false);
4166 unlock:
4167         mutex_unlock(&dev->flow_db->lock);
4168         kfree(dst);
4169
4170         return ERR_PTR(err);
4171 }
4172
4173 static u32 mlx5_ib_flow_action_flags_to_accel_xfrm_flags(u32 mlx5_flags)
4174 {
4175         u32 flags = 0;
4176
4177         if (mlx5_flags & MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA)
4178                 flags |= MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA;
4179
4180         return flags;
4181 }
4182
4183 #define MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED      MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA
4184 static struct ib_flow_action *
4185 mlx5_ib_create_flow_action_esp(struct ib_device *device,
4186                                const struct ib_flow_action_attrs_esp *attr,
4187                                struct uverbs_attr_bundle *attrs)
4188 {
4189         struct mlx5_ib_dev *mdev = to_mdev(device);
4190         struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm;
4191         struct mlx5_accel_esp_xfrm_attrs accel_attrs = {};
4192         struct mlx5_ib_flow_action *action;
4193         u64 action_flags;
4194         u64 flags;
4195         int err = 0;
4196
4197         err = uverbs_get_flags64(
4198                 &action_flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS,
4199                 ((MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED << 1) - 1));
4200         if (err)
4201                 return ERR_PTR(err);
4202
4203         flags = mlx5_ib_flow_action_flags_to_accel_xfrm_flags(action_flags);
4204
4205         /* We current only support a subset of the standard features. Only a
4206          * keymat of type AES_GCM, with icv_len == 16, iv_algo == SEQ and esn
4207          * (with overlap). Full offload mode isn't supported.
4208          */
4209         if (!attr->keymat || attr->replay || attr->encap ||
4210             attr->spi || attr->seq || attr->tfc_pad ||
4211             attr->hard_limit_pkts ||
4212             (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |
4213                              IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)))
4214                 return ERR_PTR(-EOPNOTSUPP);
4215
4216         if (attr->keymat->protocol !=
4217             IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM)
4218                 return ERR_PTR(-EOPNOTSUPP);
4219
4220         aes_gcm = &attr->keymat->keymat.aes_gcm;
4221
4222         if (aes_gcm->icv_len != 16 ||
4223             aes_gcm->iv_algo != IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ)
4224                 return ERR_PTR(-EOPNOTSUPP);
4225
4226         action = kmalloc(sizeof(*action), GFP_KERNEL);
4227         if (!action)
4228                 return ERR_PTR(-ENOMEM);
4229
4230         action->esp_aes_gcm.ib_flags = attr->flags;
4231         memcpy(&accel_attrs.keymat.aes_gcm.aes_key, &aes_gcm->aes_key,
4232                sizeof(accel_attrs.keymat.aes_gcm.aes_key));
4233         accel_attrs.keymat.aes_gcm.key_len = aes_gcm->key_len * 8;
4234         memcpy(&accel_attrs.keymat.aes_gcm.salt, &aes_gcm->salt,
4235                sizeof(accel_attrs.keymat.aes_gcm.salt));
4236         memcpy(&accel_attrs.keymat.aes_gcm.seq_iv, &aes_gcm->iv,
4237                sizeof(accel_attrs.keymat.aes_gcm.seq_iv));
4238         accel_attrs.keymat.aes_gcm.icv_len = aes_gcm->icv_len * 8;
4239         accel_attrs.keymat.aes_gcm.iv_algo = MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ;
4240         accel_attrs.keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM;
4241
4242         accel_attrs.esn = attr->esn;
4243         if (attr->flags & IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED)
4244                 accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED;
4245         if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)
4246                 accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
4247
4248         if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)
4249                 accel_attrs.action |= MLX5_ACCEL_ESP_ACTION_ENCRYPT;
4250
4251         action->esp_aes_gcm.ctx =
4252                 mlx5_accel_esp_create_xfrm(mdev->mdev, &accel_attrs, flags);
4253         if (IS_ERR(action->esp_aes_gcm.ctx)) {
4254                 err = PTR_ERR(action->esp_aes_gcm.ctx);
4255                 goto err_parse;
4256         }
4257
4258         action->esp_aes_gcm.ib_flags = attr->flags;
4259
4260         return &action->ib_action;
4261
4262 err_parse:
4263         kfree(action);
4264         return ERR_PTR(err);
4265 }
4266
4267 static int
4268 mlx5_ib_modify_flow_action_esp(struct ib_flow_action *action,
4269                                const struct ib_flow_action_attrs_esp *attr,
4270                                struct uverbs_attr_bundle *attrs)
4271 {
4272         struct mlx5_ib_flow_action *maction = to_mflow_act(action);
4273         struct mlx5_accel_esp_xfrm_attrs accel_attrs;
4274         int err = 0;
4275
4276         if (attr->keymat || attr->replay || attr->encap ||
4277             attr->spi || attr->seq || attr->tfc_pad ||
4278             attr->hard_limit_pkts ||
4279             (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |
4280                              IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS |
4281                              IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)))
4282                 return -EOPNOTSUPP;
4283
4284         /* Only the ESN value or the MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP can
4285          * be modified.
4286          */
4287         if (!(maction->esp_aes_gcm.ib_flags &
4288               IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) &&
4289             attr->flags & (IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |
4290                            IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW))
4291                 return -EINVAL;
4292
4293         memcpy(&accel_attrs, &maction->esp_aes_gcm.ctx->attrs,
4294                sizeof(accel_attrs));
4295
4296         accel_attrs.esn = attr->esn;
4297         if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)
4298                 accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
4299         else
4300                 accel_attrs.flags &= ~MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
4301
4302         err = mlx5_accel_esp_modify_xfrm(maction->esp_aes_gcm.ctx,
4303                                          &accel_attrs);
4304         if (err)
4305                 return err;
4306
4307         maction->esp_aes_gcm.ib_flags &=
4308                 ~IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;
4309         maction->esp_aes_gcm.ib_flags |=
4310                 attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;
4311
4312         return 0;
4313 }
4314
4315 static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action)
4316 {
4317         struct mlx5_ib_flow_action *maction = to_mflow_act(action);
4318
4319         switch (action->type) {
4320         case IB_FLOW_ACTION_ESP:
4321                 /*
4322                  * We only support aes_gcm by now, so we implicitly know this is
4323                  * the underline crypto.
4324                  */
4325                 mlx5_accel_esp_destroy_xfrm(maction->esp_aes_gcm.ctx);
4326                 break;
4327         case IB_FLOW_ACTION_UNSPECIFIED:
4328                 mlx5_ib_destroy_flow_action_raw(maction);
4329                 break;
4330         default:
4331                 WARN_ON(true);
4332                 break;
4333         }
4334
4335         kfree(maction);
4336         return 0;
4337 }
4338
4339 static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
4340 {
4341         struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4342         struct mlx5_ib_qp *mqp = to_mqp(ibqp);
4343         int err;
4344         u16 uid;
4345
4346         uid = ibqp->pd ?
4347                 to_mpd(ibqp->pd)->uid : 0;
4348
4349         if (mqp->flags & MLX5_IB_QP_UNDERLAY) {
4350                 mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n");
4351                 return -EOPNOTSUPP;
4352         }
4353
4354         err = mlx5_cmd_attach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
4355         if (err)
4356                 mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
4357                              ibqp->qp_num, gid->raw);
4358
4359         return err;
4360 }
4361
4362 static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
4363 {
4364         struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4365         int err;
4366         u16 uid;
4367
4368         uid = ibqp->pd ?
4369                 to_mpd(ibqp->pd)->uid : 0;
4370         err = mlx5_cmd_detach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
4371         if (err)
4372                 mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
4373                              ibqp->qp_num, gid->raw);
4374
4375         return err;
4376 }
4377
4378 static int init_node_data(struct mlx5_ib_dev *dev)
4379 {
4380         int err;
4381
4382         err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc);
4383         if (err)
4384                 return err;
4385
4386         dev->mdev->rev_id = dev->mdev->pdev->revision;
4387
4388         return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid);
4389 }
4390
4391 static ssize_t fw_pages_show(struct device *device,
4392                              struct device_attribute *attr, char *buf)
4393 {
4394         struct mlx5_ib_dev *dev =
4395                 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
4396
4397         return sprintf(buf, "%d\n", dev->mdev->priv.fw_pages);
4398 }
4399 static DEVICE_ATTR_RO(fw_pages);
4400
4401 static ssize_t reg_pages_show(struct device *device,
4402                               struct device_attribute *attr, char *buf)
4403 {
4404         struct mlx5_ib_dev *dev =
4405                 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
4406
4407         return sprintf(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
4408 }
4409 static DEVICE_ATTR_RO(reg_pages);
4410
4411 static ssize_t hca_type_show(struct device *device,
4412                              struct device_attribute *attr, char *buf)
4413 {
4414         struct mlx5_ib_dev *dev =
4415                 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
4416
4417         return sprintf(buf, "MT%d\n", dev->mdev->pdev->device);
4418 }
4419 static DEVICE_ATTR_RO(hca_type);
4420
4421 static ssize_t hw_rev_show(struct device *device,
4422                            struct device_attribute *attr, char *buf)
4423 {
4424         struct mlx5_ib_dev *dev =
4425                 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
4426
4427         return sprintf(buf, "%x\n", dev->mdev->rev_id);
4428 }
4429 static DEVICE_ATTR_RO(hw_rev);
4430
4431 static ssize_t board_id_show(struct device *device,
4432                              struct device_attribute *attr, char *buf)
4433 {
4434         struct mlx5_ib_dev *dev =
4435                 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
4436
4437         return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
4438                        dev->mdev->board_id);
4439 }
4440 static DEVICE_ATTR_RO(board_id);
4441
4442 static struct attribute *mlx5_class_attributes[] = {
4443         &dev_attr_hw_rev.attr,
4444         &dev_attr_hca_type.attr,
4445         &dev_attr_board_id.attr,
4446         &dev_attr_fw_pages.attr,
4447         &dev_attr_reg_pages.attr,
4448         NULL,
4449 };
4450
4451 static const struct attribute_group mlx5_attr_group = {
4452         .attrs = mlx5_class_attributes,
4453 };
4454
4455 static void pkey_change_handler(struct work_struct *work)
4456 {
4457         struct mlx5_ib_port_resources *ports =
4458                 container_of(work, struct mlx5_ib_port_resources,
4459                              pkey_change_work);
4460
4461         mutex_lock(&ports->devr->mutex);
4462         mlx5_ib_gsi_pkey_change(ports->gsi);
4463         mutex_unlock(&ports->devr->mutex);
4464 }
4465
4466 static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
4467 {
4468         struct mlx5_ib_qp *mqp;
4469         struct mlx5_ib_cq *send_mcq, *recv_mcq;
4470         struct mlx5_core_cq *mcq;
4471         struct list_head cq_armed_list;
4472         unsigned long flags_qp;
4473         unsigned long flags_cq;
4474         unsigned long flags;
4475
4476         INIT_LIST_HEAD(&cq_armed_list);
4477
4478         /* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
4479         spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags);
4480         list_for_each_entry(mqp, &ibdev->qp_list, qps_list) {
4481                 spin_lock_irqsave(&mqp->sq.lock, flags_qp);
4482                 if (mqp->sq.tail != mqp->sq.head) {
4483                         send_mcq = to_mcq(mqp->ibqp.send_cq);
4484                         spin_lock_irqsave(&send_mcq->lock, flags_cq);
4485                         if (send_mcq->mcq.comp &&
4486                             mqp->ibqp.send_cq->comp_handler) {
4487                                 if (!send_mcq->mcq.reset_notify_added) {
4488                                         send_mcq->mcq.reset_notify_added = 1;
4489                                         list_add_tail(&send_mcq->mcq.reset_notify,
4490                                                       &cq_armed_list);
4491                                 }
4492                         }
4493                         spin_unlock_irqrestore(&send_mcq->lock, flags_cq);
4494                 }
4495                 spin_unlock_irqrestore(&mqp->sq.lock, flags_qp);
4496                 spin_lock_irqsave(&mqp->rq.lock, flags_qp);
4497                 /* no handling is needed for SRQ */
4498                 if (!mqp->ibqp.srq) {
4499                         if (mqp->rq.tail != mqp->rq.head) {
4500                                 recv_mcq = to_mcq(mqp->ibqp.recv_cq);
4501                                 spin_lock_irqsave(&recv_mcq->lock, flags_cq);
4502                                 if (recv_mcq->mcq.comp &&
4503                                     mqp->ibqp.recv_cq->comp_handler) {
4504                                         if (!recv_mcq->mcq.reset_notify_added) {
4505                                                 recv_mcq->mcq.reset_notify_added = 1;
4506                                                 list_add_tail(&recv_mcq->mcq.reset_notify,
4507                                                               &cq_armed_list);
4508                                         }
4509                                 }
4510                                 spin_unlock_irqrestore(&recv_mcq->lock,
4511                                                        flags_cq);
4512                         }
4513                 }
4514                 spin_unlock_irqrestore(&mqp->rq.lock, flags_qp);
4515         }
4516         /*At that point all inflight post send were put to be executed as of we
4517          * lock/unlock above locks Now need to arm all involved CQs.
4518          */
4519         list_for_each_entry(mcq, &cq_armed_list, reset_notify) {
4520                 mcq->comp(mcq, NULL);
4521         }
4522         spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);
4523 }
4524
4525 static void delay_drop_handler(struct work_struct *work)
4526 {
4527         int err;
4528         struct mlx5_ib_delay_drop *delay_drop =
4529                 container_of(work, struct mlx5_ib_delay_drop,
4530                              delay_drop_work);
4531
4532         atomic_inc(&delay_drop->events_cnt);
4533
4534         mutex_lock(&delay_drop->lock);
4535         err = mlx5_core_set_delay_drop(delay_drop->dev->mdev,
4536                                        delay_drop->timeout);
4537         if (err) {
4538                 mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n",
4539                              delay_drop->timeout);
4540                 delay_drop->activate = false;
4541         }
4542         mutex_unlock(&delay_drop->lock);
4543 }
4544
4545 static void handle_general_event(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,
4546                                  struct ib_event *ibev)
4547 {
4548         u8 port = (eqe->data.port.port >> 4) & 0xf;
4549
4550         switch (eqe->sub_type) {
4551         case MLX5_GENERAL_SUBTYPE_DELAY_DROP_TIMEOUT:
4552                 if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
4553                                             IB_LINK_LAYER_ETHERNET)
4554                         schedule_work(&ibdev->delay_drop.delay_drop_work);
4555                 break;
4556         default: /* do nothing */
4557                 return;
4558         }
4559 }
4560
4561 static int handle_port_change(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,
4562                               struct ib_event *ibev)
4563 {
4564         u8 port = (eqe->data.port.port >> 4) & 0xf;
4565
4566         ibev->element.port_num = port;
4567
4568         switch (eqe->sub_type) {
4569         case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
4570         case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
4571         case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
4572                 /* In RoCE, port up/down events are handled in
4573                  * mlx5_netdev_event().
4574                  */
4575                 if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
4576                                             IB_LINK_LAYER_ETHERNET)
4577                         return -EINVAL;
4578
4579                 ibev->event = (eqe->sub_type == MLX5_PORT_CHANGE_SUBTYPE_ACTIVE) ?
4580                                 IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
4581                 break;
4582
4583         case MLX5_PORT_CHANGE_SUBTYPE_LID:
4584                 ibev->event = IB_EVENT_LID_CHANGE;
4585                 break;
4586
4587         case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
4588                 ibev->event = IB_EVENT_PKEY_CHANGE;
4589                 schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
4590                 break;
4591
4592         case MLX5_PORT_CHANGE_SUBTYPE_GUID:
4593                 ibev->event = IB_EVENT_GID_CHANGE;
4594                 break;
4595
4596         case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
4597                 ibev->event = IB_EVENT_CLIENT_REREGISTER;
4598                 break;
4599         default:
4600                 return -EINVAL;
4601         }
4602
4603         return 0;
4604 }
4605
4606 static void mlx5_ib_handle_event(struct work_struct *_work)
4607 {
4608         struct mlx5_ib_event_work *work =
4609                 container_of(_work, struct mlx5_ib_event_work, work);
4610         struct mlx5_ib_dev *ibdev;
4611         struct ib_event ibev;
4612         bool fatal = false;
4613
4614         if (work->is_slave) {
4615                 ibdev = mlx5_ib_get_ibdev_from_mpi(work->mpi);
4616                 if (!ibdev)
4617                         goto out;
4618         } else {
4619                 ibdev = work->dev;
4620         }
4621
4622         switch (work->event) {
4623         case MLX5_DEV_EVENT_SYS_ERROR:
4624                 ibev.event = IB_EVENT_DEVICE_FATAL;
4625                 mlx5_ib_handle_internal_error(ibdev);
4626                 ibev.element.port_num  = (u8)(unsigned long)work->param;
4627                 fatal = true;
4628                 break;
4629         case MLX5_EVENT_TYPE_PORT_CHANGE:
4630                 if (handle_port_change(ibdev, work->param, &ibev))
4631                         goto out;
4632                 break;
4633         case MLX5_EVENT_TYPE_GENERAL_EVENT:
4634                 handle_general_event(ibdev, work->param, &ibev);
4635                 /* fall through */
4636         default:
4637                 goto out;
4638         }
4639
4640         ibev.device = &ibdev->ib_dev;
4641
4642         if (!rdma_is_port_valid(&ibdev->ib_dev, ibev.element.port_num)) {
4643                 mlx5_ib_warn(ibdev, "warning: event on port %d\n",  ibev.element.port_num);
4644                 goto out;
4645         }
4646
4647         if (ibdev->ib_active)
4648                 ib_dispatch_event(&ibev);
4649
4650         if (fatal)
4651                 ibdev->ib_active = false;
4652 out:
4653         kfree(work);
4654 }
4655
4656 static int mlx5_ib_event(struct notifier_block *nb,
4657                          unsigned long event, void *param)
4658 {
4659         struct mlx5_ib_event_work *work;
4660
4661         work = kmalloc(sizeof(*work), GFP_ATOMIC);
4662         if (!work)
4663                 return NOTIFY_DONE;
4664
4665         INIT_WORK(&work->work, mlx5_ib_handle_event);
4666         work->dev = container_of(nb, struct mlx5_ib_dev, mdev_events);
4667         work->is_slave = false;
4668         work->param = param;
4669         work->event = event;
4670
4671         queue_work(mlx5_ib_event_wq, &work->work);
4672
4673         return NOTIFY_OK;
4674 }
4675
4676 static int mlx5_ib_event_slave_port(struct notifier_block *nb,
4677                                     unsigned long event, void *param)
4678 {
4679         struct mlx5_ib_event_work *work;
4680
4681         work = kmalloc(sizeof(*work), GFP_ATOMIC);
4682         if (!work)
4683                 return NOTIFY_DONE;
4684
4685         INIT_WORK(&work->work, mlx5_ib_handle_event);
4686         work->mpi = container_of(nb, struct mlx5_ib_multiport_info, mdev_events);
4687         work->is_slave = true;
4688         work->param = param;
4689         work->event = event;
4690         queue_work(mlx5_ib_event_wq, &work->work);
4691
4692         return NOTIFY_OK;
4693 }
4694
4695 static int set_has_smi_cap(struct mlx5_ib_dev *dev)
4696 {
4697         struct mlx5_hca_vport_context vport_ctx;
4698         int err;
4699         int port;
4700
4701         for (port = 1; port <= ARRAY_SIZE(dev->mdev->port_caps); port++) {
4702                 dev->mdev->port_caps[port - 1].has_smi = false;
4703                 if (MLX5_CAP_GEN(dev->mdev, port_type) ==
4704                     MLX5_CAP_PORT_TYPE_IB) {
4705                         if (MLX5_CAP_GEN(dev->mdev, ib_virt)) {
4706                                 err = mlx5_query_hca_vport_context(dev->mdev, 0,
4707                                                                    port, 0,
4708                                                                    &vport_ctx);
4709                                 if (err) {
4710                                         mlx5_ib_err(dev, "query_hca_vport_context for port=%d failed %d\n",
4711                                                     port, err);
4712                                         return err;
4713                                 }
4714                                 dev->mdev->port_caps[port - 1].has_smi =
4715                                         vport_ctx.has_smi;
4716                         } else {
4717                                 dev->mdev->port_caps[port - 1].has_smi = true;
4718                         }
4719                 }
4720         }
4721         return 0;
4722 }
4723
4724 static void get_ext_port_caps(struct mlx5_ib_dev *dev)
4725 {
4726         int port;
4727
4728         for (port = 1; port <= dev->num_ports; port++)
4729                 mlx5_query_ext_port_caps(dev, port);
4730 }
4731
4732 static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
4733 {
4734         struct ib_device_attr *dprops = NULL;
4735         struct ib_port_attr *pprops = NULL;
4736         int err = -ENOMEM;
4737         struct ib_udata uhw = {.inlen = 0, .outlen = 0};
4738
4739         pprops = kzalloc(sizeof(*pprops), GFP_KERNEL);
4740         if (!pprops)
4741                 goto out;
4742
4743         dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);
4744         if (!dprops)
4745                 goto out;
4746
4747         err = mlx5_ib_query_device(&dev->ib_dev, dprops, &uhw);
4748         if (err) {
4749                 mlx5_ib_warn(dev, "query_device failed %d\n", err);
4750                 goto out;
4751         }
4752
4753         err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
4754         if (err) {
4755                 mlx5_ib_warn(dev, "query_port %d failed %d\n",
4756                              port, err);
4757                 goto out;
4758         }
4759
4760         dev->mdev->port_caps[port - 1].pkey_table_len =
4761                                         dprops->max_pkeys;
4762         dev->mdev->port_caps[port - 1].gid_table_len =
4763                                         pprops->gid_tbl_len;
4764         mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n",
4765                     port, dprops->max_pkeys, pprops->gid_tbl_len);
4766
4767 out:
4768         kfree(pprops);
4769         kfree(dprops);
4770
4771         return err;
4772 }
4773
4774 static int get_port_caps(struct mlx5_ib_dev *dev, u8 port)
4775 {
4776         /* For representors use port 1, is this is the only native
4777          * port
4778          */
4779         if (dev->is_rep)
4780                 return __get_port_caps(dev, 1);
4781         return __get_port_caps(dev, port);
4782 }
4783
4784 static void destroy_umrc_res(struct mlx5_ib_dev *dev)
4785 {
4786         int err;
4787
4788         err = mlx5_mr_cache_cleanup(dev);
4789         if (err)
4790                 mlx5_ib_warn(dev, "mr cache cleanup failed\n");
4791
4792         if (dev->umrc.qp)
4793                 mlx5_ib_destroy_qp(dev->umrc.qp, NULL);
4794         if (dev->umrc.cq)
4795                 ib_free_cq(dev->umrc.cq);
4796         if (dev->umrc.pd)
4797                 ib_dealloc_pd(dev->umrc.pd);
4798 }
4799
4800 enum {
4801         MAX_UMR_WR = 128,
4802 };
4803
4804 static int create_umr_res(struct mlx5_ib_dev *dev)
4805 {
4806         struct ib_qp_init_attr *init_attr = NULL;
4807         struct ib_qp_attr *attr = NULL;
4808         struct ib_pd *pd;
4809         struct ib_cq *cq;
4810         struct ib_qp *qp;
4811         int ret;
4812
4813         attr = kzalloc(sizeof(*attr), GFP_KERNEL);
4814         init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL);
4815         if (!attr || !init_attr) {
4816                 ret = -ENOMEM;
4817                 goto error_0;
4818         }
4819
4820         pd = ib_alloc_pd(&dev->ib_dev, 0);
4821         if (IS_ERR(pd)) {
4822                 mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");
4823                 ret = PTR_ERR(pd);
4824                 goto error_0;
4825         }
4826
4827         cq = ib_alloc_cq(&dev->ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ);
4828         if (IS_ERR(cq)) {
4829                 mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");
4830                 ret = PTR_ERR(cq);
4831                 goto error_2;
4832         }
4833
4834         init_attr->send_cq = cq;
4835         init_attr->recv_cq = cq;
4836         init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
4837         init_attr->cap.max_send_wr = MAX_UMR_WR;
4838         init_attr->cap.max_send_sge = 1;
4839         init_attr->qp_type = MLX5_IB_QPT_REG_UMR;
4840         init_attr->port_num = 1;
4841         qp = mlx5_ib_create_qp(pd, init_attr, NULL);
4842         if (IS_ERR(qp)) {
4843                 mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n");
4844                 ret = PTR_ERR(qp);
4845                 goto error_3;
4846         }
4847         qp->device     = &dev->ib_dev;
4848         qp->real_qp    = qp;
4849         qp->uobject    = NULL;
4850         qp->qp_type    = MLX5_IB_QPT_REG_UMR;
4851         qp->send_cq    = init_attr->send_cq;
4852         qp->recv_cq    = init_attr->recv_cq;
4853
4854         attr->qp_state = IB_QPS_INIT;
4855         attr->port_num = 1;
4856         ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_PKEY_INDEX |
4857                                 IB_QP_PORT, NULL);
4858         if (ret) {
4859                 mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");
4860                 goto error_4;
4861         }
4862
4863         memset(attr, 0, sizeof(*attr));
4864         attr->qp_state = IB_QPS_RTR;
4865         attr->path_mtu = IB_MTU_256;
4866
4867         ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
4868         if (ret) {
4869                 mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n");
4870                 goto error_4;
4871         }
4872
4873         memset(attr, 0, sizeof(*attr));
4874         attr->qp_state = IB_QPS_RTS;
4875         ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
4876         if (ret) {
4877                 mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n");
4878                 goto error_4;
4879         }
4880
4881         dev->umrc.qp = qp;
4882         dev->umrc.cq = cq;
4883         dev->umrc.pd = pd;
4884
4885         sema_init(&dev->umrc.sem, MAX_UMR_WR);
4886         ret = mlx5_mr_cache_init(dev);
4887         if (ret) {
4888                 mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
4889                 goto error_4;
4890         }
4891
4892         kfree(attr);
4893         kfree(init_attr);
4894
4895         return 0;
4896
4897 error_4:
4898         mlx5_ib_destroy_qp(qp, NULL);
4899         dev->umrc.qp = NULL;
4900
4901 error_3:
4902         ib_free_cq(cq);
4903         dev->umrc.cq = NULL;
4904
4905 error_2:
4906         ib_dealloc_pd(pd);
4907         dev->umrc.pd = NULL;
4908
4909 error_0:
4910         kfree(attr);
4911         kfree(init_attr);
4912         return ret;
4913 }
4914
4915 static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
4916 {
4917         switch (umr_fence_cap) {
4918         case MLX5_CAP_UMR_FENCE_NONE:
4919                 return MLX5_FENCE_MODE_NONE;
4920         case MLX5_CAP_UMR_FENCE_SMALL:
4921                 return MLX5_FENCE_MODE_INITIATOR_SMALL;
4922         default:
4923                 return MLX5_FENCE_MODE_STRONG_ORDERING;
4924         }
4925 }
4926
4927 static int create_dev_resources(struct mlx5_ib_resources *devr)
4928 {
4929         struct ib_srq_init_attr attr;
4930         struct mlx5_ib_dev *dev;
4931         struct ib_device *ibdev;
4932         struct ib_cq_init_attr cq_attr = {.cqe = 1};
4933         int port;
4934         int ret = 0;
4935
4936         dev = container_of(devr, struct mlx5_ib_dev, devr);
4937         ibdev = &dev->ib_dev;
4938
4939         mutex_init(&devr->mutex);
4940
4941         devr->p0 = rdma_zalloc_drv_obj(ibdev, ib_pd);
4942         if (!devr->p0)
4943                 return -ENOMEM;
4944
4945         devr->p0->device  = ibdev;
4946         devr->p0->uobject = NULL;
4947         atomic_set(&devr->p0->usecnt, 0);
4948
4949         ret = mlx5_ib_alloc_pd(devr->p0, NULL);
4950         if (ret)
4951                 goto error0;
4952
4953         devr->c0 = rdma_zalloc_drv_obj(ibdev, ib_cq);
4954         if (!devr->c0) {
4955                 ret = -ENOMEM;
4956                 goto error1;
4957         }
4958
4959         devr->c0->device = &dev->ib_dev;
4960         atomic_set(&devr->c0->usecnt, 0);
4961
4962         ret = mlx5_ib_create_cq(devr->c0, &cq_attr, NULL);
4963         if (ret)
4964                 goto err_create_cq;
4965
4966         devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL);
4967         if (IS_ERR(devr->x0)) {
4968                 ret = PTR_ERR(devr->x0);
4969                 goto error2;
4970         }
4971         devr->x0->device = &dev->ib_dev;
4972         devr->x0->inode = NULL;
4973         atomic_set(&devr->x0->usecnt, 0);
4974         mutex_init(&devr->x0->tgt_qp_mutex);
4975         INIT_LIST_HEAD(&devr->x0->tgt_qp_list);
4976
4977         devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL);
4978         if (IS_ERR(devr->x1)) {
4979                 ret = PTR_ERR(devr->x1);
4980                 goto error3;
4981         }
4982         devr->x1->device = &dev->ib_dev;
4983         devr->x1->inode = NULL;
4984         atomic_set(&devr->x1->usecnt, 0);
4985         mutex_init(&devr->x1->tgt_qp_mutex);
4986         INIT_LIST_HEAD(&devr->x1->tgt_qp_list);
4987
4988         memset(&attr, 0, sizeof(attr));
4989         attr.attr.max_sge = 1;
4990         attr.attr.max_wr = 1;
4991         attr.srq_type = IB_SRQT_XRC;
4992         attr.ext.cq = devr->c0;
4993         attr.ext.xrc.xrcd = devr->x0;
4994
4995         devr->s0 = rdma_zalloc_drv_obj(ibdev, ib_srq);
4996         if (!devr->s0) {
4997                 ret = -ENOMEM;
4998                 goto error4;
4999         }
5000
5001         devr->s0->device        = &dev->ib_dev;
5002         devr->s0->pd            = devr->p0;
5003         devr->s0->srq_type      = IB_SRQT_XRC;
5004         devr->s0->ext.xrc.xrcd  = devr->x0;
5005         devr->s0->ext.cq        = devr->c0;
5006         ret = mlx5_ib_create_srq(devr->s0, &attr, NULL);
5007         if (ret)
5008                 goto err_create;
5009
5010         atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt);
5011         atomic_inc(&devr->s0->ext.cq->usecnt);
5012         atomic_inc(&devr->p0->usecnt);
5013         atomic_set(&devr->s0->usecnt, 0);
5014
5015         memset(&attr, 0, sizeof(attr));
5016         attr.attr.max_sge = 1;
5017         attr.attr.max_wr = 1;
5018         attr.srq_type = IB_SRQT_BASIC;
5019         devr->s1 = rdma_zalloc_drv_obj(ibdev, ib_srq);
5020         if (!devr->s1) {
5021                 ret = -ENOMEM;
5022                 goto error5;
5023         }
5024
5025         devr->s1->device        = &dev->ib_dev;
5026         devr->s1->pd            = devr->p0;
5027         devr->s1->srq_type      = IB_SRQT_BASIC;
5028         devr->s1->ext.cq        = devr->c0;
5029
5030         ret = mlx5_ib_create_srq(devr->s1, &attr, NULL);
5031         if (ret)
5032                 goto error6;
5033
5034         atomic_inc(&devr->p0->usecnt);
5035         atomic_set(&devr->s1->usecnt, 0);
5036
5037         for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) {
5038                 INIT_WORK(&devr->ports[port].pkey_change_work,
5039                           pkey_change_handler);
5040                 devr->ports[port].devr = devr;
5041         }
5042
5043         return 0;
5044
5045 error6:
5046         kfree(devr->s1);
5047 error5:
5048         mlx5_ib_destroy_srq(devr->s0, NULL);
5049 err_create:
5050         kfree(devr->s0);
5051 error4:
5052         mlx5_ib_dealloc_xrcd(devr->x1, NULL);
5053 error3:
5054         mlx5_ib_dealloc_xrcd(devr->x0, NULL);
5055 error2:
5056         mlx5_ib_destroy_cq(devr->c0, NULL);
5057 err_create_cq:
5058         kfree(devr->c0);
5059 error1:
5060         mlx5_ib_dealloc_pd(devr->p0, NULL);
5061 error0:
5062         kfree(devr->p0);
5063         return ret;
5064 }
5065
5066 static void destroy_dev_resources(struct mlx5_ib_resources *devr)
5067 {
5068         int port;
5069
5070         mlx5_ib_destroy_srq(devr->s1, NULL);
5071         kfree(devr->s1);
5072         mlx5_ib_destroy_srq(devr->s0, NULL);
5073         kfree(devr->s0);
5074         mlx5_ib_dealloc_xrcd(devr->x0, NULL);
5075         mlx5_ib_dealloc_xrcd(devr->x1, NULL);
5076         mlx5_ib_destroy_cq(devr->c0, NULL);
5077         kfree(devr->c0);
5078         mlx5_ib_dealloc_pd(devr->p0, NULL);
5079         kfree(devr->p0);
5080
5081         /* Make sure no change P_Key work items are still executing */
5082         for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
5083                 cancel_work_sync(&devr->ports[port].pkey_change_work);
5084 }
5085
5086 static u32 get_core_cap_flags(struct ib_device *ibdev,
5087                               struct mlx5_hca_vport_context *rep)
5088 {
5089         struct mlx5_ib_dev *dev = to_mdev(ibdev);
5090         enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
5091         u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
5092         u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
5093         bool raw_support = !mlx5_core_mp_enabled(dev->mdev);
5094         u32 ret = 0;
5095
5096         if (rep->grh_required)
5097                 ret |= RDMA_CORE_CAP_IB_GRH_REQUIRED;
5098
5099         if (ll == IB_LINK_LAYER_INFINIBAND)
5100                 return ret | RDMA_CORE_PORT_IBA_IB;
5101
5102         if (raw_support)
5103                 ret |= RDMA_CORE_PORT_RAW_PACKET;
5104
5105         if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
5106                 return ret;
5107
5108         if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
5109                 return ret;
5110
5111         if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)
5112                 ret |= RDMA_CORE_PORT_IBA_ROCE;
5113
5114         if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP)
5115                 ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
5116
5117         return ret;
5118 }
5119
5120 static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,
5121                                struct ib_port_immutable *immutable)
5122 {
5123         struct ib_port_attr attr;
5124         struct mlx5_ib_dev *dev = to_mdev(ibdev);
5125         enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num);
5126         struct mlx5_hca_vport_context rep = {0};
5127         int err;
5128
5129         err = ib_query_port(ibdev, port_num, &attr);
5130         if (err)
5131                 return err;
5132
5133         if (ll == IB_LINK_LAYER_INFINIBAND) {
5134                 err = mlx5_query_hca_vport_context(dev->mdev, 0, port_num, 0,
5135                                                    &rep);
5136                 if (err)
5137                         return err;
5138         }
5139
5140         immutable->pkey_tbl_len = attr.pkey_tbl_len;
5141         immutable->gid_tbl_len = attr.gid_tbl_len;
5142         immutable->core_cap_flags = get_core_cap_flags(ibdev, &rep);
5143         immutable->max_mad_size = IB_MGMT_MAD_SIZE;
5144
5145         return 0;
5146 }
5147
5148 static int mlx5_port_rep_immutable(struct ib_device *ibdev, u8 port_num,
5149                                    struct ib_port_immutable *immutable)
5150 {
5151         struct ib_port_attr attr;
5152         int err;
5153
5154         immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
5155
5156         err = ib_query_port(ibdev, port_num, &attr);
5157         if (err)
5158                 return err;
5159
5160         immutable->pkey_tbl_len = attr.pkey_tbl_len;
5161         immutable->gid_tbl_len = attr.gid_tbl_len;
5162         immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
5163
5164         return 0;
5165 }
5166
5167 static void get_dev_fw_str(struct ib_device *ibdev, char *str)
5168 {
5169         struct mlx5_ib_dev *dev =
5170                 container_of(ibdev, struct mlx5_ib_dev, ib_dev);
5171         snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%04d",
5172                  fw_rev_maj(dev->mdev), fw_rev_min(dev->mdev),
5173                  fw_rev_sub(dev->mdev));
5174 }
5175
5176 static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
5177 {
5178         struct mlx5_core_dev *mdev = dev->mdev;
5179         struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev,
5180                                                                  MLX5_FLOW_NAMESPACE_LAG);
5181         struct mlx5_flow_table *ft;
5182         int err;
5183
5184         if (!ns || !mlx5_lag_is_roce(mdev))
5185                 return 0;
5186
5187         err = mlx5_cmd_create_vport_lag(mdev);
5188         if (err)
5189                 return err;
5190
5191         ft = mlx5_create_lag_demux_flow_table(ns, 0, 0);
5192         if (IS_ERR(ft)) {
5193                 err = PTR_ERR(ft);
5194                 goto err_destroy_vport_lag;
5195         }
5196
5197         dev->flow_db->lag_demux_ft = ft;
5198         dev->lag_active = true;
5199         return 0;
5200
5201 err_destroy_vport_lag:
5202         mlx5_cmd_destroy_vport_lag(mdev);
5203         return err;
5204 }
5205
5206 static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)
5207 {
5208         struct mlx5_core_dev *mdev = dev->mdev;
5209
5210         if (dev->lag_active) {
5211                 dev->lag_active = false;
5212
5213                 mlx5_destroy_flow_table(dev->flow_db->lag_demux_ft);
5214                 dev->flow_db->lag_demux_ft = NULL;
5215
5216                 mlx5_cmd_destroy_vport_lag(mdev);
5217         }
5218 }
5219
5220 static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)
5221 {
5222         int err;
5223
5224         dev->port[port_num].roce.nb.notifier_call = mlx5_netdev_event;
5225         err = register_netdevice_notifier(&dev->port[port_num].roce.nb);
5226         if (err) {
5227                 dev->port[port_num].roce.nb.notifier_call = NULL;
5228                 return err;
5229         }
5230
5231         return 0;
5232 }
5233
5234 static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)
5235 {
5236         if (dev->port[port_num].roce.nb.notifier_call) {
5237                 unregister_netdevice_notifier(&dev->port[port_num].roce.nb);
5238                 dev->port[port_num].roce.nb.notifier_call = NULL;
5239         }
5240 }
5241
5242 static int mlx5_enable_eth(struct mlx5_ib_dev *dev)
5243 {
5244         int err;
5245
5246         err = mlx5_nic_vport_enable_roce(dev->mdev);
5247         if (err)
5248                 return err;
5249
5250         err = mlx5_eth_lag_init(dev);
5251         if (err)
5252                 goto err_disable_roce;
5253
5254         return 0;
5255
5256 err_disable_roce:
5257         mlx5_nic_vport_disable_roce(dev->mdev);
5258
5259         return err;
5260 }
5261
5262 static void mlx5_disable_eth(struct mlx5_ib_dev *dev)
5263 {
5264         mlx5_eth_lag_cleanup(dev);
5265         mlx5_nic_vport_disable_roce(dev->mdev);
5266 }
5267
5268 struct mlx5_ib_counter {
5269         const char *name;
5270         size_t offset;
5271 };
5272
5273 #define INIT_Q_COUNTER(_name)           \
5274         { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}
5275
5276 static const struct mlx5_ib_counter basic_q_cnts[] = {
5277         INIT_Q_COUNTER(rx_write_requests),
5278         INIT_Q_COUNTER(rx_read_requests),
5279         INIT_Q_COUNTER(rx_atomic_requests),
5280         INIT_Q_COUNTER(out_of_buffer),
5281 };
5282
5283 static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {
5284         INIT_Q_COUNTER(out_of_sequence),
5285 };
5286
5287 static const struct mlx5_ib_counter retrans_q_cnts[] = {
5288         INIT_Q_COUNTER(duplicate_request),
5289         INIT_Q_COUNTER(rnr_nak_retry_err),
5290         INIT_Q_COUNTER(packet_seq_err),
5291         INIT_Q_COUNTER(implied_nak_seq_err),
5292         INIT_Q_COUNTER(local_ack_timeout_err),
5293 };
5294
5295 #define INIT_CONG_COUNTER(_name)                \
5296         { .name = #_name, .offset =     \
5297                 MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}
5298
5299 static const struct mlx5_ib_counter cong_cnts[] = {
5300         INIT_CONG_COUNTER(rp_cnp_ignored),
5301         INIT_CONG_COUNTER(rp_cnp_handled),
5302         INIT_CONG_COUNTER(np_ecn_marked_roce_packets),
5303         INIT_CONG_COUNTER(np_cnp_sent),
5304 };
5305
5306 static const struct mlx5_ib_counter extended_err_cnts[] = {
5307         INIT_Q_COUNTER(resp_local_length_error),
5308         INIT_Q_COUNTER(resp_cqe_error),
5309         INIT_Q_COUNTER(req_cqe_error),
5310         INIT_Q_COUNTER(req_remote_invalid_request),
5311         INIT_Q_COUNTER(req_remote_access_errors),
5312         INIT_Q_COUNTER(resp_remote_access_errors),
5313         INIT_Q_COUNTER(resp_cqe_flush_error),
5314         INIT_Q_COUNTER(req_cqe_flush_error),
5315 };
5316
5317 #define INIT_EXT_PPCNT_COUNTER(_name)           \
5318         { .name = #_name, .offset =     \
5319         MLX5_BYTE_OFF(ppcnt_reg, \
5320                       counter_set.eth_extended_cntrs_grp_data_layout._name##_high)}
5321
5322 static const struct mlx5_ib_counter ext_ppcnt_cnts[] = {
5323         INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated),
5324 };
5325
5326 static bool is_mdev_switchdev_mode(const struct mlx5_core_dev *mdev)
5327 {
5328         return MLX5_ESWITCH_MANAGER(mdev) &&
5329                mlx5_ib_eswitch_mode(mdev->priv.eswitch) ==
5330                        MLX5_ESWITCH_OFFLOADS;
5331 }
5332
5333 static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
5334 {
5335         int num_cnt_ports;
5336         int i;
5337
5338         num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
5339
5340         for (i = 0; i < num_cnt_ports; i++) {
5341                 if (dev->port[i].cnts.set_id_valid)
5342                         mlx5_core_dealloc_q_counter(dev->mdev,
5343                                                     dev->port[i].cnts.set_id);
5344                 kfree(dev->port[i].cnts.names);
5345                 kfree(dev->port[i].cnts.offsets);
5346         }
5347 }
5348
5349 static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
5350                                     struct mlx5_ib_counters *cnts)
5351 {
5352         u32 num_counters;
5353
5354         num_counters = ARRAY_SIZE(basic_q_cnts);
5355
5356         if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt))
5357                 num_counters += ARRAY_SIZE(out_of_seq_q_cnts);
5358
5359         if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))
5360                 num_counters += ARRAY_SIZE(retrans_q_cnts);
5361
5362         if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))
5363                 num_counters += ARRAY_SIZE(extended_err_cnts);
5364
5365         cnts->num_q_counters = num_counters;
5366
5367         if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
5368                 cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);
5369                 num_counters += ARRAY_SIZE(cong_cnts);
5370         }
5371         if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
5372                 cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts);
5373                 num_counters += ARRAY_SIZE(ext_ppcnt_cnts);
5374         }
5375         cnts->names = kcalloc(num_counters, sizeof(cnts->names), GFP_KERNEL);
5376         if (!cnts->names)
5377                 return -ENOMEM;
5378
5379         cnts->offsets = kcalloc(num_counters,
5380                                 sizeof(cnts->offsets), GFP_KERNEL);
5381         if (!cnts->offsets)
5382                 goto err_names;
5383
5384         return 0;
5385
5386 err_names:
5387         kfree(cnts->names);
5388         cnts->names = NULL;
5389         return -ENOMEM;
5390 }
5391
5392 static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
5393                                   const char **names,
5394                                   size_t *offsets)
5395 {
5396         int i;
5397         int j = 0;
5398
5399         for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) {
5400                 names[j] = basic_q_cnts[i].name;
5401                 offsets[j] = basic_q_cnts[i].offset;
5402         }
5403
5404         if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {
5405                 for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) {
5406                         names[j] = out_of_seq_q_cnts[i].name;
5407                         offsets[j] = out_of_seq_q_cnts[i].offset;
5408                 }
5409         }
5410
5411         if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
5412                 for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) {
5413                         names[j] = retrans_q_cnts[i].name;
5414                         offsets[j] = retrans_q_cnts[i].offset;
5415                 }
5416         }
5417
5418         if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
5419                 for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) {
5420                         names[j] = extended_err_cnts[i].name;
5421                         offsets[j] = extended_err_cnts[i].offset;
5422                 }
5423         }
5424
5425         if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
5426                 for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {
5427                         names[j] = cong_cnts[i].name;
5428                         offsets[j] = cong_cnts[i].offset;
5429                 }
5430         }
5431
5432         if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
5433                 for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) {
5434                         names[j] = ext_ppcnt_cnts[i].name;
5435                         offsets[j] = ext_ppcnt_cnts[i].offset;
5436                 }
5437         }
5438 }
5439
5440 static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
5441 {
5442         int num_cnt_ports;
5443         int err = 0;
5444         int i;
5445         bool is_shared;
5446
5447         is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0;
5448         num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
5449
5450         for (i = 0; i < num_cnt_ports; i++) {
5451                 err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts);
5452                 if (err)
5453                         goto err_alloc;
5454
5455                 mlx5_ib_fill_counters(dev, dev->port[i].cnts.names,
5456                                       dev->port[i].cnts.offsets);
5457
5458                 err = mlx5_cmd_alloc_q_counter(dev->mdev,
5459                                                &dev->port[i].cnts.set_id,
5460                                                is_shared ?
5461                                                MLX5_SHARED_RESOURCE_UID : 0);
5462                 if (err) {
5463                         mlx5_ib_warn(dev,
5464                                      "couldn't allocate queue counter for port %d, err %d\n",
5465                                      i + 1, err);
5466                         goto err_alloc;
5467                 }
5468                 dev->port[i].cnts.set_id_valid = true;
5469         }
5470         return 0;
5471
5472 err_alloc:
5473         mlx5_ib_dealloc_counters(dev);
5474         return err;
5475 }
5476
5477 static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev,
5478                                                    u8 port_num)
5479 {
5480         return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts :
5481                                                    &dev->port[port_num].cnts;
5482 }
5483
5484 /**
5485  * mlx5_ib_get_counters_id - Returns counters id to use for device+port
5486  * @dev:        Pointer to mlx5 IB device
5487  * @port_num:   Zero based port number
5488  *
5489  * mlx5_ib_get_counters_id() Returns counters set id to use for given
5490  * device port combination in switchdev and non switchdev mode of the
5491  * parent device.
5492  */
5493 u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num)
5494 {
5495         const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
5496
5497         return cnts->set_id;
5498 }
5499
5500 static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev,
5501                                                     u8 port_num)
5502 {
5503         struct mlx5_ib_dev *dev = to_mdev(ibdev);
5504         const struct mlx5_ib_counters *cnts;
5505         bool is_switchdev = is_mdev_switchdev_mode(dev->mdev);
5506
5507         if ((is_switchdev && port_num) || (!is_switchdev && !port_num))
5508                 return NULL;
5509
5510         cnts = get_counters(dev, port_num - 1);
5511
5512         return rdma_alloc_hw_stats_struct(cnts->names,
5513                                           cnts->num_q_counters +
5514                                           cnts->num_cong_counters +
5515                                           cnts->num_ext_ppcnt_counters,
5516                                           RDMA_HW_STATS_DEFAULT_LIFESPAN);
5517 }
5518
5519 static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,
5520                                     const struct mlx5_ib_counters *cnts,
5521                                     struct rdma_hw_stats *stats,
5522                                     u16 set_id)
5523 {
5524         int outlen = MLX5_ST_SZ_BYTES(query_q_counter_out);
5525         void *out;
5526         __be32 val;
5527         int ret, i;
5528
5529         out = kvzalloc(outlen, GFP_KERNEL);
5530         if (!out)
5531                 return -ENOMEM;
5532
5533         ret = mlx5_core_query_q_counter(mdev, set_id, 0, out, outlen);
5534         if (ret)
5535                 goto free;
5536
5537         for (i = 0; i < cnts->num_q_counters; i++) {
5538                 val = *(__be32 *)(out + cnts->offsets[i]);
5539                 stats->value[i] = (u64)be32_to_cpu(val);
5540         }
5541
5542 free:
5543         kvfree(out);
5544         return ret;
5545 }
5546
5547 static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev,
5548                                             const struct mlx5_ib_counters *cnts,
5549                                             struct rdma_hw_stats *stats)
5550 {
5551         int offset = cnts->num_q_counters + cnts->num_cong_counters;
5552         int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
5553         int ret, i;
5554         void *out;
5555
5556         out = kvzalloc(sz, GFP_KERNEL);
5557         if (!out)
5558                 return -ENOMEM;
5559
5560         ret = mlx5_cmd_query_ext_ppcnt_counters(dev->mdev, out);
5561         if (ret)
5562                 goto free;
5563
5564         for (i = 0; i < cnts->num_ext_ppcnt_counters; i++)
5565                 stats->value[i + offset] =
5566                         be64_to_cpup((__be64 *)(out +
5567                                     cnts->offsets[i + offset]));
5568 free:
5569         kvfree(out);
5570         return ret;
5571 }
5572
5573 static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
5574                                 struct rdma_hw_stats *stats,
5575                                 u8 port_num, int index)
5576 {
5577         struct mlx5_ib_dev *dev = to_mdev(ibdev);
5578         const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1);
5579         struct mlx5_core_dev *mdev;
5580         int ret, num_counters;
5581         u8 mdev_port_num;
5582
5583         if (!stats)
5584                 return -EINVAL;
5585
5586         num_counters = cnts->num_q_counters +
5587                        cnts->num_cong_counters +
5588                        cnts->num_ext_ppcnt_counters;
5589
5590         /* q_counters are per IB device, query the master mdev */
5591         ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, cnts->set_id);
5592         if (ret)
5593                 return ret;
5594
5595         if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
5596                 ret =  mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats);
5597                 if (ret)
5598                         return ret;
5599         }
5600
5601         if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
5602                 mdev = mlx5_ib_get_native_port_mdev(dev, port_num,
5603                                                     &mdev_port_num);
5604                 if (!mdev) {
5605                         /* If port is not affiliated yet, its in down state
5606                          * which doesn't have any counters yet, so it would be
5607                          * zero. So no need to read from the HCA.
5608                          */
5609                         goto done;
5610                 }
5611                 ret = mlx5_lag_query_cong_counters(dev->mdev,
5612                                                    stats->value +
5613                                                    cnts->num_q_counters,
5614                                                    cnts->num_cong_counters,
5615                                                    cnts->offsets +
5616                                                    cnts->num_q_counters);
5617
5618                 mlx5_ib_put_native_port_mdev(dev, port_num);
5619                 if (ret)
5620                         return ret;
5621         }
5622
5623 done:
5624         return num_counters;
5625 }
5626
5627 static struct rdma_hw_stats *
5628 mlx5_ib_counter_alloc_stats(struct rdma_counter *counter)
5629 {
5630         struct mlx5_ib_dev *dev = to_mdev(counter->device);
5631         const struct mlx5_ib_counters *cnts =
5632                 get_counters(dev, counter->port - 1);
5633
5634         /* Q counters are in the beginning of all counters */
5635         return rdma_alloc_hw_stats_struct(cnts->names,
5636                                           cnts->num_q_counters,
5637                                           RDMA_HW_STATS_DEFAULT_LIFESPAN);
5638 }
5639
5640 static int mlx5_ib_counter_update_stats(struct rdma_counter *counter)
5641 {
5642         struct mlx5_ib_dev *dev = to_mdev(counter->device);
5643         const struct mlx5_ib_counters *cnts =
5644                 get_counters(dev, counter->port - 1);
5645
5646         return mlx5_ib_query_q_counters(dev->mdev, cnts,
5647                                         counter->stats, counter->id);
5648 }
5649
5650 static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter,
5651                                    struct ib_qp *qp)
5652 {
5653         struct mlx5_ib_dev *dev = to_mdev(qp->device);
5654         u16 cnt_set_id = 0;
5655         int err;
5656
5657         if (!counter->id) {
5658                 err = mlx5_cmd_alloc_q_counter(dev->mdev,
5659                                                &cnt_set_id,
5660                                                MLX5_SHARED_RESOURCE_UID);
5661                 if (err)
5662                         return err;
5663                 counter->id = cnt_set_id;
5664         }
5665
5666         err = mlx5_ib_qp_set_counter(qp, counter);
5667         if (err)
5668                 goto fail_set_counter;
5669
5670         return 0;
5671
5672 fail_set_counter:
5673         mlx5_core_dealloc_q_counter(dev->mdev, cnt_set_id);
5674         counter->id = 0;
5675
5676         return err;
5677 }
5678
5679 static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp)
5680 {
5681         return mlx5_ib_qp_set_counter(qp, NULL);
5682 }
5683
5684 static int mlx5_ib_counter_dealloc(struct rdma_counter *counter)
5685 {
5686         struct mlx5_ib_dev *dev = to_mdev(counter->device);
5687
5688         return mlx5_core_dealloc_q_counter(dev->mdev, counter->id);
5689 }
5690
5691 static int mlx5_ib_rn_get_params(struct ib_device *device, u8 port_num,
5692                                  enum rdma_netdev_t type,
5693                                  struct rdma_netdev_alloc_params *params)
5694 {
5695         if (type != RDMA_NETDEV_IPOIB)
5696                 return -EOPNOTSUPP;
5697
5698         return mlx5_rdma_rn_get_params(to_mdev(device)->mdev, device, params);
5699 }
5700
5701 static void delay_drop_debugfs_cleanup(struct mlx5_ib_dev *dev)
5702 {
5703         if (!dev->delay_drop.dir_debugfs)
5704                 return;
5705         debugfs_remove_recursive(dev->delay_drop.dir_debugfs);
5706         dev->delay_drop.dir_debugfs = NULL;
5707 }
5708
5709 static void cancel_delay_drop(struct mlx5_ib_dev *dev)
5710 {
5711         if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
5712                 return;
5713
5714         cancel_work_sync(&dev->delay_drop.delay_drop_work);
5715         delay_drop_debugfs_cleanup(dev);
5716 }
5717
5718 static ssize_t delay_drop_timeout_read(struct file *filp, char __user *buf,
5719                                        size_t count, loff_t *pos)
5720 {
5721         struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
5722         char lbuf[20];
5723         int len;
5724
5725         len = snprintf(lbuf, sizeof(lbuf), "%u\n", delay_drop->timeout);
5726         return simple_read_from_buffer(buf, count, pos, lbuf, len);
5727 }
5728
5729 static ssize_t delay_drop_timeout_write(struct file *filp, const char __user *buf,
5730                                         size_t count, loff_t *pos)
5731 {
5732         struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
5733         u32 timeout;
5734         u32 var;
5735
5736         if (kstrtouint_from_user(buf, count, 0, &var))
5737                 return -EFAULT;
5738
5739         timeout = min_t(u32, roundup(var, 100), MLX5_MAX_DELAY_DROP_TIMEOUT_MS *
5740                         1000);
5741         if (timeout != var)
5742                 mlx5_ib_dbg(delay_drop->dev, "Round delay drop timeout to %u usec\n",
5743                             timeout);
5744
5745         delay_drop->timeout = timeout;
5746
5747         return count;
5748 }
5749
5750 static const struct file_operations fops_delay_drop_timeout = {
5751         .owner  = THIS_MODULE,
5752         .open   = simple_open,
5753         .write  = delay_drop_timeout_write,
5754         .read   = delay_drop_timeout_read,
5755 };
5756
5757 static void delay_drop_debugfs_init(struct mlx5_ib_dev *dev)
5758 {
5759         struct dentry *root;
5760
5761         if (!mlx5_debugfs_root)
5762                 return;
5763
5764         root = debugfs_create_dir("delay_drop", dev->mdev->priv.dbg_root);
5765         dev->delay_drop.dir_debugfs = root;
5766
5767         debugfs_create_atomic_t("num_timeout_events", 0400, root,
5768                                 &dev->delay_drop.events_cnt);
5769         debugfs_create_atomic_t("num_rqs", 0400, root,
5770                                 &dev->delay_drop.rqs_cnt);
5771         debugfs_create_file("timeout", 0600, root, &dev->delay_drop,
5772                             &fops_delay_drop_timeout);
5773 }
5774
5775 static void init_delay_drop(struct mlx5_ib_dev *dev)
5776 {
5777         if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
5778                 return;
5779
5780         mutex_init(&dev->delay_drop.lock);
5781         dev->delay_drop.dev = dev;
5782         dev->delay_drop.activate = false;
5783         dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;
5784         INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler);
5785         atomic_set(&dev->delay_drop.rqs_cnt, 0);
5786         atomic_set(&dev->delay_drop.events_cnt, 0);
5787
5788         delay_drop_debugfs_init(dev);
5789 }
5790
5791 static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,
5792                                       struct mlx5_ib_multiport_info *mpi)
5793 {
5794         u8 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
5795         struct mlx5_ib_port *port = &ibdev->port[port_num];
5796         int comps;
5797         int err;
5798         int i;
5799
5800         lockdep_assert_held(&mlx5_ib_multiport_mutex);
5801
5802         mlx5_ib_cleanup_cong_debugfs(ibdev, port_num);
5803
5804         spin_lock(&port->mp.mpi_lock);
5805         if (!mpi->ibdev) {
5806                 spin_unlock(&port->mp.mpi_lock);
5807                 return;
5808         }
5809
5810         mpi->ibdev = NULL;
5811
5812         spin_unlock(&port->mp.mpi_lock);
5813         if (mpi->mdev_events.notifier_call)
5814                 mlx5_notifier_unregister(mpi->mdev, &mpi->mdev_events);
5815         mpi->mdev_events.notifier_call = NULL;
5816         mlx5_remove_netdev_notifier(ibdev, port_num);
5817         spin_lock(&port->mp.mpi_lock);
5818
5819         comps = mpi->mdev_refcnt;
5820         if (comps) {
5821                 mpi->unaffiliate = true;
5822                 init_completion(&mpi->unref_comp);
5823                 spin_unlock(&port->mp.mpi_lock);
5824
5825                 for (i = 0; i < comps; i++)
5826                         wait_for_completion(&mpi->unref_comp);
5827
5828                 spin_lock(&port->mp.mpi_lock);
5829                 mpi->unaffiliate = false;
5830         }
5831
5832         port->mp.mpi = NULL;
5833
5834         list_add_tail(&mpi->list, &mlx5_ib_unaffiliated_port_list);
5835
5836         spin_unlock(&port->mp.mpi_lock);
5837
5838         err = mlx5_nic_vport_unaffiliate_multiport(mpi->mdev);
5839
5840         mlx5_ib_dbg(ibdev, "unaffiliated port %d\n", port_num + 1);
5841         /* Log an error, still needed to cleanup the pointers and add
5842          * it back to the list.
5843          */
5844         if (err)
5845                 mlx5_ib_err(ibdev, "Failed to unaffiliate port %u\n",
5846                             port_num + 1);
5847
5848         ibdev->port[port_num].roce.last_port_state = IB_PORT_DOWN;
5849 }
5850
5851 static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,
5852                                     struct mlx5_ib_multiport_info *mpi)
5853 {
5854         u8 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
5855         int err;
5856
5857         lockdep_assert_held(&mlx5_ib_multiport_mutex);
5858
5859         spin_lock(&ibdev->port[port_num].mp.mpi_lock);
5860         if (ibdev->port[port_num].mp.mpi) {
5861                 mlx5_ib_dbg(ibdev, "port %d already affiliated.\n",
5862                             port_num + 1);
5863                 spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
5864                 return false;
5865         }
5866
5867         ibdev->port[port_num].mp.mpi = mpi;
5868         mpi->ibdev = ibdev;
5869         mpi->mdev_events.notifier_call = NULL;
5870         spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
5871
5872         err = mlx5_nic_vport_affiliate_multiport(ibdev->mdev, mpi->mdev);
5873         if (err)
5874                 goto unbind;
5875
5876         err = get_port_caps(ibdev, mlx5_core_native_port_num(mpi->mdev));
5877         if (err)
5878                 goto unbind;
5879
5880         err = mlx5_add_netdev_notifier(ibdev, port_num);
5881         if (err) {
5882                 mlx5_ib_err(ibdev, "failed adding netdev notifier for port %u\n",
5883                             port_num + 1);
5884                 goto unbind;
5885         }
5886
5887         mpi->mdev_events.notifier_call = mlx5_ib_event_slave_port;
5888         mlx5_notifier_register(mpi->mdev, &mpi->mdev_events);
5889
5890         mlx5_ib_init_cong_debugfs(ibdev, port_num);
5891
5892         return true;
5893
5894 unbind:
5895         mlx5_ib_unbind_slave_port(ibdev, mpi);
5896         return false;
5897 }
5898
5899 static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
5900 {
5901         int port_num = mlx5_core_native_port_num(dev->mdev) - 1;
5902         enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
5903                                                           port_num + 1);
5904         struct mlx5_ib_multiport_info *mpi;
5905         int err;
5906         int i;
5907
5908         if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
5909                 return 0;
5910
5911         err = mlx5_query_nic_vport_system_image_guid(dev->mdev,
5912                                                      &dev->sys_image_guid);
5913         if (err)
5914                 return err;
5915
5916         err = mlx5_nic_vport_enable_roce(dev->mdev);
5917         if (err)
5918                 return err;
5919
5920         mutex_lock(&mlx5_ib_multiport_mutex);
5921         for (i = 0; i < dev->num_ports; i++) {
5922                 bool bound = false;
5923
5924                 /* build a stub multiport info struct for the native port. */
5925                 if (i == port_num) {
5926                         mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
5927                         if (!mpi) {
5928                                 mutex_unlock(&mlx5_ib_multiport_mutex);
5929                                 mlx5_nic_vport_disable_roce(dev->mdev);
5930                                 return -ENOMEM;
5931                         }
5932
5933                         mpi->is_master = true;
5934                         mpi->mdev = dev->mdev;
5935                         mpi->sys_image_guid = dev->sys_image_guid;
5936                         dev->port[i].mp.mpi = mpi;
5937                         mpi->ibdev = dev;
5938                         mpi = NULL;
5939                         continue;
5940                 }
5941
5942                 list_for_each_entry(mpi, &mlx5_ib_unaffiliated_port_list,
5943                                     list) {
5944                         if (dev->sys_image_guid == mpi->sys_image_guid &&
5945                             (mlx5_core_native_port_num(mpi->mdev) - 1) == i) {
5946                                 bound = mlx5_ib_bind_slave_port(dev, mpi);
5947                         }
5948
5949                         if (bound) {
5950                                 dev_dbg(mpi->mdev->device,
5951                                         "removing port from unaffiliated list.\n");
5952                                 mlx5_ib_dbg(dev, "port %d bound\n", i + 1);
5953                                 list_del(&mpi->list);
5954                                 break;
5955                         }
5956                 }
5957                 if (!bound) {
5958                         get_port_caps(dev, i + 1);
5959                         mlx5_ib_dbg(dev, "no free port found for port %d\n",
5960                                     i + 1);
5961                 }
5962         }
5963
5964         list_add_tail(&dev->ib_dev_list, &mlx5_ib_dev_list);
5965         mutex_unlock(&mlx5_ib_multiport_mutex);
5966         return err;
5967 }
5968
5969 static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev)
5970 {
5971         int port_num = mlx5_core_native_port_num(dev->mdev) - 1;
5972         enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
5973                                                           port_num + 1);
5974         int i;
5975
5976         if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
5977                 return;
5978
5979         mutex_lock(&mlx5_ib_multiport_mutex);
5980         for (i = 0; i < dev->num_ports; i++) {
5981                 if (dev->port[i].mp.mpi) {
5982                         /* Destroy the native port stub */
5983                         if (i == port_num) {
5984                                 kfree(dev->port[i].mp.mpi);
5985                                 dev->port[i].mp.mpi = NULL;
5986                         } else {
5987                                 mlx5_ib_dbg(dev, "unbinding port_num: %d\n", i + 1);
5988                                 mlx5_ib_unbind_slave_port(dev, dev->port[i].mp.mpi);
5989                         }
5990                 }
5991         }
5992
5993         mlx5_ib_dbg(dev, "removing from devlist\n");
5994         list_del(&dev->ib_dev_list);
5995         mutex_unlock(&mlx5_ib_multiport_mutex);
5996
5997         mlx5_nic_vport_disable_roce(dev->mdev);
5998 }
5999
6000 ADD_UVERBS_ATTRIBUTES_SIMPLE(
6001         mlx5_ib_dm,
6002         UVERBS_OBJECT_DM,
6003         UVERBS_METHOD_DM_ALLOC,
6004         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
6005                             UVERBS_ATTR_TYPE(u64),
6006                             UA_MANDATORY),
6007         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
6008                             UVERBS_ATTR_TYPE(u16),
6009                             UA_OPTIONAL),
6010         UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
6011                              enum mlx5_ib_uapi_dm_type,
6012                              UA_OPTIONAL));
6013
6014 ADD_UVERBS_ATTRIBUTES_SIMPLE(
6015         mlx5_ib_flow_action,
6016         UVERBS_OBJECT_FLOW_ACTION,
6017         UVERBS_METHOD_FLOW_ACTION_ESP_CREATE,
6018         UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS,
6019                              enum mlx5_ib_uapi_flow_action_flags));
6020
6021 static const struct uapi_definition mlx5_ib_defs[] = {
6022 #if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
6023         UAPI_DEF_CHAIN(mlx5_ib_devx_defs),
6024         UAPI_DEF_CHAIN(mlx5_ib_flow_defs),
6025 #endif
6026
6027         UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
6028                                 &mlx5_ib_flow_action),
6029         UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm),
6030         {}
6031 };
6032
6033 static int mlx5_ib_read_counters(struct ib_counters *counters,
6034                                  struct ib_counters_read_attr *read_attr,
6035                                  struct uverbs_attr_bundle *attrs)
6036 {
6037         struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
6038         struct mlx5_read_counters_attr mread_attr = {};
6039         struct mlx5_ib_flow_counters_desc *desc;
6040         int ret, i;
6041
6042         mutex_lock(&mcounters->mcntrs_mutex);
6043         if (mcounters->cntrs_max_index > read_attr->ncounters) {
6044                 ret = -EINVAL;
6045                 goto err_bound;
6046         }
6047
6048         mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64),
6049                                  GFP_KERNEL);
6050         if (!mread_attr.out) {
6051                 ret = -ENOMEM;
6052                 goto err_bound;
6053         }
6054
6055         mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl;
6056         mread_attr.flags = read_attr->flags;
6057         ret = mcounters->read_counters(counters->device, &mread_attr);
6058         if (ret)
6059                 goto err_read;
6060
6061         /* do the pass over the counters data array to assign according to the
6062          * descriptions and indexing pairs
6063          */
6064         desc = mcounters->counters_data;
6065         for (i = 0; i < mcounters->ncounters; i++)
6066                 read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description];
6067
6068 err_read:
6069         kfree(mread_attr.out);
6070 err_bound:
6071         mutex_unlock(&mcounters->mcntrs_mutex);
6072         return ret;
6073 }
6074
6075 static int mlx5_ib_destroy_counters(struct ib_counters *counters)
6076 {
6077         struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
6078
6079         counters_clear_description(counters);
6080         if (mcounters->hw_cntrs_hndl)
6081                 mlx5_fc_destroy(to_mdev(counters->device)->mdev,
6082                                 mcounters->hw_cntrs_hndl);
6083
6084         kfree(mcounters);
6085
6086         return 0;
6087 }
6088
6089 static struct ib_counters *mlx5_ib_create_counters(struct ib_device *device,
6090                                                    struct uverbs_attr_bundle *attrs)
6091 {
6092         struct mlx5_ib_mcounters *mcounters;
6093
6094         mcounters = kzalloc(sizeof(*mcounters), GFP_KERNEL);
6095         if (!mcounters)
6096                 return ERR_PTR(-ENOMEM);
6097
6098         mutex_init(&mcounters->mcntrs_mutex);
6099
6100         return &mcounters->ibcntrs;
6101 }
6102
6103 static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
6104 {
6105         mlx5_ib_cleanup_multiport_master(dev);
6106         WARN_ON(!xa_empty(&dev->odp_mkeys));
6107         cleanup_srcu_struct(&dev->odp_srcu);
6108
6109         WARN_ON(!xa_empty(&dev->sig_mrs));
6110         WARN_ON(!bitmap_empty(dev->dm.memic_alloc_pages, MLX5_MAX_MEMIC_PAGES));
6111 }
6112
6113 static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
6114 {
6115         struct mlx5_core_dev *mdev = dev->mdev;
6116         int err;
6117         int i;
6118
6119         for (i = 0; i < dev->num_ports; i++) {
6120                 spin_lock_init(&dev->port[i].mp.mpi_lock);
6121                 rwlock_init(&dev->port[i].roce.netdev_lock);
6122                 dev->port[i].roce.dev = dev;
6123                 dev->port[i].roce.native_port_num = i + 1;
6124                 dev->port[i].roce.last_port_state = IB_PORT_DOWN;
6125         }
6126
6127         mlx5_ib_internal_fill_odp_caps(dev);
6128
6129         err = mlx5_ib_init_multiport_master(dev);
6130         if (err)
6131                 return err;
6132
6133         err = set_has_smi_cap(dev);
6134         if (err)
6135                 return err;
6136
6137         if (!mlx5_core_mp_enabled(mdev)) {
6138                 for (i = 1; i <= dev->num_ports; i++) {
6139                         err = get_port_caps(dev, i);
6140                         if (err)
6141                                 break;
6142                 }
6143         } else {
6144                 err = get_port_caps(dev, mlx5_core_native_port_num(mdev));
6145         }
6146         if (err)
6147                 goto err_mp;
6148
6149         if (mlx5_use_mad_ifc(dev))
6150                 get_ext_port_caps(dev);
6151
6152         dev->ib_dev.node_type           = RDMA_NODE_IB_CA;
6153         dev->ib_dev.local_dma_lkey      = 0 /* not supported for now */;
6154         dev->ib_dev.phys_port_cnt       = dev->num_ports;
6155         dev->ib_dev.num_comp_vectors    = mlx5_comp_vectors_count(mdev);
6156         dev->ib_dev.dev.parent          = mdev->device;
6157
6158         mutex_init(&dev->cap_mask_mutex);
6159         INIT_LIST_HEAD(&dev->qp_list);
6160         spin_lock_init(&dev->reset_flow_resource_lock);
6161         xa_init(&dev->odp_mkeys);
6162         xa_init(&dev->sig_mrs);
6163
6164         spin_lock_init(&dev->dm.lock);
6165         dev->dm.dev = mdev;
6166
6167         err = init_srcu_struct(&dev->odp_srcu);
6168         if (err)
6169                 goto err_mp;
6170
6171         return 0;
6172
6173 err_mp:
6174         mlx5_ib_cleanup_multiport_master(dev);
6175
6176         return -ENOMEM;
6177 }
6178
6179 static int mlx5_ib_stage_flow_db_init(struct mlx5_ib_dev *dev)
6180 {
6181         dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL);
6182
6183         if (!dev->flow_db)
6184                 return -ENOMEM;
6185
6186         mutex_init(&dev->flow_db->lock);
6187
6188         return 0;
6189 }
6190
6191 static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev)
6192 {
6193         kfree(dev->flow_db);
6194 }
6195
6196 static const struct ib_device_ops mlx5_ib_dev_ops = {
6197         .owner = THIS_MODULE,
6198         .driver_id = RDMA_DRIVER_MLX5,
6199         .uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION,
6200
6201         .add_gid = mlx5_ib_add_gid,
6202         .alloc_mr = mlx5_ib_alloc_mr,
6203         .alloc_mr_integrity = mlx5_ib_alloc_mr_integrity,
6204         .alloc_pd = mlx5_ib_alloc_pd,
6205         .alloc_ucontext = mlx5_ib_alloc_ucontext,
6206         .attach_mcast = mlx5_ib_mcg_attach,
6207         .check_mr_status = mlx5_ib_check_mr_status,
6208         .create_ah = mlx5_ib_create_ah,
6209         .create_counters = mlx5_ib_create_counters,
6210         .create_cq = mlx5_ib_create_cq,
6211         .create_flow = mlx5_ib_create_flow,
6212         .create_qp = mlx5_ib_create_qp,
6213         .create_srq = mlx5_ib_create_srq,
6214         .dealloc_pd = mlx5_ib_dealloc_pd,
6215         .dealloc_ucontext = mlx5_ib_dealloc_ucontext,
6216         .del_gid = mlx5_ib_del_gid,
6217         .dereg_mr = mlx5_ib_dereg_mr,
6218         .destroy_ah = mlx5_ib_destroy_ah,
6219         .destroy_counters = mlx5_ib_destroy_counters,
6220         .destroy_cq = mlx5_ib_destroy_cq,
6221         .destroy_flow = mlx5_ib_destroy_flow,
6222         .destroy_flow_action = mlx5_ib_destroy_flow_action,
6223         .destroy_qp = mlx5_ib_destroy_qp,
6224         .destroy_srq = mlx5_ib_destroy_srq,
6225         .detach_mcast = mlx5_ib_mcg_detach,
6226         .disassociate_ucontext = mlx5_ib_disassociate_ucontext,
6227         .drain_rq = mlx5_ib_drain_rq,
6228         .drain_sq = mlx5_ib_drain_sq,
6229         .enable_driver = mlx5_ib_enable_driver,
6230         .fill_res_entry = mlx5_ib_fill_res_entry,
6231         .fill_stat_entry = mlx5_ib_fill_stat_entry,
6232         .get_dev_fw_str = get_dev_fw_str,
6233         .get_dma_mr = mlx5_ib_get_dma_mr,
6234         .get_link_layer = mlx5_ib_port_link_layer,
6235         .map_mr_sg = mlx5_ib_map_mr_sg,
6236         .map_mr_sg_pi = mlx5_ib_map_mr_sg_pi,
6237         .mmap = mlx5_ib_mmap,
6238         .modify_cq = mlx5_ib_modify_cq,
6239         .modify_device = mlx5_ib_modify_device,
6240         .modify_port = mlx5_ib_modify_port,
6241         .modify_qp = mlx5_ib_modify_qp,
6242         .modify_srq = mlx5_ib_modify_srq,
6243         .poll_cq = mlx5_ib_poll_cq,
6244         .post_recv = mlx5_ib_post_recv,
6245         .post_send = mlx5_ib_post_send,
6246         .post_srq_recv = mlx5_ib_post_srq_recv,
6247         .process_mad = mlx5_ib_process_mad,
6248         .query_ah = mlx5_ib_query_ah,
6249         .query_device = mlx5_ib_query_device,
6250         .query_gid = mlx5_ib_query_gid,
6251         .query_pkey = mlx5_ib_query_pkey,
6252         .query_qp = mlx5_ib_query_qp,
6253         .query_srq = mlx5_ib_query_srq,
6254         .read_counters = mlx5_ib_read_counters,
6255         .reg_user_mr = mlx5_ib_reg_user_mr,
6256         .req_notify_cq = mlx5_ib_arm_cq,
6257         .rereg_user_mr = mlx5_ib_rereg_user_mr,
6258         .resize_cq = mlx5_ib_resize_cq,
6259
6260         INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah),
6261         INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq),
6262         INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd),
6263         INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq),
6264         INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext),
6265 };
6266
6267 static const struct ib_device_ops mlx5_ib_dev_flow_ipsec_ops = {
6268         .create_flow_action_esp = mlx5_ib_create_flow_action_esp,
6269         .modify_flow_action_esp = mlx5_ib_modify_flow_action_esp,
6270 };
6271
6272 static const struct ib_device_ops mlx5_ib_dev_ipoib_enhanced_ops = {
6273         .rdma_netdev_get_params = mlx5_ib_rn_get_params,
6274 };
6275
6276 static const struct ib_device_ops mlx5_ib_dev_sriov_ops = {
6277         .get_vf_config = mlx5_ib_get_vf_config,
6278         .get_vf_guid = mlx5_ib_get_vf_guid,
6279         .get_vf_stats = mlx5_ib_get_vf_stats,
6280         .set_vf_guid = mlx5_ib_set_vf_guid,
6281         .set_vf_link_state = mlx5_ib_set_vf_link_state,
6282 };
6283
6284 static const struct ib_device_ops mlx5_ib_dev_mw_ops = {
6285         .alloc_mw = mlx5_ib_alloc_mw,
6286         .dealloc_mw = mlx5_ib_dealloc_mw,
6287 };
6288
6289 static const struct ib_device_ops mlx5_ib_dev_xrc_ops = {
6290         .alloc_xrcd = mlx5_ib_alloc_xrcd,
6291         .dealloc_xrcd = mlx5_ib_dealloc_xrcd,
6292 };
6293
6294 static const struct ib_device_ops mlx5_ib_dev_dm_ops = {
6295         .alloc_dm = mlx5_ib_alloc_dm,
6296         .dealloc_dm = mlx5_ib_dealloc_dm,
6297         .reg_dm_mr = mlx5_ib_reg_dm_mr,
6298 };
6299
6300 static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
6301 {
6302         struct mlx5_core_dev *mdev = dev->mdev;
6303         int err;
6304
6305         dev->ib_dev.uverbs_cmd_mask     =
6306                 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT)         |
6307                 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)        |
6308                 (1ull << IB_USER_VERBS_CMD_QUERY_PORT)          |
6309                 (1ull << IB_USER_VERBS_CMD_ALLOC_PD)            |
6310                 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD)          |
6311                 (1ull << IB_USER_VERBS_CMD_CREATE_AH)           |
6312                 (1ull << IB_USER_VERBS_CMD_DESTROY_AH)          |
6313                 (1ull << IB_USER_VERBS_CMD_REG_MR)              |
6314                 (1ull << IB_USER_VERBS_CMD_REREG_MR)            |
6315                 (1ull << IB_USER_VERBS_CMD_DEREG_MR)            |
6316                 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
6317                 (1ull << IB_USER_VERBS_CMD_CREATE_CQ)           |
6318                 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ)           |
6319                 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ)          |
6320                 (1ull << IB_USER_VERBS_CMD_CREATE_QP)           |
6321                 (1ull << IB_USER_VERBS_CMD_MODIFY_QP)           |
6322                 (1ull << IB_USER_VERBS_CMD_QUERY_QP)            |
6323                 (1ull << IB_USER_VERBS_CMD_DESTROY_QP)          |
6324                 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST)        |
6325                 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST)        |
6326                 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ)          |
6327                 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ)          |
6328                 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ)           |
6329                 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ)         |
6330                 (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ)         |
6331                 (1ull << IB_USER_VERBS_CMD_OPEN_QP);
6332         dev->ib_dev.uverbs_ex_cmd_mask =
6333                 (1ull << IB_USER_VERBS_EX_CMD_QUERY_DEVICE)     |
6334                 (1ull << IB_USER_VERBS_EX_CMD_CREATE_CQ)        |
6335                 (1ull << IB_USER_VERBS_EX_CMD_CREATE_QP)        |
6336                 (1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP)        |
6337                 (1ull << IB_USER_VERBS_EX_CMD_MODIFY_CQ)        |
6338                 (1ull << IB_USER_VERBS_EX_CMD_CREATE_FLOW)      |
6339                 (1ull << IB_USER_VERBS_EX_CMD_DESTROY_FLOW);
6340
6341         if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
6342             IS_ENABLED(CONFIG_MLX5_CORE_IPOIB))
6343                 ib_set_device_ops(&dev->ib_dev,
6344                                   &mlx5_ib_dev_ipoib_enhanced_ops);
6345
6346         if (mlx5_core_is_pf(mdev))
6347                 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_sriov_ops);
6348
6349         dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));
6350
6351         if (MLX5_CAP_GEN(mdev, imaicl)) {
6352                 dev->ib_dev.uverbs_cmd_mask |=
6353                         (1ull << IB_USER_VERBS_CMD_ALLOC_MW)    |
6354                         (1ull << IB_USER_VERBS_CMD_DEALLOC_MW);
6355                 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_mw_ops);
6356         }
6357
6358         if (MLX5_CAP_GEN(mdev, xrc)) {
6359                 dev->ib_dev.uverbs_cmd_mask |=
6360                         (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) |
6361                         (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
6362                 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_xrc_ops);
6363         }
6364
6365         if (MLX5_CAP_DEV_MEM(mdev, memic) ||
6366             MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
6367             MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)
6368                 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_dm_ops);
6369
6370         if (mlx5_accel_ipsec_device_caps(dev->mdev) &
6371             MLX5_ACCEL_IPSEC_CAP_DEVICE)
6372                 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_flow_ipsec_ops);
6373         ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops);
6374
6375         if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
6376                 dev->ib_dev.driver_def = mlx5_ib_defs;
6377
6378         err = init_node_data(dev);
6379         if (err)
6380                 return err;
6381
6382         if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
6383             (MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) ||
6384              MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
6385                 mutex_init(&dev->lb.mutex);
6386
6387         dev->ib_dev.use_cq_dim = true;
6388
6389         return 0;
6390 }
6391
6392 static const struct ib_device_ops mlx5_ib_dev_port_ops = {
6393         .get_port_immutable = mlx5_port_immutable,
6394         .query_port = mlx5_ib_query_port,
6395 };
6396
6397 static int mlx5_ib_stage_non_default_cb(struct mlx5_ib_dev *dev)
6398 {
6399         ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_port_ops);
6400         return 0;
6401 }
6402
6403 static const struct ib_device_ops mlx5_ib_dev_port_rep_ops = {
6404         .get_port_immutable = mlx5_port_rep_immutable,
6405         .query_port = mlx5_ib_rep_query_port,
6406 };
6407
6408 static int mlx5_ib_stage_raw_eth_non_default_cb(struct mlx5_ib_dev *dev)
6409 {
6410         ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_port_rep_ops);
6411         return 0;
6412 }
6413
6414 static const struct ib_device_ops mlx5_ib_dev_common_roce_ops = {
6415         .create_rwq_ind_table = mlx5_ib_create_rwq_ind_table,
6416         .create_wq = mlx5_ib_create_wq,
6417         .destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table,
6418         .destroy_wq = mlx5_ib_destroy_wq,
6419         .get_netdev = mlx5_ib_get_netdev,
6420         .modify_wq = mlx5_ib_modify_wq,
6421 };
6422
6423 static int mlx5_ib_stage_common_roce_init(struct mlx5_ib_dev *dev)
6424 {
6425         u8 port_num;
6426
6427         dev->ib_dev.uverbs_ex_cmd_mask |=
6428                         (1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) |
6429                         (1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) |
6430                         (1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) |
6431                         (1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) |
6432                         (1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);
6433         ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_common_roce_ops);
6434
6435         port_num = mlx5_core_native_port_num(dev->mdev) - 1;
6436
6437         /* Register only for native ports */
6438         return mlx5_add_netdev_notifier(dev, port_num);
6439 }
6440
6441 static void mlx5_ib_stage_common_roce_cleanup(struct mlx5_ib_dev *dev)
6442 {
6443         u8 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
6444
6445         mlx5_remove_netdev_notifier(dev, port_num);
6446 }
6447
6448 static int mlx5_ib_stage_raw_eth_roce_init(struct mlx5_ib_dev *dev)
6449 {
6450         struct mlx5_core_dev *mdev = dev->mdev;
6451         enum rdma_link_layer ll;
6452         int port_type_cap;
6453         int err = 0;
6454
6455         port_type_cap = MLX5_CAP_GEN(mdev, port_type);
6456         ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
6457
6458         if (ll == IB_LINK_LAYER_ETHERNET)
6459                 err = mlx5_ib_stage_common_roce_init(dev);
6460
6461         return err;
6462 }
6463
6464 static void mlx5_ib_stage_raw_eth_roce_cleanup(struct mlx5_ib_dev *dev)
6465 {
6466         mlx5_ib_stage_common_roce_cleanup(dev);
6467 }
6468
6469 static int mlx5_ib_stage_roce_init(struct mlx5_ib_dev *dev)
6470 {
6471         struct mlx5_core_dev *mdev = dev->mdev;
6472         enum rdma_link_layer ll;
6473         int port_type_cap;
6474         int err;
6475
6476         port_type_cap = MLX5_CAP_GEN(mdev, port_type);
6477         ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
6478
6479         if (ll == IB_LINK_LAYER_ETHERNET) {
6480                 err = mlx5_ib_stage_common_roce_init(dev);
6481                 if (err)
6482                         return err;
6483
6484                 err = mlx5_enable_eth(dev);
6485                 if (err)
6486                         goto cleanup;
6487         }
6488
6489         return 0;
6490 cleanup:
6491         mlx5_ib_stage_common_roce_cleanup(dev);
6492
6493         return err;
6494 }
6495
6496 static void mlx5_ib_stage_roce_cleanup(struct mlx5_ib_dev *dev)
6497 {
6498         struct mlx5_core_dev *mdev = dev->mdev;
6499         enum rdma_link_layer ll;
6500         int port_type_cap;
6501
6502         port_type_cap = MLX5_CAP_GEN(mdev, port_type);
6503         ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
6504
6505         if (ll == IB_LINK_LAYER_ETHERNET) {
6506                 mlx5_disable_eth(dev);
6507                 mlx5_ib_stage_common_roce_cleanup(dev);
6508         }
6509 }
6510
6511 static int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev)
6512 {
6513         return create_dev_resources(&dev->devr);
6514 }
6515
6516 static void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev)
6517 {
6518         destroy_dev_resources(&dev->devr);
6519 }
6520
6521 static int mlx5_ib_stage_odp_init(struct mlx5_ib_dev *dev)
6522 {
6523         return mlx5_ib_odp_init_one(dev);
6524 }
6525
6526 static void mlx5_ib_stage_odp_cleanup(struct mlx5_ib_dev *dev)
6527 {
6528         mlx5_ib_odp_cleanup_one(dev);
6529 }
6530
6531 static const struct ib_device_ops mlx5_ib_dev_hw_stats_ops = {
6532         .alloc_hw_stats = mlx5_ib_alloc_hw_stats,
6533         .get_hw_stats = mlx5_ib_get_hw_stats,
6534         .counter_bind_qp = mlx5_ib_counter_bind_qp,
6535         .counter_unbind_qp = mlx5_ib_counter_unbind_qp,
6536         .counter_dealloc = mlx5_ib_counter_dealloc,
6537         .counter_alloc_stats = mlx5_ib_counter_alloc_stats,
6538         .counter_update_stats = mlx5_ib_counter_update_stats,
6539 };
6540
6541 static int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev)
6542 {
6543         if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) {
6544                 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_hw_stats_ops);
6545
6546                 return mlx5_ib_alloc_counters(dev);
6547         }
6548
6549         return 0;
6550 }
6551
6552 static void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev)
6553 {
6554         if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
6555                 mlx5_ib_dealloc_counters(dev);
6556 }
6557
6558 static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev)
6559 {
6560         mlx5_ib_init_cong_debugfs(dev,
6561                                   mlx5_core_native_port_num(dev->mdev) - 1);
6562         return 0;
6563 }
6564
6565 static void mlx5_ib_stage_cong_debugfs_cleanup(struct mlx5_ib_dev *dev)
6566 {
6567         mlx5_ib_cleanup_cong_debugfs(dev,
6568                                      mlx5_core_native_port_num(dev->mdev) - 1);
6569 }
6570
6571 static int mlx5_ib_stage_uar_init(struct mlx5_ib_dev *dev)
6572 {
6573         dev->mdev->priv.uar = mlx5_get_uars_page(dev->mdev);
6574         return PTR_ERR_OR_ZERO(dev->mdev->priv.uar);
6575 }
6576
6577 static void mlx5_ib_stage_uar_cleanup(struct mlx5_ib_dev *dev)
6578 {
6579         mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar);
6580 }
6581
6582 static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
6583 {
6584         int err;
6585
6586         err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
6587         if (err)
6588                 return err;
6589
6590         err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
6591         if (err)
6592                 mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
6593
6594         return err;
6595 }
6596
6597 static void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev)
6598 {
6599         mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
6600         mlx5_free_bfreg(dev->mdev, &dev->bfreg);
6601 }
6602
6603 static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
6604 {
6605         const char *name;
6606
6607         rdma_set_device_sysfs_group(&dev->ib_dev, &mlx5_attr_group);
6608         if (!mlx5_lag_is_roce(dev->mdev))
6609                 name = "mlx5_%d";
6610         else
6611                 name = "mlx5_bond_%d";
6612         return ib_register_device(&dev->ib_dev, name);
6613 }
6614
6615 static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
6616 {
6617         destroy_umrc_res(dev);
6618 }
6619
6620 static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
6621 {
6622         ib_unregister_device(&dev->ib_dev);
6623 }
6624
6625 static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)
6626 {
6627         return create_umr_res(dev);
6628 }
6629
6630 static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev)
6631 {
6632         init_delay_drop(dev);
6633
6634         return 0;
6635 }
6636
6637 static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev)
6638 {
6639         cancel_delay_drop(dev);
6640 }
6641
6642 static int mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev *dev)
6643 {
6644         dev->mdev_events.notifier_call = mlx5_ib_event;
6645         mlx5_notifier_register(dev->mdev, &dev->mdev_events);
6646         return 0;
6647 }
6648
6649 static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev)
6650 {
6651         mlx5_notifier_unregister(dev->mdev, &dev->mdev_events);
6652 }
6653
6654 static int mlx5_ib_stage_devx_init(struct mlx5_ib_dev *dev)
6655 {
6656         int uid;
6657
6658         uid = mlx5_ib_devx_create(dev, false);
6659         if (uid > 0) {
6660                 dev->devx_whitelist_uid = uid;
6661                 mlx5_ib_devx_init_event_table(dev);
6662         }
6663
6664         return 0;
6665 }
6666 static void mlx5_ib_stage_devx_cleanup(struct mlx5_ib_dev *dev)
6667 {
6668         if (dev->devx_whitelist_uid) {
6669                 mlx5_ib_devx_cleanup_event_table(dev);
6670                 mlx5_ib_devx_destroy(dev, dev->devx_whitelist_uid);
6671         }
6672 }
6673
6674 int mlx5_ib_enable_driver(struct ib_device *dev)
6675 {
6676         struct mlx5_ib_dev *mdev = to_mdev(dev);
6677         int ret;
6678
6679         ret = mlx5_ib_test_wc(mdev);
6680         mlx5_ib_dbg(mdev, "Write-Combining %s",
6681                     mdev->wc_support ? "supported" : "not supported");
6682
6683         return ret;
6684 }
6685
6686 void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
6687                       const struct mlx5_ib_profile *profile,
6688                       int stage)
6689 {
6690         /* Number of stages to cleanup */
6691         while (stage) {
6692                 stage--;
6693                 if (profile->stage[stage].cleanup)
6694                         profile->stage[stage].cleanup(dev);
6695         }
6696
6697         kfree(dev->port);
6698         ib_dealloc_device(&dev->ib_dev);
6699 }
6700
6701 void *__mlx5_ib_add(struct mlx5_ib_dev *dev,
6702                     const struct mlx5_ib_profile *profile)
6703 {
6704         int err;
6705         int i;
6706
6707         for (i = 0; i < MLX5_IB_STAGE_MAX; i++) {
6708                 if (profile->stage[i].init) {
6709                         err = profile->stage[i].init(dev);
6710                         if (err)
6711                                 goto err_out;
6712                 }
6713         }
6714
6715         dev->profile = profile;
6716         dev->ib_active = true;
6717
6718         return dev;
6719
6720 err_out:
6721         __mlx5_ib_remove(dev, profile, i);
6722
6723         return NULL;
6724 }
6725
6726 static const struct mlx5_ib_profile pf_profile = {
6727         STAGE_CREATE(MLX5_IB_STAGE_INIT,
6728                      mlx5_ib_stage_init_init,
6729                      mlx5_ib_stage_init_cleanup),
6730         STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB,
6731                      mlx5_ib_stage_flow_db_init,
6732                      mlx5_ib_stage_flow_db_cleanup),
6733         STAGE_CREATE(MLX5_IB_STAGE_CAPS,
6734                      mlx5_ib_stage_caps_init,
6735                      NULL),
6736         STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
6737                      mlx5_ib_stage_non_default_cb,
6738                      NULL),
6739         STAGE_CREATE(MLX5_IB_STAGE_ROCE,
6740                      mlx5_ib_stage_roce_init,
6741                      mlx5_ib_stage_roce_cleanup),
6742         STAGE_CREATE(MLX5_IB_STAGE_SRQ,
6743                      mlx5_init_srq_table,
6744                      mlx5_cleanup_srq_table),
6745         STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
6746                      mlx5_ib_stage_dev_res_init,
6747                      mlx5_ib_stage_dev_res_cleanup),
6748         STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
6749                      mlx5_ib_stage_dev_notifier_init,
6750                      mlx5_ib_stage_dev_notifier_cleanup),
6751         STAGE_CREATE(MLX5_IB_STAGE_ODP,
6752                      mlx5_ib_stage_odp_init,
6753                      mlx5_ib_stage_odp_cleanup),
6754         STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
6755                      mlx5_ib_stage_counters_init,
6756                      mlx5_ib_stage_counters_cleanup),
6757         STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
6758                      mlx5_ib_stage_cong_debugfs_init,
6759                      mlx5_ib_stage_cong_debugfs_cleanup),
6760         STAGE_CREATE(MLX5_IB_STAGE_UAR,
6761                      mlx5_ib_stage_uar_init,
6762                      mlx5_ib_stage_uar_cleanup),
6763         STAGE_CREATE(MLX5_IB_STAGE_BFREG,
6764                      mlx5_ib_stage_bfrag_init,
6765                      mlx5_ib_stage_bfrag_cleanup),
6766         STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
6767                      NULL,
6768                      mlx5_ib_stage_pre_ib_reg_umr_cleanup),
6769         STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
6770                      mlx5_ib_stage_devx_init,
6771                      mlx5_ib_stage_devx_cleanup),
6772         STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
6773                      mlx5_ib_stage_ib_reg_init,
6774                      mlx5_ib_stage_ib_reg_cleanup),
6775         STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
6776                      mlx5_ib_stage_post_ib_reg_umr_init,
6777                      NULL),
6778         STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
6779                      mlx5_ib_stage_delay_drop_init,
6780                      mlx5_ib_stage_delay_drop_cleanup),
6781 };
6782
6783 const struct mlx5_ib_profile raw_eth_profile = {
6784         STAGE_CREATE(MLX5_IB_STAGE_INIT,
6785                      mlx5_ib_stage_init_init,
6786                      mlx5_ib_stage_init_cleanup),
6787         STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB,
6788                      mlx5_ib_stage_flow_db_init,
6789                      mlx5_ib_stage_flow_db_cleanup),
6790         STAGE_CREATE(MLX5_IB_STAGE_CAPS,
6791                      mlx5_ib_stage_caps_init,
6792                      NULL),
6793         STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
6794                      mlx5_ib_stage_raw_eth_non_default_cb,
6795                      NULL),
6796         STAGE_CREATE(MLX5_IB_STAGE_ROCE,
6797                      mlx5_ib_stage_raw_eth_roce_init,
6798                      mlx5_ib_stage_raw_eth_roce_cleanup),
6799         STAGE_CREATE(MLX5_IB_STAGE_SRQ,
6800                      mlx5_init_srq_table,
6801                      mlx5_cleanup_srq_table),
6802         STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
6803                      mlx5_ib_stage_dev_res_init,
6804                      mlx5_ib_stage_dev_res_cleanup),
6805         STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
6806                      mlx5_ib_stage_dev_notifier_init,
6807                      mlx5_ib_stage_dev_notifier_cleanup),
6808         STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
6809                      mlx5_ib_stage_counters_init,
6810                      mlx5_ib_stage_counters_cleanup),
6811         STAGE_CREATE(MLX5_IB_STAGE_UAR,
6812                      mlx5_ib_stage_uar_init,
6813                      mlx5_ib_stage_uar_cleanup),
6814         STAGE_CREATE(MLX5_IB_STAGE_BFREG,
6815                      mlx5_ib_stage_bfrag_init,
6816                      mlx5_ib_stage_bfrag_cleanup),
6817         STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
6818                      NULL,
6819                      mlx5_ib_stage_pre_ib_reg_umr_cleanup),
6820         STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
6821                      mlx5_ib_stage_devx_init,
6822                      mlx5_ib_stage_devx_cleanup),
6823         STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
6824                      mlx5_ib_stage_ib_reg_init,
6825                      mlx5_ib_stage_ib_reg_cleanup),
6826         STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
6827                      mlx5_ib_stage_post_ib_reg_umr_init,
6828                      NULL),
6829 };
6830
6831 static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev)
6832 {
6833         struct mlx5_ib_multiport_info *mpi;
6834         struct mlx5_ib_dev *dev;
6835         bool bound = false;
6836         int err;
6837
6838         mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
6839         if (!mpi)
6840                 return NULL;
6841
6842         mpi->mdev = mdev;
6843
6844         err = mlx5_query_nic_vport_system_image_guid(mdev,
6845                                                      &mpi->sys_image_guid);
6846         if (err) {
6847                 kfree(mpi);
6848                 return NULL;
6849         }
6850
6851         mutex_lock(&mlx5_ib_multiport_mutex);
6852         list_for_each_entry(dev, &mlx5_ib_dev_list, ib_dev_list) {
6853                 if (dev->sys_image_guid == mpi->sys_image_guid)
6854                         bound = mlx5_ib_bind_slave_port(dev, mpi);
6855
6856                 if (bound) {
6857                         rdma_roce_rescan_device(&dev->ib_dev);
6858                         break;
6859                 }
6860         }
6861
6862         if (!bound) {
6863                 list_add_tail(&mpi->list, &mlx5_ib_unaffiliated_port_list);
6864                 dev_dbg(mdev->device,
6865                         "no suitable IB device found to bind to, added to unaffiliated list.\n");
6866         }
6867         mutex_unlock(&mlx5_ib_multiport_mutex);
6868
6869         return mpi;
6870 }
6871
6872 static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
6873 {
6874         const struct mlx5_ib_profile *profile;
6875         enum rdma_link_layer ll;
6876         struct mlx5_ib_dev *dev;
6877         int port_type_cap;
6878         int num_ports;
6879
6880         printk_once(KERN_INFO "%s", mlx5_version);
6881
6882         if (MLX5_ESWITCH_MANAGER(mdev) &&
6883             mlx5_ib_eswitch_mode(mdev->priv.eswitch) == MLX5_ESWITCH_OFFLOADS) {
6884                 if (!mlx5_core_mp_enabled(mdev))
6885                         mlx5_ib_register_vport_reps(mdev);
6886                 return mdev;
6887         }
6888
6889         port_type_cap = MLX5_CAP_GEN(mdev, port_type);
6890         ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
6891
6892         if (mlx5_core_is_mp_slave(mdev) && ll == IB_LINK_LAYER_ETHERNET)
6893                 return mlx5_ib_add_slave_port(mdev);
6894
6895         num_ports = max(MLX5_CAP_GEN(mdev, num_ports),
6896                         MLX5_CAP_GEN(mdev, num_vhca_ports));
6897         dev = ib_alloc_device(mlx5_ib_dev, ib_dev);
6898         if (!dev)
6899                 return NULL;
6900         dev->port = kcalloc(num_ports, sizeof(*dev->port),
6901                              GFP_KERNEL);
6902         if (!dev->port) {
6903                 ib_dealloc_device(&dev->ib_dev);
6904                 return NULL;
6905         }
6906
6907         dev->mdev = mdev;
6908         dev->num_ports = num_ports;
6909
6910         if (ll == IB_LINK_LAYER_ETHERNET && !mlx5_is_roce_enabled(mdev))
6911                 profile = &raw_eth_profile;
6912         else
6913                 profile = &pf_profile;
6914
6915         return __mlx5_ib_add(dev, profile);
6916 }
6917
6918 static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
6919 {
6920         struct mlx5_ib_multiport_info *mpi;
6921         struct mlx5_ib_dev *dev;
6922
6923         if (MLX5_ESWITCH_MANAGER(mdev) && context == mdev) {
6924                 mlx5_ib_unregister_vport_reps(mdev);
6925                 return;
6926         }
6927
6928         if (mlx5_core_is_mp_slave(mdev)) {
6929                 mpi = context;
6930                 mutex_lock(&mlx5_ib_multiport_mutex);
6931                 if (mpi->ibdev)
6932                         mlx5_ib_unbind_slave_port(mpi->ibdev, mpi);
6933                 list_del(&mpi->list);
6934                 mutex_unlock(&mlx5_ib_multiport_mutex);
6935                 kfree(mpi);
6936                 return;
6937         }
6938
6939         dev = context;
6940         __mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX);
6941 }
6942
6943 static struct mlx5_interface mlx5_ib_interface = {
6944         .add            = mlx5_ib_add,
6945         .remove         = mlx5_ib_remove,
6946         .protocol       = MLX5_INTERFACE_PROTOCOL_IB,
6947 };
6948
6949 unsigned long mlx5_ib_get_xlt_emergency_page(void)
6950 {
6951         mutex_lock(&xlt_emergency_page_mutex);
6952         return xlt_emergency_page;
6953 }
6954
6955 void mlx5_ib_put_xlt_emergency_page(void)
6956 {
6957         mutex_unlock(&xlt_emergency_page_mutex);
6958 }
6959
6960 static int __init mlx5_ib_init(void)
6961 {
6962         int err;
6963
6964         xlt_emergency_page = __get_free_page(GFP_KERNEL);
6965         if (!xlt_emergency_page)
6966                 return -ENOMEM;
6967
6968         mutex_init(&xlt_emergency_page_mutex);
6969
6970         mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0);
6971         if (!mlx5_ib_event_wq) {
6972                 free_page(xlt_emergency_page);
6973                 return -ENOMEM;
6974         }
6975
6976         mlx5_ib_odp_init();
6977
6978         err = mlx5_register_interface(&mlx5_ib_interface);
6979
6980         return err;
6981 }
6982
6983 static void __exit mlx5_ib_cleanup(void)
6984 {
6985         mlx5_unregister_interface(&mlx5_ib_interface);
6986         destroy_workqueue(mlx5_ib_event_wq);
6987         mutex_destroy(&xlt_emergency_page_mutex);
6988         free_page(xlt_emergency_page);
6989 }
6990
6991 module_init(mlx5_ib_init);
6992 module_exit(mlx5_ib_cleanup);